import test from 'node:test'; import assert from 'node:assert/strict'; import { readdirSync, readFileSync, statSync } from 'node:fs'; import { join } from 'node:path'; const TEXT_ROOTS = ['locales']; const TEXT_FILES = []; const EXTENSIONS = new Set(['.json', '.html', '.mjs']); const MOJIBAKE_PATTERNS = [ { name: 'latin1-accent', pattern: /\u00c3./g }, { name: 'stray-cp1252-prefix', pattern: /\u00c2./g }, { name: 'emoji-mojibake', pattern: /\u00f0\u0178/g }, { name: 'punctuation-mojibake', pattern: /\u00e2[\u0080-\u009f\u20ac\u0153\u2018\u2019\u201c\u201d\u2013\u2014\u2022\u2026\u201e\u2021\u02c6\u2030\u2039\u203a\u0152\u017d]/g, }, { name: 'variation-selector-mojibake', pattern: /\u00ef\u00b8/g }, { name: 'ligature-mojibake', pattern: /\u00c5[\u0080-\u017f]/g }, { name: 'replacement-character', pattern: /\ufffd/g }, ]; function collectFiles(root) { const out = []; for (const entry of readdirSync(root, { withFileTypes: true })) { const path = join(root, entry.name); if (entry.isDirectory()) { out.push(...collectFiles(path)); } else if (EXTENSIONS.has(path.slice(path.lastIndexOf('.')))) { out.push(path); } } return out; } function textFiles() { const discovered = TEXT_ROOTS.flatMap(root => collectFiles(root)); const explicit = TEXT_FILES.filter(path => statSync(path, { throwIfNoEntry: false })?.isFile()); return [...new Set([...discovered, ...explicit])].sort(); } test('locale JSON files are valid UTF-8 JSON', () => { for (const file of collectFiles('locales')) { assert.doesNotThrow(() => JSON.parse(readFileSync(file, 'utf8')), `${file} must parse as JSON`); } }); test('locale text does not contain known mojibake sequences', () => { const failures = []; for (const file of textFiles()) { const text = readFileSync(file, 'utf8'); for (const { name, pattern } of MOJIBAKE_PATTERNS) { for (const match of text.matchAll(pattern)) { const start = Math.max(0, match.index - 30); const end = Math.min(text.length, match.index + 50); failures.push(`${file}: ${name}: ${JSON.stringify(text.slice(start, end))}`); } } } assert.deepEqual(failures, []); });