import { spawnSync } from 'node:child_process';
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import fc from 'fast-check';
import { afterEach, describe, expect, it, vi } from 'vitest';
import type { EmailAddress } from '../src/content/types';
import { siteUrl } from '../src/seo';
import sitePages, { pageList } from '../src/sitePages';
import { sitemapXml } from '../src/sitemap';
import { validEmail } from '../src/test/arbitraries';
import {
htmlWithModel,
outputScenario,
type OutputScenario,
} from '../src/test/htmlArbitraries';
import {
containsAddress,
externalizeOutput,
formatViolation,
planExternalization,
type PlanContext,
type ScriptStep,
} from './externalizeInlineScripts';
import { pageFile } from './pageFiles';
const SITEMAP = 'sitemap\n';
const ROOT_PATHS = ['/work/repo', 'C:\\work\\repo'];
const ADDRESS = 'info@example.nl';
const context = (override: Partial = {}): PlanContext => ({
expected: {
pages: new Map([
['/', 'index.html'],
['/over-mij', 'over-mij.html'],
['/soorten-fotoshoots/newborn', 'soorten-fotoshoots/newborn.html'],
]),
notFound: '404.html',
},
address: ADDRESS,
sitemap: SITEMAP,
rootPaths: ROOT_PATHS,
...override,
});
const page = (script = 'boot()') =>
`tekst
`;
const validFiles = (): Map =>
new Map([
['index.html', page('boot("index")')],
['over-mij.html', page('boot("over-mij")')],
['soorten-fotoshoots/newborn.html', page('boot("newborn")')],
['404.html', page('boot("404")')],
['sitemap.xml', SITEMAP],
['assets/index-abc.js', 'console.log(1);'],
['assets/index-abc.css', 'a{color:red}'],
]);
const found = (plan: { violations: { file: string; code: string }[] }) =>
plan.violations.map((v) => `${v.file}: ${v.code}`).sort();
describe('planExternalization: pad zonder overtredingen', () => {
it('verplaatst de scripts van elke pagina, ook van 404.html', () => {
const plan = planExternalization(validFiles(), context());
expect(plan.violations).toEqual([]);
const scripts = [...plan.writes.keys()].filter((f) =>
f.startsWith('assets/inline-'),
);
expect(scripts).toHaveLength(4);
for (const file of [
'index.html',
'over-mij.html',
'soorten-fotoshoots/newborn.html',
'404.html',
]) {
expect(plan.writes.get(file)).toMatch(
/src="\/assets\/inline-[0-9a-f]{12}\.js"/,
);
}
});
it('is een no-op bij een tweede keer (al verwerkte pagina telt mee)', () => {
const files = validFiles();
const first = planExternalization(files, context());
for (const [file, content] of first.writes) files.set(file, content);
const second = planExternalization(files, context());
expect(second.writes.size).toBe(0);
expect(second.violations).toEqual([]);
});
it('schrijft een scriptbestand dat al met dezelfde inhoud bestaat niet opnieuw', () => {
const files = validFiles();
const first = planExternalization(files, context());
const name = [...first.writes.keys()].find((f) =>
f.startsWith('assets/inline-'),
)!;
files.set(name, first.writes.get(name)!);
const plan = planExternalization(files, context());
expect(plan.writes.has(name)).toBe(false);
expect(plan.violations).toEqual([]);
});
it('schrijft een gedeeld script één keer', () => {
const files = validFiles();
for (const file of ['index.html', 'over-mij.html']) {
files.set(file, page('gedeeld()'));
}
const plan = planExternalization(files, context());
const shared = [...plan.writes.values()].filter(
(c) => c === 'gedeeld()',
);
expect(shared).toHaveLength(1);
});
});
describe('planExternalization: overtredingen', () => {
it('NO_HTML: geen enkel html-bestand, zonder de pagina’s afzonderlijk te melden', () => {
const files = new Map([['sitemap.xml', SITEMAP]]);
expect(found(planExternalization(files, context()))).toEqual([
'.: NO_HTML',
]);
});
it('PAGE_MISSING: meldt alle ontbrekende pagina’s', () => {
const files = validFiles();
files.delete('over-mij.html');
files.delete('soorten-fotoshoots/newborn.html');
const plan = planExternalization(files, context());
expect(found(plan)).toEqual([
'over-mij.html: PAGE_MISSING',
'soorten-fotoshoots/newborn.html: PAGE_MISSING',
]);
expect(plan.violations[0].detail).toContain('/over-mij');
});
it('NOT_FOUND_PAGE_MISSING: 404.html ontbreekt', () => {
const files = validFiles();
files.delete('404.html');
expect(found(planExternalization(files, context()))).toEqual([
'404.html: NOT_FOUND_PAGE_MISSING',
]);
});
it('UNEXPECTED_HTML: een html-bestand buiten het register', () => {
const files = validFiles();
files.set('soorten-fotoshoots/index.html', page());
files.set('extra.html', page());
expect(found(planExternalization(files, context()))).toEqual([
'extra.html: UNEXPECTED_HTML',
'soorten-fotoshoots/index.html: UNEXPECTED_HTML',
]);
});
it('NO_SCRIPTS_MOVED: een pagina zonder uitvoerbaar of verplaatst script', () => {
const files = validFiles();
files.set(
'index.html',
'geen script
',
);
expect(found(planExternalization(files, context()))).toEqual([
'index.html: NO_SCRIPTS_MOVED',
]);
});
it('EXECUTABLE_REMAINS: de verdediging als het herschrijven iets overlaat', () => {
const scripts: ScriptStep = {
externalize: (html) => ({ html, moved: [] }),
find: () => ['geheim()'],
};
const plan = planExternalization(validFiles(), context(), scripts);
expect(found(plan)).toContain('index.html: EXECUTABLE_REMAINS');
expect(plan.violations.map(formatViolation).join('\n')).not.toContain(
'geheim',
);
});
it('UNSUPPORTED_SCRIPT en UNTERMINATED_SCRIPT: meldt de code en gooit niet', () => {
const files = validFiles();
files.set('index.html', '');
files.set('over-mij.html', '`,
moved: [{ fileName: 'inline-000000000000.js', body: html }],
}),
find: () => [],
};
const plan = planExternalization(validFiles(), context(), scripts);
expect(found(plan)).toContain(
'assets/inline-000000000000.js: COLLISION',
);
});
it('SITEMAP_MISSING en SITEMAP_DIFFERS', () => {
const missing = validFiles();
missing.delete('sitemap.xml');
expect(found(planExternalization(missing, context()))).toEqual([
'sitemap.xml: SITEMAP_MISSING',
]);
const differs = validFiles();
differs.set('sitemap.xml', SITEMAP + ' ');
expect(found(planExternalization(differs, context()))).toEqual([
'sitemap.xml: SITEMAP_DIFFERS',
]);
});
it('SOURCE_MAP_PRESENT: een .map-bestand', () => {
const files = validFiles();
files.set('assets/index-abc.js.map', '{}');
expect(found(planExternalization(files, context()))).toEqual([
'assets/index-abc.js.map: SOURCE_MAP_PRESENT',
]);
});
it.each(ROOT_PATHS)('BUILD_PATH_LEAK: %s in een tekstbestand', (root) => {
const files = validFiles();
files.set('assets/index-abc.js', `var p = "${root}/src/x.ts";`);
expect(found(planExternalization(files, context()))).toEqual([
'assets/index-abc.js: BUILD_PATH_LEAK',
]);
});
it('EMAIL_IN_OUTPUT: in html, js, css en in de inhoud van een verplaatst script', () => {
const files = validFiles();
files.set('assets/index-abc.js', `var a = "${ADDRESS}";`);
files.set('assets/index-abc.css', `a::after{content:"${ADDRESS}"}`);
files.set('over-mij.html', `${ADDRESS}
${page('boot("o")')}`);
files.set('index.html', page(`mail("${ADDRESS}")`));
const plan = planExternalization(files, context());
const inline = [...plan.writes.entries()].find(([, c]) =>
c.includes(ADDRESS),
)![0];
expect(found(plan)).toEqual(
[
'assets/index-abc.js: EMAIL_IN_OUTPUT',
'assets/index-abc.css: EMAIL_IN_OUTPUT',
'over-mij.html: EMAIL_IN_OUTPUT',
`${inline}: EMAIL_IN_OUTPUT`,
].sort(),
);
});
it('slaat de adrescontrole over zonder adres (ongeldige delen)', () => {
const files = validFiles();
files.set('assets/index-abc.js', `var a = "${ADDRESS}";`);
const plan = planExternalization(files, context({ address: null }));
expect(plan.violations).toEqual([]);
});
it('meldt alle overtredingen tegelijk, in een vast formaat zonder adres', () => {
const files = validFiles();
files.delete('404.html');
files.set('assets/index-abc.js', `"${ADDRESS}"`);
files.set('x.html', page());
const lines = planExternalization(files, context()).violations.map(
formatViolation,
);
expect(lines).toEqual(
expect.arrayContaining([
'404.html: NOT_FOUND_PAGE_MISSING: de 404-pagina ontbreekt',
'x.html: UNEXPECTED_HTML: html-bestand dat niet bij het register hoort',
'assets/index-abc.js: EMAIL_IN_OUTPUT: e-mailadres in de uitvoer',
]),
);
expect(lines.join('\n')).not.toContain(ADDRESS);
});
});
describe('containsAddress', () => {
const address = 'info@andreaslop.nl';
it.each([
['platte tekst', 'info@andreaslop.nl'],
['hoofdletters', 'INFO@AndreaSlop.NL'],
['URL-codering', 'mailto:info%40andreaslop.nl'],
['JavaScript-escape', 'info\\u0040andreaslop.nl'],
['JavaScript-escape met accolades', 'info\\u{40}andreaslop.nl'],
['hexadecimale escape', 'info\\x40andreaslop.nl'],
['decimale entiteit', 'info@andreaslop.nl'],
['hexadecimale entiteit', 'info@andreaslop.nl'],
['hexadecimale entiteit in hoofdletters', 'info@andreaslop.nl'],
['benoemde entiteit', 'info@andreaslop.nl'],
['midden in tekst', 'Mail info@andreaslop.nl!'],
])('vindt het adres als %s', (_, text) => {
expect(containsAddress(text, address)).toBe(true);
});
it.each([
['alleen de gebruiker', 'info'],
['alleen het domein', 'andreaslop.nl'],
['met een spatie ertussen', 'info @ andreaslop.nl'],
['een ander domein', 'info@andreaslop.com'],
['een andere gebruiker', 'post@andreaslop.nl'],
['een onbekende codering', 'infoAandreaslop.nl'],
['lege tekst', ''],
])('vindt het adres niet bij %s', (_, text) => {
expect(containsAddress(text, address)).toBe(false);
});
it('vindt niets bij een adres zonder @', () => {
expect(containsAddress('infoandreaslop.nl', 'infoandreaslop.nl')).toBe(
false,
);
});
});
describe('containsAddress: eigenschappen', () => {
// Onafhankelijke lijst van BR-MAIL-10
const encodings = [
'@',
'%40',
'\\u0040',
'\\u{40}',
'\\x40',
'@',
'@',
'@',
];
// Geen @, %, & of \ : een schrijfwijze kan er dus nooit in ontstaan
const safeText = fc.stringMatching(/^[a-z0-9 .<>/="'-]{0,30}$/);
const randomCase = (text: string, flips: boolean[]) =>
[...text]
.map((char, i) =>
flips[i % flips.length] ? char.toUpperCase() : char,
)
.join('');
it('vindt het adres in elke schrijfwijze op elke plaats, in elke hoofdletterstand (oracle)', () => {
fc.assert(
fc.property(
validEmail,
fc.constantFrom(...encodings),
safeText,
safeText,
fc.array(fc.boolean(), { minLength: 1, maxLength: 8 }),
(email, encoding, before, after, flips) => {
const written = randomCase(
`${email.user}${encoding}${email.domain}`,
flips,
);
expect(
containsAddress(
`${before}${written}${after}`,
`${email.user}@${email.domain}`,
),
).toBe(true);
},
),
);
});
it('markeert tekst zonder het adres nooit (oracle)', () => {
fc.assert(
fc.property(
validEmail,
safeText,
fc.constantFrom(...encodings),
(email, text, encoding) => {
const address = `${email.user}@${email.domain}`;
expect(containsAddress(text, address)).toBe(false);
// Een deel van het adres is niet genoeg
expect(
containsAddress(
`${text}${email.user}${encoding}`,
address,
),
).toBe(false);
expect(
containsAddress(
`${encoding}${email.domain}${text}`,
address,
),
).toBe(false);
},
),
);
});
});
// De drie pagina's van `context()` plus 404.html, in de volgorde van het scenario
const SCENARIO_FILES = [
'index.html',
'over-mij.html',
'soorten-fotoshoots/newborn.html',
'404.html',
];
const EXTRA_REFERENCE =
'';
function filesOf(scenario: OutputScenario, address: string) {
const files = new Map();
scenario.pages.forEach((p, i) => {
if (!p.present) return;
files.set(
SCENARIO_FILES[i],
p.content.html +
(p.reference ? EXTRA_REFERENCE : '') +
(p.leakAddress ? `${address}
` : '') +
(p.bad === 'unsupported'
? ''
: '') +
(p.bad === 'unterminated' ? '`,
);
});
const first = planExternalization(files, context());
expect(first.violations).toEqual([]);
for (const [file, content] of first.writes) {
files.set(file, content);
}
const second = planExternalization(files, context());
expect(second.writes.size).toBe(0);
expect(second.violations).toEqual([]);
}),
);
});
});
// ---------------------------------------------------------------------------
// externalizeOutput met een echte map
const temporary: string[] = [];
afterEach(() => {
for (const dir of temporary.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
});
// /web/dist/client: de repositorywortel voor de lekcontrole is /web
function outputDir() {
const root = mkdtempSync(join(tmpdir(), 'externalize-'));
temporary.push(root);
const clientDir = join(root, 'web', 'dist', 'client');
const put = (file: string, content: string) => {
mkdirSync(dirname(join(clientDir, file)), { recursive: true });
writeFileSync(join(clientDir, file), content, 'utf8');
};
for (const path of Object.keys(sitePages)) {
put(pageFile(path), page(`boot("${path}")`));
}
put('404.html', page('boot("404")'));
put('sitemap.xml', sitemapXml(pageList(sitePages), siteUrl));
put('assets/index-abc.js', 'console.log(1);');
return { root: join(root, 'web'), clientDir, put };
}
const listing = (dir: string) =>
readdirSync(dir, { recursive: true, encoding: 'utf8' }).sort();
const silent = () => ({ log: vi.fn(), error: vi.fn() });
describe('externalizeOutput', () => {
it('schrijft eerst de scriptbestanden, dan de html, en meldt één regel', () => {
const { clientDir } = outputDir();
const out = silent();
const result = externalizeOutput(clientDir, out);
expect(result.exitCode).toBe(0);
expect(result.violations).toEqual([]);
const firstHtml = result.written.findIndex((f) => f.endsWith('.html'));
expect(firstHtml).toBeGreaterThan(0);
expect(
result.written
.slice(0, firstHtml)
.every((f) => !f.endsWith('.html')),
).toBe(true);
expect(
result.written.slice(firstHtml).every((f) => f.endsWith('.html')),
).toBe(true);
const index = readFileSync(join(clientDir, 'index.html'), 'utf8');
const reference = /src="\/(assets\/inline-[0-9a-f]{12}\.js)"/.exec(
index,
)!;
expect(readFileSync(join(clientDir, reference[1]), 'utf8')).toBe(
'boot("/")',
);
expect(index).not.toContain('boot(');
expect(out.error).not.toHaveBeenCalled();
expect(out.log).toHaveBeenCalledTimes(1);
expect(out.log.mock.calls[0][0]).toContain(
`${Object.keys(sitePages).length + 1} pagina's gecontroleerd`,
);
expect(out.log.mock.calls[0][0]).toContain(
`${Object.keys(sitePages).length + 1} scripts verplaatst`,
);
});
it('doet bij een tweede keer niets', () => {
const { clientDir } = outputDir();
externalizeOutput(clientDir, silent());
const before = listing(clientDir);
const out = silent();
const result = externalizeOutput(clientDir, out);
expect(result).toEqual({ exitCode: 0, violations: [], written: [] });
expect(listing(clientDir)).toEqual(before);
expect(out.log.mock.calls[0][0]).toContain('0 scripts verplaatst');
});
it('schrijft niets en meldt alles bij een overtreding', () => {
const { clientDir, put } = outputDir();
rmSync(join(clientDir, 'over-mij.html'));
put('assets/index-abc.js.map', '{}');
const before = listing(clientDir);
const out = silent();
const result = externalizeOutput(clientDir, out);
expect(result.exitCode).toBe(1);
expect(result.written).toEqual([]);
expect(listing(clientDir)).toEqual(before);
expect(out.log).not.toHaveBeenCalled();
expect(out.error.mock.calls.map(([line]) => line)).toEqual(
result.violations.map(formatViolation),
);
expect(result.violations.map((v) => v.code).sort()).toEqual([
'PAGE_MISSING',
'SOURCE_MAP_PRESENT',
]);
});
it('vindt het e-mailadres van de site in een uitvoerbestand', () => {
const { clientDir, put } = outputDir();
put('assets/index-abc.js', 'var a = "info@andreaslop.nl";');
const result = externalizeOutput(clientDir, silent());
expect(result.violations.map(formatViolation)).toEqual([
'assets/index-abc.js: EMAIL_IN_OUTPUT: e-mailadres in de uitvoer',
]);
});
it.each([
[
'met voorwaartse slashes',
(root: string) => root.replaceAll('\\', '/'),
],
['met backslashes', (root: string) => root.replaceAll('/', '\\')],
[
'met verdubbelde backslashes',
(root: string) =>
root.replaceAll('/', '\\').replaceAll('\\', '\\\\'),
],
])('vindt het pad van de repository %s', (_, style) => {
const { clientDir, put, root } = outputDir();
put('assets/index-abc.js', `var p = "${style(root)}";`);
const result = externalizeOutput(clientDir, silent());
expect(result.violations.map((v) => v.code)).toEqual([
'BUILD_PATH_LEAK',
]);
});
it('leest alleen tekstbestanden en gaat niet af op een bestandssysteemwortel', () => {
// clientDir direct onder de wortel: '../..' is de wortel zelf en mag
// niet elk bestand met een slash als lek aanmerken
const { clientDir, put } = outputDir();
put('logo.png', 'binair');
expect(externalizeOutput(clientDir, silent()).exitCode).toBe(0);
const rootResult = externalizeOutput(join(clientDir, '..'), silent());
expect(
rootResult.violations.every((v) => v.code !== 'BUILD_PATH_LEAK'),
).toBe(true);
});
});
describe('externalizeOutput als opdrachtregel', () => {
const script = join(
dirname(fileURLToPath(import.meta.url)),
'externalizeInlineScripts.ts',
);
const run = (dir: string) =>
spawnSync(process.execPath, [script, dir], { encoding: 'utf8' });
it('geeft exitcode 0 en één regel bij een geldige uitvoer', () => {
const { clientDir } = outputDir();
const result = run(clientDir);
expect(result.status).toBe(0);
expect(result.stdout.trim().split('\n')).toHaveLength(1);
expect(existsSync(join(clientDir, 'assets'))).toBe(true);
});
it('geeft exitcode 1 en schrijft niets bij een overtreding', () => {
const { clientDir } = outputDir();
rmSync(join(clientDir, '404.html'));
const before = listing(clientDir);
const result = run(clientDir);
expect(result.status).toBe(1);
expect(result.stderr).toContain('404.html: NOT_FOUND_PAGE_MISSING');
expect(listing(clientDir)).toEqual(before);
});
});