diff --git a/aidlc-docs/features/prerendering/construction/start-migration/code/verification-log.md b/aidlc-docs/features/prerendering/construction/start-migration/code/verification-log.md index 061eaf6..31019c9 100644 --- a/aidlc-docs/features/prerendering/construction/start-migration/code/verification-log.md +++ b/aidlc-docs/features/prerendering/construction/start-migration/code/verification-log.md @@ -27,8 +27,8 @@ Notes: |---|---|---|---|---|---| | V1 | Custom `src/client.tsx` hydrates with Start; `startSentry()` before hydration | 4 | Start picks `src/client.tsx` by name (default entry `client` in `srcDirectory`); it replaces the default entry, whose body is `startTransition(() => hydrateRoot(document, ))`. `startSentry()` runs at module level before that call. | Build output `dist/client/assets/index-*.js` contains `hydrateRoot` and the Sentry loader (`sentryClient-*.js` dynamic import). Chrome (playwright-core, system Chrome, static server, no CSP yet): `/`, `/over-mij`, `/contact`, `/contact?pakket=gezin` hydrate with no console error or warning. Not proven here: that Sentry initialises (only on the test and production hostnames, so a no-op on 127.0.0.1); unit tests of `monitoring/` cover that. | OK | | V2 | `RouterProvider` in jsdom renders `shellComponent` | 5 | Yes, Router calls the root `shellComponent` (`Match.js`: `ShellComponent = route.isRoot && route.options.shellComponent`). React 19 treats `html`/`head`/`body` as singletons, so no `` lands in the RTL container (container has 0 html/head/body), but it sets `lang="nl"` on `document.documentElement` and wraps `ChakraProvider` a second time. Variant used: `renderApp` replaces the root route's `shellComponent` by a test shell (`HeadContent` + children) on its router instance; production code untouched. | Probe test (removed) printed `shellComponent: function`, `html=0 head=0 body=0`, `document.documentElement lang=nl`. All 888 tests pass with the test shell (with the real shell, 45 of 46 in `app.test.tsx` passed; the one failure was the `/$` register test). | OK | -| V3 | Inline scripts per real page | 4 (first look), 13 (final) | Per page (12 files) the inline executable scripts are: (a) router scroll restoration, 613 chars, no attributes, same body everywhere; (b) Start bootstrap `data-tsr-stream-part=""`, 916 to 1349 chars, differs per page; (c) boundary `document.currentScript.remove();/*$tsr-stream-boundary*/`, 56 chars, same everywhere. `/contact` has two more (lazy form Suspense): `` (57 chars) and the React completion runtime `$RB=[];$RV=function(a){...` (845 chars). No `$RC` seen on other pages. External: ``; + +const validFiles = (): Map => + new Map([ + ['index.html', page('boot("index")')], + ['over-mij.html', page('boot("over-mij")')], + ['soorten-fotoshoots/newborn.html', page('boot("newborn")')], + ['404.html', page('boot("404")')], + ['sitemap.xml', SITEMAP], + ['assets/index-abc.js', 'console.log(1);'], + ['assets/index-abc.css', 'a{color:red}'], + ]); + +const found = (plan: { violations: { file: string; code: string }[] }) => + plan.violations.map((v) => `${v.file}: ${v.code}`).sort(); + +describe('planExternalization: pad zonder overtredingen', () => { + it('verplaatst de scripts van elke pagina, ook van 404.html', () => { + const plan = planExternalization(validFiles(), context()); + expect(plan.violations).toEqual([]); + const scripts = [...plan.writes.keys()].filter((f) => + f.startsWith('assets/inline-'), + ); + expect(scripts).toHaveLength(4); + for (const file of [ + 'index.html', + 'over-mij.html', + 'soorten-fotoshoots/newborn.html', + '404.html', + ]) { + expect(plan.writes.get(file)).toMatch( + /src="\/assets\/inline-[0-9a-f]{12}\.js"/, + ); + } + }); + + it('is een no-op bij een tweede keer (al verwerkte pagina telt mee)', () => { + const files = validFiles(); + const first = planExternalization(files, context()); + for (const [file, content] of first.writes) files.set(file, content); + const second = planExternalization(files, context()); + expect(second.writes.size).toBe(0); + expect(second.violations).toEqual([]); + }); + + it('schrijft een scriptbestand dat al met dezelfde inhoud bestaat niet opnieuw', () => { + const files = validFiles(); + const first = planExternalization(files, context()); + const name = [...first.writes.keys()].find((f) => + f.startsWith('assets/inline-'), + )!; + files.set(name, first.writes.get(name)!); + const plan = planExternalization(files, context()); + expect(plan.writes.has(name)).toBe(false); + expect(plan.violations).toEqual([]); + }); + + it('schrijft een gedeeld script één keer', () => { + const files = validFiles(); + for (const file of ['index.html', 'over-mij.html']) { + files.set(file, page('gedeeld()')); + } + const plan = planExternalization(files, context()); + const shared = [...plan.writes.values()].filter( + (c) => c === 'gedeeld()', + ); + expect(shared).toHaveLength(1); + }); +}); + +describe('planExternalization: overtredingen', () => { + it('NO_HTML: geen enkel html-bestand, zonder de pagina’s afzonderlijk te melden', () => { + const files = new Map([['sitemap.xml', SITEMAP]]); + expect(found(planExternalization(files, context()))).toEqual([ + '.: NO_HTML', + ]); + }); + + it('PAGE_MISSING: meldt alle ontbrekende pagina’s', () => { + const files = validFiles(); + files.delete('over-mij.html'); + files.delete('soorten-fotoshoots/newborn.html'); + const plan = planExternalization(files, context()); + expect(found(plan)).toEqual([ + 'over-mij.html: PAGE_MISSING', + 'soorten-fotoshoots/newborn.html: PAGE_MISSING', + ]); + expect(plan.violations[0].detail).toContain('/over-mij'); + }); + + it('NOT_FOUND_PAGE_MISSING: 404.html ontbreekt', () => { + const files = validFiles(); + files.delete('404.html'); + expect(found(planExternalization(files, context()))).toEqual([ + '404.html: NOT_FOUND_PAGE_MISSING', + ]); + }); + + it('UNEXPECTED_HTML: een html-bestand buiten het register', () => { + const files = validFiles(); + files.set('soorten-fotoshoots/index.html', page()); + files.set('extra.html', page()); + expect(found(planExternalization(files, context()))).toEqual([ + 'extra.html: UNEXPECTED_HTML', + 'soorten-fotoshoots/index.html: UNEXPECTED_HTML', + ]); + }); + + it('NO_SCRIPTS_MOVED: een pagina zonder uitvoerbaar of verplaatst script', () => { + const files = validFiles(); + files.set( + 'index.html', + '

geen script

', + ); + expect(found(planExternalization(files, context()))).toEqual([ + 'index.html: NO_SCRIPTS_MOVED', + ]); + }); + + it('EXECUTABLE_REMAINS: de verdediging als het herschrijven iets overlaat', () => { + const scripts: ScriptStep = { + externalize: (html) => ({ html, moved: [] }), + find: () => ['geheim()'], + }; + const plan = planExternalization(validFiles(), context(), scripts); + expect(found(plan)).toContain('index.html: EXECUTABLE_REMAINS'); + expect(plan.violations.map(formatViolation).join('\n')).not.toContain( + 'geheim', + ); + }); + + it('UNSUPPORTED_SCRIPT en UNTERMINATED_SCRIPT: meldt de code en gooit niet', () => { + const files = validFiles(); + files.set('index.html', ''); + files.set('over-mij.html', '`, + moved: [{ fileName: 'inline-000000000000.js', body: html }], + }), + find: () => [], + }; + const plan = planExternalization(validFiles(), context(), scripts); + expect(found(plan)).toContain( + 'assets/inline-000000000000.js: COLLISION', + ); + }); + + it('SITEMAP_MISSING en SITEMAP_DIFFERS', () => { + const missing = validFiles(); + missing.delete('sitemap.xml'); + expect(found(planExternalization(missing, context()))).toEqual([ + 'sitemap.xml: SITEMAP_MISSING', + ]); + const differs = validFiles(); + differs.set('sitemap.xml', SITEMAP + ' '); + expect(found(planExternalization(differs, context()))).toEqual([ + 'sitemap.xml: SITEMAP_DIFFERS', + ]); + }); + + it('SOURCE_MAP_PRESENT: een .map-bestand', () => { + const files = validFiles(); + files.set('assets/index-abc.js.map', '{}'); + expect(found(planExternalization(files, context()))).toEqual([ + 'assets/index-abc.js.map: SOURCE_MAP_PRESENT', + ]); + }); + + it.each(ROOT_PATHS)('BUILD_PATH_LEAK: %s in een tekstbestand', (root) => { + const files = validFiles(); + files.set('assets/index-abc.js', `var p = "${root}/src/x.ts";`); + expect(found(planExternalization(files, context()))).toEqual([ + 'assets/index-abc.js: BUILD_PATH_LEAK', + ]); + }); + + it('EMAIL_IN_OUTPUT: in html, js, css en in de inhoud van een verplaatst script', () => { + const files = validFiles(); + files.set('assets/index-abc.js', `var a = "${ADDRESS}";`); + files.set('assets/index-abc.css', `a::after{content:"${ADDRESS}"}`); + files.set('over-mij.html', `

${ADDRESS}

${page('boot("o")')}`); + files.set('index.html', page(`mail("${ADDRESS}")`)); + const plan = planExternalization(files, context()); + const inline = [...plan.writes.entries()].find(([, c]) => + c.includes(ADDRESS), + )![0]; + expect(found(plan)).toEqual( + [ + 'assets/index-abc.js: EMAIL_IN_OUTPUT', + 'assets/index-abc.css: EMAIL_IN_OUTPUT', + 'over-mij.html: EMAIL_IN_OUTPUT', + `${inline}: EMAIL_IN_OUTPUT`, + ].sort(), + ); + }); + + it('slaat de adrescontrole over zonder adres (ongeldige delen)', () => { + const files = validFiles(); + files.set('assets/index-abc.js', `var a = "${ADDRESS}";`); + const plan = planExternalization(files, context({ address: null })); + expect(plan.violations).toEqual([]); + }); + + it('meldt alle overtredingen tegelijk, in een vast formaat zonder adres', () => { + const files = validFiles(); + files.delete('404.html'); + files.set('assets/index-abc.js', `"${ADDRESS}"`); + files.set('x.html', page()); + const lines = planExternalization(files, context()).violations.map( + formatViolation, + ); + expect(lines).toEqual( + expect.arrayContaining([ + '404.html: NOT_FOUND_PAGE_MISSING: de 404-pagina ontbreekt', + 'x.html: UNEXPECTED_HTML: html-bestand dat niet bij het register hoort', + 'assets/index-abc.js: EMAIL_IN_OUTPUT: e-mailadres in de uitvoer', + ]), + ); + expect(lines.join('\n')).not.toContain(ADDRESS); + }); +}); + +describe('containsAddress', () => { + const address = 'info@andreaslop.nl'; + + it.each([ + ['platte tekst', 'info@andreaslop.nl'], + ['hoofdletters', 'INFO@AndreaSlop.NL'], + ['URL-codering', 'mailto:info%40andreaslop.nl'], + ['JavaScript-escape', 'info\\u0040andreaslop.nl'], + ['JavaScript-escape met accolades', 'info\\u{40}andreaslop.nl'], + ['hexadecimale escape', 'info\\x40andreaslop.nl'], + ['decimale entiteit', 'info@andreaslop.nl'], + ['hexadecimale entiteit', 'info@andreaslop.nl'], + ['hexadecimale entiteit in hoofdletters', 'info@andreaslop.nl'], + ['benoemde entiteit', 'info@andreaslop.nl'], + ['midden in tekst', 'Mail info@andreaslop.nl!'], + ])('vindt het adres als %s', (_, text) => { + expect(containsAddress(text, address)).toBe(true); + }); + + it.each([ + ['alleen de gebruiker', 'info'], + ['alleen het domein', 'andreaslop.nl'], + ['met een spatie ertussen', 'info @ andreaslop.nl'], + ['een ander domein', 'info@andreaslop.com'], + ['een andere gebruiker', 'post@andreaslop.nl'], + ['een onbekende codering', 'infoAandreaslop.nl'], + ['lege tekst', ''], + ])('vindt het adres niet bij %s', (_, text) => { + expect(containsAddress(text, address)).toBe(false); + }); + + it('vindt niets bij een adres zonder @', () => { + expect(containsAddress('infoandreaslop.nl', 'infoandreaslop.nl')).toBe( + false, + ); + }); +}); + +describe('containsAddress: eigenschappen', () => { + // Onafhankelijke lijst van BR-MAIL-10 + const encodings = [ + '@', + '%40', + '\\u0040', + '\\u{40}', + '\\x40', + '@', + '@', + '@', + ]; + // Geen @, %, & of \ : een schrijfwijze kan er dus nooit in ontstaan + const safeText = fc.stringMatching(/^[a-z0-9 .<>/="'-]{0,30}$/); + const randomCase = (text: string, flips: boolean[]) => + [...text] + .map((char, i) => + flips[i % flips.length] ? char.toUpperCase() : char, + ) + .join(''); + + it('vindt het adres in elke schrijfwijze op elke plaats, in elke hoofdletterstand (oracle)', () => { + fc.assert( + fc.property( + validEmail, + fc.constantFrom(...encodings), + safeText, + safeText, + fc.array(fc.boolean(), { minLength: 1, maxLength: 8 }), + (email, encoding, before, after, flips) => { + const written = randomCase( + `${email.user}${encoding}${email.domain}`, + flips, + ); + expect( + containsAddress( + `${before}${written}${after}`, + `${email.user}@${email.domain}`, + ), + ).toBe(true); + }, + ), + ); + }); + + it('markeert tekst zonder het adres nooit (oracle)', () => { + fc.assert( + fc.property( + validEmail, + safeText, + fc.constantFrom(...encodings), + (email, text, encoding) => { + const address = `${email.user}@${email.domain}`; + expect(containsAddress(text, address)).toBe(false); + // Een deel van het adres is niet genoeg + expect( + containsAddress( + `${text}${email.user}${encoding}`, + address, + ), + ).toBe(false); + expect( + containsAddress( + `${encoding}${email.domain}${text}`, + address, + ), + ).toBe(false); + }, + ), + ); + }); +}); + +// De drie pagina's van `context()` plus 404.html, in de volgorde van het scenario +const SCENARIO_FILES = [ + 'index.html', + 'over-mij.html', + 'soorten-fotoshoots/newborn.html', + '404.html', +]; +const EXTRA_REFERENCE = + ''; + +function filesOf(scenario: OutputScenario, address: string) { + const files = new Map(); + scenario.pages.forEach((p, i) => { + if (!p.present) return; + files.set( + SCENARIO_FILES[i], + p.content.html + + (p.reference ? EXTRA_REFERENCE : '') + + (p.leakAddress ? `

${address}

` : '') + + (p.bad === 'unsupported' + ? '' + : '') + + (p.bad === 'unterminated' ? '`, + ); + }); + const first = planExternalization(files, context()); + expect(first.violations).toEqual([]); + for (const [file, content] of first.writes) { + files.set(file, content); + } + const second = planExternalization(files, context()); + expect(second.writes.size).toBe(0); + expect(second.violations).toEqual([]); + }), + ); + }); +}); + +// --------------------------------------------------------------------------- +// externalizeOutput met een echte map + +const temporary: string[] = []; +afterEach(() => { + for (const dir of temporary.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +// /web/dist/client: de repositorywortel voor de lekcontrole is /web +function outputDir() { + const root = mkdtempSync(join(tmpdir(), 'externalize-')); + temporary.push(root); + const clientDir = join(root, 'web', 'dist', 'client'); + const put = (file: string, content: string) => { + mkdirSync(dirname(join(clientDir, file)), { recursive: true }); + writeFileSync(join(clientDir, file), content, 'utf8'); + }; + for (const path of Object.keys(sitePages)) { + put(pageFile(path), page(`boot("${path}")`)); + } + put('404.html', page('boot("404")')); + put('sitemap.xml', sitemapXml(pageList(sitePages), siteUrl)); + put('assets/index-abc.js', 'console.log(1);'); + return { root: join(root, 'web'), clientDir, put }; +} + +const listing = (dir: string) => + readdirSync(dir, { recursive: true, encoding: 'utf8' }).sort(); + +const silent = () => ({ log: vi.fn(), error: vi.fn() }); + +describe('externalizeOutput', () => { + it('schrijft eerst de scriptbestanden, dan de html, en meldt één regel', () => { + const { clientDir } = outputDir(); + const out = silent(); + const result = externalizeOutput(clientDir, out); + + expect(result.exitCode).toBe(0); + expect(result.violations).toEqual([]); + const firstHtml = result.written.findIndex((f) => f.endsWith('.html')); + expect(firstHtml).toBeGreaterThan(0); + expect( + result.written + .slice(0, firstHtml) + .every((f) => !f.endsWith('.html')), + ).toBe(true); + expect( + result.written.slice(firstHtml).every((f) => f.endsWith('.html')), + ).toBe(true); + + const index = readFileSync(join(clientDir, 'index.html'), 'utf8'); + const reference = /src="\/(assets\/inline-[0-9a-f]{12}\.js)"/.exec( + index, + )!; + expect(readFileSync(join(clientDir, reference[1]), 'utf8')).toBe( + 'boot("/")', + ); + expect(index).not.toContain('boot('); + expect(out.error).not.toHaveBeenCalled(); + expect(out.log).toHaveBeenCalledTimes(1); + expect(out.log.mock.calls[0][0]).toContain( + `${Object.keys(sitePages).length + 1} pagina's gecontroleerd`, + ); + expect(out.log.mock.calls[0][0]).toContain( + `${Object.keys(sitePages).length + 1} scripts verplaatst`, + ); + }); + + it('doet bij een tweede keer niets', () => { + const { clientDir } = outputDir(); + externalizeOutput(clientDir, silent()); + const before = listing(clientDir); + const out = silent(); + const result = externalizeOutput(clientDir, out); + expect(result).toEqual({ exitCode: 0, violations: [], written: [] }); + expect(listing(clientDir)).toEqual(before); + expect(out.log.mock.calls[0][0]).toContain('0 scripts verplaatst'); + }); + + it('schrijft niets en meldt alles bij een overtreding', () => { + const { clientDir, put } = outputDir(); + rmSync(join(clientDir, 'over-mij.html')); + put('assets/index-abc.js.map', '{}'); + const before = listing(clientDir); + const out = silent(); + const result = externalizeOutput(clientDir, out); + + expect(result.exitCode).toBe(1); + expect(result.written).toEqual([]); + expect(listing(clientDir)).toEqual(before); + expect(out.log).not.toHaveBeenCalled(); + expect(out.error.mock.calls.map(([line]) => line)).toEqual( + result.violations.map(formatViolation), + ); + expect(result.violations.map((v) => v.code).sort()).toEqual([ + 'PAGE_MISSING', + 'SOURCE_MAP_PRESENT', + ]); + }); + + it('vindt het e-mailadres van de site in een uitvoerbestand', () => { + const { clientDir, put } = outputDir(); + put('assets/index-abc.js', 'var a = "info@andreaslop.nl";'); + const result = externalizeOutput(clientDir, silent()); + expect(result.violations.map(formatViolation)).toEqual([ + 'assets/index-abc.js: EMAIL_IN_OUTPUT: e-mailadres in de uitvoer', + ]); + }); + + it.each([ + [ + 'met voorwaartse slashes', + (root: string) => root.replaceAll('\\', '/'), + ], + ['met backslashes', (root: string) => root.replaceAll('/', '\\')], + [ + 'met verdubbelde backslashes', + (root: string) => + root.replaceAll('/', '\\').replaceAll('\\', '\\\\'), + ], + ])('vindt het pad van de repository %s', (_, style) => { + const { clientDir, put, root } = outputDir(); + put('assets/index-abc.js', `var p = "${style(root)}";`); + const result = externalizeOutput(clientDir, silent()); + expect(result.violations.map((v) => v.code)).toEqual([ + 'BUILD_PATH_LEAK', + ]); + }); + + it('leest alleen tekstbestanden en gaat niet af op een bestandssysteemwortel', () => { + // clientDir direct onder de wortel: '../..' is de wortel zelf en mag + // niet elk bestand met een slash als lek aanmerken + const { clientDir, put } = outputDir(); + put('logo.png', 'binair'); + expect(externalizeOutput(clientDir, silent()).exitCode).toBe(0); + const rootResult = externalizeOutput(join(clientDir, '..'), silent()); + expect( + rootResult.violations.every((v) => v.code !== 'BUILD_PATH_LEAK'), + ).toBe(true); + }); +}); + +describe('externalizeOutput als opdrachtregel', () => { + const script = join( + dirname(fileURLToPath(import.meta.url)), + 'externalizeInlineScripts.ts', + ); + const run = (dir: string) => + spawnSync(process.execPath, [script, dir], { encoding: 'utf8' }); + + it('geeft exitcode 0 en één regel bij een geldige uitvoer', () => { + const { clientDir } = outputDir(); + const result = run(clientDir); + expect(result.status).toBe(0); + expect(result.stdout.trim().split('\n')).toHaveLength(1); + expect(existsSync(join(clientDir, 'assets'))).toBe(true); + }); + + it('geeft exitcode 1 en schrijft niets bij een overtreding', () => { + const { clientDir } = outputDir(); + rmSync(join(clientDir, '404.html')); + const before = listing(clientDir); + const result = run(clientDir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('404.html: NOT_FOUND_PAGE_MISSING'); + expect(listing(clientDir)).toEqual(before); + }); +}); diff --git a/web/config/externalizeInlineScripts.ts b/web/config/externalizeInlineScripts.ts new file mode 100644 index 0000000..476947c --- /dev/null +++ b/web/config/externalizeInlineScripts.ts @@ -0,0 +1,363 @@ +// Stap na `vite build` die de inline scripts van Start in dist/client naar +// bestanden verplaatst en de uitvoer controleert. +// +// Waarom (BR-BLD-12): de CSP staat alleen script-src 'self' en Umami toe, en +// Start zet inline scripts in elke geprerenderde pagina. Zonder deze stap +// blokkeert de browser ze. De CSP blijft ongewijzigd. +// +// Alles wordt eerst in het geheugen gepland (planExternalization, puur). Pas +// als er geen enkele overtreding is, schrijft de runner: eerst de +// scriptbestanden, daarna de aangepaste HTML (BR-SCR-12, SECURITY-15). Een +// mislukte controle laat dist/client dus ongemoeid. +import { + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join, parse, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { emailAddress } from '../src/lib/emailAddress.ts'; +import { siteUrl } from '../src/seo.ts'; +import siteContent from '../src/content/siteContent.ts'; +import { sitemapXml } from '../src/sitemap.ts'; +import sitePages, { pageList } from '../src/sitePages.ts'; +import { + ScriptError, + externalizeInlineScripts, + findExecutableInlineScripts, +} from './inlineScripts.ts'; +import { NOT_FOUND_FILE, pageFile } from './pageFiles.ts'; + +export type ViolationCode = + | 'NO_HTML' + | 'PAGE_MISSING' + | 'NOT_FOUND_PAGE_MISSING' + | 'UNEXPECTED_HTML' + | 'NO_SCRIPTS_MOVED' + | 'EXECUTABLE_REMAINS' + | 'UNSUPPORTED_SCRIPT' + | 'UNTERMINATED_SCRIPT' + | 'COLLISION' + | 'EMAIL_IN_OUTPUT' + | 'SITEMAP_MISSING' + | 'SITEMAP_DIFFERS' + | 'SOURCE_MAP_PRESENT' + | 'BUILD_PATH_LEAK'; + +export interface Violation { + file: string; + code: ViolationCode; + // Nooit een scriptinhoud en nooit het e-mailadres (BR-SCR-21) + detail: string; +} + +export interface ExpectedFileSet { + // Paginapad uit het register naar zijn bestand + pages: Map; + notFound: string; +} + +export interface PlanContext { + expected: ExpectedFileSet; + // Het samengestelde adres, of null als de delen ongeldig zijn + address: string | null; + sitemap: string; + // De repositorywortel in alle schrijfwijzen die in tekst kunnen staan + rootPaths: string[]; +} + +export interface OutputPlan { + // Relatief pad naar nieuwe inhoud + writes: Map; + violations: Violation[]; +} + +// De twee functies van de scriptstap als parameter, zodat de verdediging +// EXECUTABLE_REMAINS getest kan worden: met de echte functies kan ze niet +// optreden (BR-SCR-11). +export interface ScriptStep { + externalize: typeof externalizeInlineScripts; + find: typeof findExecutableInlineScripts; +} + +const defaultScriptStep: ScriptStep = { + externalize: externalizeInlineScripts, + find: findExecutableInlineScripts, +}; + +const ASSETS_DIR = 'assets'; +const SITEMAP_FILE = 'sitemap.xml'; +const MOVED_REFERENCE = /src="\/assets\/inline-[0-9a-f]{12}\.js"/; + +// Hoe een '@' in een bestand kan staan zonder dat het adres leesbaar is: +// platte tekst, URL-codering, JavaScript-escapes en HTML-entiteiten +const AT_ENCODINGS = [ + '@', + '%40', + '\\u0040', + '\\u{40}', + '\\x40', + '@', + '@', + '@', +]; + +// Staat het adres in een van de schrijfwijzen in de tekst? Hoofdletterongevoelig. +export function containsAddress(text: string, address: string): boolean { + const at = address.indexOf('@'); + if (at === -1) return false; + const haystack = text.toLowerCase(); + const user = address.slice(0, at).toLowerCase(); + const domain = address.slice(at + 1).toLowerCase(); + return AT_ENCODINGS.some((encoding) => + haystack.includes(`${user}${encoding}${domain}`), + ); +} + +const isHtml = (file: string) => file.endsWith('.html'); + +// Een ScriptError heeft het bericht `CODE: detail`; de runner zet er zelf de +// code en het bestand voor +const detailOf = (error: ScriptError) => + error.message.slice(error.code.length + 2); + +export function formatViolation({ file, code, detail }: Violation): string { + return `${file}: ${code}: ${detail}`; +} + +// De beslissingen van de runner zonder bestandssysteem. `files` bevat alle +// tekstbestanden (en eventuele .map-bestanden) onder dist/client op relatief +// pad met voorwaartse slashes. +export function planExternalization( + files: ReadonlyMap, + context: PlanContext, + scripts: ScriptStep = defaultScriptStep, +): OutputPlan { + const violations: Violation[] = []; + const writes = new Map(); + const add = (file: string, code: ViolationCode, detail: string) => + violations.push({ file, code, detail }); + + const htmlFiles = [...files.keys()].filter(isHtml).sort(); + const expectedFiles = new Set([ + ...context.expected.pages.values(), + context.expected.notFound, + ]); + + if (htmlFiles.length === 0) { + // De ontbrekende pagina's afzonderlijk melden zou alleen herhalen + add('.', 'NO_HTML', 'geen .html-bestand in de uitvoermap'); + } else { + for (const [path, file] of context.expected.pages) { + if (!files.has(file)) { + add(file, 'PAGE_MISSING', `pagina ${path} ontbreekt`); + } + } + if (!files.has(context.expected.notFound)) { + add( + context.expected.notFound, + 'NOT_FOUND_PAGE_MISSING', + 'de 404-pagina ontbreekt', + ); + } + } + for (const file of htmlFiles) { + if (!expectedFiles.has(file)) { + add( + file, + 'UNEXPECTED_HTML', + 'html-bestand dat niet bij het register hoort', + ); + } + } + + for (const file of htmlFiles) { + if (!expectedFiles.has(file)) continue; + const html = files.get(file)!; + + let result; + try { + result = scripts.externalize(html); + } catch (error) { + if (!(error instanceof ScriptError)) throw error; + add(file, error.code, detailOf(error)); + continue; + } + + const remaining = scripts.find(result.html).length; + if (remaining > 0) { + add( + file, + 'EXECUTABLE_REMAINS', + `${remaining} uitvoerbaar inline script blijft over`, + ); + } + // Een al verwerkte pagina telt mee, dus twee keer draaien mag + if (!MOVED_REFERENCE.test(result.html)) { + add( + file, + 'NO_SCRIPTS_MOVED', + 'geen verwijzing naar een verplaatst inline script', + ); + } + + for (const { fileName, body } of result.moved) { + const path = `${ASSETS_DIR}/${fileName}`; + const existing = writes.get(path) ?? files.get(path); + if (existing !== undefined && existing !== body) { + add(path, 'COLLISION', 'bestaat al met andere inhoud'); + } else if (existing === undefined) { + writes.set(path, body); + } + } + if (result.html !== html) writes.set(file, result.html); + } + + // De uitvoer zoals hij na het schrijven is: ook de verplaatste scripts + const finalFiles = new Map(files); + for (const [path, content] of writes) finalFiles.set(path, content); + + for (const [file, content] of finalFiles) { + if (file.endsWith('.map')) { + add(file, 'SOURCE_MAP_PRESENT', 'sourcemap in de uitvoer'); + } + if (context.address && containsAddress(content, context.address)) { + add(file, 'EMAIL_IN_OUTPUT', 'e-mailadres in de uitvoer'); + } + if (context.rootPaths.some((root) => content.includes(root))) { + add(file, 'BUILD_PATH_LEAK', 'pad van de repository in de uitvoer'); + } + } + + const sitemap = files.get(SITEMAP_FILE); + if (sitemap === undefined) { + add(SITEMAP_FILE, 'SITEMAP_MISSING', 'bestand ontbreekt'); + } else if (sitemap !== context.sitemap) { + add( + SITEMAP_FILE, + 'SITEMAP_DIFFERS', + 'inhoud wijkt af van het pagina-register', + ); + } + + return { writes, violations }; +} + +// Alleen deze bestanden worden gelezen en op adres en pad doorzocht +const TEXT_EXTENSIONS = [ + '.html', + '.js', + '.css', + '.xml', + '.txt', + '.json', + '.webmanifest', + '.map', +]; + +function readOutput(clientDir: string): Map { + const files = new Map(); + const entries = readdirSync(clientDir, { + recursive: true, + encoding: 'utf8', + }) + .map((entry) => entry.replaceAll('\\', '/')) + .filter((entry) => TEXT_EXTENSIONS.some((ext) => entry.endsWith(ext))) + .sort(); + for (const entry of entries) { + const path = join(clientDir, entry); + if (statSync(path).isFile()) + files.set(entry, readFileSync(path, 'utf8')); + } + return files; +} + +// De repositorywortel in de schrijfwijzen waarin hij in tekst kan staan: +// met / , met \ en met verdubbelde \ zoals in een JSON- of JS-string +function rootPathsOf(root: string): string[] { + // De wortel van het bestandssysteem zou alles als lek aanmerken + if (parse(root).root === root) return []; + return [ + ...new Set([ + root, + root.replaceAll('\\', '/'), + root.replaceAll('/', '\\'), + root.replaceAll('/', '\\').replaceAll('\\', '\\\\'), + ]), + ]; +} + +function contextFor(clientDir: string): PlanContext { + return { + expected: { + pages: new Map( + Object.keys(sitePages).map((path) => [path, pageFile(path)]), + ), + notFound: NOT_FOUND_FILE, + }, + address: emailAddress(siteContent.email), + sitemap: sitemapXml(pageList(sitePages), siteUrl), + rootPaths: rootPathsOf(resolve(clientDir, '../..')), + }; +} + +export interface RunResult { + exitCode: 0 | 1; + violations: Violation[]; + written: string[]; +} + +interface Log { + log: (message: string) => void; + error: (message: string) => void; +} + +// Leest de map, plant, en schrijft alleen zonder overtredingen. Geeft de +// exitcode terug in plaats van hem te zetten, zodat tests hem kunnen lezen. +export function externalizeOutput( + clientDir: string, + out: Log = console, +): RunResult { + const context = contextFor(clientDir); + const { writes, violations } = planExternalization( + readOutput(clientDir), + context, + ); + + if (violations.length > 0) { + for (const violation of violations) { + out.error(formatViolation(violation)); + } + return { exitCode: 1, violations, written: [] }; + } + + // Scriptbestanden eerst: een afgebroken run laat nooit HTML achter die + // naar een bestand verwijst dat niet bestaat + const paths = [...writes.keys()].sort( + (a, b) => Number(isHtml(a)) - Number(isHtml(b)), + ); + for (const path of paths) { + const target = join(clientDir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, writes.get(path)!, 'utf8'); + } + + const moved = paths.filter((path) => !isHtml(path)).length; + out.log( + `Inline scripts verplaatst: ${context.expected.pages.size + 1} pagina's gecontroleerd, ${moved} scripts verplaatst, ${paths.length} bestanden geschreven`, + ); + return { exitCode: 0, violations, written: paths }; +} + +// Draait alleen als het bestand zelf wordt gestart (pnpm build), niet als een +// test het importeert. Optioneel argument: de map, standaard dist/client. +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + process.exitCode = externalizeOutput( + resolve(process.argv[2] ?? 'dist/client'), + ).exitCode; +} diff --git a/web/config/inlineScripts.test.ts b/web/config/inlineScripts.test.ts new file mode 100644 index 0000000..4815750 --- /dev/null +++ b/web/config/inlineScripts.test.ts @@ -0,0 +1,507 @@ +import { createHash } from 'node:crypto'; +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import { + canonicalHtmlWithModel, + htmlFragments, + htmlWithModel, + scriptBody, +} from '../src/test/htmlArbitraries'; +import { + ScriptError, + externalizeInlineScripts, + findExecutableInlineScripts, +} from './inlineScripts'; + +// Onafhankelijk van de implementatie: de naam volgens BR-SCR-07 +const fileOf = (body: string) => + `inline-${createHash('sha256').update(body, 'utf8').digest('hex').slice(0, 12)}.js`; + +const referenceOf = (attributes: string, body: string) => + ``; + +// Een door de stap geplaatste verwijzing: ', + ); + expect(result.html).toBe(referenceOf(' data-tsr-stream-part=""', 'A')); + expect(result.moved).toEqual([{ fileName: fileOf('A'), body: 'A' }]); + }); + + it('laat een extern module-script met async ongemoeid', () => { + const html = ''; + expect(externalizeInlineScripts(html)).toEqual({ html, moved: [] }); + }); + + it('verplaatst een kaal script en kent het bestand de inhoud toe', () => { + const result = externalizeInlineScripts( + '', + ); + expect(result.html).toBe( + referenceOf('', 'document.currentScript.remove()'), + ); + expect(result.moved).toHaveLength(1); + }); + + it.each([ + [ + 'een JSON-LD-blok', + '', + ], + [ + 'een style-body', + '', + ], + ['commentaar', ''], + ['een attribuutwaarde', '
'], + ])('laat %s ongemoeid', (_, html) => { + expect(externalizeInlineScripts(html)).toEqual({ html, moved: [] }); + }); + + it('laat al het andere byte voor byte staan, ook regeleinden', () => { + const html = + '\r\n\r\n\r\n' + + '' + + '\r\n\r\n'; + expect(externalizeInlineScripts(html).html).toBe( + html.replace('', referenceOf('', 'go()')), + ); + }); + + it('hasht de inhoud zoals geschreven, zonder trimmen', () => { + const a = externalizeInlineScripts(''); + const b = externalizeInlineScripts(''); + expect(a.moved[0].fileName).not.toBe(b.moved[0].fileName); + expect(b.moved[0].body).toBe(' x() '); + expect(b.moved[0].fileName).toBe(fileOf(' x() ')); + }); + + it('geeft gelijke inhoud één bestand, op volgorde van eerste voorkomen', () => { + const result = externalizeInlineScripts( + '', + ); + expect(result.moved.map((m) => m.body)).toEqual(['b()', 'a()']); + expect(result.html).toBe( + referenceOf('', 'b()') + + referenceOf('', 'a()') + + referenceOf('', 'b()'), + ); + }); +}); + +describe('classificatie (BR-SCR-04)', () => { + it.each([ + ' type="text/javascript"', + ' type="TEXT/JavaScript"', + ' type=" text/javascript "', + ' type="application/javascript"', + ' type="application/ecmascript"', + ' type="application/x-ecmascript"', + ' type="application/x-javascript"', + ' type="text/javascript1.0"', + ' type="text/javascript1.5"', + ' type="text/ecmascript"', + ' type="text/jscript"', + ' type="text/livescript"', + ' type="text/x-ecmascript"', + ' type="text/x-javascript"', + ' type=""', + ' type', + ' type="module"', + ' type="Module"', + ' language="vbscript"', + ])('verplaatst een script met %s', (attributes) => { + const html = `go()`; + expect(externalizeInlineScripts(html).html).toBe( + referenceOf(attributes, 'go()'), + ); + expect(findExecutableInlineScripts(html)).toEqual(['go()']); + }); + + it.each([ + ' type="application/ld+json"', + ' type="application/json"', + ' type="importmap"', + ' type="speculationrules"', + ' type="text/plain"', + ' type="text/javascript2.0"', + ' type="jscript"', + ])('laat een datablok met %s staan', (attributes) => { + const html = `{"a":1}`; + expect(externalizeInlineScripts(html)).toEqual({ html, moved: [] }); + expect(findExecutableInlineScripts(html)).toEqual([]); + }); + + it.each([ + ['src met waarde', ''], + ['src zonder waarde', ''], + ['lege src', ''], + ['src in hoofdletters', ''], + ])('laat een extern script staan: %s', (_, html) => { + expect(externalizeInlineScripts(html)).toEqual({ html, moved: [] }); + expect(findExecutableInlineScripts(html)).toEqual([]); + }); + + it('laat een leeg script staan, ook met alleen witruimte', () => { + const html = ''; + expect(externalizeInlineScripts(html)).toEqual({ html, moved: [] }); + expect(findExecutableInlineScripts(html)).toEqual([]); + }); + + it('leest alleen het eerste type-attribuut, zoals een browser', () => { + const html = ''; + expect(externalizeInlineScripts(html).moved).toEqual([]); + }); +}); + +describe('niet-ondersteunde vormen (BR-SCR-05)', () => { + it.each([ + '', + '', + '', + ])('(a) weigert een klassiek script met async of defer: %s', (html) => { + fails(html, 'UNSUPPORTED_SCRIPT'); + }); + + it('(a) staat async op een module-script wel toe', () => { + expect( + externalizeInlineScripts( + '', + ).moved, + ).toHaveLength(1); + }); + + it('(b) weigert een commentaaropener in de body', () => { + fails('', 'UNSUPPORTED_SCRIPT'); + }); + + it('(b) weigert ook in een module-script', () => { + fails('', 'UNSUPPORTED_SCRIPT'); + }); + + it.each([ + "import a from './x.js';", + 'import a from "../x.js";', + "import './side.js';", + "const m = await import('./x.js');", + 'const m = import( "../x.js" );', + "export * from './x.js';", + "import a from '.';", + ])('(c) weigert een module met relatieve import: %s', (body) => { + fails(``, 'UNSUPPORTED_SCRIPT'); + }); + + it.each([ + "import a from '/assets/x.js';", + "import a from 'https://cdn.test/x.js';", + "import a from 'pakket';", + 'const from = 1; const s = "x./y";', + ])('(c) staat een module zonder relatieve import toe: %s', (body) => { + expect( + externalizeInlineScripts(``) + .moved, + ).toHaveLength(1); + }); + + it('(c) geldt niet voor een klassiek script', () => { + expect( + externalizeInlineScripts("").moved, + ).toHaveLength(1); + }); + + it('noemt de regel en de code, nooit de inhoud van het script', () => { + const error = fails( + '

\n

\n', + 'UNSUPPORTED_SCRIPT', + ); + expect(error.message).toContain('regel 3'); + expect(error.message).not.toContain('geheim'); + }); + + it('controleert alleen scripts die verplaatst zouden worden', () => { + const html = + '' + + ''; + expect(externalizeInlineScripts(html).moved).toEqual([]); + }); +}); + +describe('scriptgrenzen en scanner (BR-SCR-02, 03)', () => { + it('weigert een script zonder eindtag', () => { + const error = fails('

{ + fails('go()go()', () => { + const html = ''; + expect(findExecutableInlineScripts(html)).toEqual([ + 'a = ""; b = 1;', + ]); + expect(findExecutableInlineScripts('')).toEqual([ + 'a', + ]); + expect(findExecutableInlineScripts('`, + ); + expect(result.html).toBe(referenceOf(attributes, 'go()')); + }); + + it('behoudt een afsluitende / in de tag en leest de naam hoofdletterongevoelig', () => { + expect(externalizeInlineScripts('').html).toBe( + referenceOf(' /', 'go()'), + ); + expect( + externalizeInlineScripts('') + .html, + ).toBe(referenceOf(' TYPE="Module"', 'go()')); + }); + + it('een ongeldig <-teken is tekst', () => { + const html = '

1 < 2

<3>'; + expect(findExecutableInlineScripts(html)).toEqual(['go()', 'x']); + }); + + it('slaat tekstelementen over tot hun eindtag, ook zonder eindtag', () => { + expect( + findExecutableInlineScripts( + 'a<script>x</script>', + ), + ).toEqual(['y']); + expect( + findExecutableInlineScripts('', + ), + ).toEqual(['y']); + }); + + it('slaat commentaar over, ook en een niet-afgesloten commentaar', () => { + expect(findExecutableInlineScripts('')).toEqual([ + 'x', + ]); + expect(findExecutableInlineScripts('')).toEqual( + ['x'], + ); + expect(findExecutableInlineScripts(' en zoals in een browser sluiten + const closing = html.indexOf('-->', lt + 2); + pos = closing === -1 ? length : closing + 3; + continue; + } + if (next === '!' || next === '?' || next === '/') { + const closing = html.indexOf('>', lt + 2); + pos = closing === -1 ? length : closing + 1; + continue; + } + if (!isLetter(next)) { + pos = lt + 1; + continue; + } + + const tag = readStartTag(html, lt); + if (!tag) break; + const bodyStart = tag.close + 1; + + if (tag.name === 'script') { + scriptEnd.lastIndex = bodyStart; + const endTag = scriptEnd.exec(html); + let body: string; + let end: number; + let terminated: boolean; + if (!endTag) { + body = html.slice(bodyStart); + end = length; + terminated = false; + } else { + body = html.slice(bodyStart, endTag.index); + const closing = html.indexOf('>', endTag.index); + terminated = closing !== -1; + end = terminated ? closing + 1 : length; + } + const { kind, module } = classify(tag.attributes, body); + found.push({ + start: lt, + end, + attributeText: html.slice(tag.attributeStart, tag.close), + body, + terminated, + kind, + module, + asyncOrDefer: + tag.attributes.has('async') || tag.attributes.has('defer'), + }); + pos = end; + continue; + } + + if (RAW_TEXT_ELEMENTS.has(tag.name)) { + const pattern = endTagPattern(tag.name); + pattern.lastIndex = bodyStart; + const endTag = pattern.exec(html); + pos = endTag ? endTag.index : length; + continue; + } + + pos = bodyStart; + } + return found; +} + +// Een relatieve importspecifier ('./x', '../x') zou na het verplaatsen tegen +// /assets/ oplossen in plaats van tegen de pagina +const RELATIVE_IMPORT = + /\bfrom\s*['"`]\.{1,2}(?:\/|(?=['"`]))|\bimport\s*\(?\s*['"`]\.{1,2}(?:\/|(?=['"`]))/; + +function lineOf(html: string, index: number): number { + return html.slice(0, index).split('\n').length; +} + +// Vormen die we niet veilig kunnen verplaatsen (BR-SCR-05): liever een +// mislukte build dan stil ander gedrag als Start zijn uitvoer wijzigt. +function unsupportedReason(script: ScriptElement): string | null { + if (!script.module && script.asyncOrDefer) { + return 'async of defer op een klassiek script'; + } + if (script.body.includes('', + '', + '', + '', + '', + ) + .map((html) => ({ kind: 'comment' as const, html })); + +const rawTextPiece: fc.Arbitrary = fc + .constantFrom( + '', + '', + '', + 'a <script>x()</script>', + '', + '<script>x()</script>', + '', + ) + .map((html) => ({ kind: 'rawText' as const, html })); + +const attributeDecoyPiece: fc.Arbitrary = fc + .constantFrom( + '
t
', + "t", + '', + '

w()>t

', + ) + .map((html) => ({ kind: 'attributeDecoy' as const, html })); + +// Complete markup with no script in it. Text pieces never end in '<', so a +// piece cannot glue onto the next one. +const markupPiece: fc.Arbitrary = fc + .constantFrom( + '', + '', + '
text
', + '

1 < 2 and 3 > 2

', + '
', + '', + '', + 'plain text', + '

', + ) + .map((html) => ({ kind: 'markup' as const, html })); + +// Everything the step must leave alone, whatever surrounds it +export const decoyPiece: fc.Arbitrary = fc.oneof( + externalPiece, + emptyPiece, + dataPiece, + commentPiece, + rawTextPiece, + attributeDecoyPiece, + markupPiece, +); + +export const htmlPiece: fc.Arbitrary = fc.oneof( + { arbitrary: executablePiece, weight: 3 }, + { arbitrary: decoyPiece, weight: 4 }, +); + +const canonicalHtmlPiece: fc.Arbitrary = fc.oneof( + { arbitrary: canonicalExecutablePiece, weight: 3 }, + { arbitrary: decoyPiece, weight: 4 }, +); + +const separator = fc.constantFrom('', '\n', ' '); + +function modelOf(pieces: fc.Arbitrary): fc.Arbitrary { + return fc + .tuple(fc.array(pieces, { maxLength: 12 }), separator) + .map(([list, join]) => ({ + html: list.map((piece) => piece.html).join(join), + pieces: list, + separator: join, + })); +} + +// A document and what each part of it is +export const htmlWithModel: fc.Arbitrary = modelOf(htmlPiece); + +// The same, with executable scripts written as `` +// only, so that putting the moved bodies back restores the input exactly +export const canonicalHtmlWithModel: fc.Arbitrary = + modelOf(canonicalHtmlPiece); + +// Independent complete fragments, for split-and-join properties: each one +// parses the same alone as inside a larger document +export const htmlFragments: fc.Arbitrary = fc.array(htmlPiece, { + maxLength: 10, +}); + +// A build output for the runner: which expected pages exist and what is wrong +// with them, plus files that do not belong. The test turns this into a file +// map and computes the expected violations from the same description. +export type BadScript = 'none' | 'unsupported' | 'unterminated'; + +export interface PageScenario { + present: boolean; + content: HtmlModel; + // Already processed: references a moved script without executable ones + reference: boolean; + bad: BadScript; + // The assembled address sits in the page text + leakAddress: boolean; +} + +export interface ExtraFile { + name: string; + address: boolean; + rootPath: boolean; +} + +export interface OutputScenario { + // index.html, over-mij.html, soorten-fotoshoots/newborn.html, 404.html + pages: PageScenario[]; + unexpected: string[]; + extras: ExtraFile[]; + sourceMap: boolean; + sitemap: 'ok' | 'missing' | 'differs'; +} + +const pageScenario: fc.Arbitrary = fc.record({ + present: fc.boolean(), + content: htmlWithModel, + reference: fc.boolean(), + bad: fc.constantFrom( + 'none', + 'none', + 'none', + 'unsupported', + 'unterminated', + ), + leakAddress: fc.boolean(), +}); + +const extraFile: fc.Arbitrary = fc.record({ + name: fc.constantFrom( + 'assets/app.js', + 'assets/app.css', + 'robots.txt', + 'data.json', + 'manifest.webmanifest', + 'feed.xml', + ), + address: fc.boolean(), + rootPath: fc.boolean(), +}); + +export const outputScenario: fc.Arbitrary = fc.record({ + pages: fc.tuple(pageScenario, pageScenario, pageScenario, pageScenario), + unexpected: fc.uniqueArray( + fc.constantFrom( + 'extra.html', + 'soorten-fotoshoots/index.html', + 'a/b.html', + ), + { maxLength: 2 }, + ), + extras: fc.uniqueArray(extraFile, { + selector: (file) => file.name, + maxLength: 3, + }), + sourceMap: fc.boolean(), + sitemap: fc.constantFrom('ok', 'ok', 'missing', 'differs'), +});