From 0c5d8b8f40bde0a07a7ce40fb3ddcd2e6118d09a Mon Sep 17 00:00:00 2001 From: Sluijsens Date: Sat, 31 Oct 2026 18:30:30 +0100 Subject: [PATCH] Compresses responses in the preview server The baseline ran on vite preview, which compresses, so measurements could not be compared without this. Also records that the real servers need compression. Co-Authored-By: Claude Opus 5.5 --- aidlc-docs/features/prerendering/audit.md | 12 ++ .../functional-design/business-rules.md | 2 +- .../inception/plans/execution-plan.md | 2 +- .../inception/requirements/requirements.md | 2 + web/config/previewServer.test.ts | 160 +++++++++++++++++- web/config/previewServer.ts | 57 ++++++- 6 files changed, 220 insertions(+), 15 deletions(-) diff --git a/aidlc-docs/features/prerendering/audit.md b/aidlc-docs/features/prerendering/audit.md index c9c2ab6..21b8ccb 100644 --- a/aidlc-docs/features/prerendering/audit.md +++ b/aidlc-docs/features/prerendering/audit.md @@ -215,3 +215,15 @@ **Context**: Code Generation Part 2, unit 1. Permission ends when this feature ends. --- + +## Code Generation - Lighthouse Comparison Finding + +**Timestamp**: 2026-10-31T14:15:01Z + +**User Input**: "(none: finding by the main session)" + +**AI Response**: "Step 24 measured /over-mij at 0.77 with LCP 6.2 s against the baseline 0.86 and 3.5 s. Root cause in the Lighthouse reports: the new preview server does not compress, while vite preview at the baseline did (script transfer 667 kB versus 202 kB; document 94 kB uncompressed, about 16 kB gzipped). FCP was already better at 1.66 s versus 2.60 s. Extended BR-SRV-08 with gzip. Separately, curl showed test.andreaslop.nl returns HTML and JavaScript without Content-Encoding and serverConfig.ts has no compression rules, so FR-PR-23 was added to unit 3. Lighthouse also reported React error #418 on /404.html; its root cause goes to Steps 26 and 27." + +**Context**: Code Generation unit 1, after Step 24. + +--- diff --git a/aidlc-docs/features/prerendering/construction/start-migration/functional-design/business-rules.md b/aidlc-docs/features/prerendering/construction/start-migration/functional-design/business-rules.md index 17d933c..1a35ab3 100644 --- a/aidlc-docs/features/prerendering/construction/start-migration/functional-design/business-rules.md +++ b/aidlc-docs/features/prerendering/construction/start-migration/functional-design/business-rules.md @@ -211,7 +211,7 @@ Only what Lighthouse and the browser tests need. Trailing slash and `/404` are f | BR-SRV-05 | Anything else gives `404.html` with status 404. | FR-PR-14 | | BR-SRV-06 | The path is percent-decoded once. A path that fails to decode, contains a NUL or a backslash, or has a `..` segment gives `404.html` with 404. The returned `file` never contains `..` and is never absolute, so it cannot leave `clientDir` (SECURITY-11 misuse case). | NFR-PR-SEC-01 | | BR-SRV-07 | Marked "final rule in unit 3 Infrastructure Design": a path with a trailing slash (interim: `404.html` with 404, no redirect in unit 1, so `{ redirect }` is not produced); the path `/404` (interim: `404.html` with status 404, not 200); `/index` and `/index.html`. | FR-PR-16 | -| BR-SRV-08 | `startPreviewServer(clientDir, port)` answers `GET` and `HEAD`; other methods get 405. Every response, also 404, carries `securityHeaders('production')`. HTML responses send `Cache-Control: no-cache`. The content type comes from the file extension (html, js, css, svg, png, jpg, webp, woff2, ico, xml, txt, json). It binds to `127.0.0.1`. `pnpm preview` runs it with default port 4173, accepts the `--port` and `--strictPort` arguments that `lighthouserc.cjs` passes (or that command changes with it), and prints a line matching its `startServerReadyPattern`. | NFR-PR-SEC-01; D10 | +| BR-SRV-08 | `startPreviewServer(clientDir, port)` answers `GET` and `HEAD`; other methods get 405. Every response, also 404, carries `securityHeaders('production')`. HTML responses send `Cache-Control: no-cache`. Text responses (HTML, JavaScript, CSS, SVG, JSON, XML, TXT) are gzip-compressed when the request accepts gzip, with `Vary: Accept-Encoding`, because the baseline ran on `vite preview`, which compresses; without this the measurements cannot be compared (found in Step 24: script transfer 667 kB versus 202 kB). The content type comes from the file extension (html, js, css, svg, png, jpg, webp, woff2, ico, xml, txt, json). It binds to `127.0.0.1`. `pnpm preview` runs it with default port 4173, accepts the `--port` and `--strictPort` arguments that `lighthouserc.cjs` passes (or that command changes with it), and prints a line matching its `startServerReadyPattern`. | NFR-PR-SEC-01; D10 | ### BR-TST: Tests diff --git a/aidlc-docs/features/prerendering/inception/plans/execution-plan.md b/aidlc-docs/features/prerendering/inception/plans/execution-plan.md index 0662d1a..74eadef 100644 --- a/aidlc-docs/features/prerendering/inception/plans/execution-plan.md +++ b/aidlc-docs/features/prerendering/inception/plans/execution-plan.md @@ -152,7 +152,7 @@ Text alternative: after Workflow Planning come Application Design, Functional De |---|---|---|---| | 1. Start migration and script step | Start setup, entry and root document, head tags, post-build script step and guard, hydration of all pages, Sentry and Umami, tests, CI output path and preview | FR-PR-01 to 08, 19, 20, 22; catch-all route of FR-PR-14 and 15; NFR-PR-P-03, P-04, S-02, SEC-01 to 04, Q-01 to 05 | None | | 2. Fade-in, no JavaScript, first photo | `Reveal`, `DeferredImage`, mobile menu without JavaScript, carousels and form fallback, photo preload | FR-PR-09 to 13, 21 | Unit 1 | -| 3. Hosting and targets | URLs without a redirect, real 404, nginx and IIS configuration, upload order, Lighthouse targets | FR-PR-14 to 18; NFR-PR-P-01, P-02, S-01 | Units 1 and 2 | +| 3. Hosting and targets | URLs without a redirect, real 404, nginx and IIS configuration, upload order, Lighthouse targets | FR-PR-14 to 18, 23; NFR-PR-P-01, P-02, S-01 | Units 1 and 2 | **Decision gate after unit 1 (NFR-PR-P-03)**: bundle size and Lighthouse scores are measured against master and shown to the owner. If they got worse, the owner decides whether to continue with Start before units 2 and 3 start. diff --git a/aidlc-docs/features/prerendering/inception/requirements/requirements.md b/aidlc-docs/features/prerendering/inception/requirements/requirements.md index 5041ac6..229e292 100644 --- a/aidlc-docs/features/prerendering/inception/requirements/requirements.md +++ b/aidlc-docs/features/prerendering/inception/requirements/requirements.md @@ -91,6 +91,8 @@ Text alternative: Start renders each page to HTML, a post-build step moves inlin - **FR-PR-17**: Only the client output is deployed to test and production. The Node server output from the build is never uploaded. - **FR-PR-18**: During a production upload, a visitor never receives HTML that refers to a file not yet present. The existing order (assets first, then HTML, then cleanup) must cover all HTML files instead of only `index.html`. HTML stays uncached and hashed assets stay long-cached. +- **FR-PR-23**: nginx and IIS serve text responses (HTML, JavaScript, CSS, SVG, JSON, XML) compressed. Measured on 2026-10-31: `test.andreaslop.nl` returned HTML and JavaScript without `Content-Encoding` and the generated server configuration has no compression rules; the cause on the test host (nginx or its reverse proxy) is settled in Infrastructure Design. Added by the main session as a consequence of the goal; the owner can overrule it. + ### Inline scripts - **FR-PR-19**: A post-build step moves every executable inline script in the generated HTML to a file in `/assets`, preserving execution order. JSON-LD data blocks stay inline, provided a browser test confirms the CSP does not block them. diff --git a/web/config/previewServer.test.ts b/web/config/previewServer.test.ts index 25c00f4..fc8402b 100644 --- a/web/config/previewServer.test.ts +++ b/web/config/previewServer.test.ts @@ -5,6 +5,7 @@ import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { dirname, join, posix, win32 } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { gunzipSync } from 'node:zlib'; import fc from 'fast-check'; import { afterEach, describe, expect, it } from 'vitest'; import { @@ -16,6 +17,7 @@ import { } from '../src/test/requestArbitraries'; import { securityHeaders } from './securityHeaders'; import { + acceptsGzip, contentTypeOf, resolveRequest, startPreviewServer, @@ -263,27 +265,42 @@ describe('contentTypeOf', () => { }); }); +// Groot genoeg voor meerdere stukken in de stroom, en ook niet-ASCII +const GROOT_SCRIPT = 'const tekst = "héllo wörld ✓";\n'.repeat(5000); +// Geen geldige UTF-8: de bytes moeten ongewijzigd aankomen +const BINARY = Buffer.from(Array.from({ length: 2000 }, (_, i) => i % 256)); +const GZIP = { 'Accept-Encoding': 'gzip, deflate, br' }; + interface Answer { status: number; headers: Record; body: string; + // De ontvangen bytes, voor gzip en binaire bestanden + raw: Buffer; } // node:http in plaats van fetch: die laat het pad ongemoeid, ook '..' -const ask = (port: number, path: string, method = 'GET'): Promise => +const ask = ( + port: number, + path: string, + method = 'GET', + headers: Record = {}, +): Promise => new Promise((resolve, reject) => { const req = request( - { host: '127.0.0.1', port, path, method }, + { host: '127.0.0.1', port, path, method, headers }, (response) => { const chunks: Buffer[] = []; response.on('data', (chunk: Buffer) => chunks.push(chunk)); - response.on('end', () => + response.on('end', () => { + const raw = Buffer.concat(chunks); resolve({ status: response.statusCode ?? 0, headers: response.headers, - body: Buffer.concat(chunks).toString('utf8'), - }), - ); + body: raw.toString('utf8'), + raw, + }); + }); }, ); req.on('error', reject); @@ -313,6 +330,9 @@ function siteDir(): string { writeFileSync(join(clientDir, 'over-mij.html'), '

over mij

'); writeFileSync(join(clientDir, '404.html'), '

niet gevonden

'); writeFileSync(join(clientDir, 'assets', 'app.js'), 'boot()'); + writeFileSync(join(clientDir, 'assets', 'groot.js'), GROOT_SCRIPT); + writeFileSync(join(clientDir, 'logo.png'), BINARY); + writeFileSync(join(clientDir, 'font.woff2'), BINARY); // Buiten clientDir: mag nooit bereikbaar zijn writeFileSync(join(root, 'dist', 'geheim.txt'), 'geheim'); return clientDir; @@ -435,6 +455,134 @@ describe('startPreviewServer', () => { }); }); +describe('acceptsGzip', () => { + it('leest Accept-Encoding met gewichten en sterretje', () => { + expect(acceptsGzip(undefined)).toBe(false); + expect(acceptsGzip('')).toBe(false); + expect(acceptsGzip('gzip')).toBe(true); + expect(acceptsGzip('GZIP, deflate')).toBe(true); + expect(acceptsGzip('br;q=1.0, gzip;q=0.5')).toBe(true); + expect(acceptsGzip('gzip;q=0')).toBe(false); + expect(acceptsGzip('gzip; q=0.0, *')).toBe(false); + expect(acceptsGzip('*')).toBe(true); + expect(acceptsGzip('*;q=0')).toBe(false); + expect(acceptsGzip('deflate, br')).toBe(false); + expect(acceptsGzip('identity')).toBe(false); + }); +}); + +describe('startPreviewServer met compressie (BR-SRV-08)', () => { + it('comprimeert tekst als gzip wordt geaccepteerd en geeft dezelfde bytes terug', async () => { + const port = await serve(siteDir()); + const answer = await ask(port, '/assets/groot.js', 'GET', GZIP); + expect(answer.status).toBe(200); + expect(answer.headers['content-encoding']).toBe('gzip'); + expect(answer.headers['vary']).toBe('Accept-Encoding'); + expect(answer.headers['content-type']).toBe( + 'text/javascript; charset=utf-8', + ); + // Een stroom: geen Content-Length, wel chunked + expect(answer.headers['content-length']).toBeUndefined(); + expect(answer.headers['transfer-encoding']).toBe('chunked'); + expect(answer.raw.length).toBeLessThan(GROOT_SCRIPT.length / 10); + expect(gunzipSync(answer.raw).toString('utf8')).toBe(GROOT_SCRIPT); + }); + + it('stuurt tekst ongecomprimeerd zonder gzip in Accept-Encoding', async () => { + const port = await serve(siteDir()); + const variants: Record[] = [ + {}, + { 'Accept-Encoding': 'identity' }, + { 'Accept-Encoding': 'gzip;q=0' }, + ]; + for (const headers of variants) { + const answer = await ask(port, '/assets/groot.js', 'GET', headers); + expect(answer.headers['content-encoding']).toBeUndefined(); + expect(answer.headers['vary']).toBe('Accept-Encoding'); + expect(answer.headers['content-length']).toBe( + String(Buffer.byteLength(GROOT_SCRIPT)), + ); + expect(answer.body).toBe(GROOT_SCRIPT); + } + }); + + it('comprimeert elk teksttype, met het type van de extensie', async () => { + const clientDir = siteDir(); + for (const name of ['a.svg', 'a.json', 'a.xml', 'a.txt', 'a.css']) { + writeFileSync(join(clientDir, name), `inhoud van ${name}`); + } + const port = await serve(clientDir); + for (const name of [ + 'a.svg', + 'a.json', + 'a.xml', + 'a.txt', + 'a.css', + 'over-mij.html', + ]) { + const answer = await ask(port, `/${name}`, 'GET', GZIP); + expect(answer.headers['content-encoding'], name).toBe('gzip'); + expect(answer.headers['content-type'], name).toBe( + contentTypeOf(name), + ); + expect(gunzipSync(answer.raw).toString('utf8'), name).toBe( + name === 'over-mij.html' + ? '

over mij

' + : `inhoud van ${name}`, + ); + } + }); + + it('laat afbeeldingen en lettertypen ongemoeid', async () => { + const port = await serve(siteDir()); + for (const path of ['/logo.png', '/font.woff2']) { + const answer = await ask(port, path, 'GET', GZIP); + expect(answer.status, path).toBe(200); + expect(answer.headers['content-encoding'], path).toBeUndefined(); + expect(answer.headers['vary'], path).toBeUndefined(); + expect(answer.headers['content-length'], path).toBe( + String(BINARY.length), + ); + expect(answer.raw.equals(BINARY), path).toBe(true); + } + }); + + it('comprimeert ook de 404-pagina, met status 404', async () => { + const port = await serve(siteDir()); + const answer = await ask(port, '/nope', 'GET', GZIP); + expect(answer.status).toBe(404); + expect(answer.headers['content-encoding']).toBe('gzip'); + expect(answer.headers['vary']).toBe('Accept-Encoding'); + expect(gunzipSync(answer.raw).toString('utf8')).toBe( + '

niet gevonden

', + ); + const plain = await ask(port, '/nope'); + expect(plain.status).toBe(404); + expect(plain.headers['content-encoding']).toBeUndefined(); + expect(plain.body).toBe('

niet gevonden

'); + }); + + it('stuurt bij HEAD de gzip-headers maar geen inhoud', async () => { + const port = await serve(siteDir()); + const answer = await ask(port, '/assets/groot.js', 'HEAD', GZIP); + expect(answer.status).toBe(200); + expect(answer.raw.length).toBe(0); + expect(answer.headers['content-encoding']).toBe('gzip'); + expect(answer.headers['vary']).toBe('Accept-Encoding'); + expect(answer.headers['content-length']).toBeUndefined(); + }); + + it('houdt de securityheaders ook op gecomprimeerde antwoorden', async () => { + const port = await serve(siteDir()); + const { headers } = await ask(port, '/over-mij', 'GET', GZIP); + for (const [name, value] of Object.entries( + securityHeaders('production'), + )) { + expect(headers[name.toLowerCase()], name).toBe(value); + } + }); +}); + describe('previewServer als opdrachtregel', () => { const script = join( dirname(fileURLToPath(import.meta.url)), diff --git a/web/config/previewServer.ts b/web/config/previewServer.ts index cfc3402..fb37de4 100644 --- a/web/config/previewServer.ts +++ b/web/config/previewServer.ts @@ -14,6 +14,7 @@ import { } from 'node:http'; import { extname, join, resolve, sep, win32 } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { createGzip } from 'node:zlib'; import { NOT_FOUND_FILE, pageFile } from './pageFiles.ts'; import { securityHeaders } from './securityHeaders.ts'; @@ -92,6 +93,41 @@ const CONTENT_TYPES: Record = { export const contentTypeOf = (file: string): string => CONTENT_TYPES[extname(file).toLowerCase()] ?? 'application/octet-stream'; +// Tekst wordt gecomprimeerd, afbeeldingen en lettertypen niet (die zijn het al). +// Waarom: de nulmeting draaide op `vite preview`, dat comprimeert; zonder dit +// zijn de Lighthouse-metingen niet te vergelijken (scripts: 667 kB tegen +// 202 kB, gevonden in stap 24). Productie comprimeert op de host (BR-SRV-08). +const COMPRESSIBLE = new Set([ + '.html', + '.js', + '.css', + '.svg', + '.json', + '.xml', + '.txt', +]); + +export const isCompressible = (file: string): boolean => + COMPRESSIBLE.has(extname(file).toLowerCase()); + +// Staat gzip in Accept-Encoding? 'gzip;q=0' weigert het, '*' staat het toe, +// tenzij gzip zelf genoemd is +export function acceptsGzip(header: string | undefined): boolean { + if (!header) return false; + let gzip: number | undefined; + let star: number | undefined; + for (const part of header.split(',')) { + const [name, ...params] = part.trim().toLowerCase().split(';'); + const q = params + .map((param) => /^\s*q\s*=\s*([0-9.]+)\s*$/.exec(param)) + .find(Boolean); + const weight = q ? Number(q[1]) : 1; + if (name === 'gzip') gzip = weight; + else if (name === '*') star = weight; + } + return (gzip ?? star ?? 0) > 0; +} + // Alleen het pad: query en hash horen niet bij het bestand. Bewust niet via // `new URL`, want die haalt '..' al weg en dan ziet resolveRequest het niet. const pathOf = (target: string) => target.split(/[?#]/, 1)[0]; @@ -151,15 +187,22 @@ export function startPreviewServer( } const headers: Record = { 'Content-Type': contentTypeOf(file), - 'Content-Length': statSync(target).size, }; if (file.endsWith('.html')) headers['Cache-Control'] = 'no-cache'; - send( - response, - status, - headers, - request.method === 'HEAD' ? null : createReadStream(target), - ); + // Gzip als stroom: de lengte is vooraf niet bekend, dus dan geen + // Content-Length (chunked); ook niet bij HEAD, dat geen inhoud stuurt + const compressible = isCompressible(file); + const gzip = + compressible && acceptsGzip(request.headers['accept-encoding']); + if (compressible) headers['Vary'] = 'Accept-Encoding'; + if (gzip) headers['Content-Encoding'] = 'gzip'; + else headers['Content-Length'] = statSync(target).size; + let body: NodeJS.ReadableStream | null = null; + if (request.method !== 'HEAD') { + body = createReadStream(target); + if (gzip) body = body.pipe(createGzip()); + } + send(response, status, headers, body); }; return new Promise((resolvePromise, reject) => {