From 54ddc13f57c5bc4e88b5b47c330b7104238e7410 Mon Sep 17 00:00:00 2001 From: AmanTahiliani Date: Wed, 29 Jul 2026 22:41:11 -0400 Subject: [PATCH] feat(#91): add release fidelity evidence gate --- .gitignore | 1 + docs/testing.md | 55 ++++++++++++++ package.json | 7 +- playwright.fidelity.config.ts | 37 ++++++++++ scripts/release-fidelity/generate.mjs | 86 ++++++++++++++++++++++ scripts/release-fidelity/generate.test.mjs | 27 +++++++ scripts/release-fidelity/verify.mjs | 41 +++++++++++ tests/release-fidelity/capture.spec.ts | 32 ++++++++ 8 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 playwright.fidelity.config.ts create mode 100644 scripts/release-fidelity/generate.mjs create mode 100644 scripts/release-fidelity/generate.test.mjs create mode 100644 scripts/release-fidelity/verify.mjs create mode 100644 tests/release-fidelity/capture.spec.ts diff --git a/.gitignore b/.gitignore index ec9187c..abcf11c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ node_modules/ /test-results/ /playwright-report/ /blob-report/ +/release-fidelity/ /playwright/.cache/ /playwright/.auth/ diff --git a/docs/testing.md b/docs/testing.md index 402e0dd..f7a5981 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -57,3 +57,58 @@ npm run test:visual:prod:update ``` Snapshots live under `tests/visual/__snapshots__/`. + +## Release Fidelity and Owner Review + +Visual regression and mockup fidelity are deliberately separate gates. `npm run test:visual:prod` verifies the candidate against committed regression snapshots; it does not establish that the current UI matches the approved product design. + +After the approved references are available at `docs/product//mockups/`, make an offline review packet. For v0.4.0 the packet pairs `weekend-between-races` and `weekend-live` at the 1280×800 desktop reference viewport, plus `weekend-between-sessions-mobile` at the 390×844 mobile reference viewport. The capture configuration uses the same seeded SQLite data and unreachable OpenF1 endpoint as the visual suite. + +```bash +export RELEASE_FIDELITY_VERSION=v0.4.0 +npm run release:fidelity:capture +npm run release:fidelity:packet +``` + +Open `release-fidelity/v0.4.0/index.html` and review every approved-mockup/candidate pair. The owner, not an automated tool, records a decision in the committed file `docs/release/owner-reviews/v0.4.0.md`: + +```md +# Owner Fidelity Sign-off: v0.4.0 + +- Version: v0.4.0 +- Candidate commit: +- Reviewed by: +- Reviewed on: YYYY-MM-DD +- Decision: approved +``` + +Do not create the file or use `approved` until the owner has reviewed the packet. Once it is committed, the release gate can verify the evidence and decision: + +```bash +npm run release:fidelity:verify +``` + +The verifier checks that the packet exists and the sign-off exists in `HEAD`, names the current candidate commit, and has the required owner fields. It intentionally cannot assess visual fidelity or create approval. + +For a release candidate, `npm run release:fidelity:gate` runs production visual regression first, then capture, packet generation, and owner-evidence verification in that order. It will remain red until the owner has committed the sign-off. + +## Deployment and Rollback + +Build and preserve a SHA-256 record with the deployable binary. Verify the staged binary before replacing the running one: + +```bash +mkdir -p dist +go build -trimpath -o dist/box-box ./cmd/main.go +sha256sum dist/box-box | tee dist/box-box.sha256 +sha256sum -c dist/box-box.sha256 +``` + +Record the current production binary and its SHA before deployment. If owner review rejects the release or deployment fails, restore that saved binary, then verify the restored SHA is byte-identical to the pre-deployment record: + +```bash +sha256sum /srv/box-box/box-box +install -m 0755 /srv/box-box/backups/box-box.previous /srv/box-box/box-box +sha256sum -c /srv/box-box/backups/box-box.previous.sha256 +``` + +Restart and health-check the service using the deployment environment's normal procedure. Keep the candidate SHA, prior SHA, fidelity packet path, and owner sign-off path with the release record. diff --git a/package.json b/package.json index 1669ec5..fe62c24 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,12 @@ "test:visual": "playwright test --config playwright.visual.config.ts", "test:visual:prod": "playwright test --config playwright.visual.prod.config.ts", "test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots", - "test:visual:prod:update": "playwright test --config playwright.visual.prod.config.ts --update-snapshots" + "test:visual:prod:update": "playwright test --config playwright.visual.prod.config.ts --update-snapshots", + "release:fidelity:capture": "playwright test --config playwright.fidelity.config.ts", + "release:fidelity:packet": "node scripts/release-fidelity/generate.mjs", + "release:fidelity:verify": "node scripts/release-fidelity/verify.mjs", + "release:fidelity:gate": "npm run test:visual:prod && npm run release:fidelity:capture && npm run release:fidelity:packet && npm run release:fidelity:verify", + "test:release-fidelity": "node --test scripts/release-fidelity/*.test.mjs" }, "type": "commonjs", "devDependencies": { diff --git a/playwright.fidelity.config.ts b/playwright.fidelity.config.ts new file mode 100644 index 0000000..ca8ac60 --- /dev/null +++ b/playwright.fidelity.config.ts @@ -0,0 +1,37 @@ +import { defineConfig } from '@playwright/test' +import { VIEWPORTS } from './tests/visual/helpers' + +const E2E_DB = '.playwright/boxbox-fidelity.db' +const API_PORT = process.env.BOXBOX_API_PORT ?? '18080' +const WEB_PORT = process.env.BOXBOX_WEB_PORT ?? '15173' + +// Candidate captures are evidence for owner review, not regression baselines. +export default defineConfig({ + testDir: './tests/release-fidelity', + fullyParallel: true, + forbidOnly: !!process.env.CI, + reporter: process.env.CI ? 'github' : 'list', + use: { + baseURL: `http://localhost:${WEB_PORT}`, + colorScheme: 'dark', + trace: 'on-first-retry', + }, + projects: [ + { name: 'desktop', use: { browserName: 'chromium', viewport: VIEWPORTS.desktop } }, + { name: 'mobile', use: { browserName: 'chromium', viewport: VIEWPORTS.mobile } }, + ], + webServer: [ + { + command: `go run ./scripts/seed-e2e-db/main.go --db ${E2E_DB} && BOXBOX_DISABLE_LIVE=1 BOXBOX_OPENF1_BASE_URL=http://127.0.0.1:9 go run ./cmd/main.go --web --db ${E2E_DB} --port ${API_PORT}`, + url: `http://localhost:${API_PORT}/api/v1/race-hub?session_key=9472`, + reuseExistingServer: false, + timeout: 120_000, + }, + { + command: `BOXBOX_API_PORT=${API_PORT} npm run dev --prefix frontend -- --port ${WEB_PORT} --strictPort`, + url: `http://localhost:${WEB_PORT}`, + reuseExistingServer: false, + timeout: 120_000, + }, + ], +}) diff --git a/scripts/release-fidelity/generate.mjs b/scripts/release-fidelity/generate.mjs new file mode 100644 index 0000000..6a1d4aa --- /dev/null +++ b/scripts/release-fidelity/generate.mjs @@ -0,0 +1,86 @@ +import { readdir, readFile, stat, writeFile } from 'node:fs/promises' +import { join, relative, resolve } from 'node:path' + +const references = [ + { viewport: 'desktop', name: 'weekend-between-races' }, + { viewport: 'desktop', name: 'weekend-live' }, + { viewport: 'mobile', name: 'weekend-between-sessions-mobile' }, +] + +function option(name, fallback) { + const index = process.argv.indexOf(`--${name}`) + return index === -1 ? fallback : process.argv[index + 1] +} + +async function filesBelow(directory) { + const entries = await readdir(directory, { withFileTypes: true }) + const files = await Promise.all(entries.map(async (entry) => { + const path = join(directory, entry.name) + return entry.isDirectory() ? filesBelow(path) : [path] + })) + return files.flat() +} + +async function locateMockup(root, name) { + const preferred = join(root, `${name}.png`) + try { + await stat(preferred) + return preferred + } catch { + const matches = (await filesBelow(root)).filter((path) => path.endsWith(`${name}.png`)) + return matches.length === 1 ? matches[0] : null + } +} + +function imageData(path) { + return readFile(path).then((data) => `data:image/png;base64,${data.toString('base64')}`) +} + +export async function generatePacket({ version, evidence, mockups }) { + const candidateRoot = join(evidence, 'candidate') + const pairs = [] + const missing = [] + + try { + await stat(mockups) + } catch { + throw new Error(`Approved mockups directory is missing: ${mockups}. It is supplied by the product reference integration.`) + } + + for (const { viewport, name } of references) { + const candidate = join(candidateRoot, viewport, `${name}.png`) + const reference = await locateMockup(mockups, name) + try { + await stat(candidate) + } catch { + missing.push(`candidate: ${candidate}`) + } + if (!reference) missing.push(`mockup: ${join(mockups, `${name}.png`)}`) + if (reference) pairs.push({ viewport, name, candidate, reference }) + } + + if (missing.length) { + throw new Error(`Release-fidelity packet is incomplete:\n${missing.join('\n')}`) + } + + const cards = await Promise.all(pairs.map(async ({ viewport, name, candidate, reference }) => ` +

${viewport}: ${name}

+
Approved mockup
+
Candidate
+

Reference: ${relative(process.cwd(), reference)}

`)) + const summary = `# Release Fidelity Evidence: ${version}\n\n- Candidate screenshots: candidate/\n- Approved mockups: ${relative(process.cwd(), mockups)}\n- Review: human owner decision required; this packet does not approve the release.\n\nOpen index.html for side-by-side evidence.\n` + const html = `${version} release-fidelity review

${version} Release-Fidelity Review

Visual regression is a separate automated gate. This packet is evidence for a subjective owner review and is not approval.

${cards.join('')}` + await writeFile(join(evidence, 'summary.md'), summary) + await writeFile(join(evidence, 'index.html'), html) +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const version = option('version', process.env.RELEASE_FIDELITY_VERSION ?? 'v0.4.0') + const output = process.env.RELEASE_FIDELITY_OUTPUT ?? 'release-fidelity' + const evidence = resolve(option('evidence', join(output, version))) + const mockups = resolve(option('mockups', join('docs/product', version, 'mockups'))) + generatePacket({ version, evidence, mockups }).then( + () => console.log(`Release-fidelity packet: ${join(evidence, 'index.html')}`), + (error) => { console.error(error.message); process.exitCode = 1 }, + ) +} diff --git a/scripts/release-fidelity/generate.test.mjs b/scripts/release-fidelity/generate.test.mjs new file mode 100644 index 0000000..42a6188 --- /dev/null +++ b/scripts/release-fidelity/generate.test.mjs @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict' +import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { generatePacket } from './generate.mjs' + +test('generates a self-contained side-by-side packet for desktop and mobile', async () => { + const root = await mkdtemp(join(tmpdir(), 'boxbox-fidelity-')) + const evidence = join(root, 'evidence') + const mockups = join(root, 'mockups') + const references = [ + ['desktop', 'weekend-between-races'], + ['desktop', 'weekend-live'], + ['mobile', 'weekend-between-sessions-mobile'], + ] + for (const [viewport, name] of references) { + await mkdir(join(evidence, 'candidate', viewport), { recursive: true }) + await mkdir(mockups, { recursive: true }) + await writeFile(join(evidence, 'candidate', viewport, `${name}.png`), 'candidate') + await writeFile(join(mockups, `${name}.png`), 'mockup') + } + + await generatePacket({ version: 'v-test', evidence, mockups }) + assert.match(await readFile(join(evidence, 'index.html'), 'utf8'), /Approved mockup/) + assert.match(await readFile(join(evidence, 'summary.md'), 'utf8'), /human owner decision required/) +}) diff --git a/scripts/release-fidelity/verify.mjs b/scripts/release-fidelity/verify.mjs new file mode 100644 index 0000000..35759d9 --- /dev/null +++ b/scripts/release-fidelity/verify.mjs @@ -0,0 +1,41 @@ +import { readFile } from 'node:fs/promises' +import { execFileSync } from 'node:child_process' +import { join, resolve } from 'node:path' + +function option(name, fallback) { + const index = process.argv.indexOf(`--${name}`) + return index === -1 ? fallback : process.argv[index + 1] +} + +function git(...args) { + return execFileSync('git', args, { encoding: 'utf8' }).trim() +} + +const version = option('version', process.env.RELEASE_FIDELITY_VERSION ?? 'v0.4.0') +const output = process.env.RELEASE_FIDELITY_OUTPUT ?? 'release-fidelity' +const evidence = resolve(option('evidence', join(output, version))) +const signoff = option('signoff', join('docs/release/owner-reviews', `${version}.md`)) +const required = ['index.html', 'summary.md'] + +try { + for (const file of required) await readFile(join(evidence, file)) + git('cat-file', '-e', `HEAD:${signoff}`) + const text = await readFile(signoff, 'utf8') + const commit = git('rev-parse', 'HEAD') + const fields = [ + ['Version', version], + ['Candidate commit', commit], + ['Reviewed by', '.+'], + ['Reviewed on', '\\d{4}-\\d{2}-\\d{2}'], + ['Decision', 'approved'], + ] + for (const [name, value] of fields) { + if (!new RegExp(`^- ${name}: ${value}$`, 'm').test(text)) { + throw new Error(`${signoff} must contain "- ${name}: ${value}"`) + } + } + console.log(`Release-fidelity evidence and committed owner approval verified for ${version}.`) +} catch (error) { + console.error(`Release-fidelity gate blocked: ${error.message}`) + process.exitCode = 1 +} diff --git a/tests/release-fidelity/capture.spec.ts b/tests/release-fidelity/capture.spec.ts new file mode 100644 index 0000000..e7c09be --- /dev/null +++ b/tests/release-fidelity/capture.spec.ts @@ -0,0 +1,32 @@ +import { mkdir } from 'node:fs/promises' +import { join } from 'node:path' +import { test, type Page } from '@playwright/test' +import { + gotoCommandCenterReady, + gotoLiveEmptyReady, +} from '../visual/helpers' + +const version = process.env.RELEASE_FIDELITY_VERSION ?? 'v0.4.0' +const output = process.env.RELEASE_FIDELITY_OUTPUT ?? 'release-fidelity' + +async function capture(page: Page, name: string, project: string): Promise { + const directory = join(output, version, 'candidate', project) + await mkdir(directory, { recursive: true }) + await page.screenshot({ fullPage: true, path: join(directory, `${name}.png`) }) +} + +test.describe('release-fidelity candidate captures', () => { + test('approved screen set', async ({ page }, testInfo) => { + const project = testInfo.project.name + + if (project === 'desktop') { + await gotoCommandCenterReady(page) + await capture(page, 'weekend-between-races', project) + await gotoLiveEmptyReady(page) + await capture(page, 'weekend-live', project) + } else { + await gotoCommandCenterReady(page) + await capture(page, 'weekend-between-sessions-mobile', project) + } + }) +})