test: retain bounded browser launch evidence on failure

scripts/qualify-runtime-cold-build.tsadded+134-0

Changes to scripts/qualify-runtime-cold-build.ts

		@@ -0,0 +1,134 @@
	1	+import assert from 'node:assert/strict'
	2	+import { cp, mkdir, readFile, writeFile } from 'node:fs/promises'
	3	+import { join, resolve } from 'node:path'
	4	+import process from 'node:process'
	5	+import { captureColdBuild, coldBuildQualificationTimeout } from '../tests/helpers/runtime-cold-build-supervisor'
	6	+
	7	+// Exercise actual build/test initialization across the former 20-second hook.
	8	+// Tool witnesses forward to real CMake/CTest and retain actual child settlement.
	9	+interface ToolEvent { event: 'start' | 'spawn' | 'exit', stage: string, groupId: number, pid: number, exitCode?: number, elapsedMs?: number }
	10	+assert(process.platform !== 'win32', 'Cold build qualification requires POSIX process groups')
	11	+assert(process.argv[2], 'Provide a new owned evidence directory')
	12	+const reportingFault = process.argv[3] === '--interrupt-before-capture-report'
	13	+assert(process.argv.length <= 4 && (!process.argv[3] || reportingFault), 'Unknown qualification mode')
	14	+const repository = resolve(import.meta.dir, '..'), evidence = resolve(process.argv[2]!)
	15	+await mkdir(evidence)
	16	+if (reportingFault) await mkdir(join(evidence, 'capture-result.json'))
	17	+const project = join(evidence, 'project'), tools = join(evidence, 'tools'), eventsPath = join(evidence, 'tool-events.ndjson')
	18	+await mkdir(join(project, 'firmware'), { recursive: true })
	19	+await mkdir(tools)
	20	+await cp(join(repository, 'firmware/runtime'), join(project, 'firmware/runtime'), { recursive: true })
	21	+await writeFile(join(project, 'bunfig.toml'), '[test]\npreload = []\n')
	22	+const cmakePath = join(project, 'firmware/runtime/CMakeLists.txt'), cmake = await readFile(cmakePath, 'utf8')
	23	+assert(cmake.startsWith('cmake_minimum_required(VERSION 3.20)'), 'Expected recorded runtime CMake entrypoint')
	24	+await writeFile(cmakePath, cmake.replace('cmake_minimum_required(VERSION 3.20)', 'cmake_minimum_required(VERSION 3.20)\nexecute_process(COMMAND "${CMAKE_COMMAND}" -E sleep 21 COMMAND_ERROR_IS_FATAL ANY)'))
	25	+assert(!await Bun.file(join(project, 'storage/runtime-build/host/CMakeCache.txt')).exists(), 'Qualification must start cold')
	26	+assert(!/\s/.test(process.execPath), 'Tool witness needs a single executable shebang path')
	27	+for (const tool of ['cmake', 'ctest']) {
	28	+ const executable = Bun.which(tool)
	29	+ assert(executable, `Missing native tool: ${tool}`)
	30	+ const source = String.raw`#!${process.execPath}
	31	+import { appendFileSync } from 'node:fs'
	32	+const stage = ${JSON.stringify(tool)} === 'ctest' ? 'contracts' : process.argv[2] === '-S' ? 'configure' : 'compile'
	33	+const began = performance.now()
	34	+const owner = { stage, groupId: process.pid, pid: process.pid }
	35	+appendFileSync(${JSON.stringify(eventsPath)}, JSON.stringify({ ...owner, event: 'start' }) + '\n')
	36	+const child = Bun.spawn([${JSON.stringify(executable)}, ...process.argv.slice(2)], { stdin: 'ignore', stdout: 'inherit', stderr: 'inherit' })
	37	+const event = { stage, groupId: process.pid, pid: child.pid }
	38	+appendFileSync(${JSON.stringify(eventsPath)}, JSON.stringify({ ...event, event: 'spawn' }) + '\n')
	39	+if (${reportingFault} && stage === 'configure') {
	40	+ process.kill(child.pid, 0)
	41	+ process.kill(process.ppid, 'SIGKILL')
	42	+}
	43	+const exitCode = await child.exited
	44	+appendFileSync(${JSON.stringify(eventsPath)}, JSON.stringify({ ...event, event: 'exit', exitCode, elapsedMs: performance.now() - began }) + '\n')
	45	+process.exit(exitCode)
	46	+`
	47	+ await writeFile(join(tools, tool), source, { mode: 0o700 })
	48	+}
	49	+const pattern = process.platform === 'linux' ? 'kernel-confirmed full input FIFO' : 'separate supervised boots'
	50	+const buildCommandTimeoutMs = 120000, selectedTestTimeoutMs = process.platform === 'linux' ? 20000 : 5000
	51	+const qualificationTimeoutMs = coldBuildQualificationTimeout(buildCommandTimeoutMs, selectedTestTimeoutMs)
	52	+let result: Awaited<ReturnType<typeof captureColdBuild>> | undefined, failure: unknown
	53	+const began = performance.now()
	54	+try {
	55	+ result = await captureColdBuild([process.execPath, 'test', join(repository, 'tests/unit/runtime.test.ts'), '--test-name-pattern', pattern], { cwd: project, env: { ...process.env, PATH: `${tools}:${process.env.PATH ?? ''}` }, timeoutMs: qualificationTimeoutMs })
	56	+ failure = result.failure
	57	+}
	58	+catch (error) { failure = error }
	59	+// Retain the capture outcome even if a later cleanup observation itself fails.
	60	+const reportingErrors: { stage: string, code: string | null }[] = []
	61	+try {
	62	+ await writeFile(join(evidence, 'capture-result.json'), `${JSON.stringify({ exitCode: result?.exitCode ?? null, elapsedMs: performance.now() - began, failure: failure instanceof Error ? failure.message : failure ? String(failure) : null }, null, 2)}\n`)
	63	+}
	64	+catch (error) {
	65	+ failure ??= error
	66	+ reportingErrors.push({ stage: 'capture-result', code: (error as NodeJS.ErrnoException).code ?? null })
	67	+}
	68	+let eventText = ''
	69	+try { eventText = await readFile(eventsPath, 'utf8') }
	70	+catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error }
	71	+const events: ToolEvent[] = []
	72	+for (const line of eventText.split('\n').filter(Boolean)) {
	73	+ try { events.push(JSON.parse(line)) }
	74	+ catch (error) { failure ??= error }
	75	+}
	76	+const groups = [...new Set(events.map(event => event.groupId))]
	77	+assert(groups.every(group => Number.isSafeInteger(group) && group > 1), 'Tool witnesses must identify owned process groups')
	78	+const probeErrors: { group: number, code: string }[] = []
	79	+const cleanupErrors: { group: number, code: string | null }[] = []
	80	+function alive(group: number): boolean {
	81	+ try { process.kill(-group, 0); return true }
	82	+ catch (error) {
	83	+ const code = (error as NodeJS.ErrnoException).code
	84	+ if (code === 'ESRCH') return false
	85	+ if (code === 'EPERM') {
	86	+ if (probeErrors.length < 20) probeErrors.push({ group, code })
	87	+ return true // Unknown membership remains live until an actual absence probe.
	88	+ }
	89	+ failure ??= error
	90	+ if (cleanupErrors.length < 20) cleanupErrors.push({ group, code: code ?? null })
	91	+ return true
	92	+ }
	93	+}
	94	+const beforeCleanup = groups.filter(alive)
	95	+for (const group of beforeCleanup) {
	96	+ try { process.kill(-group, 'SIGKILL') }
	97	+ catch (error) {
	98	+ if ((error as NodeJS.ErrnoException).code !== 'ESRCH') {
	99	+ failure ??= error
	100	+ cleanupErrors.push({ group, code: (error as NodeJS.ErrnoException).code ?? null })
	101	+ }
	102	+ }
	103	+}
	104	+const cleanupDeadline = performance.now() + 5000
	105	+while (groups.some(alive) && performance.now() < cleanupDeadline) await Bun.sleep(25)
	106	+const afterCleanup = groups.filter(alive)
	107	+await writeFile(join(evidence, 'stdout.log'), result?.stdout ?? '')
	108	+await writeFile(join(evidence, 'stderr.log'), result?.stderr ?? '')
	109	+const report = { evidence: reportingFault ? 'controlled-cold-native-report-refusal' : 'controlled-cold-native-build', platform: process.platform, bun: Bun.version, pattern, reportingFault, configureDelaySeconds: 21, buildCommandTimeoutMs, selectedTestTimeoutMs, qualificationTimeoutMs, subjectExitCode: result?.exitCode ?? null, captureAvailable: result !== undefined, capture: result ? { pid: result.pid, deadlineReached: result.deadlineReached, stopRequested: result.stopRequested, rootCleanupRequired: result.rootCleanupRequired, rootGroupRemaining: result.rootGroupRemaining, probeErrors: result.probeErrors } : null, elapsedMs: performance.now() - began, events, beforeCleanup, afterCleanup, probeErrors, cleanupErrors, reportingErrors, failure: failure instanceof Error ? failure.message : failure ? String(failure) : null, physicalFlightEnabled: false }
	110	+await writeFile(join(evidence, 'receipt.json'), `${JSON.stringify(report, null, 2)}\n`)
	111	+assert.equal(afterCleanup.length, 0, 'Owned tool groups must settle before any evidence removal')
	112	+assert.equal(cleanupErrors.length, 0, 'Unexpected tool cleanup errors must remain qualification failures')
	113	+assert(result, 'Actual parent capture must be available')
	114	+assert.equal(result.rootGroupRemaining, false, 'The actual parent process group must settle')
	115	+if (reportingFault) {
	116	+ assert.notEqual(result.exitCode, 0, 'The actual test parent must have been interrupted')
	117	+ assert(failure instanceof Error && (failure as NodeJS.ErrnoException).code === 'EISDIR', 'Retain the actual report refusal')
	118	+ assert(reportingErrors.some(error => error.stage === 'capture-result' && error.code === 'EISDIR'))
	119	+ assert(beforeCleanup.length > 0, 'Reporting refusal must encounter a real live detached tool group')
	120	+}
	121	+else {
	122	+ if (failure) throw failure
	123	+ assert.equal(result.exitCode, 0, 'Original selected native runtime test must pass')
	124	+ assert.equal(result.rootCleanupRequired, false, 'A successful qualification must require no parent group cleanup')
	125	+ assert.match(result.stderr, /1 pass/)
	126	+ assert.match(result.stderr, /0 fail/)
	127	+ assert.equal(beforeCleanup.length, 0, 'A successful build must require no controller cleanup')
	128	+ const exits = events.filter(event => event.event === 'exit')
	129	+ assert.deepEqual(exits.map(event => event.stage), ['configure', 'compile', 'contracts'])
	130	+ assert(exits.every(event => event.exitCode === 0), 'Real CMake and CTest must exit cleanly')
	131	+ assert(exits[0]!.elapsedMs! >= 21000, 'Actual configure must cross the former setup hook')
	132	+ assert(await Bun.file(join(project, 'storage/runtime-build/host/drone_runtime_host')).exists(), 'Real compiler output must exist')
	133	+}
	134	+console.log(JSON.stringify(report))