ReviewOS

also looking at this

stacks/drone

test: qualify partial host image writes and recovery

#57
Merged chris wants to merge codex/runtime-file-write-faults into main
9 files +311 -7
tests/fixtures/updates/host-write-limit.tsadded+155-0
Changes to tests/fixtures/updates/host-write-limit.ts
@@ -0,0 +1,155 @@
1import assert from 'node:assert/strict'
2import { randomBytes } from 'node:crypto'
3import { mkdir, readFile, writeFile } from 'node:fs/promises'
4import { basename, dirname, join, resolve } from 'node:path'
5import { buildRuntime } from '../../../resources/functions/drone/runtime/build'
6import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment'
7import { createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store'
8import { runBoundedHostProcess } from '../../../resources/functions/host/process'
9import { createHostSourceProject } from './host-project'
10import { digest, repository, target } from './repository'
11
12const fileLimitBytes = 1048576
13const clock = { source: 'independent-resource-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') }
14
15function errorCodes(error: unknown) {
16 const pending = [error], seen = new Set<unknown>(), codes: string[] = []
17 while (pending.length && seen.size < 32) {
18 const item = pending.shift()
19 if (!item || typeof item !== 'object' || seen.has(item)) continue
20 seen.add(item)
21 const code = Object.getOwnPropertyDescriptor(item, 'code')?.value
22 if ((typeof code === 'string' || typeof code === 'number') && String(code).length <= 64) codes.push(String(code))
23 const cause = Object.getOwnPropertyDescriptor(item, 'cause')?.value
24 if (cause !== undefined) pending.push(cause)
25 const errors = Object.getOwnPropertyDescriptor(item, 'errors')?.value
26 if (Array.isArray(errors)) pending.push(...errors.slice(0, 32))
27 }
28 return codes
29}
30
31async function fixtureInput(inputPath: string) {
32 assert.equal(basename(inputPath), 'worker-input.json')
33 const root = dirname(resolve(inputPath))
34 const input = await Bun.file(inputPath).json()
35 const marker = await Bun.file(join(root, 'fixture.json')).json()
36 assert.equal(marker.kind, 'host-file-write-limit-v1'); assert.equal(input.nonce, marker.nonce)
37 assert.match(input.nonce, /^[a-f0-9]{32}$/)
38 assert.equal(input.reference.directory, join(root, 'store'))
39 assert.equal(input.reference.enrollmentDirectory, join(root, 'enrollment'))
40 assert.equal(input.project, join(root, 'project')); assert.equal(input.repository, join(root, 'repository'))
41 return input
42}
43
44async function limitedTrial(inputPath: string) {
45 assert.equal(process.env.DRONE_TEST_FILE_LIMIT, String(fileLimitBytes))
46 const input = await fixtureInput(inputPath)
47 const phases: string[] = []
48 let caught = false, codes: string[] = []
49 try {
50 await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, onProgress(event) {
51 assert.ok(phases.length < 512); phases.push(event.phase)
52 } })
53 }
54 catch (error) { caught = true; codes = errorCodes(error) }
55 assert.equal(caught, true, 'The actual kernel file limit must reject the trial')
56 return { errorCodes: codes, phases }
57}
58
59/** Real signed public coordinator, actual short OS write, retained bytes and fresh recovery. */
60export async function qualifyHostFileWriteLimit(directory: string) {
61 directory = resolve(directory)
62 await mkdir(directory, { mode: 0o700 })
63 const nonce = randomBytes(16).toString('hex')
64 await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-file-write-limit-v1', nonce }), { mode: 0o600 })
65 const launcher = join(directory, 'file-write-limit')
66 const compiled = await runBoundedHostProcess(['c++', '-std=c++17', '-O2', join(import.meta.dir, 'file-write-limit.cpp'), '-o', launcher], { timeoutMs: 10000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Host file-limit launcher build' })
67 assert.equal(compiled.exitCode, 0, compiled.stderr)
68 const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true })
69 const built = await buildRuntime({ project }), image = await readFile(built.artifact)
70 assert.ok(image.length > fileLimitBytes && image.length < 4 * fileLimitBytes)
71 const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } })
72 const enrollmentDirectory = join(directory, 'enrollment')
73 const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory })
74 const store = join(directory, 'store'), created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: store })
75 const reference = { directory: store, enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 }
76 const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
77 const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id })
78 assert.equal(good.knownGood!.description.releaseSequence, 7)
79 await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } })
80 const next = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
81 assert.equal(next.store.releaseFloor, 9)
82 await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true })
83 const config = join(directory, 'worker.bunfig.toml'), input = join(directory, 'worker-input.json')
84 await writeFile(config, 'preload=[]\n', { mode: 0o600 })
85 await writeFile(input, JSON.stringify({ nonce, reference, project, repository: source, candidateId: next.candidate!.id }), { mode: 0o600 })
86 const workerEnv = { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }
87 const command = [launcher, String(fileLimitBytes), process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, 'limited', input]
88 const worker = await runBoundedHostProcess(command, { cwd: directory, env: workerEnv, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Actual host image short-write trial' })
89 await writeFile(join(directory, 'worker.stdout'), worker.stdout)
90 await writeFile(join(directory, 'worker.stderr'), worker.stderr)
91 await writeFile(join(directory, 'worker.json'), JSON.stringify({ exitCode: worker.exitCode }))
92 assert.equal(worker.exitCode, 0, worker.stderr)
93 assert.equal(worker.stderr, `FILE_LIMIT_ACTIVE ${fileLimitBytes}\n`)
94 const rejected = JSON.parse(worker.stdout)
95 assert.ok(Array.isArray(rejected.errorCodes) && rejected.errorCodes.length <= 32 && rejected.errorCodes.every((code: unknown) => typeof code === 'string' && code.length <= 64) && rejected.errorCodes.includes('EFBIG'))
96 assert.ok(Array.isArray(rejected.phases) && rejected.phases.length <= 512 && rejected.phases.every((phase: unknown) => typeof phase === 'string' && phase.length <= 64))
97 assert.ok(rejected.phases.includes('slot-selected'))
98 assert.ok(!rejected.phases.includes('slot-image-written') && !rejected.phases.includes('runtime-started') && !rejected.phases.includes('runtime-input-committed'))
99 const failed = await inspectRuntimeFirmwareSlots(reference)
100 await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2))
101 assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed)
102 assert.equal(failed.trial!.phase, 'failed'); assert.equal(failed.trial!.health, null)
103 assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'maintenance')
104 assert.equal(failed.store.releaseFloor, 9)
105 const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version
106 assert.equal(version('root'), 2); assert.equal(version('targets'), 3)
107 const partialPath = join(store, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath)
108 assert.equal(partial.length, fileLimitBytes); assert.deepEqual(partial, image.subarray(0, fileLimitBytes))
109 const oldImage = join(store, 'host-slots', good.knownGood!.id, 'image.bin')
110 assert.deepEqual(await readFile(oldImage), image)
111 async function unrestricted(mode: 'recover' | 'fresh') {
112 const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, input], { cwd: directory, env: { ...workerEnv, DRONE_TEST_FILE_LIMIT: undefined }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Fresh host write-limit ${mode}` })
113 await writeFile(join(directory, `${mode}.stdout`), result.stdout)
114 await writeFile(join(directory, `${mode}.stderr`), result.stderr)
115 await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode }))
116 assert.equal(result.exitCode, 0, result.stderr)
117 assert.equal(result.stderr, '')
118 assert.equal(result.stdout, `${mode === 'recover' ? 'RECOVERY_COMPLETED' : 'FRESH_TRIAL_COMPLETED'}\n`)
119 }
120 await unrestricted('recover')
121 const recovered = await inspectRuntimeFirmwareSlots(reference)
122 await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2))
123 assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood)
124 assert.equal(recovered.store.releaseFloor, 9)
125 assert.deepEqual(recovered.store.metadata, failed.store.metadata)
126 await unrestricted('fresh')
127 const fresh = await inspectRuntimeFirmwareSlots(reference)
128 await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2))
129 assert.notEqual(fresh.knownGood!.id, failed.trial!.id)
130 assert.equal(fresh.knownGood!.description.releaseSequence, 9)
131 assert.equal(fresh.knownGood!.health!.cleanProcessExit, true)
132 assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId)
133 assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId)
134 assert.deepEqual(await readFile(oldImage), image); assert.deepEqual(await readFile(partialPath), partial)
135 assert.deepEqual(await readFile(join(store, 'host-slots', fresh.knownGood!.id, 'image.bin')), image)
136 const result = { evidence: 'host-slot-actual-short-write-v1', sourceRevision, authorizedImageSha256: signed.imageHash, fileLimitBytes, partialImageBytes: partial.length, partialImageSha256: digest(partial), authorizedImageBytes: image.length, errorCodes: rejected.errorCodes as string[], acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
137 await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2))
138 return result
139}
140
141if (import.meta.main) {
142 try {
143 assert.equal(process.argv.length, 4)
144 const mode = process.argv[2], inputPath = resolve(process.argv[3]!)
145 if (mode === 'limited') console.log(JSON.stringify(await limitedTrial(inputPath)))
146 else if (mode === 'recover' || mode === 'fresh') {
147 assert.equal(process.env.DRONE_TEST_FILE_LIMIT, undefined)
148 const input = await fixtureInput(inputPath)
149 if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') }
150 else { await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') }
151 }
152 else { assert.equal(mode, 'qualify'); console.log(JSON.stringify(await qualifyHostFileWriteLimit(inputPath))) }
153 }
154 catch (error) { console.error('Host file-limit fixture failed', error); process.exitCode = 1 }
155}