ReviewOS

also looking at this

stacks/drone

test: exercise full host filesystem recovery

#58
Merged chris wants to merge codex/runtime-full-volume-recovery into main
13 files +644 -5
tests/fixtures/updates/host-write-limit.tsadded+140-0
Changes to tests/fixtures/updates/host-write-limit.ts
@@ -0,0 +1,140 @@
1import assert from 'node:assert/strict'
2import { randomBytes } from 'node:crypto'
3import { mkdir, readFile, writeFile } from 'node:fs/promises'
4import { basename, dirname, join, resolve } from 'node:path'
5import { buildRuntime } from '../../../resources/functions/drone/runtime/build'
6import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment'
7import { createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store'
8import { runBoundedHostProcess } from '../../../resources/functions/host/process'
9import { createHostSourceProject } from './host-project'
10import { digest, repository, target } from './repository'
11import { storageErrorCodes } from './storage-errors'
12
13const fileLimitBytes = 1048576
14const clock = { source: 'independent-resource-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') }
15
16async function fixtureInput(inputPath: string) {
17 assert.equal(basename(inputPath), 'worker-input.json')
18 const root = dirname(resolve(inputPath))
19 const input = await Bun.file(inputPath).json()
20 const marker = await Bun.file(join(root, 'fixture.json')).json()
21 assert.equal(marker.kind, 'host-file-write-limit-v1'); assert.equal(input.nonce, marker.nonce)
22 assert.match(input.nonce, /^[a-f0-9]{32}$/)
23 assert.equal(input.reference.directory, join(root, 'store'))
24 assert.equal(input.reference.enrollmentDirectory, join(root, 'enrollment'))
25 assert.equal(input.project, join(root, 'project')); assert.equal(input.repository, join(root, 'repository'))
26 return input
27}
28
29async function limitedTrial(inputPath: string) {
30 assert.equal(process.env.DRONE_TEST_FILE_LIMIT, String(fileLimitBytes))
31 const input = await fixtureInput(inputPath)
32 const phases: string[] = []
33 let caught = false, codes: string[] = []
34 try {
35 await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, onProgress(event) {
36 assert.ok(phases.length < 512); phases.push(event.phase)
37 } })
38 }
39 catch (error) { caught = true; codes = storageErrorCodes(error) }
40 assert.equal(caught, true, 'The actual kernel file limit must reject the trial')
41 return { errorCodes: codes, phases }
42}
43
44/** Real signed public coordinator, actual short OS write, retained bytes and fresh recovery. */
45export async function qualifyHostFileWriteLimit(directory: string) {
46 directory = resolve(directory)
47 await mkdir(directory, { mode: 0o700 })
48 const nonce = randomBytes(16).toString('hex')
49 await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-file-write-limit-v1', nonce }), { mode: 0o600 })
50 const launcher = join(directory, 'file-write-limit')
51 const compiled = await runBoundedHostProcess(['c++', '-std=c++17', '-O2', join(import.meta.dir, 'file-write-limit.cpp'), '-o', launcher], { timeoutMs: 10000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Host file-limit launcher build' })
52 assert.equal(compiled.exitCode, 0, compiled.stderr)
53 const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true })
54 const built = await buildRuntime({ project }), image = await readFile(built.artifact)
55 assert.ok(image.length > fileLimitBytes && image.length < 4 * fileLimitBytes)
56 const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } })
57 const enrollmentDirectory = join(directory, 'enrollment')
58 const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory })
59 const store = join(directory, 'store'), created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: store })
60 const reference = { directory: store, enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 }
61 const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
62 const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id })
63 assert.equal(good.knownGood!.description.releaseSequence, 7)
64 await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } })
65 const next = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
66 assert.equal(next.store.releaseFloor, 9)
67 await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true })
68 const config = join(directory, 'worker.bunfig.toml'), input = join(directory, 'worker-input.json')
69 await writeFile(config, 'preload=[]\n', { mode: 0o600 })
70 await writeFile(input, JSON.stringify({ nonce, reference, project, repository: source, candidateId: next.candidate!.id }), { mode: 0o600 })
71 const workerEnv = { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }
72 const command = [launcher, String(fileLimitBytes), process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, 'limited', input]
73 const worker = await runBoundedHostProcess(command, { cwd: directory, env: workerEnv, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Actual host image short-write trial' })
74 await writeFile(join(directory, 'worker.stdout'), worker.stdout)
75 await writeFile(join(directory, 'worker.stderr'), worker.stderr)
76 await writeFile(join(directory, 'worker.json'), JSON.stringify({ exitCode: worker.exitCode }))
77 assert.equal(worker.exitCode, 0, worker.stderr)
78 assert.equal(worker.stderr, `FILE_LIMIT_ACTIVE ${fileLimitBytes}\n`)
79 const rejected = JSON.parse(worker.stdout)
80 assert.ok(Array.isArray(rejected.errorCodes) && rejected.errorCodes.length <= 32 && rejected.errorCodes.every((code: unknown) => typeof code === 'string' && code.length <= 64) && rejected.errorCodes.includes('EFBIG'))
81 assert.ok(Array.isArray(rejected.phases) && rejected.phases.length <= 512 && rejected.phases.every((phase: unknown) => typeof phase === 'string' && phase.length <= 64))
82 assert.ok(rejected.phases.includes('slot-selected'))
83 assert.ok(!rejected.phases.includes('slot-image-written') && !rejected.phases.includes('runtime-started') && !rejected.phases.includes('runtime-input-committed'))
84 const failed = await inspectRuntimeFirmwareSlots(reference)
85 await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2))
86 assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed)
87 assert.equal(failed.trial!.phase, 'failed'); assert.equal(failed.trial!.health, null)
88 assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'maintenance')
89 assert.equal(failed.store.releaseFloor, 9)
90 const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version
91 assert.equal(version('root'), 2); assert.equal(version('targets'), 3)
92 const partialPath = join(store, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath)
93 assert.equal(partial.length, fileLimitBytes); assert.deepEqual(partial, image.subarray(0, fileLimitBytes))
94 const oldImage = join(store, 'host-slots', good.knownGood!.id, 'image.bin')
95 assert.deepEqual(await readFile(oldImage), image)
96 async function unrestricted(mode: 'recover' | 'fresh') {
97 const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, input], { cwd: directory, env: { ...workerEnv, DRONE_TEST_FILE_LIMIT: undefined }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Fresh host write-limit ${mode}` })
98 await writeFile(join(directory, `${mode}.stdout`), result.stdout)
99 await writeFile(join(directory, `${mode}.stderr`), result.stderr)
100 await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode }))
101 assert.equal(result.exitCode, 0, result.stderr)
102 assert.equal(result.stderr, '')
103 assert.equal(result.stdout, `${mode === 'recover' ? 'RECOVERY_COMPLETED' : 'FRESH_TRIAL_COMPLETED'}\n`)
104 }
105 await unrestricted('recover')
106 const recovered = await inspectRuntimeFirmwareSlots(reference)
107 await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2))
108 assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood)
109 assert.equal(recovered.store.releaseFloor, 9)
110 assert.deepEqual(recovered.store.metadata, failed.store.metadata)
111 await unrestricted('fresh')
112 const fresh = await inspectRuntimeFirmwareSlots(reference)
113 await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2))
114 assert.notEqual(fresh.knownGood!.id, failed.trial!.id)
115 assert.equal(fresh.knownGood!.description.releaseSequence, 9)
116 assert.equal(fresh.knownGood!.health!.cleanProcessExit, true)
117 assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId)
118 assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId)
119 assert.deepEqual(await readFile(oldImage), image); assert.deepEqual(await readFile(partialPath), partial)
120 assert.deepEqual(await readFile(join(store, 'host-slots', fresh.knownGood!.id, 'image.bin')), image)
121 const result = { evidence: 'host-slot-actual-short-write-v1', sourceRevision, authorizedImageSha256: signed.imageHash, fileLimitBytes, partialImageBytes: partial.length, partialImageSha256: digest(partial), authorizedImageBytes: image.length, errorCodes: rejected.errorCodes as string[], acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
122 await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2))
123 return result
124}
125
126if (import.meta.main) {
127 try {
128 assert.equal(process.argv.length, 4)
129 const mode = process.argv[2], inputPath = resolve(process.argv[3]!)
130 if (mode === 'limited') console.log(JSON.stringify(await limitedTrial(inputPath)))
131 else if (mode === 'recover' || mode === 'fresh') {
132 assert.equal(process.env.DRONE_TEST_FILE_LIMIT, undefined)
133 const input = await fixtureInput(inputPath)
134 if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') }
135 else { await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') }
136 }
137 else { assert.equal(mode, 'qualify'); console.log(JSON.stringify(await qualifyHostFileWriteLimit(inputPath))) }
138 }
139 catch (error) { console.error('Host file-limit fixture failed', error); process.exitCode = 1 }
140}