ReviewOS

also looking at this

stacks/drone

test: exercise full host filesystem recovery

#58
Merged chris wants to merge codex/runtime-full-volume-recovery into main
13 files +644 -5
tests/fixtures/updates/host-volume-full.tsadded+235-0
Changes to tests/fixtures/updates/host-volume-full.ts
@@ -0,0 +1,235 @@
1import assert from 'node:assert/strict'
2import { randomBytes } from 'node:crypto'
3import { closeSync, constants, ftruncateSync, lstatSync, openSync, readFileSync, readdirSync, readlinkSync, statfsSync, writeSync } from 'node:fs'
4import { cp, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'
5import { basename, dirname, join, resolve } from 'node:path'
6import { buildRuntime } from '../../../resources/functions/drone/runtime/build'
7import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment'
8import { abandonRuntimeFirmwareTrial, createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store'
9import { runBoundedHostProcess } from '../../../resources/functions/host/process'
10import { createHostSourceProject } from './host-project'
11import { digest, repository, target } from './repository'
12import { storageErrorCodes } from './storage-errors'
13
14const volumeBytes = 16777216, reserveBytes = 1048576
15const clock = { source: 'independent-full-volume-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') }
16
17function privateNamespace() {
18 assert.equal(process.platform, 'linux'); assert.equal(process.env.RUNTIME_HOST_ENOSPC_TEST, '1')
19 assert.equal(process.getuid?.(), 0)
20 const current = readlinkSync('/proc/self/ns/mnt')
21 assert.notEqual(current, readlinkSync('/proc/1/ns/mnt'), 'Use an actual private mount namespace')
22 const mounts = readFileSync('/proc/self/mountinfo', 'utf8')
23 assert.ok(mounts.length > 0 && mounts.length < 2097152)
24 for (const line of mounts.trim().split('\n')) {
25 const [fields, filesystem] = line.split(' - ')
26 assert.ok(fields && filesystem)
27 assert.ok(!fields.split(' ').slice(6).some(field => /^(shared|master|propagate_from):/.test(field)), 'Require recursively private mount propagation before creating a filesystem')
28 }
29 return current
30}
31
32function checkedVolume(volume: string) {
33 const identity = lstatSync(volume), stats = statfsSync(volume)
34 assert.ok(identity.isDirectory() && !identity.isSymbolicLink() && identity.uid === process.getuid?.() && (identity.mode & 0o777) === 0o700)
35 assert.equal(stats.type, 0x01021994, 'Require actual Linux tmpfs')
36 assert.ok(stats.bsize >= 4096 && stats.bsize <= 65536 && reserveBytes % stats.bsize === 0)
37 assert.equal(stats.blocks * stats.bsize, volumeBytes)
38 return { device: identity.dev, blockBytes: stats.bsize, capacityBytes: stats.blocks * stats.bsize, freeBytes: stats.bfree * stats.bsize }
39}
40
41function object(value: unknown) {
42 assert.ok(value && typeof value === 'object' && !Array.isArray(value))
43 return value as Record<string, unknown>
44}
45function text(value: unknown) { assert.equal(typeof value, 'string'); return value as string }
46function identity(value: unknown, length: 32 | 64) { const result = text(value); assert.match(result, new RegExp(`^[a-f0-9]{${length}}$`)); return result }
47
48async function workerInput(path: string) {
49 const namespace = privateNamespace(), root = dirname(resolve(path)), volume = join(root, 'volume')
50 assert.equal(basename(path), 'worker-input.json')
51 const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json())
52 assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce)
53 assert.equal(item.namespace, namespace); identity(item.nonce, 32)
54 assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce)
55 checkedVolume(volume)
56 const reference = object(item.reference)
57 assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment'))
58 assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository'))
59 return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: {
60 directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64),
61 } }
62}
63
64function fillVolume(volume: string) {
65 const before = checkedVolume(volume), path = join(volume, 'filler.bin')
66 const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
67 let written = 0, full = false
68 try {
69 const bytes = Buffer.alloc(65536, 0xa5)
70 while (written <= volumeBytes) {
71 try { const count = writeSync(fd, bytes); assert.ok(count > 0); written += count }
72 catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break }
73 }
74 assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0)
75 assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes)
76 const after = checkedVolume(volume)
77 assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device)
78 return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' }
79 }
80 finally { closeSync(fd) }
81}
82
83async function limited(inputPath: string) {
84 const input = await workerInput(inputPath), phases: string[] = []
85 let rejected = false, codes: string[] = []
86 try {
87 await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) {
88 assert.ok(phases.length < 512); phases.push(event.phase)
89 if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2))
90 } })
91 }
92 catch (error) { rejected = true; codes = storageErrorCodes(error) }
93 assert.equal(rejected, true, 'Actual filesystem exhaustion must reject confirmation')
94 return { errorCodes: codes, phases, volumeAfterFailure: checkedVolume(input.volume) }
95}
96
97function volumeFiles(root: string) {
98 const records: { path: string, byteLength: number, sha256: string }[] = []
99 let total = 0
100 function walk(path: string, relative: string) {
101 for (const name of readdirSync(path).sort()) {
102 const child = join(path, name), next = relative ? `${relative}/${name}` : name, stat = lstatSync(child)
103 assert.ok(!stat.isSymbolicLink())
104 if (stat.isDirectory()) walk(child, next)
105 else {
106 assert.ok(stat.isFile() && stat.nlink === 1 && stat.size <= volumeBytes)
107 assert.ok(records.length < 512); total += stat.size; assert.ok(total <= volumeBytes)
108 records.push({ path: next, byteLength: stat.size, sha256: digest(readFileSync(child)) })
109 }
110 }
111 }
112 walk(root, '')
113 return records
114}
115
116async function archiveVolume(volume: string, output: string) {
117 const before = volumeFiles(volume)
118 await cp(volume, output, { recursive: true, errorOnExist: true, force: false })
119 assert.deepEqual(volumeFiles(output), before)
120 await writeFile(`${output}.json`, JSON.stringify(before, null, 2))
121}
122
123/** Full bounded tmpfs, genuine ENOSPC/SQLite failure, and fresh public recovery. */
124export async function qualifyHostFullVolume(directory: string) {
125 const namespace = privateNamespace()
126 directory = resolve(directory)
127 await mkdir(directory, { mode: 0o700 })
128 const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex')
129 await mkdir(volume, { mode: 0o700 })
130 await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 })
131 async function command(argv: string[], label: string, timeoutMs = 10000) {
132 const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label })
133 assert.equal(result.exitCode, 0, result.stderr)
134 return result
135 }
136 await command(['mount', '--no-mtab', '-t', 'tmpfs', '-o', `size=${volumeBytes},mode=0700,nosuid,nodev,noexec`, 'drone-runtime-test', volume], 'Private bounded update filesystem mount')
137 try {
138 const mounted = checkedVolume(volume)
139 assert.notEqual(mounted.device, lstatSync(directory).dev)
140 await writeFile(join(volume, 'nonce'), nonce, { mode: 0o600 })
141 const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true })
142 const built = await buildRuntime({ project }), image = await readFile(built.artifact)
143 assert.ok(image.length > reserveBytes && image.length < 4 * reserveBytes)
144 const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } })
145 const enrollmentDirectory = join(directory, 'enrollment')
146 const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory })
147 const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(volume, 'store') })
148 const reference = { directory: join(volume, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 }
149 const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
150 const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id })
151 assert.equal(good.knownGood!.description.releaseSequence, 7)
152 await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } })
153 const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
154 await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true })
155 const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml')
156 const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id }
157 await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 })
158 async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') {
159 const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` })
160 await writeFile(join(directory, `${mode}.stdout`), result.stdout)
161 await writeFile(join(directory, `${mode}.stderr`), result.stderr)
162 await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode }))
163 assert.equal(result.exitCode, 0, result.stderr); assert.equal(result.stderr, '')
164 return result.stdout
165 }
166 const rejection = object(JSON.parse(await worker('limited')))
167 assert.ok(Array.isArray(rejection.errorCodes) && rejection.errorCodes.every(code => typeof code === 'string' && code.length <= 64) && rejection.errorCodes.length <= 32)
168 assert.ok(rejection.errorCodes.includes('ENOSPC') && rejection.errorCodes.includes('SQLITE_FULL'))
169 assert.ok(Array.isArray(rejection.phases) && rejection.phases.includes('slot-selected'))
170 assert.ok(!rejection.phases.includes('slot-image-written') && !rejection.phases.includes('runtime-started') && !rejection.phases.includes('runtime-input-committed'))
171 const fullVolume = checkedVolume(volume)
172 assert.equal(fullVolume.freeBytes, 0)
173 await writeFile(join(directory, 'full-filesystem.json'), JSON.stringify(fullVolume, null, 2))
174 await archiveVolume(volume, join(directory, 'full-volume'))
175 await unlink(join(volume, 'filler.bin'))
176 const failed = await inspectRuntimeFirmwareSlots(reference)
177 await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2))
178 assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed)
179 assert.equal(failed.trial!.phase, 'selected'); assert.equal(failed.trial!.health, null)
180 assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'pending')
181 assert.equal(failed.store.releaseFloor, 9)
182 const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version
183 assert.equal(version('root'), 2); assert.equal(version('targets'), 3)
184 const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath)
185 assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length))
186 const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin')
187 assert.deepEqual(await readFile(previousImage), image)
188 await archiveVolume(volume, join(directory, 'failed-volume'))
189 await assert.rejects(abandonRuntimeFirmwareTrial({ ...reference, trialId: failed.trial!.id }), /Resolve metadata/)
190 await writeFile(inputPath, JSON.stringify({ ...input, trialId: failed.trial!.id }))
191 assert.equal(await worker('recover'), 'RECOVERY_COMPLETED\n')
192 const recovered = await inspectRuntimeFirmwareSlots(reference)
193 await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2))
194 assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood)
195 assert.equal(recovered.store.attempt!.phase, 'recovered'); assert.equal(recovered.store.releaseFloor, 9)
196 assert.deepEqual(recovered.store.metadata, failed.store.metadata)
197 assert.equal(await worker('abandon'), 'ABANDON_COMPLETED\n')
198 const abandoned = await inspectRuntimeFirmwareSlots(reference)
199 await writeFile(join(directory, 'abandoned-slots.json'), JSON.stringify(abandoned, null, 2))
200 assert.equal(abandoned.trial!.phase, 'abandoned'); assert.equal(abandoned.trial!.health, null)
201 assert.deepEqual(abandoned.knownGood, good.knownGood)
202 assert.equal(await worker('fresh'), 'FRESH_TRIAL_COMPLETED\n')
203 const fresh = await inspectRuntimeFirmwareSlots(reference)
204 await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2))
205 assert.equal(fresh.knownGood!.description.releaseSequence, 9); assert.equal(fresh.knownGood!.health!.cleanProcessExit, true)
206 assert.notEqual(fresh.knownGood!.id, failed.trial!.id)
207 assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId)
208 assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId)
209 assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image)
210 assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image)
211 const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
212 await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2))
213 return result
214 }
215 finally {
216 try { await archiveVolume(volume, join(directory, 'final-volume')) }
217 finally { await command(['umount', '--no-mtab', volume], 'Private bounded update filesystem unmount') }
218 }
219}
220
221if (import.meta.main) {
222 try {
223 assert.equal(process.argv.length, 4)
224 const mode = process.argv[2], path = resolve(process.argv[3]!)
225 if (mode === 'qualify') console.log(JSON.stringify(await qualifyHostFullVolume(path)))
226 else if (mode === 'limited') console.log(JSON.stringify(await limited(path)))
227 else {
228 const input = await workerInput(path)
229 if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') }
230 else if (mode === 'abandon') { assert.ok(input.trialId); await abandonRuntimeFirmwareTrial({ ...input.reference, trialId: input.trialId }); console.log('ABANDON_COMPLETED') }
231 else { assert.equal(mode, 'fresh'); await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') }
232 }
233 }
234 catch (error) { console.error('Host full-volume fixture failed', error); process.exitCode = 1 }
235}