ReviewOS

also looking at this

stacks/drone

test: qualify zero-byte host image write failures

#61
Merged chris wants to merge codex/runtime-host-zero-write into main
5 files +50 -27
tests/fixtures/updates/host-volume-full.tsmodified+19-12
Changes to tests/fixtures/updates/host-volume-full.ts
@@ -51,17 +51,20 @@ async function workerInput(path: string) {
5151 const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json())
5252 assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce)
5353 assert.equal(item.namespace, namespace); identity(item.nonce, 32)
54 assert.ok(item.imageWriteBudgetBytes === 0 || item.imageWriteBudgetBytes === reserveBytes)
55 assert.equal(item.imageWriteBudgetBytes, marker.imageWriteBudgetBytes)
5456 assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce)
5557 checkedVolume(volume)
5658 const reference = object(item.reference)
5759 assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment'))
5860 assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository'))
59 return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: {
61 return { root, volume, imageWriteBudgetBytes: item.imageWriteBudgetBytes, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: {
6062 directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64),
6163 } }
6264}
6365
64function fillVolume(volume: string) {
66function fillVolume(volume: string, imageWriteBudgetBytes: number) {
67 assert.ok(imageWriteBudgetBytes === 0 || imageWriteBudgetBytes === reserveBytes)
6568 const before = checkedVolume(volume), path = join(volume, 'filler.bin')
6669 const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
6770 let written = 0, full = false
@@ -72,10 +75,11 @@ function fillVolume(volume: string) {
7275 catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break }
7376 }
7477 assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0)
75 assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes)
78 assert.ok(written > reserveBytes)
79 if (imageWriteBudgetBytes > 0) ftruncateSync(fd, written - imageWriteBudgetBytes)
7680 const after = checkedVolume(volume)
77 assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device)
78 return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' }
81 assert.equal(after.freeBytes, imageWriteBudgetBytes); assert.equal(after.device, before.device)
82 return { before, after, written, fillerBytes: written - imageWriteBudgetBytes, fillError: 'ENOSPC' }
7983 }
8084 finally { closeSync(fd) }
8185}
@@ -86,7 +90,7 @@ async function limited(inputPath: string) {
8690 try {
8791 await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) {
8892 assert.ok(phases.length < 512); phases.push(event.phase)
89 if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2))
93 if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume, input.imageWriteBudgetBytes), null, 2))
9094 } })
9195 }
9296 catch (error) { rejected = true; codes = storageErrorCodes(error) }
@@ -121,13 +125,14 @@ async function archiveVolume(volume: string, output: string) {
121125}
122126
123127/** Full bounded tmpfs, genuine ENOSPC/SQLite failure, and fresh public recovery. */
124export async function qualifyHostFullVolume(directory: string) {
128export async function qualifyHostFullVolume(directory: string, imageWriteBudgetBytes: 0 | 1048576 = reserveBytes) {
125129 const namespace = privateNamespace()
130 assert.ok(imageWriteBudgetBytes === 0 || imageWriteBudgetBytes === reserveBytes)
126131 directory = resolve(directory)
127132 await mkdir(directory, { mode: 0o700 })
128133 const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex')
129134 await mkdir(volume, { mode: 0o700 })
130 await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 })
135 await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace, imageWriteBudgetBytes }), { mode: 0o600 })
131136 async function command(argv: string[], label: string, timeoutMs = 10000) {
132137 const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label })
133138 assert.equal(result.exitCode, 0, result.stderr)
@@ -153,7 +158,7 @@ export async function qualifyHostFullVolume(directory: string) {
153158 const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
154159 await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true })
155160 const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml')
156 const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id }
161 const input = { nonce, namespace, imageWriteBudgetBytes, reference, project, repository: source, candidateId: candidate.candidate!.id }
157162 await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 })
158163 async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') {
159164 const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` })
@@ -177,12 +182,14 @@ export async function qualifyHostFullVolume(directory: string) {
177182 await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2))
178183 assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed)
179184 assert.equal(failed.trial!.phase, 'selected'); assert.equal(failed.trial!.health, null)
180 assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'pending')
185 assert.equal(failed.trial!.integrity, imageWriteBudgetBytes === 0 ? 'unavailable' : 'mismatch')
186 assert.equal(failed.store.attempt!.phase, 'pending')
181187 assert.equal(failed.store.releaseFloor, 9)
182188 const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version
183189 assert.equal(version('root'), 2); assert.equal(version('targets'), 3)
184190 const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath)
185 assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length))
191 assert.equal(partial.length, imageWriteBudgetBytes); assert.ok(partial.length < image.length)
192 assert.deepEqual(partial, image.subarray(0, partial.length))
186193 const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin')
187194 assert.deepEqual(await readFile(previousImage), image)
188195 await archiveVolume(volume, join(directory, 'failed-volume'))
@@ -208,7 +215,7 @@ export async function qualifyHostFullVolume(directory: string) {
208215 assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId)
209216 assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image)
210217 assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image)
211 const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
218 const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, imageWriteBudgetBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
212219 await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2))
213220 return result
214221 }