test: retain execution lifetime failure observations

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

.github/workflows/ci.ymlmodified+1-1

Changes to .github/workflows/ci.yml

		⋯464 linesShow the 464 lines above this hunk@@ -465,7 +465,7 @@ jobs:
465	465	- run: bun install --frozen-lockfile
466	466	- run: cp .env.example .env
467	467	- run: ./buddy key:generate
468		- - name: Qualify actual ENOSPC and fresh runtime recovery in a private filesystem
	468	+ - name: Qualify actual zero-byte and partial ENOSPC writes with fresh runtime recovery
469	469	shell: bash
470	470	run: |
471	471	set -euo pipefail

docs/adr/0031-full-host-filesystem-fixture.mdmodified+9-4

Changes to docs/adr/0031-full-host-filesystem-fixture.md

		⋯12 linesShow the 12 lines above this hunk@@ -13,8 +13,12 @@ Keep signed repositories and the owned build outside the bounded filesystem.
13	13	Place the actual protocol store, candidates and slots inside it. Confirm a real
14	14	release-7 host session, prepare release 9, then publish new signed root/role
15	15	metadata. At the public slot-selection progress boundary, fill an owned sibling
16		-file through real writes until ENOSPC and release only 1 MiB of its allocation.
17		-The larger signed image must make actual partial progress before ENOSPC. Leave
	16	+file through real writes until ENOSPC. Run two independent campaigns: release
	17	+only 1 MiB of its allocation for a partial image write, or leave zero free bytes
	18	+for failure before any image bytes are written. Pin the selected budget in the
	19	+private fixture marker and worker input, and read it back from the filesystem
	20	+before the copy. Require exactly the selected number of retained image bytes,
	21	+including an existing empty image file and its SHA-256 in the zero-byte case. Leave
18	22	the filesystem full until the trial process has exited, including its attempts
19	23	to record failure. Never replace filesystem or SQLite methods with faults.
20	24
		⋯2 linesShow the 2 lines above this hunk@@ -23,7 +27,7 @@ Require the previous known-good image and accepted metadata/security floors to
23	27	survive without a new health confirmation. A pending attempt and selected slot
24	28	must undergo explicit fresh-process metadata recovery and exact-slot abandonment
25	29	before an unrestricted fresh trial can confirm a different slot through actual
26		-new boot/session evidence. Preserve failed partial bytes and the older image.
	30	+new boot/session evidence. Preserve failed partial or empty files and the older image.
27	31
28	32	Copy and hash the full volume before freeing the filler or opening it for
29	33	inspection, then archive later retained states before unmounting the volatile
		⋯5 linesShow the 5 lines above this hunk@@ -35,6 +39,7 @@ no positive filesystem-exhaustion evidence.
35	39	Linux documents tmpfs capacity limits and loss of contents on unmount in
36	40	[tmpfs(5)](https://man7.org/linux/man-pages/man5/tmpfs.5.html). This is real logical
37	41	filesystem exhaustion on a memory-backed filesystem. It does not qualify storage
38		-device exhaustion, persistent-media durability, EDQUOT, EIO, sync failures or
	42	+device exhaustion, a write syscall returning zero without an error,
	43	+persistent-media durability, EDQUOT, EIO, sync failures or
39	44	electrical power loss. Those remain separate acceptance requirements. The fixture
40	45	adds no production storage layer, privileged application command or dependency.

docs/runtime-updates.mdmodified+9-5

Changes to docs/runtime-updates.md

		⋯644 linesShow the 644 lines above this hunk@@ -645,9 +645,13 @@ EIO, sync and physical power-loss campaigns open.
645	645
646	646	The opt-in [full-filesystem qualification](adr/0031-full-host-filesystem-fixture.md)
647	647	uses a separate private Linux mount namespace and a new bounded tmpfs for the
648		-actual store, candidates and slots. It targets real ENOSPC during the image copy
649		-and SQLite failure-state persistence, then exercises explicit metadata recovery,
	648	+actual store, candidates and slots. Separate cases leave either 1 MiB or zero
	649	+free bytes after actual ENOSPC filling. They require exactly a 1 MiB authenticated
	650	+prefix or an existing empty image file, with no runtime launch or health claim.
	651	+Both target real ENOSPC during the image copy and SQLite failure-state
	652	+persistence, then exercise explicit metadata recovery,
650	653	exact-slot abandonment and a fresh host trial in new processes. Retained file
651		-archives are copied and hashed before unmount. This additional campaign is in
652		-qualification; a local skip is not positive ENOSPC evidence. Physical-media,
653		-EDQUOT, EIO, sync and power-loss acceptance remain separate.
	654	+archives are copied and hashed before unmount. CI executes both cases; a local
	655	+skip is not positive ENOSPC evidence. Failure before any image bytes are written
	656	+does not establish behavior for a write syscall that returns zero without an
	657	+error. Physical-media, EDQUOT, EIO, sync and power-loss acceptance remain separate.

tests/fixtures/updates/host-volume-full.tsmodified+17-11

Changes to tests/fixtures/updates/host-volume-full.ts

		⋯50 linesShow the 50 lines above this hunk@@ -51,17 +51,20 @@ async function workerInput(path: string) {
51	51	const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json())
52	52	assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce)
53	53	assert.equal(item.namespace, namespace); identity(item.nonce, 32)
	54	+ assert.ok(item.imageWriteBudgetBytes === 0 || item.imageWriteBudgetBytes === reserveBytes)
	55	+ assert.equal(item.imageWriteBudgetBytes, marker.imageWriteBudgetBytes)
54	56	assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce)
55	57	checkedVolume(volume)
56	58	const reference = object(item.reference)
57	59	assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment'))
58	60	assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository'))
59		- return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: {
	61	+ return { root, volume, imageWriteBudgetBytes: item.imageWriteBudgetBytes, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: {
60	62	directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64),
61	63	} }
62	64	}
63	65
64		-function fillVolume(volume: string) {
	66	+function fillVolume(volume: string, imageWriteBudgetBytes: number) {
	67	+ assert.ok(imageWriteBudgetBytes === 0 || imageWriteBudgetBytes === reserveBytes)
65	68	const before = checkedVolume(volume), path = join(volume, 'filler.bin')
66	69	const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
67	70	let written = 0, full = false
		⋯4 linesShow the 4 lines above this hunk@@ -72,10 +75,11 @@ function fillVolume(volume: string) {
72	75	catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break }
73	76	}
74	77	assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0)
75		- assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes)
	78	+ assert.ok(written > reserveBytes)
	79	+ if (imageWriteBudgetBytes > 0) ftruncateSync(fd, written - imageWriteBudgetBytes)
76	80	const after = checkedVolume(volume)
77		- assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device)
78		- return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' }
	81	+ assert.equal(after.freeBytes, imageWriteBudgetBytes); assert.equal(after.device, before.device)
	82	+ return { before, after, written, fillerBytes: written - imageWriteBudgetBytes, fillError: 'ENOSPC' }
79	83	}
80	84	finally { closeSync(fd) }
81	85	}
		⋯4 linesShow the 4 lines above this hunk@@ -86,7 +90,7 @@ async function limited(inputPath: string) {
86	90	try {
87	91	await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) {
88	92	assert.ok(phases.length < 512); phases.push(event.phase)
89		- if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2))
	93	+ if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume, input.imageWriteBudgetBytes), null, 2))
90	94	} })
91	95	}
92	96	catch (error) { rejected = true; codes = storageErrorCodes(error) }
		⋯28 linesShow the 28 lines above this hunk@@ -121,13 +125,14 @@ async function archiveVolume(volume: string, output: string) {
121	125	}
122	126
123	127	/** Full bounded tmpfs, genuine ENOSPC/SQLite failure, and fresh public recovery. */
124		-export async function qualifyHostFullVolume(directory: string) {
	128	+export async function qualifyHostFullVolume(directory: string, imageWriteBudgetBytes: 0 | 1048576 = reserveBytes) {
125	129	const namespace = privateNamespace()
	130	+ assert.ok(imageWriteBudgetBytes === 0 || imageWriteBudgetBytes === reserveBytes)
126	131	directory = resolve(directory)
127	132	await mkdir(directory, { mode: 0o700 })
128	133	const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex')
129	134	await mkdir(volume, { mode: 0o700 })
130		- await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 })
	135	+ await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace, imageWriteBudgetBytes }), { mode: 0o600 })
131	136	async function command(argv: string[], label: string, timeoutMs = 10000) {
132	137	const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label })
133	138	assert.equal(result.exitCode, 0, result.stderr)
		⋯19 linesShow the 19 lines above this hunk@@ -153,7 +158,7 @@ export async function qualifyHostFullVolume(directory: string) {
153	158	const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })
154	159	await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true })
155	160	const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml')
156		- const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id }
	161	+ const input = { nonce, namespace, imageWriteBudgetBytes, reference, project, repository: source, candidateId: candidate.candidate!.id }
157	162	await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 })
158	163	async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') {
159	164	const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` })
		⋯22 linesShow the 22 lines above this hunk@@ -182,7 +187,8 @@ export async function qualifyHostFullVolume(directory: string) {
182	187	const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version
183	188	assert.equal(version('root'), 2); assert.equal(version('targets'), 3)
184	189	const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath)
185		- assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length))
	190	+ assert.equal(partial.length, imageWriteBudgetBytes); assert.ok(partial.length < image.length)
	191	+ assert.deepEqual(partial, image.subarray(0, partial.length))
186	192	const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin')
187	193	assert.deepEqual(await readFile(previousImage), image)
188	194	await archiveVolume(volume, join(directory, 'failed-volume'))
		⋯19 linesShow the 19 lines above this hunk@@ -208,7 +214,7 @@ export async function qualifyHostFullVolume(directory: string) {
208	214	assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId)
209	215	assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image)
210	216	assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image)
211		- const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
	217	+ const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, imageWriteBudgetBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed }
212	218	await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2))
213	219	return result
214	220	}

tests/integration/runtime-host-storage.test.tsmodified+8-5

Changes to tests/integration/runtime-host-storage.test.ts

		@@ -1,21 +1,24 @@
1	1	import { expect, test } from 'bun:test'
2		-import { mkdtemp, rm } from 'node:fs/promises'
	2	+import { mkdir, mkdtemp, rm } from 'node:fs/promises'
3	3	import { tmpdir } from 'node:os'
4	4	import { join, resolve } from 'node:path'
5	5	import { qualifyHostFullVolume } from '../fixtures/updates/host-volume-full'
6	6
7	7	// Actual privileged Linux namespace qualification only. A skip is not ENOSPC evidence.
8		-test.skipIf(process.env.RUNTIME_HOST_ENOSPC_TEST !== '1')('a full host filesystem preserves accepted trust and requires explicit recovery before fresh confirmation', async () => {
	8	+for (const imageWriteBudgetBytes of [1048576, 0] as const) test.skipIf(process.env.RUNTIME_HOST_ENOSPC_TEST !== '1')(`a full host filesystem with ${imageWriteBudgetBytes} image bytes available requires explicit recovery before fresh confirmation`, async () => {
9	9	const evidence = process.env.RUNTIME_HOST_ENOSPC_EVIDENCE
10	10	const temporary = evidence ? undefined : await mkdtemp(join(tmpdir(), 'drone-host-enospc-'))
11		- const root = evidence ? resolve(evidence) : join(temporary!, 'campaign')
	11	+ if (evidence) await mkdir(resolve(evidence), { recursive: true, mode: 0o700 })
	12	+ const root = evidence ? join(resolve(evidence), imageWriteBudgetBytes === 0 ? 'zero-write' : 'short-write') : join(temporary!, 'campaign')
12	13	let completed = false
13	14	try {
14		- const result = await qualifyHostFullVolume(root)
	15	+ const result = await qualifyHostFullVolume(root, imageWriteBudgetBytes)
15	16	expect(result.filesystemType).toBe('tmpfs')
16	17	expect(result.filesystemBytes).toBe(16777216)
17	18	expect(result.freeBytesAfterFailure).toBe(0)
18		- expect(result.partialImageBytes).toBeGreaterThan(0)
	19	+ expect(result.imageWriteBudgetBytes).toBe(imageWriteBudgetBytes)
	20	+ expect(result.partialImageBytes).toBe(imageWriteBudgetBytes)
	21	+ if (imageWriteBudgetBytes === 0) expect(result.partialImageSha256).toBe('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
19	22	expect(result.partialImageBytes).toBeLessThan(result.authorizedImageBytes)
20	23	expect(result.errorCodes).toContain('ENOSPC')
21	24	expect(result.errorCodes).toContain('SQLITE_FULL')