also looking at this
test: qualify partial host image writes and recovery
#57
9 files
+311
-7
| @@ -0,0 +1,155 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { randomBytes } from 'node:crypto' | |
| 3 | import { mkdir, readFile, writeFile } from 'node:fs/promises' | |
| 4 | import { basename, dirname, join, resolve } from 'node:path' | |
| 5 | import { buildRuntime } from '../../../resources/functions/drone/runtime/build' | |
| 6 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 7 | import { createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 8 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 9 | import { createHostSourceProject } from './host-project' | |
| 10 | import { digest, repository, target } from './repository' | |
| 11 | ||
| 12 | const fileLimitBytes = 1048576 | |
| 13 | const clock = { source: 'independent-resource-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 14 | ||
| 15 | function errorCodes(error: unknown) { | |
| 16 | const pending = [error], seen = new Set<unknown>(), codes: string[] = [] | |
| 17 | while (pending.length && seen.size < 32) { | |
| 18 | const item = pending.shift() | |
| 19 | if (!item || typeof item !== 'object' || seen.has(item)) continue | |
| 20 | seen.add(item) | |
| 21 | const code = Object.getOwnPropertyDescriptor(item, 'code')?.value | |
| 22 | if ((typeof code === 'string' || typeof code === 'number') && String(code).length <= 64) codes.push(String(code)) | |
| 23 | const cause = Object.getOwnPropertyDescriptor(item, 'cause')?.value | |
| 24 | if (cause !== undefined) pending.push(cause) | |
| 25 | const errors = Object.getOwnPropertyDescriptor(item, 'errors')?.value | |
| 26 | if (Array.isArray(errors)) pending.push(...errors.slice(0, 32)) | |
| 27 | } | |
| 28 | return codes | |
| 29 | } | |
| 30 | ||
| 31 | async function fixtureInput(inputPath: string) { | |
| 32 | assert.equal(basename(inputPath), 'worker-input.json') | |
| 33 | const root = dirname(resolve(inputPath)) | |
| 34 | const input = await Bun.file(inputPath).json() | |
| 35 | const marker = await Bun.file(join(root, 'fixture.json')).json() | |
| 36 | assert.equal(marker.kind, 'host-file-write-limit-v1'); assert.equal(input.nonce, marker.nonce) | |
| 37 | assert.match(input.nonce, /^[a-f0-9]{32}$/) | |
| 38 | assert.equal(input.reference.directory, join(root, 'store')) | |
| 39 | assert.equal(input.reference.enrollmentDirectory, join(root, 'enrollment')) | |
| 40 | assert.equal(input.project, join(root, 'project')); assert.equal(input.repository, join(root, 'repository')) | |
| 41 | return input | |
| 42 | } | |
| 43 | ||
| 44 | async function limitedTrial(inputPath: string) { | |
| 45 | assert.equal(process.env.DRONE_TEST_FILE_LIMIT, String(fileLimitBytes)) | |
| 46 | const input = await fixtureInput(inputPath) | |
| 47 | const phases: string[] = [] | |
| 48 | let caught = false, codes: string[] = [] | |
| 49 | try { | |
| 50 | await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, onProgress(event) { | |
| 51 | assert.ok(phases.length < 512); phases.push(event.phase) | |
| 52 | } }) | |
| 53 | } | |
| 54 | catch (error) { caught = true; codes = errorCodes(error) } | |
| 55 | assert.equal(caught, true, 'The actual kernel file limit must reject the trial') | |
| 56 | return { errorCodes: codes, phases } | |
| 57 | } | |
| 58 | ||
| 59 | /** Real signed public coordinator, actual short OS write, retained bytes and fresh recovery. */ | |
| 60 | export async function qualifyHostFileWriteLimit(directory: string) { | |
| 61 | directory = resolve(directory) | |
| 62 | await mkdir(directory, { mode: 0o700 }) | |
| 63 | const nonce = randomBytes(16).toString('hex') | |
| 64 | await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-file-write-limit-v1', nonce }), { mode: 0o600 }) | |
| 65 | const launcher = join(directory, 'file-write-limit') | |
| 66 | const compiled = await runBoundedHostProcess(['c++', '-std=c++17', '-O2', join(import.meta.dir, 'file-write-limit.cpp'), '-o', launcher], { timeoutMs: 10000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Host file-limit launcher build' }) | |
| 67 | assert.equal(compiled.exitCode, 0, compiled.stderr) | |
| 68 | const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true }) | |
| 69 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 70 | assert.ok(image.length > fileLimitBytes && image.length < 4 * fileLimitBytes) | |
| 71 | const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } }) | |
| 72 | const enrollmentDirectory = join(directory, 'enrollment') | |
| 73 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 74 | const store = join(directory, 'store'), created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: store }) | |
| 75 | const reference = { directory: store, enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 76 | const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 77 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id }) | |
| 78 | assert.equal(good.knownGood!.description.releaseSequence, 7) | |
| 79 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } }) | |
| 80 | const next = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 81 | assert.equal(next.store.releaseFloor, 9) | |
| 82 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 83 | const config = join(directory, 'worker.bunfig.toml'), input = join(directory, 'worker-input.json') | |
| 84 | await writeFile(config, 'preload=[]\n', { mode: 0o600 }) | |
| 85 | await writeFile(input, JSON.stringify({ nonce, reference, project, repository: source, candidateId: next.candidate!.id }), { mode: 0o600 }) | |
| 86 | const workerEnv = { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } | |
| 87 | const command = [launcher, String(fileLimitBytes), process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, 'limited', input] | |
| 88 | const worker = await runBoundedHostProcess(command, { cwd: directory, env: workerEnv, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Actual host image short-write trial' }) | |
| 89 | await writeFile(join(directory, 'worker.stdout'), worker.stdout) | |
| 90 | await writeFile(join(directory, 'worker.stderr'), worker.stderr) | |
| 91 | await writeFile(join(directory, 'worker.json'), JSON.stringify({ exitCode: worker.exitCode })) | |
| 92 | assert.equal(worker.exitCode, 0, worker.stderr) | |
| 93 | assert.equal(worker.stderr, `FILE_LIMIT_ACTIVE ${fileLimitBytes}\n`) | |
| 94 | const rejected = JSON.parse(worker.stdout) | |
| 95 | assert.ok(Array.isArray(rejected.errorCodes) && rejected.errorCodes.length <= 32 && rejected.errorCodes.every((code: unknown) => typeof code === 'string' && code.length <= 64) && rejected.errorCodes.includes('EFBIG')) | |
| 96 | assert.ok(Array.isArray(rejected.phases) && rejected.phases.length <= 512 && rejected.phases.every((phase: unknown) => typeof phase === 'string' && phase.length <= 64)) | |
| 97 | assert.ok(rejected.phases.includes('slot-selected')) | |
| 98 | assert.ok(!rejected.phases.includes('slot-image-written') && !rejected.phases.includes('runtime-started') && !rejected.phases.includes('runtime-input-committed')) | |
| 99 | const failed = await inspectRuntimeFirmwareSlots(reference) | |
| 100 | await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2)) | |
| 101 | assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed) | |
| 102 | assert.equal(failed.trial!.phase, 'failed'); assert.equal(failed.trial!.health, null) | |
| 103 | assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'maintenance') | |
| 104 | assert.equal(failed.store.releaseFloor, 9) | |
| 105 | const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version | |
| 106 | assert.equal(version('root'), 2); assert.equal(version('targets'), 3) | |
| 107 | const partialPath = join(store, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath) | |
| 108 | assert.equal(partial.length, fileLimitBytes); assert.deepEqual(partial, image.subarray(0, fileLimitBytes)) | |
| 109 | const oldImage = join(store, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 110 | assert.deepEqual(await readFile(oldImage), image) | |
| 111 | async function unrestricted(mode: 'recover' | 'fresh') { | |
| 112 | const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, input], { cwd: directory, env: { ...workerEnv, DRONE_TEST_FILE_LIMIT: undefined }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Fresh host write-limit ${mode}` }) | |
| 113 | await writeFile(join(directory, `${mode}.stdout`), result.stdout) | |
| 114 | await writeFile(join(directory, `${mode}.stderr`), result.stderr) | |
| 115 | await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode })) | |
| 116 | assert.equal(result.exitCode, 0, result.stderr) | |
| 117 | assert.equal(result.stderr, '') | |
| 118 | assert.equal(result.stdout, `${mode === 'recover' ? 'RECOVERY_COMPLETED' : 'FRESH_TRIAL_COMPLETED'}\n`) | |
| 119 | } | |
| 120 | await unrestricted('recover') | |
| 121 | const recovered = await inspectRuntimeFirmwareSlots(reference) | |
| 122 | await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2)) | |
| 123 | assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood) | |
| 124 | assert.equal(recovered.store.releaseFloor, 9) | |
| 125 | assert.deepEqual(recovered.store.metadata, failed.store.metadata) | |
| 126 | await unrestricted('fresh') | |
| 127 | const fresh = await inspectRuntimeFirmwareSlots(reference) | |
| 128 | await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2)) | |
| 129 | assert.notEqual(fresh.knownGood!.id, failed.trial!.id) | |
| 130 | assert.equal(fresh.knownGood!.description.releaseSequence, 9) | |
| 131 | assert.equal(fresh.knownGood!.health!.cleanProcessExit, true) | |
| 132 | assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId) | |
| 133 | assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId) | |
| 134 | assert.deepEqual(await readFile(oldImage), image); assert.deepEqual(await readFile(partialPath), partial) | |
| 135 | assert.deepEqual(await readFile(join(store, 'host-slots', fresh.knownGood!.id, 'image.bin')), image) | |
| 136 | const result = { evidence: 'host-slot-actual-short-write-v1', sourceRevision, authorizedImageSha256: signed.imageHash, fileLimitBytes, partialImageBytes: partial.length, partialImageSha256: digest(partial), authorizedImageBytes: image.length, errorCodes: rejected.errorCodes as string[], acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed } | |
| 137 | await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) | |
| 138 | return result | |
| 139 | } | |
| 140 | ||
| 141 | if (import.meta.main) { | |
| 142 | try { | |
| 143 | assert.equal(process.argv.length, 4) | |
| 144 | const mode = process.argv[2], inputPath = resolve(process.argv[3]!) | |
| 145 | if (mode === 'limited') console.log(JSON.stringify(await limitedTrial(inputPath))) | |
| 146 | else if (mode === 'recover' || mode === 'fresh') { | |
| 147 | assert.equal(process.env.DRONE_TEST_FILE_LIMIT, undefined) | |
| 148 | const input = await fixtureInput(inputPath) | |
| 149 | if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') } | |
| 150 | else { await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') } | |
| 151 | } | |
| 152 | else { assert.equal(mode, 'qualify'); console.log(JSON.stringify(await qualifyHostFileWriteLimit(inputPath))) } | |
| 153 | } | |
| 154 | catch (error) { console.error('Host file-limit fixture failed', error); process.exitCode = 1 } | |
| 155 | } | |