also looking at this
test: exercise full host filesystem recovery
#58
13 files
+644
-5
| @@ -0,0 +1,235 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { randomBytes } from 'node:crypto' | |
| 3 | import { closeSync, constants, ftruncateSync, lstatSync, openSync, readFileSync, readdirSync, readlinkSync, statfsSync, writeSync } from 'node:fs' | |
| 4 | import { cp, mkdir, readFile, unlink, writeFile } from 'node:fs/promises' | |
| 5 | import { basename, dirname, join, resolve } from 'node:path' | |
| 6 | import { buildRuntime } from '../../../resources/functions/drone/runtime/build' | |
| 7 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 8 | import { abandonRuntimeFirmwareTrial, createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 9 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 10 | import { createHostSourceProject } from './host-project' | |
| 11 | import { digest, repository, target } from './repository' | |
| 12 | import { storageErrorCodes } from './storage-errors' | |
| 13 | ||
| 14 | const volumeBytes = 16777216, reserveBytes = 1048576 | |
| 15 | const clock = { source: 'independent-full-volume-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 16 | ||
| 17 | function privateNamespace() { | |
| 18 | assert.equal(process.platform, 'linux'); assert.equal(process.env.RUNTIME_HOST_ENOSPC_TEST, '1') | |
| 19 | assert.equal(process.getuid?.(), 0) | |
| 20 | const current = readlinkSync('/proc/self/ns/mnt') | |
| 21 | assert.notEqual(current, readlinkSync('/proc/1/ns/mnt'), 'Use an actual private mount namespace') | |
| 22 | const mounts = readFileSync('/proc/self/mountinfo', 'utf8') | |
| 23 | assert.ok(mounts.length > 0 && mounts.length < 2097152) | |
| 24 | for (const line of mounts.trim().split('\n')) { | |
| 25 | const [fields, filesystem] = line.split(' - ') | |
| 26 | assert.ok(fields && filesystem) | |
| 27 | assert.ok(!fields.split(' ').slice(6).some(field => /^(shared|master|propagate_from):/.test(field)), 'Require recursively private mount propagation before creating a filesystem') | |
| 28 | } | |
| 29 | return current | |
| 30 | } | |
| 31 | ||
| 32 | function checkedVolume(volume: string) { | |
| 33 | const identity = lstatSync(volume), stats = statfsSync(volume) | |
| 34 | assert.ok(identity.isDirectory() && !identity.isSymbolicLink() && identity.uid === process.getuid?.() && (identity.mode & 0o777) === 0o700) | |
| 35 | assert.equal(stats.type, 0x01021994, 'Require actual Linux tmpfs') | |
| 36 | assert.ok(stats.bsize >= 4096 && stats.bsize <= 65536 && reserveBytes % stats.bsize === 0) | |
| 37 | assert.equal(stats.blocks * stats.bsize, volumeBytes) | |
| 38 | return { device: identity.dev, blockBytes: stats.bsize, capacityBytes: stats.blocks * stats.bsize, freeBytes: stats.bfree * stats.bsize } | |
| 39 | } | |
| 40 | ||
| 41 | function object(value: unknown) { | |
| 42 | assert.ok(value && typeof value === 'object' && !Array.isArray(value)) | |
| 43 | return value as Record<string, unknown> | |
| 44 | } | |
| 45 | function text(value: unknown) { assert.equal(typeof value, 'string'); return value as string } | |
| 46 | function identity(value: unknown, length: 32 | 64) { const result = text(value); assert.match(result, new RegExp(`^[a-f0-9]{${length}}$`)); return result } | |
| 47 | ||
| 48 | async function workerInput(path: string) { | |
| 49 | const namespace = privateNamespace(), root = dirname(resolve(path)), volume = join(root, 'volume') | |
| 50 | assert.equal(basename(path), 'worker-input.json') | |
| 51 | const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json()) | |
| 52 | assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce) | |
| 53 | assert.equal(item.namespace, namespace); identity(item.nonce, 32) | |
| 54 | assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce) | |
| 55 | checkedVolume(volume) | |
| 56 | const reference = object(item.reference) | |
| 57 | assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment')) | |
| 58 | assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository')) | |
| 59 | return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: { | |
| 60 | directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64), | |
| 61 | } } | |
| 62 | } | |
| 63 | ||
| 64 | function fillVolume(volume: string) { | |
| 65 | const before = checkedVolume(volume), path = join(volume, 'filler.bin') | |
| 66 | const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) | |
| 67 | let written = 0, full = false | |
| 68 | try { | |
| 69 | const bytes = Buffer.alloc(65536, 0xa5) | |
| 70 | while (written <= volumeBytes) { | |
| 71 | try { const count = writeSync(fd, bytes); assert.ok(count > 0); written += count } | |
| 72 | catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break } | |
| 73 | } | |
| 74 | assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0) | |
| 75 | assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes) | |
| 76 | const after = checkedVolume(volume) | |
| 77 | assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device) | |
| 78 | return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' } | |
| 79 | } | |
| 80 | finally { closeSync(fd) } | |
| 81 | } | |
| 82 | ||
| 83 | async function limited(inputPath: string) { | |
| 84 | const input = await workerInput(inputPath), phases: string[] = [] | |
| 85 | let rejected = false, codes: string[] = [] | |
| 86 | try { | |
| 87 | await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) { | |
| 88 | assert.ok(phases.length < 512); phases.push(event.phase) | |
| 89 | if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2)) | |
| 90 | } }) | |
| 91 | } | |
| 92 | catch (error) { rejected = true; codes = storageErrorCodes(error) } | |
| 93 | assert.equal(rejected, true, 'Actual filesystem exhaustion must reject confirmation') | |
| 94 | return { errorCodes: codes, phases, volumeAfterFailure: checkedVolume(input.volume) } | |
| 95 | } | |
| 96 | ||
| 97 | function volumeFiles(root: string) { | |
| 98 | const records: { path: string, byteLength: number, sha256: string }[] = [] | |
| 99 | let total = 0 | |
| 100 | function walk(path: string, relative: string) { | |
| 101 | for (const name of readdirSync(path).sort()) { | |
| 102 | const child = join(path, name), next = relative ? `${relative}/${name}` : name, stat = lstatSync(child) | |
| 103 | assert.ok(!stat.isSymbolicLink()) | |
| 104 | if (stat.isDirectory()) walk(child, next) | |
| 105 | else { | |
| 106 | assert.ok(stat.isFile() && stat.nlink === 1 && stat.size <= volumeBytes) | |
| 107 | assert.ok(records.length < 512); total += stat.size; assert.ok(total <= volumeBytes) | |
| 108 | records.push({ path: next, byteLength: stat.size, sha256: digest(readFileSync(child)) }) | |
| 109 | } | |
| 110 | } | |
| 111 | } | |
| 112 | walk(root, '') | |
| 113 | return records | |
| 114 | } | |
| 115 | ||
| 116 | async function archiveVolume(volume: string, output: string) { | |
| 117 | const before = volumeFiles(volume) | |
| 118 | await cp(volume, output, { recursive: true, errorOnExist: true, force: false }) | |
| 119 | assert.deepEqual(volumeFiles(output), before) | |
| 120 | await writeFile(`${output}.json`, JSON.stringify(before, null, 2)) | |
| 121 | } | |
| 122 | ||
| 123 | /** Full bounded tmpfs, genuine ENOSPC/SQLite failure, and fresh public recovery. */ | |
| 124 | export async function qualifyHostFullVolume(directory: string) { | |
| 125 | const namespace = privateNamespace() | |
| 126 | directory = resolve(directory) | |
| 127 | await mkdir(directory, { mode: 0o700 }) | |
| 128 | const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex') | |
| 129 | await mkdir(volume, { mode: 0o700 }) | |
| 130 | await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 }) | |
| 131 | async function command(argv: string[], label: string, timeoutMs = 10000) { | |
| 132 | const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label }) | |
| 133 | assert.equal(result.exitCode, 0, result.stderr) | |
| 134 | return result | |
| 135 | } | |
| 136 | await command(['mount', '--no-mtab', '-t', 'tmpfs', '-o', `size=${volumeBytes},mode=0700,nosuid,nodev,noexec`, 'drone-runtime-test', volume], 'Private bounded update filesystem mount') | |
| 137 | try { | |
| 138 | const mounted = checkedVolume(volume) | |
| 139 | assert.notEqual(mounted.device, lstatSync(directory).dev) | |
| 140 | await writeFile(join(volume, 'nonce'), nonce, { mode: 0o600 }) | |
| 141 | const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true }) | |
| 142 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 143 | assert.ok(image.length > reserveBytes && image.length < 4 * reserveBytes) | |
| 144 | const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } }) | |
| 145 | const enrollmentDirectory = join(directory, 'enrollment') | |
| 146 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 147 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(volume, 'store') }) | |
| 148 | const reference = { directory: join(volume, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 149 | const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 150 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id }) | |
| 151 | assert.equal(good.knownGood!.description.releaseSequence, 7) | |
| 152 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } }) | |
| 153 | const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 154 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 155 | const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml') | |
| 156 | const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id } | |
| 157 | await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 }) | |
| 158 | async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') { | |
| 159 | const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` }) | |
| 160 | await writeFile(join(directory, `${mode}.stdout`), result.stdout) | |
| 161 | await writeFile(join(directory, `${mode}.stderr`), result.stderr) | |
| 162 | await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode })) | |
| 163 | assert.equal(result.exitCode, 0, result.stderr); assert.equal(result.stderr, '') | |
| 164 | return result.stdout | |
| 165 | } | |
| 166 | const rejection = object(JSON.parse(await worker('limited'))) | |
| 167 | assert.ok(Array.isArray(rejection.errorCodes) && rejection.errorCodes.every(code => typeof code === 'string' && code.length <= 64) && rejection.errorCodes.length <= 32) | |
| 168 | assert.ok(rejection.errorCodes.includes('ENOSPC') && rejection.errorCodes.includes('SQLITE_FULL')) | |
| 169 | assert.ok(Array.isArray(rejection.phases) && rejection.phases.includes('slot-selected')) | |
| 170 | assert.ok(!rejection.phases.includes('slot-image-written') && !rejection.phases.includes('runtime-started') && !rejection.phases.includes('runtime-input-committed')) | |
| 171 | const fullVolume = checkedVolume(volume) | |
| 172 | assert.equal(fullVolume.freeBytes, 0) | |
| 173 | await writeFile(join(directory, 'full-filesystem.json'), JSON.stringify(fullVolume, null, 2)) | |
| 174 | await archiveVolume(volume, join(directory, 'full-volume')) | |
| 175 | await unlink(join(volume, 'filler.bin')) | |
| 176 | const failed = await inspectRuntimeFirmwareSlots(reference) | |
| 177 | await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2)) | |
| 178 | assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed) | |
| 179 | assert.equal(failed.trial!.phase, 'selected'); assert.equal(failed.trial!.health, null) | |
| 180 | assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'pending') | |
| 181 | assert.equal(failed.store.releaseFloor, 9) | |
| 182 | const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version | |
| 183 | assert.equal(version('root'), 2); assert.equal(version('targets'), 3) | |
| 184 | const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath) | |
| 185 | assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length)) | |
| 186 | const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 187 | assert.deepEqual(await readFile(previousImage), image) | |
| 188 | await archiveVolume(volume, join(directory, 'failed-volume')) | |
| 189 | await assert.rejects(abandonRuntimeFirmwareTrial({ ...reference, trialId: failed.trial!.id }), /Resolve metadata/) | |
| 190 | await writeFile(inputPath, JSON.stringify({ ...input, trialId: failed.trial!.id })) | |
| 191 | assert.equal(await worker('recover'), 'RECOVERY_COMPLETED\n') | |
| 192 | const recovered = await inspectRuntimeFirmwareSlots(reference) | |
| 193 | await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2)) | |
| 194 | assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood) | |
| 195 | assert.equal(recovered.store.attempt!.phase, 'recovered'); assert.equal(recovered.store.releaseFloor, 9) | |
| 196 | assert.deepEqual(recovered.store.metadata, failed.store.metadata) | |
| 197 | assert.equal(await worker('abandon'), 'ABANDON_COMPLETED\n') | |
| 198 | const abandoned = await inspectRuntimeFirmwareSlots(reference) | |
| 199 | await writeFile(join(directory, 'abandoned-slots.json'), JSON.stringify(abandoned, null, 2)) | |
| 200 | assert.equal(abandoned.trial!.phase, 'abandoned'); assert.equal(abandoned.trial!.health, null) | |
| 201 | assert.deepEqual(abandoned.knownGood, good.knownGood) | |
| 202 | assert.equal(await worker('fresh'), 'FRESH_TRIAL_COMPLETED\n') | |
| 203 | const fresh = await inspectRuntimeFirmwareSlots(reference) | |
| 204 | await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2)) | |
| 205 | assert.equal(fresh.knownGood!.description.releaseSequence, 9); assert.equal(fresh.knownGood!.health!.cleanProcessExit, true) | |
| 206 | assert.notEqual(fresh.knownGood!.id, failed.trial!.id) | |
| 207 | assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId) | |
| 208 | assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId) | |
| 209 | assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image) | |
| 210 | assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image) | |
| 211 | const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed } | |
| 212 | await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) | |
| 213 | return result | |
| 214 | } | |
| 215 | finally { | |
| 216 | try { await archiveVolume(volume, join(directory, 'final-volume')) } | |
| 217 | finally { await command(['umount', '--no-mtab', volume], 'Private bounded update filesystem unmount') } | |
| 218 | } | |
| 219 | } | |
| 220 | ||
| 221 | if (import.meta.main) { | |
| 222 | try { | |
| 223 | assert.equal(process.argv.length, 4) | |
| 224 | const mode = process.argv[2], path = resolve(process.argv[3]!) | |
| 225 | if (mode === 'qualify') console.log(JSON.stringify(await qualifyHostFullVolume(path))) | |
| 226 | else if (mode === 'limited') console.log(JSON.stringify(await limited(path))) | |
| 227 | else { | |
| 228 | const input = await workerInput(path) | |
| 229 | if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') } | |
| 230 | else if (mode === 'abandon') { assert.ok(input.trialId); await abandonRuntimeFirmwareTrial({ ...input.reference, trialId: input.trialId }); console.log('ABANDON_COMPLETED') } | |
| 231 | else { assert.equal(mode, 'fresh'); await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') } | |
| 232 | } | |
| 233 | } | |
| 234 | catch (error) { console.error('Host full-volume fixture failed', error); process.exitCode = 1 } | |
| 235 | } | |