| @@ -0,0 +1,234 @@ |
| 1 | +import assert from 'node:assert/strict' |
| 2 | +import { randomBytes } from 'node:crypto' |
| 3 | +import { closeSync, constants, ftruncateSync, lstatSync, openSync, readFileSync, readdirSync, readlinkSync, statfsSync, writeSync } from 'node:fs' |
| 4 | +import { cp, mkdir, readFile, unlink, writeFile } from 'node:fs/promises' |
| 5 | +import { basename, dirname, join, resolve } from 'node:path' |
| 6 | +import { buildRuntime } from '../../../resources/functions/drone/runtime/build' |
| 7 | +import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' |
| 8 | +import { abandonRuntimeFirmwareTrial, createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' |
| 9 | +import { runBoundedHostProcess } from '../../../resources/functions/host/process' |
| 10 | +import { createHostSourceProject } from './host-project' |
| 11 | +import { digest, repository, target } from './repository' |
| 12 | +import { storageErrorCodes } from './storage-errors' |
| 13 | + |
| 14 | +const volumeBytes = 16777216, reserveBytes = 1048576 |
| 15 | +const clock = { source: 'independent-full-volume-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } |
| 16 | + |
| 17 | +function privateNamespace() { |
| 18 | + assert.equal(process.platform, 'linux'); assert.equal(process.env.RUNTIME_HOST_ENOSPC_TEST, '1') |
| 19 | + assert.equal(process.getuid?.(), 0) |
| 20 | + const current = readlinkSync('/proc/self/ns/mnt') |
| 21 | + assert.notEqual(current, readlinkSync('/proc/1/ns/mnt'), 'Use an actual private mount namespace') |
| 22 | + const mounts = readFileSync('/proc/self/mountinfo', 'utf8') |
| 23 | + assert.ok(mounts.length > 0 && mounts.length < 2097152) |
| 24 | + for (const line of mounts.trim().split('\n')) { |
| 25 | + const [fields, filesystem] = line.split(' - ') |
| 26 | + assert.ok(fields && filesystem) |
| 27 | + assert.ok(!fields.split(' ').slice(6).some(field => /^(shared|master|propagate_from):/.test(field)), 'Require recursively private mount propagation before creating a filesystem') |
| 28 | + } |
| 29 | + return current |
| 30 | +} |
| 31 | + |
| 32 | +function checkedVolume(volume: string) { |
| 33 | + const identity = lstatSync(volume), stats = statfsSync(volume) |
| 34 | + assert.ok(identity.isDirectory() && !identity.isSymbolicLink() && identity.uid === process.getuid?.() && (identity.mode & 0o777) === 0o700) |
| 35 | + assert.equal(stats.type, 0x01021994, 'Require actual Linux tmpfs') |
| 36 | + assert.ok(stats.bsize >= 4096 && stats.bsize <= 65536 && reserveBytes % stats.bsize === 0) |
| 37 | + assert.equal(stats.blocks * stats.bsize, volumeBytes) |
| 38 | + return { device: identity.dev, blockBytes: stats.bsize, capacityBytes: stats.blocks * stats.bsize, freeBytes: stats.bfree * stats.bsize } |
| 39 | +} |
| 40 | + |
| 41 | +function object(value: unknown) { |
| 42 | + assert.ok(value && typeof value === 'object' && !Array.isArray(value)) |
| 43 | + return value as Record<string, unknown> |
| 44 | +} |
| 45 | +function text(value: unknown) { assert.equal(typeof value, 'string'); return value as string } |
| 46 | +function identity(value: unknown, length: 32 | 64) { const result = text(value); assert.match(result, new RegExp(`^[a-f0-9]{${length}}$`)); return result } |
| 47 | + |
| 48 | +async function workerInput(path: string) { |
| 49 | + const namespace = privateNamespace(), root = dirname(resolve(path)), volume = join(root, 'volume') |
| 50 | + assert.equal(basename(path), 'worker-input.json') |
| 51 | + const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json()) |
| 52 | + assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce) |
| 53 | + assert.equal(item.namespace, namespace); identity(item.nonce, 32) |
| 54 | + assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce) |
| 55 | + checkedVolume(volume) |
| 56 | + const reference = object(item.reference) |
| 57 | + assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment')) |
| 58 | + assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository')) |
| 59 | + return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: { |
| 60 | + directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64), |
| 61 | + } } |
| 62 | +} |
| 63 | + |
| 64 | +function fillVolume(volume: string) { |
| 65 | + const before = checkedVolume(volume), path = join(volume, 'filler.bin') |
| 66 | + const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) |
| 67 | + let written = 0, full = false |
| 68 | + try { |
| 69 | + const bytes = Buffer.alloc(65536, 0xa5) |
| 70 | + while (written <= volumeBytes) { |
| 71 | + try { const count = writeSync(fd, bytes); assert.ok(count > 0); written += count } |
| 72 | + catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break } |
| 73 | + } |
| 74 | + assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0) |
| 75 | + assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes) |
| 76 | + const after = checkedVolume(volume) |
| 77 | + assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device) |
| 78 | + return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' } |
| 79 | + } |
| 80 | + finally { closeSync(fd) } |
| 81 | +} |
| 82 | + |
| 83 | +async function limited(inputPath: string) { |
| 84 | + const input = await workerInput(inputPath), phases: string[] = [] |
| 85 | + let rejected = false, codes: string[] = [] |
| 86 | + try { |
| 87 | + await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) { |
| 88 | + assert.ok(phases.length < 512); phases.push(event.phase) |
| 89 | + if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2)) |
| 90 | + } }) |
| 91 | + } |
| 92 | + catch (error) { rejected = true; codes = storageErrorCodes(error) } |
| 93 | + assert.equal(rejected, true, 'Actual filesystem exhaustion must reject confirmation') |
| 94 | + return { errorCodes: codes, phases, volumeAfterFailure: checkedVolume(input.volume) } |
| 95 | +} |
| 96 | + |
| 97 | +function volumeFiles(root: string) { |
| 98 | + const records: { path: string, byteLength: number, sha256: string }[] = [] |
| 99 | + let total = 0 |
| 100 | + function walk(path: string, relative: string) { |
| 101 | + for (const name of readdirSync(path).sort()) { |
| 102 | + const child = join(path, name), next = relative ? `${relative}/${name}` : name, stat = lstatSync(child) |
| 103 | + assert.ok(!stat.isSymbolicLink()) |
| 104 | + if (stat.isDirectory()) walk(child, next) |
| 105 | + else { |
| 106 | + assert.ok(stat.isFile() && stat.nlink === 1 && stat.size <= volumeBytes) |
| 107 | + assert.ok(records.length < 512); total += stat.size; assert.ok(total <= volumeBytes) |
| 108 | + records.push({ path: next, byteLength: stat.size, sha256: digest(readFileSync(child)) }) |
| 109 | + } |
| 110 | + } |
| 111 | + } |
| 112 | + walk(root, '') |
| 113 | + return records |
| 114 | +} |
| 115 | + |
| 116 | +async function archiveVolume(volume: string, output: string) { |
| 117 | + const before = volumeFiles(volume) |
| 118 | + await cp(volume, output, { recursive: true, errorOnExist: true, force: false }) |
| 119 | + assert.deepEqual(volumeFiles(output), before) |
| 120 | + await writeFile(`${output}.json`, JSON.stringify(before, null, 2)) |
| 121 | +} |
| 122 | + |
| 123 | + |
| 124 | +export async function qualifyHostFullVolume(directory: string) { |
| 125 | + const namespace = privateNamespace() |
| 126 | + directory = resolve(directory) |
| 127 | + await mkdir(directory, { mode: 0o700 }) |
| 128 | + const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex') |
| 129 | + await mkdir(volume, { mode: 0o700 }) |
| 130 | + await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 }) |
| 131 | + async function command(argv: string[], label: string, timeoutMs = 10000) { |
| 132 | + const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label }) |
| 133 | + assert.equal(result.exitCode, 0, result.stderr) |
| 134 | + return result |
| 135 | + } |
| 136 | + await command(['mount', '--no-mtab', '-t', 'tmpfs', '-o', `size=${volumeBytes},mode=0700,nosuid,nodev,noexec`, 'drone-runtime-test', volume], 'Private bounded update filesystem mount') |
| 137 | + try { |
| 138 | + const mounted = checkedVolume(volume) |
| 139 | + assert.notEqual(mounted.device, lstatSync(directory).dev) |
| 140 | + await writeFile(join(volume, 'nonce'), nonce, { mode: 0o600 }) |
| 141 | + const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true }) |
| 142 | + const built = await buildRuntime({ project }), image = await readFile(built.artifact) |
| 143 | + assert.ok(image.length > reserveBytes && image.length < 4 * reserveBytes) |
| 144 | + const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } }) |
| 145 | + const enrollmentDirectory = join(directory, 'enrollment') |
| 146 | + const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) |
| 147 | + const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(volume, 'store') }) |
| 148 | + const reference = { directory: join(volume, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } |
| 149 | + const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) |
| 150 | + const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id }) |
| 151 | + assert.equal(good.knownGood!.description.releaseSequence, 7) |
| 152 | + await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } }) |
| 153 | + const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) |
| 154 | + await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) |
| 155 | + const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml') |
| 156 | + const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id } |
| 157 | + await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 }) |
| 158 | + async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') { |
| 159 | + const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` }) |
| 160 | + await writeFile(join(directory, `${mode}.stdout`), result.stdout) |
| 161 | + await writeFile(join(directory, `${mode}.stderr`), result.stderr) |
| 162 | + await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode })) |
| 163 | + assert.equal(result.exitCode, 0, result.stderr); assert.equal(result.stderr, '') |
| 164 | + return result.stdout |
| 165 | + } |
| 166 | + const rejection = object(JSON.parse(await worker('limited'))) |
| 167 | + assert.ok(Array.isArray(rejection.errorCodes) && rejection.errorCodes.every(code => typeof code === 'string' && code.length <= 64) && rejection.errorCodes.length <= 32) |
| 168 | + assert.ok(rejection.errorCodes.includes('ENOSPC') && rejection.errorCodes.includes('SQLITE_FULL')) |
| 169 | + assert.ok(Array.isArray(rejection.phases) && rejection.phases.includes('slot-selected')) |
| 170 | + assert.ok(!rejection.phases.includes('slot-image-written') && !rejection.phases.includes('runtime-started') && !rejection.phases.includes('runtime-input-committed')) |
| 171 | + assert.equal(checkedVolume(volume).freeBytes, 0) |
| 172 | + await writeFile(join(directory, 'full-volume.json'), JSON.stringify(checkedVolume(volume), null, 2)) |
| 173 | + await archiveVolume(volume, join(directory, 'full-volume')) |
| 174 | + await unlink(join(volume, 'filler.bin')) |
| 175 | + const failed = await inspectRuntimeFirmwareSlots(reference) |
| 176 | + await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2)) |
| 177 | + assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed) |
| 178 | + assert.equal(failed.trial!.phase, 'selected'); assert.equal(failed.trial!.health, null) |
| 179 | + assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'pending') |
| 180 | + assert.equal(failed.store.releaseFloor, 9) |
| 181 | + const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version |
| 182 | + assert.equal(version('root'), 2); assert.equal(version('targets'), 3) |
| 183 | + const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath) |
| 184 | + assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length)) |
| 185 | + const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin') |
| 186 | + assert.deepEqual(await readFile(previousImage), image) |
| 187 | + await archiveVolume(volume, join(directory, 'failed-volume')) |
| 188 | + await assert.rejects(abandonRuntimeFirmwareTrial({ ...reference, trialId: failed.trial!.id }), /Resolve metadata/) |
| 189 | + await writeFile(inputPath, JSON.stringify({ ...input, trialId: failed.trial!.id })) |
| 190 | + assert.equal(await worker('recover'), 'RECOVERY_COMPLETED\n') |
| 191 | + const recovered = await inspectRuntimeFirmwareSlots(reference) |
| 192 | + await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2)) |
| 193 | + assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood) |
| 194 | + assert.equal(recovered.store.attempt!.phase, 'recovered'); assert.equal(recovered.store.releaseFloor, 9) |
| 195 | + assert.deepEqual(recovered.store.metadata, failed.store.metadata) |
| 196 | + assert.equal(await worker('abandon'), 'ABANDON_COMPLETED\n') |
| 197 | + const abandoned = await inspectRuntimeFirmwareSlots(reference) |
| 198 | + await writeFile(join(directory, 'abandoned-slots.json'), JSON.stringify(abandoned, null, 2)) |
| 199 | + assert.equal(abandoned.trial!.phase, 'abandoned'); assert.equal(abandoned.trial!.health, null) |
| 200 | + assert.deepEqual(abandoned.knownGood, good.knownGood) |
| 201 | + assert.equal(await worker('fresh'), 'FRESH_TRIAL_COMPLETED\n') |
| 202 | + const fresh = await inspectRuntimeFirmwareSlots(reference) |
| 203 | + await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2)) |
| 204 | + assert.equal(fresh.knownGood!.description.releaseSequence, 9); assert.equal(fresh.knownGood!.health!.cleanProcessExit, true) |
| 205 | + assert.notEqual(fresh.knownGood!.id, failed.trial!.id) |
| 206 | + assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId) |
| 207 | + assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId) |
| 208 | + assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image) |
| 209 | + assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image) |
| 210 | + const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: 0, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed } |
| 211 | + await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) |
| 212 | + return result |
| 213 | + } |
| 214 | + finally { |
| 215 | + try { await archiveVolume(volume, join(directory, 'final-volume')) } |
| 216 | + finally { await command(['umount', '--no-mtab', volume], 'Private bounded update filesystem unmount') } |
| 217 | + } |
| 218 | +} |
| 219 | + |
| 220 | +if (import.meta.main) { |
| 221 | + try { |
| 222 | + assert.equal(process.argv.length, 4) |
| 223 | + const mode = process.argv[2], path = resolve(process.argv[3]!) |
| 224 | + if (mode === 'qualify') console.log(JSON.stringify(await qualifyHostFullVolume(path))) |
| 225 | + else if (mode === 'limited') console.log(JSON.stringify(await limited(path))) |
| 226 | + else { |
| 227 | + const input = await workerInput(path) |
| 228 | + if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') } |
| 229 | + else if (mode === 'abandon') { assert.ok(input.trialId); await abandonRuntimeFirmwareTrial({ ...input.reference, trialId: input.trialId }); console.log('ABANDON_COMPLETED') } |
| 230 | + else { assert.equal(mode, 'fresh'); await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') } |
| 231 | + } |
| 232 | + } |
| 233 | + catch (error) { console.error('Host full-volume fixture failed', error); process.exitCode = 1 } |
| 234 | +} |