also looking at this
test: exercise full host filesystem recovery
#58
13 files
+644
-5
| @@ -32,7 +32,27 @@ jobs: | ||
| 32 | 32 | - run: python3 -m unittest discover -s tests/hardware |
| 33 | 33 | - name: Verify a kernel-confirmed full input pipe on Linux |
| 34 | 34 | run: bun test tests/unit/runtime.test.ts --test-name-pattern 'kernel-confirmed full input FIFO' |
| 35 | - name: Verify an actual partial host image write and fresh-process recovery | |
| 36 | env: | |
| 37 | RUNTIME_HOST_FILE_LIMIT_EVIDENCE: ${{ runner.temp }}/runtime-host-file-limit-evidence | |
| 38 | run: bun test tests/unit/updates-host-writes.test.ts | |
| 39 | - uses: actions/upload-artifact@v4 | |
| 40 | if: always() | |
| 41 | with: | |
| 42 | name: runtime-host-file-limit-evidence | |
| 43 | if-no-files-found: ignore | |
| 44 | retention-days: 7 | |
| 45 | path: ${{ runner.temp }}/runtime-host-file-limit-evidence | |
| 35 | 46 | - run: ./buddy test |
| 47 | env: | |
| 48 | RUNTIME_HOST_FILE_LIMIT_EVIDENCE: ${{ runner.temp }}/runtime-host-file-limit-full-evidence | |
| 49 | - uses: actions/upload-artifact@v4 | |
| 50 | if: always() | |
| 51 | with: | |
| 52 | name: runtime-host-file-limit-full-evidence | |
| 53 | if-no-files-found: ignore | |
| 54 | retention-days: 7 | |
| 55 | path: ${{ runner.temp }}/runtime-host-file-limit-full-evidence | |
| 36 | 56 | - run: bun run runtime:cli |
| 37 | 57 | - name: Verify signed test-image staging through the portable CLI outside the checkout |
| 38 | 58 | run: bun tests/fixtures/updates/qualify.ts "$RUNNER_TEMP/runtime-update-qualification" "$GITHUB_WORKSPACE/storage/runtime-build/cli/drone" |
| @@ -414,6 +434,26 @@ jobs: | ||
| 414 | 434 | - run: bun install --frozen-lockfile |
| 415 | 435 | - run: cp .env.example .env |
| 416 | 436 | - run: ./buddy key:generate |
| 437 | - name: Qualify actual ENOSPC and fresh runtime recovery in a private filesystem | |
| 438 | shell: bash | |
| 439 | run: | | |
| 440 | set -euo pipefail | |
| 441 | sudo --non-interactive env RUNTIME_HOST_ENOSPC_TEST=1 RUNTIME_HOST_ENOSPC_EVIDENCE="$RUNNER_TEMP/runtime-host-enospc" unshare --mount --propagation private --fork "$(command -v bun)" test ./tests/integration/runtime-host-storage.test.ts | |
| 442 | - name: Make the owned filesystem campaign artifacts readable by the runner | |
| 443 | if: always() | |
| 444 | shell: bash | |
| 445 | run: | | |
| 446 | set -euo pipefail | |
| 447 | if sudo --non-interactive test -d "$RUNNER_TEMP/runtime-host-enospc"; then | |
| 448 | sudo --non-interactive chown -R --no-dereference "$(id -u):$(id -g)" "$RUNNER_TEMP/runtime-host-enospc" | |
| 449 | fi | |
| 450 | - uses: actions/upload-artifact@v4 | |
| 451 | if: always() | |
| 452 | with: | |
| 453 | name: runtime-host-enospc-evidence | |
| 454 | if-no-files-found: ignore | |
| 455 | retention-days: 7 | |
| 456 | path: ${{ runner.temp }}/runtime-host-enospc | |
| 417 | 457 | - name: Qualify accepted purge with real process and mount isolation |
| 418 | 458 | run: sudo --non-interactive env PUBLIC_SAFETY_PURGE_NAMESPACE_TEST=1 unshare --fork --pid --mount-proc "$(command -v bun)" test ./tests/integration/public-safety-purge.test.ts |
| 419 | 459 | firmware: |
| @@ -0,0 +1,36 @@ | ||
| 1 | # Host storage fault fixtures | |
| 2 | ||
| 3 | Status: qualification in progress for #42. Physical deployment remains disabled. | |
| 4 | ||
| 5 | Exercise storage failures through the existing public update coordinator with | |
| 6 | real signed repositories and separate processes. Keep the enrollment, protocol | |
| 7 | store, owned candidates, host slots and current-policy checks in the path. | |
| 8 | Assertions read public status and actual retained files. No filesystem method or | |
| 9 | SDK verifier is replaced. | |
| 10 | ||
| 11 | The first campaign uses POSIX `RLIMIT_FSIZE` on macOS and Linux. A test-only C++17 | |
| 12 | launcher sets and reads back the child's hard and soft limit, ignores `SIGXFSZ` | |
| 13 | so an over-limit write can report `EFBIG`, then executes Bun. This small native | |
| 14 | test exception supplies the kernel resource control that the TypeScript harness | |
| 15 | needs. It is not linked into the application or board runtime. Stacks' bounded | |
| 16 | process capture owns command execution and output. | |
| 17 | ||
| 18 | A disposable signed source checkout adds initialized padding to its host image. | |
| 19 | It retains the neutral kernel behavior and forces the authentic executable past | |
| 20 | the file-size limit. Prepare candidates and the prior known-good image before | |
| 21 | the limited process starts. New signed metadata is accepted during the limited | |
| 22 | trial. The host-slot copy must then make real partial progress and fail before | |
| 23 | runtime launch or health confirmation. | |
| 24 | ||
| 25 | Retain the partial slot file and prior known-good bytes. Verify metadata and | |
| 26 | release floors after rejection and a fresh recovery process. A later unrestricted | |
| 27 | trial needs a new slot and actual fresh runtime health; it cannot complete the | |
| 28 | failed slot by marker inspection. The large padded image is a host fixture, | |
| 29 | without a board memory or timing qualification claim. | |
| 30 | ||
| 31 | This establishes a process-local file-size-limit failure, including an actual | |
| 32 | short image write. It does not establish zero-return writes, ENOSPC, EDQUOT, EIO, | |
| 33 | sync failure, full-medium behavior or electrical power-loss durability. Those | |
| 34 | remain explicit parts of the full storage campaign. Authenticated maintenance, | |
| 35 | board installation, boot recovery and hardware anti-rollback retain their own | |
| 36 | acceptance gates. | |
| @@ -0,0 +1,40 @@ | ||
| 1 | # Full host filesystem qualification | |
| 2 | ||
| 3 | Status: implementation in progress for #42. Physical deployment remains disabled. | |
| 4 | ||
| 5 | Extend the public update-coordinator storage campaign with actual Linux ENOSPC. | |
| 6 | Run explicitly inside a private mount namespace, verified against PID 1's mount | |
| 7 | namespace, with kernel mountinfo readback rejecting shared or slave propagation. | |
| 8 | Mount a new 16 MiB tmpfs only at an exclusively created private fixture | |
| 9 | directory. Read back the filesystem type, capacity, ownership and mount identity. | |
| 10 | No block device, existing mount or physical vehicle is involved. | |
| 11 | ||
| 12 | Keep signed repositories and the owned build outside the bounded filesystem. | |
| 13 | Place the actual protocol store, candidates and slots inside it. Confirm a real | |
| 14 | release-7 host session, prepare release 9, then publish new signed root/role | |
| 15 | metadata. At the public slot-selection progress boundary, fill an owned sibling | |
| 16 | file through real writes until ENOSPC and release only 1 MiB of its allocation. | |
| 17 | The larger signed image must make actual partial progress before ENOSPC. Leave | |
| 18 | the filesystem full until the trial process has exited, including its attempts | |
| 19 | to record failure. Never replace filesystem or SQLite methods with faults. | |
| 20 | ||
| 21 | Inspect retained bytes and public status after removing only the owned filler. | |
| 22 | Require the previous known-good image and accepted metadata/security floors to | |
| 23 | survive without a new health confirmation. A pending attempt and selected slot | |
| 24 | must undergo explicit fresh-process metadata recovery and exact-slot abandonment | |
| 25 | before an unrestricted fresh trial can confirm a different slot through actual | |
| 26 | new boot/session evidence. Preserve failed partial bytes and the older image. | |
| 27 | ||
| 28 | Copy and hash the full volume before freeing the filler or opening it for | |
| 29 | inspection, then archive later retained states before unmounting the volatile | |
| 30 | filesystem. Keep failure observations, status snapshots | |
| 31 | and process results. CI retains these synthetic artifacts for seven days. | |
| 32 | The opt-in Linux qualification must execute in CI; a skipped local test supplies | |
| 33 | no positive filesystem-exhaustion evidence. | |
| 34 | ||
| 35 | Linux documents tmpfs capacity limits and loss of contents on unmount in | |
| 36 | [tmpfs(5)](https://man7.org/linux/man-pages/man5/tmpfs.5.html). This is real logical | |
| 37 | filesystem exhaustion on a memory-backed filesystem. It does not qualify storage | |
| 38 | device exhaustion, persistent-media durability, EDQUOT, EIO, sync failures or | |
| 39 | electrical power loss. Those remain separate acceptance requirements. The fixture | |
| 40 | adds no production storage layer, privileged application command or dependency. | |
| @@ -287,7 +287,14 @@ | ||
| 287 | 287 | "tests/unit/updates-host-policy.test.ts", |
| 288 | 288 | "tests/unit/updates-host-slots-cli.test.ts", |
| 289 | 289 | "tests/fixtures/updates/qualify-host-slots.ts", |
| 290 | "tests/fixtures/updates/host-slot-interruptions.ts" | |
| 290 | "tests/fixtures/updates/host-slot-interruptions.ts", | |
| 291 | "tests/fixtures/updates/host-write-limit.ts", | |
| 292 | "tests/fixtures/updates/file-write-limit.cpp", | |
| 293 | "tests/unit/updates-host-writes.test.ts", | |
| 294 | "docs/adr/0030-host-storage-fault-fixtures.md", | |
| 295 | "tests/fixtures/updates/host-volume-full.ts", | |
| 296 | "tests/integration/runtime-host-storage.test.ts", | |
| 297 | "docs/adr/0031-full-host-filesystem-fixture.md" | |
| 291 | 298 | ], |
| 292 | 299 | "remaining": [ |
| 293 | 300 | "Live runtime/sensor/estimator/authority/mission/payload correlation and measured clock relations", |
| @@ -182,8 +182,10 @@ cuts do not cover every write/sync boundary or actual physical power loss. | ||
| 182 | 182 | The metadata format is bounded to 256 attempts, 10,000 metadata advances and a 128 MiB |
| 183 | 183 | state database. Each attempt allows 64 inputs, 32 MiB of metadata and 512 KiB per |
| 184 | 184 | file. Exhaustion fails closed; history is not automatically deleted or compacted. |
| 185 | Capacity migration, the full storage-fault campaign, authenticated repair/key | |
| 186 | revocation, durable slot coordination and session-bound health remain in #42. | |
| 185 | Capacity migration, the full storage-fault campaign and authenticated repair/key | |
| 186 | revocation remain in #42. [Owned host slots](#owned-host-slots-and-observed-health) | |
| 187 | below add durable host roles and observed boot/session health. Physical board | |
| 188 | selection, boot health and recovery still require their separate qualification. | |
| 187 | 189 | |
| 188 | 190 | ## Protected signed-image preparation |
| 189 | 191 | |
| @@ -581,3 +583,23 @@ An actual CMake output-overflow case also stops before runtime start, retains | ||
| 581 | 583 | release floor 9 through recovery, then observes a fresh healthy trial after the |
| 582 | 584 | private build fault is removed. Build tools use native bounded process capture |
| 583 | 585 | with the existing 120-second command deadline and 2 MB per-stream limits. |
| 586 | ||
| 587 | The [host storage fixture](adr/0030-host-storage-fault-fixtures.md) also runs an | |
| 588 | actual signed large host image under a read-back 1 MiB POSIX file-size limit. | |
| 589 | The slot copy retains exactly 1 MiB of the authenticated prefix before `EFBIG`, | |
| 590 | with no trial runtime launch or health confirmation. New root/target metadata and | |
| 591 | release floor 9 survive, and the prior known-good image remains intact. Recovery | |
| 592 | and the later unrestricted trial run in separate fresh Bun processes. Only the | |
| 593 | new slot can become known-good through actual fresh runtime health; the failed | |
| 594 | partial file stays retained. CI keeps the synthetic campaign artifacts for seven | |
| 595 | days. This file-size-limit evidence leaves zero-return writes, full-medium/quota, | |
| 596 | EIO, sync and physical power-loss campaigns open. | |
| 597 | ||
| 598 | The opt-in [full-filesystem qualification](adr/0031-full-host-filesystem-fixture.md) | |
| 599 | uses a separate private Linux mount namespace and a new bounded tmpfs for the | |
| 600 | actual store, candidates and slots. It targets real ENOSPC during the image copy | |
| 601 | and SQLite failure-state persistence, then exercises explicit metadata recovery, | |
| 602 | exact-slot abandonment and a fresh host trial in new processes. Retained file | |
| 603 | archives are copied and hashed before unmount. This additional campaign is in | |
| 604 | qualification; a local skip is not positive ENOSPC evidence. Physical-media, | |
| 605 | EDQUOT, EIO, sync and power-loss acceptance remain separate. | |
| @@ -0,0 +1,28 @@ | ||
| 1 | // Test-only inherited POSIX file-size limit. No application or board linkage. | |
| 2 | #include <cerrno> | |
| 3 | #include <csignal> | |
| 4 | #include <cstdio> | |
| 5 | #include <cstdlib> | |
| 6 | #include <cstring> | |
| 7 | #include <sys/resource.h> | |
| 8 | #include <unistd.h> | |
| 9 | ||
| 10 | int main(int argc, char** argv) { | |
| 11 | if (argc < 3 || !argv[1][0] || std::strlen(argv[1]) > 8 | |
| 12 | || std::strspn(argv[1], "0123456789") != std::strlen(argv[1])) return 125; | |
| 13 | errno = 0; | |
| 14 | char* end = nullptr; | |
| 15 | const auto requested = std::strtoull(argv[1], &end, 10); | |
| 16 | if (errno || !end || *end || requested < 1 || requested > 16777216) return 125; | |
| 17 | const rlim_t bytes = static_cast<rlim_t>(requested); | |
| 18 | const rlimit limit{bytes, bytes}; | |
| 19 | if (setrlimit(RLIMIT_FSIZE, &limit) != 0 || std::signal(SIGXFSZ, SIG_IGN) == SIG_ERR) return 125; | |
| 20 | rlimit observed{}; | |
| 21 | if (getrlimit(RLIMIT_FSIZE, &observed) != 0 || observed.rlim_cur != bytes || observed.rlim_max != bytes) return 125; | |
| 22 | if (setenv("DRONE_TEST_FILE_LIMIT", argv[1], 1) != 0) return 125; | |
| 23 | std::fprintf(stderr, "FILE_LIMIT_ACTIVE %llu\n", requested); | |
| 24 | std::fflush(stderr); | |
| 25 | execvp(argv[2], argv + 2); | |
| 26 | std::perror("File-limit test exec"); | |
| 27 | return 127; | |
| 28 | } | |
| @@ -1,12 +1,18 @@ | ||
| 1 | 1 | import { cp, mkdir, readFile, writeFile } from 'node:fs/promises' |
| 2 | 2 | import { join } from 'node:path' |
| 3 | 3 | |
| 4 | /** Actual disposable source repo with explicit stop-reply or configure-output faults. */ | |
| 5 | export async function createHostSourceProject(root: string, options: { fault?: 'stop-session' | 'configure-output' } = {}) { | |
| 4 | /** Actual disposable source repo with explicit faults and optional host-only file padding. */ | |
| 5 | export async function createHostSourceProject(root: string, options: { fault?: 'stop-session' | 'configure-output', imagePadding?: true } = {}) { | |
| 6 | 6 | const project = join(root, 'project') |
| 7 | 7 | await mkdir(project, { mode: 0o700 }) |
| 8 | 8 | await mkdir(join(project, 'firmware'), { mode: 0o700 }) |
| 9 | 9 | await cp(join(import.meta.dir, '../../../firmware/runtime'), join(project, 'firmware/runtime'), { recursive: true }) |
| 10 | if (options.imagePadding) { | |
| 11 | const path = join(project, 'firmware/runtime/src/host.cpp'), source = await readFile(path, 'utf8') | |
| 12 | const entry = 'int main(int argc, char** argv) {' | |
| 13 | if (source.split(entry).length !== 2) throw new Error('Owned host entry no longer matches the large-image fixture') | |
| 14 | await writeFile(path, source.replace(entry, 'struct TestImagePadding { unsigned char first; unsigned char bytes[2 * 1024 * 1024]; unsigned char last; };\nvolatile TestImagePadding test_image_padding{1, {}, 1};\n' + entry + '\n if (test_image_padding.first != 1 || test_image_padding.last != 1) return 2;')) | |
| 15 | } | |
| 10 | 16 | if (options.fault === 'stop-session') { |
| 11 | 17 | const path = join(project, 'firmware/runtime/src/host.cpp'), source = await readFile(path, 'utf8') |
| 12 | 18 | const original = 'drone::encode(runtime.status(now), bytes);' |
| @@ -0,0 +1,235 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { randomBytes } from 'node:crypto' | |
| 3 | import { closeSync, constants, ftruncateSync, lstatSync, openSync, readFileSync, readdirSync, readlinkSync, statfsSync, writeSync } from 'node:fs' | |
| 4 | import { cp, mkdir, readFile, unlink, writeFile } from 'node:fs/promises' | |
| 5 | import { basename, dirname, join, resolve } from 'node:path' | |
| 6 | import { buildRuntime } from '../../../resources/functions/drone/runtime/build' | |
| 7 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 8 | import { abandonRuntimeFirmwareTrial, createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 9 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 10 | import { createHostSourceProject } from './host-project' | |
| 11 | import { digest, repository, target } from './repository' | |
| 12 | import { storageErrorCodes } from './storage-errors' | |
| 13 | ||
| 14 | const volumeBytes = 16777216, reserveBytes = 1048576 | |
| 15 | const clock = { source: 'independent-full-volume-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 16 | ||
| 17 | function privateNamespace() { | |
| 18 | assert.equal(process.platform, 'linux'); assert.equal(process.env.RUNTIME_HOST_ENOSPC_TEST, '1') | |
| 19 | assert.equal(process.getuid?.(), 0) | |
| 20 | const current = readlinkSync('/proc/self/ns/mnt') | |
| 21 | assert.notEqual(current, readlinkSync('/proc/1/ns/mnt'), 'Use an actual private mount namespace') | |
| 22 | const mounts = readFileSync('/proc/self/mountinfo', 'utf8') | |
| 23 | assert.ok(mounts.length > 0 && mounts.length < 2097152) | |
| 24 | for (const line of mounts.trim().split('\n')) { | |
| 25 | const [fields, filesystem] = line.split(' - ') | |
| 26 | assert.ok(fields && filesystem) | |
| 27 | assert.ok(!fields.split(' ').slice(6).some(field => /^(shared|master|propagate_from):/.test(field)), 'Require recursively private mount propagation before creating a filesystem') | |
| 28 | } | |
| 29 | return current | |
| 30 | } | |
| 31 | ||
| 32 | function checkedVolume(volume: string) { | |
| 33 | const identity = lstatSync(volume), stats = statfsSync(volume) | |
| 34 | assert.ok(identity.isDirectory() && !identity.isSymbolicLink() && identity.uid === process.getuid?.() && (identity.mode & 0o777) === 0o700) | |
| 35 | assert.equal(stats.type, 0x01021994, 'Require actual Linux tmpfs') | |
| 36 | assert.ok(stats.bsize >= 4096 && stats.bsize <= 65536 && reserveBytes % stats.bsize === 0) | |
| 37 | assert.equal(stats.blocks * stats.bsize, volumeBytes) | |
| 38 | return { device: identity.dev, blockBytes: stats.bsize, capacityBytes: stats.blocks * stats.bsize, freeBytes: stats.bfree * stats.bsize } | |
| 39 | } | |
| 40 | ||
| 41 | function object(value: unknown) { | |
| 42 | assert.ok(value && typeof value === 'object' && !Array.isArray(value)) | |
| 43 | return value as Record<string, unknown> | |
| 44 | } | |
| 45 | function text(value: unknown) { assert.equal(typeof value, 'string'); return value as string } | |
| 46 | function identity(value: unknown, length: 32 | 64) { const result = text(value); assert.match(result, new RegExp(`^[a-f0-9]{${length}}$`)); return result } | |
| 47 | ||
| 48 | async function workerInput(path: string) { | |
| 49 | const namespace = privateNamespace(), root = dirname(resolve(path)), volume = join(root, 'volume') | |
| 50 | assert.equal(basename(path), 'worker-input.json') | |
| 51 | const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json()) | |
| 52 | assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce) | |
| 53 | assert.equal(item.namespace, namespace); identity(item.nonce, 32) | |
| 54 | assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce) | |
| 55 | checkedVolume(volume) | |
| 56 | const reference = object(item.reference) | |
| 57 | assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment')) | |
| 58 | assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository')) | |
| 59 | return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: { | |
| 60 | directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64), | |
| 61 | } } | |
| 62 | } | |
| 63 | ||
| 64 | function fillVolume(volume: string) { | |
| 65 | const before = checkedVolume(volume), path = join(volume, 'filler.bin') | |
| 66 | const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) | |
| 67 | let written = 0, full = false | |
| 68 | try { | |
| 69 | const bytes = Buffer.alloc(65536, 0xa5) | |
| 70 | while (written <= volumeBytes) { | |
| 71 | try { const count = writeSync(fd, bytes); assert.ok(count > 0); written += count } | |
| 72 | catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break } | |
| 73 | } | |
| 74 | assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0) | |
| 75 | assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes) | |
| 76 | const after = checkedVolume(volume) | |
| 77 | assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device) | |
| 78 | return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' } | |
| 79 | } | |
| 80 | finally { closeSync(fd) } | |
| 81 | } | |
| 82 | ||
| 83 | async function limited(inputPath: string) { | |
| 84 | const input = await workerInput(inputPath), phases: string[] = [] | |
| 85 | let rejected = false, codes: string[] = [] | |
| 86 | try { | |
| 87 | await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) { | |
| 88 | assert.ok(phases.length < 512); phases.push(event.phase) | |
| 89 | if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2)) | |
| 90 | } }) | |
| 91 | } | |
| 92 | catch (error) { rejected = true; codes = storageErrorCodes(error) } | |
| 93 | assert.equal(rejected, true, 'Actual filesystem exhaustion must reject confirmation') | |
| 94 | return { errorCodes: codes, phases, volumeAfterFailure: checkedVolume(input.volume) } | |
| 95 | } | |
| 96 | ||
| 97 | function volumeFiles(root: string) { | |
| 98 | const records: { path: string, byteLength: number, sha256: string }[] = [] | |
| 99 | let total = 0 | |
| 100 | function walk(path: string, relative: string) { | |
| 101 | for (const name of readdirSync(path).sort()) { | |
| 102 | const child = join(path, name), next = relative ? `${relative}/${name}` : name, stat = lstatSync(child) | |
| 103 | assert.ok(!stat.isSymbolicLink()) | |
| 104 | if (stat.isDirectory()) walk(child, next) | |
| 105 | else { | |
| 106 | assert.ok(stat.isFile() && stat.nlink === 1 && stat.size <= volumeBytes) | |
| 107 | assert.ok(records.length < 512); total += stat.size; assert.ok(total <= volumeBytes) | |
| 108 | records.push({ path: next, byteLength: stat.size, sha256: digest(readFileSync(child)) }) | |
| 109 | } | |
| 110 | } | |
| 111 | } | |
| 112 | walk(root, '') | |
| 113 | return records | |
| 114 | } | |
| 115 | ||
| 116 | async function archiveVolume(volume: string, output: string) { | |
| 117 | const before = volumeFiles(volume) | |
| 118 | await cp(volume, output, { recursive: true, errorOnExist: true, force: false }) | |
| 119 | assert.deepEqual(volumeFiles(output), before) | |
| 120 | await writeFile(`${output}.json`, JSON.stringify(before, null, 2)) | |
| 121 | } | |
| 122 | ||
| 123 | /** Full bounded tmpfs, genuine ENOSPC/SQLite failure, and fresh public recovery. */ | |
| 124 | export async function qualifyHostFullVolume(directory: string) { | |
| 125 | const namespace = privateNamespace() | |
| 126 | directory = resolve(directory) | |
| 127 | await mkdir(directory, { mode: 0o700 }) | |
| 128 | const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex') | |
| 129 | await mkdir(volume, { mode: 0o700 }) | |
| 130 | await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 }) | |
| 131 | async function command(argv: string[], label: string, timeoutMs = 10000) { | |
| 132 | const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label }) | |
| 133 | assert.equal(result.exitCode, 0, result.stderr) | |
| 134 | return result | |
| 135 | } | |
| 136 | await command(['mount', '--no-mtab', '-t', 'tmpfs', '-o', `size=${volumeBytes},mode=0700,nosuid,nodev,noexec`, 'drone-runtime-test', volume], 'Private bounded update filesystem mount') | |
| 137 | try { | |
| 138 | const mounted = checkedVolume(volume) | |
| 139 | assert.notEqual(mounted.device, lstatSync(directory).dev) | |
| 140 | await writeFile(join(volume, 'nonce'), nonce, { mode: 0o600 }) | |
| 141 | const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true }) | |
| 142 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 143 | assert.ok(image.length > reserveBytes && image.length < 4 * reserveBytes) | |
| 144 | const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } }) | |
| 145 | const enrollmentDirectory = join(directory, 'enrollment') | |
| 146 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 147 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(volume, 'store') }) | |
| 148 | const reference = { directory: join(volume, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 149 | const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 150 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id }) | |
| 151 | assert.equal(good.knownGood!.description.releaseSequence, 7) | |
| 152 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } }) | |
| 153 | const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 154 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 155 | const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml') | |
| 156 | const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id } | |
| 157 | await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 }) | |
| 158 | async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') { | |
| 159 | const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` }) | |
| 160 | await writeFile(join(directory, `${mode}.stdout`), result.stdout) | |
| 161 | await writeFile(join(directory, `${mode}.stderr`), result.stderr) | |
| 162 | await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode })) | |
| 163 | assert.equal(result.exitCode, 0, result.stderr); assert.equal(result.stderr, '') | |
| 164 | return result.stdout | |
| 165 | } | |
| 166 | const rejection = object(JSON.parse(await worker('limited'))) | |
| 167 | assert.ok(Array.isArray(rejection.errorCodes) && rejection.errorCodes.every(code => typeof code === 'string' && code.length <= 64) && rejection.errorCodes.length <= 32) | |
| 168 | assert.ok(rejection.errorCodes.includes('ENOSPC') && rejection.errorCodes.includes('SQLITE_FULL')) | |
| 169 | assert.ok(Array.isArray(rejection.phases) && rejection.phases.includes('slot-selected')) | |
| 170 | assert.ok(!rejection.phases.includes('slot-image-written') && !rejection.phases.includes('runtime-started') && !rejection.phases.includes('runtime-input-committed')) | |
| 171 | const fullVolume = checkedVolume(volume) | |
| 172 | assert.equal(fullVolume.freeBytes, 0) | |
| 173 | await writeFile(join(directory, 'full-filesystem.json'), JSON.stringify(fullVolume, null, 2)) | |
| 174 | await archiveVolume(volume, join(directory, 'full-volume')) | |
| 175 | await unlink(join(volume, 'filler.bin')) | |
| 176 | const failed = await inspectRuntimeFirmwareSlots(reference) | |
| 177 | await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2)) | |
| 178 | assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed) | |
| 179 | assert.equal(failed.trial!.phase, 'selected'); assert.equal(failed.trial!.health, null) | |
| 180 | assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'pending') | |
| 181 | assert.equal(failed.store.releaseFloor, 9) | |
| 182 | const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version | |
| 183 | assert.equal(version('root'), 2); assert.equal(version('targets'), 3) | |
| 184 | const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath) | |
| 185 | assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length)) | |
| 186 | const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 187 | assert.deepEqual(await readFile(previousImage), image) | |
| 188 | await archiveVolume(volume, join(directory, 'failed-volume')) | |
| 189 | await assert.rejects(abandonRuntimeFirmwareTrial({ ...reference, trialId: failed.trial!.id }), /Resolve metadata/) | |
| 190 | await writeFile(inputPath, JSON.stringify({ ...input, trialId: failed.trial!.id })) | |
| 191 | assert.equal(await worker('recover'), 'RECOVERY_COMPLETED\n') | |
| 192 | const recovered = await inspectRuntimeFirmwareSlots(reference) | |
| 193 | await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2)) | |
| 194 | assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood) | |
| 195 | assert.equal(recovered.store.attempt!.phase, 'recovered'); assert.equal(recovered.store.releaseFloor, 9) | |
| 196 | assert.deepEqual(recovered.store.metadata, failed.store.metadata) | |
| 197 | assert.equal(await worker('abandon'), 'ABANDON_COMPLETED\n') | |
| 198 | const abandoned = await inspectRuntimeFirmwareSlots(reference) | |
| 199 | await writeFile(join(directory, 'abandoned-slots.json'), JSON.stringify(abandoned, null, 2)) | |
| 200 | assert.equal(abandoned.trial!.phase, 'abandoned'); assert.equal(abandoned.trial!.health, null) | |
| 201 | assert.deepEqual(abandoned.knownGood, good.knownGood) | |
| 202 | assert.equal(await worker('fresh'), 'FRESH_TRIAL_COMPLETED\n') | |
| 203 | const fresh = await inspectRuntimeFirmwareSlots(reference) | |
| 204 | await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2)) | |
| 205 | assert.equal(fresh.knownGood!.description.releaseSequence, 9); assert.equal(fresh.knownGood!.health!.cleanProcessExit, true) | |
| 206 | assert.notEqual(fresh.knownGood!.id, failed.trial!.id) | |
| 207 | assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId) | |
| 208 | assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId) | |
| 209 | assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image) | |
| 210 | assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image) | |
| 211 | const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed } | |
| 212 | await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) | |
| 213 | return result | |
| 214 | } | |
| 215 | finally { | |
| 216 | try { await archiveVolume(volume, join(directory, 'final-volume')) } | |
| 217 | finally { await command(['umount', '--no-mtab', volume], 'Private bounded update filesystem unmount') } | |
| 218 | } | |
| 219 | } | |
| 220 | ||
| 221 | if (import.meta.main) { | |
| 222 | try { | |
| 223 | assert.equal(process.argv.length, 4) | |
| 224 | const mode = process.argv[2], path = resolve(process.argv[3]!) | |
| 225 | if (mode === 'qualify') console.log(JSON.stringify(await qualifyHostFullVolume(path))) | |
| 226 | else if (mode === 'limited') console.log(JSON.stringify(await limited(path))) | |
| 227 | else { | |
| 228 | const input = await workerInput(path) | |
| 229 | if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') } | |
| 230 | else if (mode === 'abandon') { assert.ok(input.trialId); await abandonRuntimeFirmwareTrial({ ...input.reference, trialId: input.trialId }); console.log('ABANDON_COMPLETED') } | |
| 231 | else { assert.equal(mode, 'fresh'); await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') } | |
| 232 | } | |
| 233 | } | |
| 234 | catch (error) { console.error('Host full-volume fixture failed', error); process.exitCode = 1 } | |
| 235 | } | |
| @@ -0,0 +1,140 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { randomBytes } from 'node:crypto' | |
| 3 | import { mkdir, readFile, writeFile } from 'node:fs/promises' | |
| 4 | import { basename, dirname, join, resolve } from 'node:path' | |
| 5 | import { buildRuntime } from '../../../resources/functions/drone/runtime/build' | |
| 6 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 7 | import { createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 8 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 9 | import { createHostSourceProject } from './host-project' | |
| 10 | import { digest, repository, target } from './repository' | |
| 11 | import { storageErrorCodes } from './storage-errors' | |
| 12 | ||
| 13 | const fileLimitBytes = 1048576 | |
| 14 | const clock = { source: 'independent-resource-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 15 | ||
| 16 | async function fixtureInput(inputPath: string) { | |
| 17 | assert.equal(basename(inputPath), 'worker-input.json') | |
| 18 | const root = dirname(resolve(inputPath)) | |
| 19 | const input = await Bun.file(inputPath).json() | |
| 20 | const marker = await Bun.file(join(root, 'fixture.json')).json() | |
| 21 | assert.equal(marker.kind, 'host-file-write-limit-v1'); assert.equal(input.nonce, marker.nonce) | |
| 22 | assert.match(input.nonce, /^[a-f0-9]{32}$/) | |
| 23 | assert.equal(input.reference.directory, join(root, 'store')) | |
| 24 | assert.equal(input.reference.enrollmentDirectory, join(root, 'enrollment')) | |
| 25 | assert.equal(input.project, join(root, 'project')); assert.equal(input.repository, join(root, 'repository')) | |
| 26 | return input | |
| 27 | } | |
| 28 | ||
| 29 | async function limitedTrial(inputPath: string) { | |
| 30 | assert.equal(process.env.DRONE_TEST_FILE_LIMIT, String(fileLimitBytes)) | |
| 31 | const input = await fixtureInput(inputPath) | |
| 32 | const phases: string[] = [] | |
| 33 | let caught = false, codes: string[] = [] | |
| 34 | try { | |
| 35 | await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, onProgress(event) { | |
| 36 | assert.ok(phases.length < 512); phases.push(event.phase) | |
| 37 | } }) | |
| 38 | } | |
| 39 | catch (error) { caught = true; codes = storageErrorCodes(error) } | |
| 40 | assert.equal(caught, true, 'The actual kernel file limit must reject the trial') | |
| 41 | return { errorCodes: codes, phases } | |
| 42 | } | |
| 43 | ||
| 44 | /** Real signed public coordinator, actual short OS write, retained bytes and fresh recovery. */ | |
| 45 | export async function qualifyHostFileWriteLimit(directory: string) { | |
| 46 | directory = resolve(directory) | |
| 47 | await mkdir(directory, { mode: 0o700 }) | |
| 48 | const nonce = randomBytes(16).toString('hex') | |
| 49 | await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-file-write-limit-v1', nonce }), { mode: 0o600 }) | |
| 50 | const launcher = join(directory, 'file-write-limit') | |
| 51 | const compiled = await runBoundedHostProcess(['c++', '-std=c++17', '-O2', join(import.meta.dir, 'file-write-limit.cpp'), '-o', launcher], { timeoutMs: 10000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Host file-limit launcher build' }) | |
| 52 | assert.equal(compiled.exitCode, 0, compiled.stderr) | |
| 53 | const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true }) | |
| 54 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 55 | assert.ok(image.length > fileLimitBytes && image.length < 4 * fileLimitBytes) | |
| 56 | const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } }) | |
| 57 | const enrollmentDirectory = join(directory, 'enrollment') | |
| 58 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 59 | const store = join(directory, 'store'), created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: store }) | |
| 60 | const reference = { directory: store, enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 61 | const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 62 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id }) | |
| 63 | assert.equal(good.knownGood!.description.releaseSequence, 7) | |
| 64 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } }) | |
| 65 | const next = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 66 | assert.equal(next.store.releaseFloor, 9) | |
| 67 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 68 | const config = join(directory, 'worker.bunfig.toml'), input = join(directory, 'worker-input.json') | |
| 69 | await writeFile(config, 'preload=[]\n', { mode: 0o600 }) | |
| 70 | await writeFile(input, JSON.stringify({ nonce, reference, project, repository: source, candidateId: next.candidate!.id }), { mode: 0o600 }) | |
| 71 | const workerEnv = { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } | |
| 72 | const command = [launcher, String(fileLimitBytes), process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, 'limited', input] | |
| 73 | const worker = await runBoundedHostProcess(command, { cwd: directory, env: workerEnv, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: 'Actual host image short-write trial' }) | |
| 74 | await writeFile(join(directory, 'worker.stdout'), worker.stdout) | |
| 75 | await writeFile(join(directory, 'worker.stderr'), worker.stderr) | |
| 76 | await writeFile(join(directory, 'worker.json'), JSON.stringify({ exitCode: worker.exitCode })) | |
| 77 | assert.equal(worker.exitCode, 0, worker.stderr) | |
| 78 | assert.equal(worker.stderr, `FILE_LIMIT_ACTIVE ${fileLimitBytes}\n`) | |
| 79 | const rejected = JSON.parse(worker.stdout) | |
| 80 | assert.ok(Array.isArray(rejected.errorCodes) && rejected.errorCodes.length <= 32 && rejected.errorCodes.every((code: unknown) => typeof code === 'string' && code.length <= 64) && rejected.errorCodes.includes('EFBIG')) | |
| 81 | assert.ok(Array.isArray(rejected.phases) && rejected.phases.length <= 512 && rejected.phases.every((phase: unknown) => typeof phase === 'string' && phase.length <= 64)) | |
| 82 | assert.ok(rejected.phases.includes('slot-selected')) | |
| 83 | assert.ok(!rejected.phases.includes('slot-image-written') && !rejected.phases.includes('runtime-started') && !rejected.phases.includes('runtime-input-committed')) | |
| 84 | const failed = await inspectRuntimeFirmwareSlots(reference) | |
| 85 | await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2)) | |
| 86 | assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed) | |
| 87 | assert.equal(failed.trial!.phase, 'failed'); assert.equal(failed.trial!.health, null) | |
| 88 | assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'maintenance') | |
| 89 | assert.equal(failed.store.releaseFloor, 9) | |
| 90 | const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version | |
| 91 | assert.equal(version('root'), 2); assert.equal(version('targets'), 3) | |
| 92 | const partialPath = join(store, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath) | |
| 93 | assert.equal(partial.length, fileLimitBytes); assert.deepEqual(partial, image.subarray(0, fileLimitBytes)) | |
| 94 | const oldImage = join(store, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 95 | assert.deepEqual(await readFile(oldImage), image) | |
| 96 | async function unrestricted(mode: 'recover' | 'fresh') { | |
| 97 | const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, input], { cwd: directory, env: { ...workerEnv, DRONE_TEST_FILE_LIMIT: undefined }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Fresh host write-limit ${mode}` }) | |
| 98 | await writeFile(join(directory, `${mode}.stdout`), result.stdout) | |
| 99 | await writeFile(join(directory, `${mode}.stderr`), result.stderr) | |
| 100 | await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode })) | |
| 101 | assert.equal(result.exitCode, 0, result.stderr) | |
| 102 | assert.equal(result.stderr, '') | |
| 103 | assert.equal(result.stdout, `${mode === 'recover' ? 'RECOVERY_COMPLETED' : 'FRESH_TRIAL_COMPLETED'}\n`) | |
| 104 | } | |
| 105 | await unrestricted('recover') | |
| 106 | const recovered = await inspectRuntimeFirmwareSlots(reference) | |
| 107 | await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2)) | |
| 108 | assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood) | |
| 109 | assert.equal(recovered.store.releaseFloor, 9) | |
| 110 | assert.deepEqual(recovered.store.metadata, failed.store.metadata) | |
| 111 | await unrestricted('fresh') | |
| 112 | const fresh = await inspectRuntimeFirmwareSlots(reference) | |
| 113 | await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2)) | |
| 114 | assert.notEqual(fresh.knownGood!.id, failed.trial!.id) | |
| 115 | assert.equal(fresh.knownGood!.description.releaseSequence, 9) | |
| 116 | assert.equal(fresh.knownGood!.health!.cleanProcessExit, true) | |
| 117 | assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId) | |
| 118 | assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId) | |
| 119 | assert.deepEqual(await readFile(oldImage), image); assert.deepEqual(await readFile(partialPath), partial) | |
| 120 | assert.deepEqual(await readFile(join(store, 'host-slots', fresh.knownGood!.id, 'image.bin')), image) | |
| 121 | const result = { evidence: 'host-slot-actual-short-write-v1', sourceRevision, authorizedImageSha256: signed.imageHash, fileLimitBytes, partialImageBytes: partial.length, partialImageSha256: digest(partial), authorizedImageBytes: image.length, errorCodes: rejected.errorCodes as string[], acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed } | |
| 122 | await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) | |
| 123 | return result | |
| 124 | } | |
| 125 | ||
| 126 | if (import.meta.main) { | |
| 127 | try { | |
| 128 | assert.equal(process.argv.length, 4) | |
| 129 | const mode = process.argv[2], inputPath = resolve(process.argv[3]!) | |
| 130 | if (mode === 'limited') console.log(JSON.stringify(await limitedTrial(inputPath))) | |
| 131 | else if (mode === 'recover' || mode === 'fresh') { | |
| 132 | assert.equal(process.env.DRONE_TEST_FILE_LIMIT, undefined) | |
| 133 | const input = await fixtureInput(inputPath) | |
| 134 | if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') } | |
| 135 | else { await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') } | |
| 136 | } | |
| 137 | else { assert.equal(mode, 'qualify'); console.log(JSON.stringify(await qualifyHostFileWriteLimit(inputPath))) } | |
| 138 | } | |
| 139 | catch (error) { console.error('Host file-limit fixture failed', error); process.exitCode = 1 } | |
| 140 | } | |
| @@ -0,0 +1,16 @@ | ||
| 1 | /** Bounded own error descriptors only, including actual SQLite/OS aggregate causes. */ | |
| 2 | export function storageErrorCodes(error: unknown) { | |
| 3 | const pending = [error], seen = new Set<unknown>(), codes: string[] = [] | |
| 4 | while (pending.length && seen.size < 32) { | |
| 5 | const item = pending.shift() | |
| 6 | if (!item || typeof item !== 'object' || seen.has(item)) continue | |
| 7 | seen.add(item) | |
| 8 | const code = Object.getOwnPropertyDescriptor(item, 'code')?.value | |
| 9 | if ((typeof code === 'string' || typeof code === 'number') && String(code).length <= 64) codes.push(String(code)) | |
| 10 | const cause = Object.getOwnPropertyDescriptor(item, 'cause')?.value | |
| 11 | if (cause !== undefined) pending.push(cause) | |
| 12 | const errors = Object.getOwnPropertyDescriptor(item, 'errors')?.value | |
| 13 | if (Array.isArray(errors)) pending.push(...errors.slice(0, 32)) | |
| 14 | } | |
| 15 | return codes | |
| 16 | } | |
| @@ -0,0 +1,36 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdtemp, rm } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join, resolve } from 'node:path' | |
| 5 | import { qualifyHostFullVolume } from '../fixtures/updates/host-volume-full' | |
| 6 | ||
| 7 | // Actual privileged Linux namespace qualification only. A skip is not ENOSPC evidence. | |
| 8 | test.skipIf(process.env.RUNTIME_HOST_ENOSPC_TEST !== '1')('a full host filesystem preserves accepted trust and requires explicit recovery before fresh confirmation', async () => { | |
| 9 | const evidence = process.env.RUNTIME_HOST_ENOSPC_EVIDENCE | |
| 10 | const temporary = evidence ? undefined : await mkdtemp(join(tmpdir(), 'drone-host-enospc-')) | |
| 11 | const root = evidence ? resolve(evidence) : join(temporary!, 'campaign') | |
| 12 | let completed = false | |
| 13 | try { | |
| 14 | const result = await qualifyHostFullVolume(root) | |
| 15 | expect(result.filesystemType).toBe('tmpfs') | |
| 16 | expect(result.filesystemBytes).toBe(16777216) | |
| 17 | expect(result.freeBytesAfterFailure).toBe(0) | |
| 18 | expect(result.partialImageBytes).toBeGreaterThan(0) | |
| 19 | expect(result.partialImageBytes).toBeLessThan(result.authorizedImageBytes) | |
| 20 | expect(result.errorCodes).toContain('ENOSPC') | |
| 21 | expect(result.errorCodes).toContain('SQLITE_FULL') | |
| 22 | expect(result.interruptedPhase).toBe('selected') | |
| 23 | expect(result.interruptedAttempt).toBe('pending') | |
| 24 | expect(result.acceptedRootVersion).toBe(2) | |
| 25 | expect(result.acceptedTargetsVersion).toBe(3) | |
| 26 | expect(result.releaseFloor).toBe(9) | |
| 27 | expect(result.abandonedPhase).toBe('abandoned') | |
| 28 | expect(result.freshBootId).not.toBe(result.previousBootId) | |
| 29 | expect(result.physicalDeploymentAllowed).toBe(false) | |
| 30 | completed = true | |
| 31 | } | |
| 32 | finally { | |
| 33 | if (completed && temporary) await rm(temporary, { recursive: true, force: true }) | |
| 34 | else console.log(`Host full-volume campaign retained at ${root}`) | |
| 35 | } | |
| 36 | }, 180000) | |
| @@ -33,6 +33,7 @@ | ||
| 33 | 33 | "../resources/functions/drone/updates/**/*.ts", |
| 34 | 34 | "./unit/updates*.test.ts", |
| 35 | 35 | "./fixtures/updates/**/*.ts", |
| 36 | "./integration/runtime-host-storage.test.ts", | |
| 36 | 37 | "./unit/diagnostics.test.ts", |
| 37 | 38 | "./fixtures/diagnostics/**/*.ts", |
| 38 | 39 | "../resources/functions/drone/missions/**/*.ts", |
| @@ -0,0 +1,32 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdir, mkdtemp, rm } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join, resolve } from 'node:path' | |
| 5 | import { qualifyHostFileWriteLimit } from '../fixtures/updates/host-write-limit' | |
| 6 | ||
| 7 | // Public coordinator plus actual OS writes catches lost floors or false health | |
| 8 | // after a partial image copy. Physical media and other errno classes are separate. | |
| 9 | test('an actual short host-image write preserves accepted trust and needs fresh runtime confirmation after recovery', async () => { | |
| 10 | const evidence = process.env.RUNTIME_HOST_FILE_LIMIT_EVIDENCE | |
| 11 | const root = evidence ? resolve(evidence) : await mkdtemp(join(tmpdir(), 'drone-host-write-limit-')) | |
| 12 | if (evidence) await mkdir(root, { mode: 0o700 }) | |
| 13 | let completed = false | |
| 14 | try { | |
| 15 | const result = await qualifyHostFileWriteLimit(join(root, 'campaign')) | |
| 16 | expect(result.fileLimitBytes).toBe(1048576) | |
| 17 | expect(result.partialImageBytes).toBe(1048576) | |
| 18 | expect(result.authorizedImageBytes).toBeGreaterThan(result.partialImageBytes) | |
| 19 | expect(result.errorCodes).toContain('EFBIG') | |
| 20 | expect(result.acceptedRootVersion).toBe(2) | |
| 21 | expect(result.acceptedTargetsVersion).toBe(3) | |
| 22 | expect(result.releaseFloor).toBe(9) | |
| 23 | expect(result.freshBootId).not.toBe(result.previousBootId) | |
| 24 | expect(result.physicalDeploymentAllowed).toBe(false) | |
| 25 | completed = true | |
| 26 | } | |
| 27 | finally { | |
| 28 | if (completed && !evidence) await rm(root, { recursive: true, force: true }) | |
| 29 | else if (completed) console.log(`Host file-limit campaign retained at ${root}`) | |
| 30 | else console.error(`Failed host file-limit campaign retained at ${root}`) | |
| 31 | } | |
| 32 | }, 120000) | |