also looking at this
feat: continue unresolved runtime update maintenance
#59
21 files
+1097
-40
| @@ -43,9 +43,21 @@ jobs: | ||
| 43 | 43 | if-no-files-found: ignore |
| 44 | 44 | retention-days: 7 |
| 45 | 45 | path: ${{ runner.temp }}/runtime-host-file-limit-evidence |
| 46 | - name: Verify explicit maintenance handoff interruption and fresh-process recovery | |
| 47 | env: | |
| 48 | RUNTIME_MAINTENANCE_EVIDENCE: ${{ runner.temp }}/runtime-maintenance-evidence | |
| 49 | run: bun test tests/unit/updates-maintenance-cuts.test.ts | |
| 50 | - uses: actions/upload-artifact@v4 | |
| 51 | if: always() | |
| 52 | with: | |
| 53 | name: runtime-maintenance-evidence | |
| 54 | if-no-files-found: ignore | |
| 55 | retention-days: 7 | |
| 56 | path: ${{ runner.temp }}/runtime-maintenance-evidence | |
| 46 | 57 | - run: ./buddy test |
| 47 | 58 | env: |
| 48 | 59 | RUNTIME_HOST_FILE_LIMIT_EVIDENCE: ${{ runner.temp }}/runtime-host-file-limit-full-evidence |
| 60 | RUNTIME_MAINTENANCE_EVIDENCE: ${{ runner.temp }}/runtime-maintenance-full-evidence | |
| 49 | 61 | - uses: actions/upload-artifact@v4 |
| 50 | 62 | if: always() |
| 51 | 63 | with: |
| @@ -53,6 +65,13 @@ jobs: | ||
| 53 | 65 | if-no-files-found: ignore |
| 54 | 66 | retention-days: 7 |
| 55 | 67 | path: ${{ runner.temp }}/runtime-host-file-limit-full-evidence |
| 68 | - uses: actions/upload-artifact@v4 | |
| 69 | if: always() | |
| 70 | with: | |
| 71 | name: runtime-maintenance-full-evidence | |
| 72 | if-no-files-found: ignore | |
| 73 | retention-days: 7 | |
| 74 | path: ${{ runner.temp }}/runtime-maintenance-full-evidence | |
| 56 | 75 | - run: bun run runtime:cli |
| 57 | 76 | - name: Verify signed test-image staging through the portable CLI outside the checkout |
| 58 | 77 | run: bun tests/fixtures/updates/qualify.ts "$RUNNER_TEMP/runtime-update-qualification" "$GITHUB_WORKSPACE/storage/runtime-build/cli/drone" |
| @@ -68,6 +87,18 @@ jobs: | ||
| 68 | 87 | run: bun tests/fixtures/updates/qualify-candidates.ts "$RUNNER_TEMP/runtime-candidates-qualification" "$GITHUB_WORKSPACE/storage/runtime-build/cli/drone" |
| 69 | 88 | - name: Verify host slots, observed health and interrupted recovery with a copied standalone CLI |
| 70 | 89 | run: bun tests/fixtures/updates/qualify-host-slots.ts "$RUNNER_TEMP/runtime-host-slots-qualification" "$GITHUB_WORKSPACE/storage/runtime-build/cli/drone" "$GITHUB_WORKSPACE" |
| 90 | - name: Verify explicit maintenance and format-4 runtime health through a copied CLI | |
| 91 | run: bun tests/fixtures/updates/qualify-maintenance.ts "$RUNNER_TEMP/runtime-maintenance-cli" "$GITHUB_WORKSPACE/storage/runtime-build/cli/drone" | |
| 92 | - uses: actions/upload-artifact@v4 | |
| 93 | if: always() | |
| 94 | with: | |
| 95 | name: runtime-maintenance-cli-evidence | |
| 96 | if-no-files-found: ignore | |
| 97 | include-hidden-files: true | |
| 98 | retention-days: 7 | |
| 99 | path: | | |
| 100 | ${{ runner.temp }}/runtime-maintenance-cli | |
| 101 | !${{ runner.temp }}/runtime-maintenance-cli/drone | |
| 71 | 102 | - uses: actions/upload-artifact@v4 |
| 72 | 103 | if: always() |
| 73 | 104 | with: |
| @@ -434,6 +465,26 @@ jobs: | ||
| 434 | 465 | - run: bun install --frozen-lockfile |
| 435 | 466 | - run: cp .env.example .env |
| 436 | 467 | - run: ./buddy key:generate |
| 468 | - name: Qualify actual ENOSPC and fresh runtime recovery in a private filesystem | |
| 469 | shell: bash | |
| 470 | run: | | |
| 471 | set -euo pipefail | |
| 472 | sudo --non-interactive env RUNTIME_HOST_ENOSPC_TEST=1 RUNTIME_HOST_ENOSPC_EVIDENCE="$RUNNER_TEMP/runtime-host-enospc" unshare --mount --propagation private --fork "$(command -v bun)" test ./tests/integration/runtime-host-storage.test.ts | |
| 473 | - name: Make the owned filesystem campaign artifacts readable by the runner | |
| 474 | if: always() | |
| 475 | shell: bash | |
| 476 | run: | | |
| 477 | set -euo pipefail | |
| 478 | if sudo --non-interactive test -d "$RUNNER_TEMP/runtime-host-enospc"; then | |
| 479 | sudo --non-interactive chown -R --no-dereference "$(id -u):$(id -g)" "$RUNNER_TEMP/runtime-host-enospc" | |
| 480 | fi | |
| 481 | - uses: actions/upload-artifact@v4 | |
| 482 | if: always() | |
| 483 | with: | |
| 484 | name: runtime-host-enospc-evidence | |
| 485 | if-no-files-found: ignore | |
| 486 | retention-days: 7 | |
| 487 | path: ${{ runner.temp }}/runtime-host-enospc | |
| 437 | 488 | - name: Qualify accepted purge with real process and mount isolation |
| 438 | 489 | run: sudo --non-interactive env PUBLIC_SAFETY_PURGE_NAMESPACE_TEST=1 unshare --fork --pid --mount-proc "$(command -v bun)" test ./tests/integration/public-safety-purge.test.ts |
| 439 | 490 | firmware: |
| @@ -0,0 +1,23 @@ | ||
| 1 | import { defineCommand } from '@stacksjs/cli' | |
| 2 | import { printAsync } from '../../resources/functions/cli/output' | |
| 3 | import { storeCommandFailure, storeCommandReference, storeReferenceFlags } from '../../resources/functions/drone/updates/command' | |
| 4 | import { continueRuntimeTrustMaintenance } from '../../resources/functions/drone/updates/store' | |
| 5 | ||
| 6 | export default defineCommand({ | |
| 7 | name: 'updates:maintain <directory>', | |
| 8 | description: 'Explicitly retain an unresolved attempt and require fresh signed metadata verification without resetting trust', | |
| 9 | options: { | |
| 10 | ...storeReferenceFlags, | |
| 11 | '--attempt <id>': 'Exact unresolved attempt identity from independently pinned store readback', | |
| 12 | '--acknowledge-unresolved-inputs': 'Acknowledge that prior inputs remain unresolved and retained for investigation', | |
| 13 | '--repository <path>': 'Bounded offline metadata repository for full fresh verification', | |
| 14 | '--trusted-system-clock-source <name>': 'Audit label for the independently managed trusted system clock, not an attestation', | |
| 15 | }, | |
| 16 | async handle(options, directory) { | |
| 17 | try { | |
| 18 | if (options.acknowledgeUnresolvedInputs !== true || typeof options.attempt !== 'string' || typeof options.repository !== 'string' || typeof options.trustedSystemClockSource !== 'string') throw new Error('Explicit attempt, acknowledgement, repository and trusted clock source required') | |
| 19 | await printAsync(JSON.stringify(await continueRuntimeTrustMaintenance({ ...storeCommandReference(options, directory), repository: options.repository, previousAttemptId: options.attempt, acknowledgeUnresolvedInputs: true, clock: { source: options.trustedSystemClockSource, read: () => new Date() } }))) | |
| 20 | } | |
| 21 | catch { await storeCommandFailure() } | |
| 22 | }, | |
| 23 | }) | |
| @@ -27,6 +27,7 @@ import UpdatesStoreInit from '../app/Commands/UpdatesStoreInit' | ||
| 27 | 27 | import UpdatesStore from '../app/Commands/UpdatesStore' |
| 28 | 28 | import UpdatesRefresh from '../app/Commands/UpdatesRefresh' |
| 29 | 29 | import UpdatesRecover from '../app/Commands/UpdatesRecover' |
| 30 | import UpdatesMaintain from '../app/Commands/UpdatesMaintain' | |
| 30 | 31 | import UpdatesPrepare from '../app/Commands/UpdatesPrepare' |
| 31 | 32 | import UpdatesReserve from '../app/Commands/UpdatesReserve' |
| 32 | 33 | import UpdatesCandidates from '../app/Commands/UpdatesCandidates' |
| @@ -105,6 +106,7 @@ UpdatesStoreInit(app) | ||
| 105 | 106 | UpdatesStore(app) |
| 106 | 107 | UpdatesRefresh(app) |
| 107 | 108 | UpdatesRecover(app) |
| 109 | UpdatesMaintain(app) | |
| 108 | 110 | UpdatesPrepare(app) |
| 109 | 111 | UpdatesReserve(app) |
| 110 | 112 | UpdatesCandidates(app) |
| @@ -0,0 +1,40 @@ | ||
| 1 | # Full host filesystem qualification | |
| 2 | ||
| 3 | Status: implementation in progress for #42. Physical deployment remains disabled. | |
| 4 | ||
| 5 | Extend the public update-coordinator storage campaign with actual Linux ENOSPC. | |
| 6 | Run explicitly inside a private mount namespace, verified against PID 1's mount | |
| 7 | namespace, with kernel mountinfo readback rejecting shared or slave propagation. | |
| 8 | Mount a new 16 MiB tmpfs only at an exclusively created private fixture | |
| 9 | directory. Read back the filesystem type, capacity, ownership and mount identity. | |
| 10 | No block device, existing mount or physical vehicle is involved. | |
| 11 | ||
| 12 | Keep signed repositories and the owned build outside the bounded filesystem. | |
| 13 | Place the actual protocol store, candidates and slots inside it. Confirm a real | |
| 14 | release-7 host session, prepare release 9, then publish new signed root/role | |
| 15 | metadata. At the public slot-selection progress boundary, fill an owned sibling | |
| 16 | file through real writes until ENOSPC and release only 1 MiB of its allocation. | |
| 17 | The larger signed image must make actual partial progress before ENOSPC. Leave | |
| 18 | the filesystem full until the trial process has exited, including its attempts | |
| 19 | to record failure. Never replace filesystem or SQLite methods with faults. | |
| 20 | ||
| 21 | Inspect retained bytes and public status after removing only the owned filler. | |
| 22 | Require the previous known-good image and accepted metadata/security floors to | |
| 23 | survive without a new health confirmation. A pending attempt and selected slot | |
| 24 | must undergo explicit fresh-process metadata recovery and exact-slot abandonment | |
| 25 | before an unrestricted fresh trial can confirm a different slot through actual | |
| 26 | new boot/session evidence. Preserve failed partial bytes and the older image. | |
| 27 | ||
| 28 | Copy and hash the full volume before freeing the filler or opening it for | |
| 29 | inspection, then archive later retained states before unmounting the volatile | |
| 30 | filesystem. Keep failure observations, status snapshots | |
| 31 | and process results. CI retains these synthetic artifacts for seven days. | |
| 32 | The opt-in Linux qualification must execute in CI; a skipped local test supplies | |
| 33 | no positive filesystem-exhaustion evidence. | |
| 34 | ||
| 35 | Linux documents tmpfs capacity limits and loss of contents on unmount in | |
| 36 | [tmpfs(5)](https://man7.org/linux/man-pages/man5/tmpfs.5.html). This is real logical | |
| 37 | filesystem exhaustion on a memory-backed filesystem. It does not qualify storage | |
| 38 | device exhaustion, persistent-media durability, EDQUOT, EIO, sync failures or | |
| 39 | electrical power loss. Those remain separate acceptance requirements. The fixture | |
| 40 | adds no production storage layer, privileged application command or dependency. | |
| @@ -0,0 +1,56 @@ | ||
| 1 | # Explicit continuation of unresolved update attempts | |
| 2 | ||
| 3 | Status: implementation in progress for #42. Physical deployment remains disabled. | |
| 4 | ||
| 5 | A metadata attempt can retain a verified root/timestamp/snapshot prefix and then | |
| 6 | reject later input. Ordinary recovery cannot turn an unknown verifier, crypto or | |
| 7 | resource error into proof of invalid input. Replaying that exact input can fail | |
| 8 | again after the repository has been corrected, leaving no supported fresh path. | |
| 9 | ||
| 10 | Add an explicit local maintenance continuation. The caller supplies independently | |
| 11 | retained enrollment/store pins, the exact unresolved attempt ID, an independently | |
| 12 | trusted current clock, and acknowledgement that the original inputs remain | |
| 13 | unresolved. Existing protected ownership and exclusive-writer checks apply. This | |
| 14 | is an operator action, never automatic error classification or silent retry. | |
| 15 | ||
| 16 | Format 4 adds immutable predecessor/successor handoffs. In one SQLite transaction, | |
| 17 | retain the original attempt unchanged, create a new pending attempt at the current | |
| 18 | committed generation, record the handoff, and retain the new clock observation. | |
| 19 | Exactly one unresolved successor remains active. Older readers reject format 4 | |
| 20 | before rewriting either database. The writer database stays format 1. Formats | |
| 21 | 1/2/3 migrate atomically while preserving all metadata, inputs, floors, candidates, | |
| 22 | slots and their independent store identity. This remains the local protocol | |
| 23 | exception in [ADR 28](0028-runtime-trust-store.md), with native Stacks CLI entrypoints. | |
| 24 | ||
| 25 | Every prior input, accepted observation, phase and release remains retained. A | |
| 26 | handoff says only that the operator requested fresh verification; it does not say | |
| 27 | the prior input was invalid, recovered or successfully verified. Public status | |
| 28 | exposes the chain and bounded input digests/counts for investigation. Chains are | |
| 29 | forward-only, bounded by the existing attempt limit and checked against attempt | |
| 30 | ordering, generations and clocks. There is no automatic deletion or compaction. | |
| 31 | ||
| 32 | The successor uses the existing signed metadata verifier and persistence path | |
| 33 | from the latest committed root and high-water state. No trusted root, metadata | |
| 34 | epoch, release floor or clock floor is reset. New root continuity and role checks | |
| 35 | still follow [TUF's client workflow](https://theupdateframework.github.io/specification/latest/#detailed-client-workflow). | |
| 36 | Only a full fresh metadata verification completes the successor. It neither | |
| 37 | downloads an image nor changes installed/known-good roles. Candidate selection, | |
| 38 | rollback and boot confirmation still require their own fresh-policy checks. | |
| 39 | Unverified predecessor inputs remain retained; the handoff does not automatically | |
| 40 | replay or authenticate them. This differs from historical recovery, which replays | |
| 41 | the active attempt's exact durable inputs. Neither path can decrease a committed | |
| 42 | security floor. | |
| 43 | ||
| 44 | A rejected or interrupted successor remains pending/maintenance and blocks normal | |
| 45 | operations. Historical recovery addresses that active successor only and cannot | |
| 46 | promote a retained predecessor. An explicit continuation can itself be continued | |
| 47 | after another rejection, preserving the complete chain and all security floors. | |
| 48 | Crashes at handoff/input/metadata boundaries and actual concurrent writers need | |
| 49 | qualification. Missing or corrupt state continues to fail closed. | |
| 50 | ||
| 51 | This supplies a supported path after unresolved repository input, including | |
| 52 | authenticated key rotation through existing root continuity. It does not invent | |
| 53 | authority to replace a lost root, decrease a poisoned clock floor, repair corrupt | |
| 54 | history or bypass revoked/expired metadata. Independent recovery authority and | |
| 55 | trusted-clock repair still need their own design and acceptance. Physical board | |
| 56 | installation, anti-rollback and electrical power-loss qualification remain open. | |
| @@ -291,7 +291,19 @@ | ||
| 291 | 291 | "tests/fixtures/updates/host-write-limit.ts", |
| 292 | 292 | "tests/fixtures/updates/file-write-limit.cpp", |
| 293 | 293 | "tests/unit/updates-host-writes.test.ts", |
| 294 | "docs/adr/0030-host-storage-fault-fixtures.md" | |
| 294 | "docs/adr/0030-host-storage-fault-fixtures.md", | |
| 295 | "tests/fixtures/updates/host-volume-full.ts", | |
| 296 | "tests/integration/runtime-host-storage.test.ts", | |
| 297 | "docs/adr/0031-full-host-filesystem-fixture.md", | |
| 298 | "docs/adr/0032-runtime-maintenance-continuations.md", | |
| 299 | "app/Commands/UpdatesMaintain.ts", | |
| 300 | "tests/unit/updates-maintenance.test.ts", | |
| 301 | "tests/unit/updates-maintenance-cli.test.ts", | |
| 302 | "tests/unit/updates-maintenance-cuts.test.ts", | |
| 303 | "tests/unit/updates-maintenance-host.test.ts", | |
| 304 | "tests/fixtures/updates/maintenance-scenario.ts", | |
| 305 | "tests/fixtures/updates/maintenance-interruptions.ts", | |
| 306 | "tests/fixtures/updates/qualify-maintenance.ts" | |
| 295 | 307 | ], |
| 296 | 308 | "remaining": [ |
| 297 | 309 | "Live runtime/sensor/estimator/authority/mission/payload correlation and measured clock relations", |
| @@ -182,8 +182,58 @@ cuts do not cover every write/sync boundary or actual physical power loss. | ||
| 182 | 182 | The metadata format is bounded to 256 attempts, 10,000 metadata advances and a 128 MiB |
| 183 | 183 | state database. Each attempt allows 64 inputs, 32 MiB of metadata and 512 KiB per |
| 184 | 184 | file. Exhaustion fails closed; history is not automatically deleted or compacted. |
| 185 | Capacity migration, the full storage-fault campaign, authenticated repair/key | |
| 186 | revocation, durable slot coordination and session-bound health remain in #42. | |
| 185 | Capacity migration, the full storage-fault campaign and authenticated repair/key | |
| 186 | revocation remain in #42. [Owned host slots](#owned-host-slots-and-observed-health) | |
| 187 | below add durable host roles and observed boot/session health. Physical board | |
| 188 | selection, boot health and recovery still require their separate qualification. | |
| 189 | ||
| 190 | ### Explicit maintenance continuation | |
| 191 | ||
| 192 | A corrected repository does not make an earlier rejected input valid. If ordinary | |
| 193 | `updates:recover` cannot finish, explicitly retain that unresolved attempt and | |
| 194 | require a new full signed metadata verification: | |
| 195 | ||
| 196 | ```sh | |
| 197 | ./buddy updates:maintain /path/to/store \ | |
| 198 | --enrollment /path/to/trust \ | |
| 199 | --enrollment-sha256 "$INDEPENDENTLY_RETAINED_ENROLLMENT_SHA256" \ | |
| 200 | --store-sha256 "$INDEPENDENTLY_RETAINED_STORE_SHA256" \ | |
| 201 | --attempt "$EXACT_UNRESOLVED_ATTEMPT_ID" \ | |
| 202 | --acknowledge-unresolved-inputs \ | |
| 203 | --repository /path/to/corrected-offline-repository \ | |
| 204 | --trusted-system-clock-source operator-managed-utc-clock | |
| 205 | ``` | |
| 206 | ||
| 207 | The same command is in the standalone CLI. It requires the exact active attempt | |
| 208 | and explicit acknowledgement, and retains the original phase, inputs, accepted | |
| 209 | metadata and security floors. It does not mark the earlier failure recovered or | |
| 210 | invalid. Public store status exposes `maintenanceHandoffs`; each entry includes | |
| 211 | the predecessor's bounded input digest/counts, unchanged outcome, successor ID | |
| 212 | and committed starting generation. The latest `attempt` also exposes its input | |
| 213 | digest/counts and accepted metadata count. | |
| 214 | ||
| 215 | Format 4 adds the handoff and new pending attempt atomically. Existing formats | |
| 216 | 1/2/3 migrate without resetting their independent store identity or security | |
| 217 | history. The writer database remains format 1. Older readers reject format 4 | |
| 218 | before rewriting either database. A pending or rejected successor blocks normal | |
| 219 | updates. Historical recovery acts on the active successor only; another explicit | |
| 220 | continuation can retain it and start a further successor within the same bounded | |
| 221 | history. No handoff deletes, reclassifies or silently drops earlier evidence. | |
| 222 | ||
| 223 | Fresh verification starts at the latest committed root, role epochs and floors. | |
| 224 | Unverified predecessor input stays retained and is not automatically replayed by | |
| 225 | this operation. New root continuity and every current-policy check still apply. | |
| 226 | The command does not download an image, change slot roles or confirm health. | |
| 227 | An active candidate or interrupted host slot still needs its ordinary explicit | |
| 228 | retirement or exact-slot abandonment before a replacement can proceed. | |
| 229 | Subsequent selection and rollback still require their own current verification | |
| 230 | and observed runtime session. Existing known-good files remain retained. | |
| 231 | ||
| 232 | This supports corrected metadata and authenticated key rotation through existing | |
| 233 | root continuity. It cannot replace a lost root, reduce an erroneous clock floor, | |
| 234 | repair corrupt history or authorize expired/revoked images. Those maintenance and | |
| 235 | physical acceptance gates remain open. See the | |
| 236 | [continuation decision](adr/0032-runtime-maintenance-continuations.md). | |
| 187 | 237 | |
| 188 | 238 | ## Protected signed-image preparation |
| 189 | 239 | |
| @@ -592,3 +642,12 @@ new slot can become known-good through actual fresh runtime health; the failed | ||
| 592 | 642 | partial file stays retained. CI keeps the synthetic campaign artifacts for seven |
| 593 | 643 | days. This file-size-limit evidence leaves zero-return writes, full-medium/quota, |
| 594 | 644 | EIO, sync and physical power-loss campaigns open. |
| 645 | ||
| 646 | The opt-in [full-filesystem qualification](adr/0031-full-host-filesystem-fixture.md) | |
| 647 | uses a separate private Linux mount namespace and a new bounded tmpfs for the | |
| 648 | actual store, candidates and slots. It targets real ENOSPC during the image copy | |
| 649 | and SQLite failure-state persistence, then exercises explicit metadata recovery, | |
| 650 | exact-slot abandonment and a fresh host trial in new processes. Retained file | |
| 651 | archives are copied and hashed before unmount. This additional campaign is in | |
| 652 | qualification; a local skip is not positive ENOSPC evidence. Physical-media, | |
| 653 | EDQUOT, EIO, sync and power-loss acceptance remain separate. | |
| @@ -38,6 +38,9 @@ const slotSchema = [ | ||
| 38 | 38 | 'CREATE TABLE host_roles (id INTEGER PRIMARY KEY CHECK(id = 1), installed_id TEXT, good_id TEXT, FOREIGN KEY(installed_id) REFERENCES host_slots(id), FOREIGN KEY(good_id) REFERENCES host_slots(id)) STRICT', |
| 39 | 39 | "CREATE TABLE host_inputs (slot_id TEXT NOT NULL, ordinal INTEGER NOT NULL CHECK(ordinal BETWEEN 0 AND 255), kind TEXT NOT NULL CHECK(kind IN ('tx', 'rx')), bytes BLOB NOT NULL CHECK(length(bytes) BETWEEN 1 AND 160), PRIMARY KEY(slot_id, ordinal), FOREIGN KEY(slot_id) REFERENCES host_slots(id)) STRICT", |
| 40 | 40 | ] |
| 41 | const maintenanceSchema = [ | |
| 42 | 'CREATE TABLE maintenance_handoffs (previous_attempt_id TEXT PRIMARY KEY, next_attempt_id TEXT NOT NULL UNIQUE, generation INTEGER NOT NULL, CHECK(previous_attempt_id != next_attempt_id), FOREIGN KEY(previous_attempt_id) REFERENCES attempts(id), FOREIGN KEY(next_attempt_id) REFERENCES attempts(id), FOREIGN KEY(generation) REFERENCES metadata(generation)) STRICT', | |
| 43 | ] | |
| 41 | 44 | |
| 42 | 45 | interface TrustReference { enrollmentDirectory: string, enrollmentSha256: string } |
| 43 | 46 | interface StoreReference extends TrustReference { directory: string, storeSha256: string } |
| @@ -45,6 +48,15 @@ interface Binding { schema: 'drone-runtime-store-v1', storeId: string, enrollmen | ||
| 45 | 48 | interface State { store_id: string, enrollment_sha256: string, generation: number, release_floor: number, clock_floor_ms: number } |
| 46 | 49 | interface MetadataRow { generation: number, attempt_id: string | null, ordinal: number, role: string, version: number, epoch: number, bytes: Uint8Array, sha256: string } |
| 47 | 50 | interface Attempt { id: string, base_generation: number, started_ms: number, clock_source: string, phase: 'pending' | 'verified' | 'recovered' | 'maintenance', accepted_release: number | null } |
| 51 | export interface RuntimeUpdateAttemptSummary { | |
| 52 | id: string | |
| 53 | phase: Attempt['phase'] | |
| 54 | clockSource: string | |
| 55 | startedMs: number | |
| 56 | inputRecords: number | |
| 57 | inputSha256: string | |
| 58 | metadataRecords: number | |
| 59 | } | |
| 48 | 60 | export interface RuntimeTrustStoreSummary { |
| 49 | 61 | schema: 'drone-runtime-store-status-v1' |
| 50 | 62 | storeId: string |
| @@ -56,7 +68,8 @@ export interface RuntimeTrustStoreSummary { | ||
| 56 | 68 | clockFloorMs: number |
| 57 | 69 | retainedMetadataRecords: number |
| 58 | 70 | metadata: ReadonlyArray<Readonly<{ role: string, version: number, sha256: string }>> |
| 59 | attempt: Readonly<{ id: string, phase: Attempt['phase'], clockSource: string, startedMs: number }> | null | |
| 71 | attempt: Readonly<RuntimeUpdateAttemptSummary> | null | |
| 72 | maintenanceHandoffs: ReadonlyArray<Readonly<{ previousAttempt: Readonly<RuntimeUpdateAttemptSummary>, nextAttemptId: string, generation: number }>> | |
| 60 | 73 | physicalDeploymentAllowed: false |
| 61 | 74 | } |
| 62 | 75 | |
| @@ -86,8 +99,8 @@ function openDatabase(path: string, expectedSchema: string[], allowCandidates = | ||
| 86 | 99 | // Authenticate the protocol format before any configuration that could |
| 87 | 100 | // rewrite a foreign database's journal/header format. |
| 88 | 101 | const version = databaseVersion(db) |
| 89 | const additions = allowCandidates && version === 3 ? [...candidateSchema, ...slotSchema] : allowCandidates && version === 2 ? candidateSchema : [] | |
| 90 | checkSchema(db, [...expectedSchema, ...additions], allowCandidates && (version === 2 || version === 3) ? version : 1) | |
| 102 | const additions = allowCandidates && version === 4 ? [...candidateSchema, ...slotSchema, ...maintenanceSchema] : allowCandidates && version === 3 ? [...candidateSchema, ...slotSchema] : allowCandidates && version === 2 ? candidateSchema : [] | |
| 103 | checkSchema(db, [...expectedSchema, ...additions], allowCandidates && [2, 3, 4].includes(version) ? version : 1) | |
| 91 | 104 | if (Object.values(db.query('PRAGMA journal_mode').get() as Record<string, unknown>)[0] !== 'delete') throw new Error('Unexpected update-store journal format') |
| 92 | 105 | configure(db) |
| 93 | 106 | return db |
| @@ -136,6 +149,35 @@ function checkedMetadataBytes(row: MetadataRow) { | ||
| 136 | 149 | if (!/^[a-z0-9][a-z0-9._-]{0,179}$/i.test(row.role) || sha256(bytes) !== row.sha256 || signed.version !== row.version || signed._type !== kind) throw new Error('Protected metadata integrity mismatch') |
| 137 | 150 | return bytes |
| 138 | 151 | } |
| 152 | function activeAttempts(db: Database) { | |
| 153 | const retained = databaseVersion(db) === 4 ? 'AND id NOT IN (SELECT previous_attempt_id FROM maintenance_handoffs)' : '' | |
| 154 | return db.query<Attempt, []>(`SELECT * FROM attempts WHERE phase IN ('pending', 'maintenance') ${retained} ORDER BY rowid`).all() | |
| 155 | } | |
| 156 | function attemptObservation(db: Database, row: Attempt): Readonly<RuntimeUpdateAttemptSummary> { | |
| 157 | const inputs = db.query<InputRow, [string]>('SELECT * FROM inputs WHERE attempt_id = ? ORDER BY ordinal').all(row.id) | |
| 158 | if (inputs.length > 64 || inputs.some((input, index) => input.ordinal !== index || (input.status === 200 ? !input.bytes || sha256(input.bytes) !== input.sha256 : input.bytes !== null || input.sha256 !== null))) throw new Error('Invalid retained update input history') | |
| 159 | for (const input of inputs) safePath(input.path) | |
| 160 | const metadata = db.query<MetadataRow, [string]>('SELECT * FROM metadata WHERE attempt_id = ? ORDER BY ordinal').all(row.id) | |
| 161 | if (metadata.length > 64 || metadata.some((item, index) => item.ordinal !== index)) throw new Error('Invalid retained metadata observation history') | |
| 162 | for (const item of metadata) checkedMetadataBytes(item) | |
| 163 | const digest = sha256(Buffer.from(JSON.stringify(inputs.map(input => [input.ordinal, input.path, input.status, input.sha256])))) | |
| 164 | return Object.freeze({ id: row.id, phase: row.phase, clockSource: row.clock_source, startedMs: row.started_ms, inputRecords: inputs.length, inputSha256: digest, metadataRecords: metadata.length }) | |
| 165 | } | |
| 166 | function maintenanceHandoffs(db: Database): RuntimeTrustStoreSummary['maintenanceHandoffs'] { | |
| 167 | if (databaseVersion(db) !== 4) return Object.freeze([]) | |
| 168 | const attempts = db.query<Attempt & { position: number }, []>('SELECT rowid AS position, * FROM attempts').all() | |
| 169 | if (attempts.length > 256) throw new Error('Update attempt history budget exceeded') | |
| 170 | const index = new Map(attempts.map(row => [row.id, row])) | |
| 171 | const rows = db.query<{ previous_attempt_id: string, next_attempt_id: string, generation: number }, []>('SELECT * FROM maintenance_handoffs ORDER BY rowid').all() | |
| 172 | if (rows.length > 255) throw new Error('Maintenance handoff history budget exceeded') | |
| 173 | return Object.freeze(rows.map(row => { | |
| 174 | const previous = index.get(row.previous_attempt_id), next = index.get(row.next_attempt_id) | |
| 175 | if (!previous || !next || !['pending', 'maintenance'].includes(previous.phase) || previous.position >= next.position || previous.started_ms > next.started_ms || row.generation !== next.base_generation || row.generation < previous.base_generation) throw new Error('Invalid retained maintenance handoff') | |
| 176 | const preceding = db.query<{ generation: number }, [number]>('SELECT coalesce(max(m.generation), 0) AS generation FROM metadata m LEFT JOIN attempts a ON a.id = m.attempt_id WHERE m.attempt_id IS NULL OR a.rowid < ?').get(next.position)! | |
| 177 | if (row.generation !== preceding.generation) throw new Error('Maintenance handoff skipped committed metadata history') | |
| 178 | return Object.freeze({ previousAttempt: attemptObservation(db, previous), nextAttemptId: next.id, generation: row.generation }) | |
| 179 | })) | |
| 180 | } | |
| 139 | 181 | function summary(db: Database, binding: Binding, storeSha256: string): Readonly<RuntimeTrustStoreSummary> { |
| 140 | 182 | const current = state(db, binding), metadata = activeMetadata(db, current.generation) |
| 141 | 183 | const anchor = db.query<MetadataRow, []>('SELECT * FROM metadata WHERE generation = 0').get() |
| @@ -143,11 +185,13 @@ function summary(db: Database, binding: Binding, storeSha256: string): Readonly< | ||
| 143 | 185 | for (const row of [anchor, ...metadata]) checkedMetadataBytes(row) |
| 144 | 186 | const counts = db.query<{ attempts: number, metadata: number, inputs: number }, []>('SELECT (SELECT count(*) FROM attempts) AS attempts, (SELECT count(*) FROM metadata) AS metadata, (SELECT count(*) FROM inputs) AS inputs').get()! |
| 145 | 187 | if (counts.attempts > 256 || counts.metadata !== current.generation + 1 || counts.metadata > maximumGeneration + 1 || counts.inputs > 16384) throw new Error('Update-store history budget exceeded or metadata history is incomplete') |
| 146 | const clockQuery = databaseVersion(db) === 3 ? 'max(coalesce((SELECT max(started_ms) FROM attempts), 0), coalesce((SELECT max(checked_ms) FROM host_slots), 0))' : 'coalesce((SELECT max(started_ms) FROM attempts), 0)' | |
| 188 | const handoffs = maintenanceHandoffs(db) | |
| 189 | const clockQuery = databaseVersion(db) >= 3 ? 'max(coalesce((SELECT max(started_ms) FROM attempts), 0), coalesce((SELECT max(checked_ms) FROM host_slots), 0))' : 'coalesce((SELECT max(started_ms) FROM attempts), 0)' | |
| 147 | 190 | const floors = db.query<{ clock: number, release: number, generation: number }, []>(`SELECT ${clockQuery} AS clock, coalesce((SELECT max(accepted_release) FROM attempts), 0) AS release, (SELECT max(generation) FROM metadata) AS generation`).get()! |
| 148 | if (current.clock_floor_ms !== floors.clock || current.release_floor < floors.release || current.generation !== floors.generation || db.query<{ count: number }, []>("SELECT count(*) AS count FROM attempts WHERE phase IN ('pending', 'maintenance')").get()!.count > 1) throw new Error('Inconsistent update-store floors or attempts') | |
| 149 | 191 | const attempt = db.query<Attempt, []>('SELECT * FROM attempts ORDER BY rowid DESC LIMIT 1').get() |
| 150 | return Object.freeze({ schema: 'drone-runtime-store-status-v1', storeId: binding.storeId, storeSha256, enrollmentSha256: binding.enrollmentSha256, generation: current.generation, epoch: metadata.find(row => row.role === 'root')!.epoch, releaseFloor: current.release_floor, clockFloorMs: current.clock_floor_ms, retainedMetadataRecords: counts.metadata, metadata: Object.freeze(metadata.map(row => Object.freeze({ role: row.role, version: row.version, sha256: row.sha256 }))), attempt: attempt ? Object.freeze({ id: attempt.id, phase: attempt.phase, clockSource: attempt.clock_source, startedMs: attempt.started_ms }) : null, physicalDeploymentAllowed: false }) | |
| 192 | const unresolved = activeAttempts(db) | |
| 193 | if (current.clock_floor_ms !== floors.clock || current.release_floor < floors.release || current.generation !== floors.generation || unresolved.length > 1 || (unresolved.length === 1 && unresolved[0]!.id !== attempt?.id)) throw new Error('Inconsistent update-store floors or attempts') | |
| 194 | return Object.freeze({ schema: 'drone-runtime-store-status-v1', storeId: binding.storeId, storeSha256, enrollmentSha256: binding.enrollmentSha256, generation: current.generation, epoch: metadata.find(row => row.role === 'root')!.epoch, releaseFloor: current.release_floor, clockFloorMs: current.clock_floor_ms, retainedMetadataRecords: counts.metadata, metadata: Object.freeze(metadata.map(row => Object.freeze({ role: row.role, version: row.version, sha256: row.sha256 }))), attempt: attempt ? attemptObservation(db, attempt) : null, maintenanceHandoffs: handoffs, physicalDeploymentAllowed: false }) | |
| 151 | 195 | } |
| 152 | 196 | |
| 153 | 197 | async function withStore<T>(options: StoreReference, operation: (db: Database, binding: Binding, trusted: Readonly<RuntimeTrustSummary>, path: string) => T | Promise<T>): Promise<T> { |
| @@ -181,10 +225,10 @@ async function withStore<T>(options: StoreReference, operation: (db: Database, b | ||
| 181 | 225 | summary(db, binding, options.storeSha256) |
| 182 | 226 | for (const row of candidateRows(db)) ownedDescription(db, row, trusted) |
| 183 | 227 | if (readdirSync(path).includes('host-slots')) { |
| 184 | if (databaseVersion(db) !== 3) throw new Error('Host slot files require an owned format3 store') | |
| 228 | if (databaseVersion(db) < 3) throw new Error('Host slot files require an owned format3 or later store') | |
| 185 | 229 | protectedDirectory(join(path, 'host-slots')) |
| 186 | 230 | } |
| 187 | if (databaseVersion(db) === 3) slotsSummary(db, binding, trusted, path, options.storeSha256) | |
| 231 | if (databaseVersion(db) >= 3) slotsSummary(db, binding, trusted, path, options.storeSha256) | |
| 188 | 232 | return await operation(db, binding, trusted, path) |
| 189 | 233 | } |
| 190 | 234 | finally { |
| @@ -233,7 +277,7 @@ export interface RuntimeUpdateClock { | ||
| 233 | 277 | } |
| 234 | 278 | |
| 235 | 279 | export interface RuntimeUpdateProgress { |
| 236 | phase: 'input-committed' | 'metadata-committed' | 'image-written' | 'candidate-completed' | 'candidate-reserved' | 'candidate-recorded' | 'candidate-retired' | 'slot-selected' | 'slot-image-written' | 'runtime-started' | 'runtime-input-committed' | 'final-clock-committed' | 'runtime-confirmed' | |
| 280 | phase: 'input-committed' | 'metadata-committed' | 'image-written' | 'candidate-completed' | 'candidate-reserved' | 'candidate-recorded' | 'candidate-retired' | 'slot-selected' | 'slot-image-written' | 'runtime-started' | 'runtime-input-committed' | 'final-clock-committed' | 'runtime-confirmed' | 'maintenance-handoff-committed' | |
| 237 | 281 | ordinal: number |
| 238 | 282 | } |
| 239 | 283 | type ProgressObserver = (event: Readonly<RuntimeUpdateProgress>) => void | Promise<void> |
| @@ -300,11 +344,15 @@ interface VerifiedSelection { | ||
| 300 | 344 | path: string |
| 301 | 345 | } |
| 302 | 346 | |
| 303 | async function refreshAndComplete<T>(options: RefreshOptions, complete: (selection: VerifiedSelection) => Promise<T>, before?: (db: Database) => void): Promise<{ store: Readonly<RuntimeTrustStoreSummary>, value: T }> { | |
| 347 | async function refreshAndComplete<T>(options: RefreshOptions, complete: (selection: VerifiedSelection) => Promise<T>, before?: (db: Database) => void, previousAttemptId?: string): Promise<{ store: Readonly<RuntimeTrustStoreSummary>, value: T }> { | |
| 304 | 348 | options = { ...options } |
| 305 | 349 | return withStore(options, async (db, binding, trusted, path) => { |
| 306 | 350 | before?.(db) |
| 307 | if (db.query("SELECT id FROM attempts WHERE phase IN ('pending', 'maintenance')").get()) throw new Error('An unresolved update attempt requires recovery') | |
| 351 | const unresolved = activeAttempts(db) | |
| 352 | if (previousAttemptId) { | |
| 353 | if (unresolved.length !== 1 || unresolved[0]!.id !== previousAttemptId) throw new Error('Exact unresolved update attempt required for maintenance continuation') | |
| 354 | } | |
| 355 | else if (unresolved.length) throw new Error('An unresolved update attempt requires recovery') | |
| 308 | 356 | const current = state(db, binding), now = clockReading(options.clock, current.clock_floor_ms) |
| 309 | 357 | if (db.query<{ count: number }, []>('SELECT count(*) AS count FROM attempts').get()!.count >= 256) throw new Error('Update attempt history limit reached') |
| 310 | 358 | const source = realpathSync(options.repository); directory(source) |
| @@ -312,9 +360,24 @@ async function refreshAndComplete<T>(options: RefreshOptions, complete: (selecti | ||
| 312 | 360 | let requests = 0, received = 0, observations = 0 |
| 313 | 361 | try { |
| 314 | 362 | db.transaction(() => { |
| 363 | if (previousAttemptId && databaseVersion(db) !== 4) { | |
| 364 | const version = databaseVersion(db) | |
| 365 | if (version < 2) for (const sql of candidateSchema) db.exec(sql) | |
| 366 | if (version < 3) { | |
| 367 | for (const sql of slotSchema) db.exec(sql) | |
| 368 | db.exec('INSERT INTO host_roles VALUES(1, NULL, NULL)') | |
| 369 | } | |
| 370 | for (const sql of maintenanceSchema) db.exec(sql) | |
| 371 | db.exec('PRAGMA user_version=4') | |
| 372 | } | |
| 315 | 373 | db.query("INSERT INTO attempts VALUES(?, ?, ?, ?, 'pending', NULL)").run(attempt, current.generation, now.time.getTime(), now.source) |
| 316 | 374 | db.query('UPDATE store_state SET clock_floor_ms = ? WHERE id = 1').run(now.time.getTime()) |
| 375 | if (previousAttemptId) db.query('INSERT INTO maintenance_handoffs VALUES(?, ?, ?)').run(previousAttemptId, attempt, current.generation) | |
| 317 | 376 | })() |
| 377 | if (previousAttemptId) { | |
| 378 | checkSchema(db, [...schema, ...candidateSchema, ...slotSchema, ...maintenanceSchema], 4) | |
| 379 | await options.onProgress?.(Object.freeze({ phase: 'maintenance-handoff-committed', ordinal: 0 })) | |
| 380 | } | |
| 318 | 381 | const fetcher = { |
| 319 | 382 | async downloadBytes(url: string, limit: number) { |
| 320 | 383 | const address = new URL(url), path = address.pathname.slice('/metadata/'.length) |
| @@ -370,6 +433,13 @@ export async function refreshRuntimeTrustStore(options: RefreshOptions): Promise | ||
| 370 | 433 | return (await refreshAndComplete(options, async () => undefined)).store |
| 371 | 434 | } |
| 372 | 435 | |
| 436 | /** Explicitly retain an unresolved attempt and require a fresh fully verified successor. No trust/floor reset. */ | |
| 437 | export async function continueRuntimeTrustMaintenance(options: RefreshOptions & { previousAttemptId: string, acknowledgeUnresolvedInputs: true }): Promise<Readonly<RuntimeTrustStoreSummary>> { | |
| 438 | options = { ...options } | |
| 439 | if (options.acknowledgeUnresolvedInputs !== true || typeof options.previousAttemptId !== 'string' || !/^[a-f0-9]{32}$/.test(options.previousAttemptId)) throw new Error('Explicit acknowledgement and exact unresolved attempt identity required') | |
| 440 | return (await refreshAndComplete(options, async () => undefined, undefined, options.previousAttemptId)).store | |
| 441 | } | |
| 442 | ||
| 373 | 443 | export interface PreparedRuntimeFirmware { |
| 374 | 444 | schema: 'drone-runtime-prepared-image-v1' |
| 375 | 445 | store: Readonly<RuntimeTrustStoreSummary> |
| @@ -427,7 +497,7 @@ export interface RuntimeFirmwareCandidate { | ||
| 427 | 497 | } |
| 428 | 498 | export interface RuntimeFirmwareCandidates { |
| 429 | 499 | schema: 'drone-runtime-candidates-v1' |
| 430 | stateFormat: 1 | 2 | 3 | |
| 500 | stateFormat: 1 | 2 | 3 | 4 | |
| 431 | 501 | store: Readonly<RuntimeTrustStoreSummary> |
| 432 | 502 | retainedCandidates: number |
| 433 | 503 | /** Owned declarations only. Retired file integrity is not inspected or authenticated for current use. */ |
| @@ -511,7 +581,7 @@ function candidatesSummary(db: Database, binding: Binding, trusted: Readonly<Run | ||
| 511 | 581 | candidate = Object.freeze({ ...recorded, integrity: row.phase === 'reserved' ? 'incomplete' : candidateIntegrity(db, row.generation, directory, description), currentTrustValidated: false }) |
| 512 | 582 | } |
| 513 | 583 | } |
| 514 | return Object.freeze({ schema: 'drone-runtime-candidates-v1', stateFormat: databaseVersion(db) as 1 | 2 | 3, store: summary(db, binding, storeSha256), retainedCandidates: rows.length, history: Object.freeze(history), candidate, physicalDeploymentAllowed: false }) | |
| 584 | return Object.freeze({ schema: 'drone-runtime-candidates-v1', stateFormat: databaseVersion(db) as 1 | 2 | 3 | 4, store: summary(db, binding, storeSha256), retainedCandidates: rows.length, history: Object.freeze(history), candidate, physicalDeploymentAllowed: false }) | |
| 515 | 585 | } |
| 516 | 586 | |
| 517 | 587 | /** Inspect owned host candidates. Recorded-byte integrity is not fresh selection or boot authorization. */ |
| @@ -579,7 +649,7 @@ export interface RuntimeHostSlot { | ||
| 579 | 649 | } |
| 580 | 650 | export interface RuntimeFirmwareSlots { |
| 581 | 651 | schema: 'drone-runtime-host-slots-v1' |
| 582 | stateFormat: 1 | 2 | 3 | |
| 652 | stateFormat: 1 | 2 | 3 | 4 | |
| 583 | 653 | store: Readonly<RuntimeTrustStoreSummary> |
| 584 | 654 | installed: Readonly<RuntimeHostSlot> | null |
| 585 | 655 | knownGood: Readonly<RuntimeHostSlot> | null |
| @@ -639,7 +709,7 @@ function slotsSummary(db: Database, binding: Binding, trusted: Readonly<RuntimeT | ||
| 639 | 709 | slots.set(row.id, Object.freeze({ id: row.id, candidateId: row.candidate_id, phase: row.phase, clockCheckedMs: row.checked_ms, confirmedClockMs: row.completed_ms, description, health: slotHealth(db, row), integrity, currentTrustValidated: false })) |
| 640 | 710 | } |
| 641 | 711 | let installed: Readonly<RuntimeHostSlot> | null = null, knownGood: Readonly<RuntimeHostSlot> | null = null |
| 642 | if (databaseVersion(db) === 3) { | |
| 712 | if (databaseVersion(db) >= 3) { | |
| 643 | 713 | const roles = db.query<{ id: number, installed_id: string | null, good_id: string | null }, []>('SELECT * FROM host_roles').all() |
| 644 | 714 | if (roles.length !== 1 || roles[0]!.id !== 1) throw new Error('Invalid host slot roles') |
| 645 | 715 | const role = roles[0]! |
| @@ -647,7 +717,7 @@ function slotsSummary(db: Database, binding: Binding, trusted: Readonly<RuntimeT | ||
| 647 | 717 | knownGood = role.good_id ? slots.get(role.good_id) ?? null : null |
| 648 | 718 | if ((role.installed_id && !installed) || (role.good_id && (!knownGood || knownGood.phase !== 'confirmed')) || installed?.phase === 'selected') throw new Error('Invalid installed or known-good host role') |
| 649 | 719 | } |
| 650 | return Object.freeze({ schema: 'drone-runtime-host-slots-v1', stateFormat: databaseVersion(db) as 1 | 2 | 3, store: summary(db, binding, storeSha256), installed, knownGood, trial: rows.length ? slots.get(rows.at(-1)!.id)! : null, retainedSlots: rows.length, physicalDeploymentAllowed: false }) | |
| 720 | return Object.freeze({ schema: 'drone-runtime-host-slots-v1', stateFormat: databaseVersion(db) as 1 | 2 | 3 | 4, store: summary(db, binding, storeSha256), installed, knownGood, trial: rows.length ? slots.get(rows.at(-1)!.id)! : null, retainedSlots: rows.length, physicalDeploymentAllowed: false }) | |
| 651 | 721 | } |
| 652 | 722 | |
| 653 | 723 | /** Historical host role/evidence inspection. This never authorizes use under current trust. */ |
| @@ -778,7 +848,7 @@ async function runHostTrial(options: RefreshOptions & { project: string }, candi | ||
| 778 | 848 | if (hostRows(db).some(row => row.phase === 'selected' || row.phase === 'running')) throw new Error('Interrupted host trial requires explicit recovery') |
| 779 | 849 | if (hostRows(db).length >= 128) throw new Error('Host slot history budget reached') |
| 780 | 850 | if (!candidateId) { |
| 781 | if (databaseVersion(db) !== 3) throw new Error('No retained known-good host image for rollback') | |
| 851 | if (databaseVersion(db) < 3) throw new Error('No retained known-good host image for rollback') | |
| 782 | 852 | const role = db.query<{ good_id: string | null }, []>('SELECT good_id FROM host_roles WHERE id = 1').get() |
| 783 | 853 | const previous = hostRows(db).find(row => row.id === role?.good_id && row.phase === 'confirmed') |
| 784 | 854 | if (!previous) throw new Error('No retained known-good host image for rollback') |
| @@ -802,7 +872,7 @@ function causedBy(error: unknown, marker: Error) { | ||
| 802 | 872 | export async function recoverRuntimeTrustStore(options: StoreReference): Promise<Readonly<RuntimeTrustStoreSummary>> { |
| 803 | 873 | options = { ...options } |
| 804 | 874 | return withStore(options, async (db, binding, trusted) => { |
| 805 | const pending = db.query<Attempt, []>("SELECT * FROM attempts WHERE phase IN ('pending', 'maintenance')").all() | |
| 875 | const pending = activeAttempts(db) | |
| 806 | 876 | if (pending.length !== 1) throw new Error('Exactly one unresolved attempt is required for recovery') |
| 807 | 877 | const attempt = pending[0]!, current = state(db, binding) |
| 808 | 878 | const inputs = db.query<InputRow, [string]>('SELECT * FROM inputs WHERE attempt_id = ? ORDER BY ordinal').all(attempt.id) |
| @@ -0,0 +1,235 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { randomBytes } from 'node:crypto' | |
| 3 | import { closeSync, constants, ftruncateSync, lstatSync, openSync, readFileSync, readdirSync, readlinkSync, statfsSync, writeSync } from 'node:fs' | |
| 4 | import { cp, mkdir, readFile, unlink, writeFile } from 'node:fs/promises' | |
| 5 | import { basename, dirname, join, resolve } from 'node:path' | |
| 6 | import { buildRuntime } from '../../../resources/functions/drone/runtime/build' | |
| 7 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 8 | import { abandonRuntimeFirmwareTrial, createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 9 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 10 | import { createHostSourceProject } from './host-project' | |
| 11 | import { digest, repository, target } from './repository' | |
| 12 | import { storageErrorCodes } from './storage-errors' | |
| 13 | ||
| 14 | const volumeBytes = 16777216, reserveBytes = 1048576 | |
| 15 | const clock = { source: 'independent-full-volume-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 16 | ||
| 17 | function privateNamespace() { | |
| 18 | assert.equal(process.platform, 'linux'); assert.equal(process.env.RUNTIME_HOST_ENOSPC_TEST, '1') | |
| 19 | assert.equal(process.getuid?.(), 0) | |
| 20 | const current = readlinkSync('/proc/self/ns/mnt') | |
| 21 | assert.notEqual(current, readlinkSync('/proc/1/ns/mnt'), 'Use an actual private mount namespace') | |
| 22 | const mounts = readFileSync('/proc/self/mountinfo', 'utf8') | |
| 23 | assert.ok(mounts.length > 0 && mounts.length < 2097152) | |
| 24 | for (const line of mounts.trim().split('\n')) { | |
| 25 | const [fields, filesystem] = line.split(' - ') | |
| 26 | assert.ok(fields && filesystem) | |
| 27 | assert.ok(!fields.split(' ').slice(6).some(field => /^(shared|master|propagate_from):/.test(field)), 'Require recursively private mount propagation before creating a filesystem') | |
| 28 | } | |
| 29 | return current | |
| 30 | } | |
| 31 | ||
| 32 | function checkedVolume(volume: string) { | |
| 33 | const identity = lstatSync(volume), stats = statfsSync(volume) | |
| 34 | assert.ok(identity.isDirectory() && !identity.isSymbolicLink() && identity.uid === process.getuid?.() && (identity.mode & 0o777) === 0o700) | |
| 35 | assert.equal(stats.type, 0x01021994, 'Require actual Linux tmpfs') | |
| 36 | assert.ok(stats.bsize >= 4096 && stats.bsize <= 65536 && reserveBytes % stats.bsize === 0) | |
| 37 | assert.equal(stats.blocks * stats.bsize, volumeBytes) | |
| 38 | return { device: identity.dev, blockBytes: stats.bsize, capacityBytes: stats.blocks * stats.bsize, freeBytes: stats.bfree * stats.bsize } | |
| 39 | } | |
| 40 | ||
| 41 | function object(value: unknown) { | |
| 42 | assert.ok(value && typeof value === 'object' && !Array.isArray(value)) | |
| 43 | return value as Record<string, unknown> | |
| 44 | } | |
| 45 | function text(value: unknown) { assert.equal(typeof value, 'string'); return value as string } | |
| 46 | function identity(value: unknown, length: 32 | 64) { const result = text(value); assert.match(result, new RegExp(`^[a-f0-9]{${length}}$`)); return result } | |
| 47 | ||
| 48 | async function workerInput(path: string) { | |
| 49 | const namespace = privateNamespace(), root = dirname(resolve(path)), volume = join(root, 'volume') | |
| 50 | assert.equal(basename(path), 'worker-input.json') | |
| 51 | const item = object(await Bun.file(path).json()), marker = object(await Bun.file(join(root, 'fixture.json')).json()) | |
| 52 | assert.equal(marker.kind, 'host-full-volume-v1'); assert.equal(item.nonce, marker.nonce) | |
| 53 | assert.equal(item.namespace, namespace); identity(item.nonce, 32) | |
| 54 | assert.equal(readFileSync(join(volume, 'nonce'), 'utf8'), item.nonce) | |
| 55 | checkedVolume(volume) | |
| 56 | const reference = object(item.reference) | |
| 57 | assert.equal(reference.directory, join(volume, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment')) | |
| 58 | assert.equal(item.project, join(root, 'project')); assert.equal(item.repository, join(root, 'repository')) | |
| 59 | return { root, volume, project: text(item.project), repository: text(item.repository), candidateId: identity(item.candidateId, 32), trialId: item.trialId === undefined ? undefined : identity(item.trialId, 32), reference: { | |
| 60 | directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: identity(reference.enrollmentSha256, 64), storeSha256: identity(reference.storeSha256, 64), | |
| 61 | } } | |
| 62 | } | |
| 63 | ||
| 64 | function fillVolume(volume: string) { | |
| 65 | const before = checkedVolume(volume), path = join(volume, 'filler.bin') | |
| 66 | const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) | |
| 67 | let written = 0, full = false | |
| 68 | try { | |
| 69 | const bytes = Buffer.alloc(65536, 0xa5) | |
| 70 | while (written <= volumeBytes) { | |
| 71 | try { const count = writeSync(fd, bytes); assert.ok(count > 0); written += count } | |
| 72 | catch (error) { assert.ok(storageErrorCodes(error).includes('ENOSPC')); full = true; break } | |
| 73 | } | |
| 74 | assert.equal(full, true); assert.equal(checkedVolume(volume).freeBytes, 0) | |
| 75 | assert.ok(written > reserveBytes); ftruncateSync(fd, written - reserveBytes) | |
| 76 | const after = checkedVolume(volume) | |
| 77 | assert.equal(after.freeBytes, reserveBytes); assert.equal(after.device, before.device) | |
| 78 | return { before, after, written, fillerBytes: written - reserveBytes, fillError: 'ENOSPC' } | |
| 79 | } | |
| 80 | finally { closeSync(fd) } | |
| 81 | } | |
| 82 | ||
| 83 | async function limited(inputPath: string) { | |
| 84 | const input = await workerInput(inputPath), phases: string[] = [] | |
| 85 | let rejected = false, codes: string[] = [] | |
| 86 | try { | |
| 87 | await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock, async onProgress(event) { | |
| 88 | assert.ok(phases.length < 512); phases.push(event.phase) | |
| 89 | if (event.phase === 'slot-selected') await writeFile(join(input.root, 'fill.json'), JSON.stringify(fillVolume(input.volume), null, 2)) | |
| 90 | } }) | |
| 91 | } | |
| 92 | catch (error) { rejected = true; codes = storageErrorCodes(error) } | |
| 93 | assert.equal(rejected, true, 'Actual filesystem exhaustion must reject confirmation') | |
| 94 | return { errorCodes: codes, phases, volumeAfterFailure: checkedVolume(input.volume) } | |
| 95 | } | |
| 96 | ||
| 97 | function volumeFiles(root: string) { | |
| 98 | const records: { path: string, byteLength: number, sha256: string }[] = [] | |
| 99 | let total = 0 | |
| 100 | function walk(path: string, relative: string) { | |
| 101 | for (const name of readdirSync(path).sort()) { | |
| 102 | const child = join(path, name), next = relative ? `${relative}/${name}` : name, stat = lstatSync(child) | |
| 103 | assert.ok(!stat.isSymbolicLink()) | |
| 104 | if (stat.isDirectory()) walk(child, next) | |
| 105 | else { | |
| 106 | assert.ok(stat.isFile() && stat.nlink === 1 && stat.size <= volumeBytes) | |
| 107 | assert.ok(records.length < 512); total += stat.size; assert.ok(total <= volumeBytes) | |
| 108 | records.push({ path: next, byteLength: stat.size, sha256: digest(readFileSync(child)) }) | |
| 109 | } | |
| 110 | } | |
| 111 | } | |
| 112 | walk(root, '') | |
| 113 | return records | |
| 114 | } | |
| 115 | ||
| 116 | async function archiveVolume(volume: string, output: string) { | |
| 117 | const before = volumeFiles(volume) | |
| 118 | await cp(volume, output, { recursive: true, errorOnExist: true, force: false }) | |
| 119 | assert.deepEqual(volumeFiles(output), before) | |
| 120 | await writeFile(`${output}.json`, JSON.stringify(before, null, 2)) | |
| 121 | } | |
| 122 | ||
| 123 | /** Full bounded tmpfs, genuine ENOSPC/SQLite failure, and fresh public recovery. */ | |
| 124 | export async function qualifyHostFullVolume(directory: string) { | |
| 125 | const namespace = privateNamespace() | |
| 126 | directory = resolve(directory) | |
| 127 | await mkdir(directory, { mode: 0o700 }) | |
| 128 | const volume = join(directory, 'volume'), nonce = randomBytes(16).toString('hex') | |
| 129 | await mkdir(volume, { mode: 0o700 }) | |
| 130 | await writeFile(join(directory, 'fixture.json'), JSON.stringify({ kind: 'host-full-volume-v1', nonce, namespace }), { mode: 0o600 }) | |
| 131 | async function command(argv: string[], label: string, timeoutMs = 10000) { | |
| 132 | const result = await runBoundedHostProcess(argv, { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs, stdoutBytes: 16384, stderrBytes: 16384, label }) | |
| 133 | assert.equal(result.exitCode, 0, result.stderr) | |
| 134 | return result | |
| 135 | } | |
| 136 | await command(['mount', '--no-mtab', '-t', 'tmpfs', '-o', `size=${volumeBytes},mode=0700,nosuid,nodev,noexec`, 'drone-runtime-test', volume], 'Private bounded update filesystem mount') | |
| 137 | try { | |
| 138 | const mounted = checkedVolume(volume) | |
| 139 | assert.notEqual(mounted.device, lstatSync(directory).dev) | |
| 140 | await writeFile(join(volume, 'nonce'), nonce, { mode: 0o600 }) | |
| 141 | const { project, sourceRevision } = await createHostSourceProject(directory, { imagePadding: true }) | |
| 142 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 143 | assert.ok(image.length > reserveBytes && image.length < 4 * reserveBytes) | |
| 144 | const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision } }) | |
| 145 | const enrollmentDirectory = join(directory, 'enrollment') | |
| 146 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 147 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(volume, 'store') }) | |
| 148 | const reference = { directory: join(volume, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 149 | const first = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 150 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: first.candidate!.id }) | |
| 151 | assert.equal(good.knownGood!.description.releaseSequence, 7) | |
| 152 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 } }) | |
| 153 | const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 154 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 155 | const inputPath = join(directory, 'worker-input.json'), config = join(directory, 'worker.bunfig.toml') | |
| 156 | const input = { nonce, namespace, reference, project, repository: source, candidateId: candidate.candidate!.id } | |
| 157 | await writeFile(inputPath, JSON.stringify(input), { mode: 0o600 }); await writeFile(config, 'preload=[]\n', { mode: 0o600 }) | |
| 158 | async function worker(mode: 'limited' | 'recover' | 'abandon' | 'fresh') { | |
| 159 | const result = await runBoundedHostProcess([process.execPath, '--no-env-file', `--config=${config}`, import.meta.path, mode, inputPath], { cwd: directory, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' }, timeoutMs: 60000, stdoutBytes: 16384, stderrBytes: 16384, label: `Full update filesystem ${mode}` }) | |
| 160 | await writeFile(join(directory, `${mode}.stdout`), result.stdout) | |
| 161 | await writeFile(join(directory, `${mode}.stderr`), result.stderr) | |
| 162 | await writeFile(join(directory, `${mode}.json`), JSON.stringify({ exitCode: result.exitCode })) | |
| 163 | assert.equal(result.exitCode, 0, result.stderr); assert.equal(result.stderr, '') | |
| 164 | return result.stdout | |
| 165 | } | |
| 166 | const rejection = object(JSON.parse(await worker('limited'))) | |
| 167 | assert.ok(Array.isArray(rejection.errorCodes) && rejection.errorCodes.every(code => typeof code === 'string' && code.length <= 64) && rejection.errorCodes.length <= 32) | |
| 168 | assert.ok(rejection.errorCodes.includes('ENOSPC') && rejection.errorCodes.includes('SQLITE_FULL')) | |
| 169 | assert.ok(Array.isArray(rejection.phases) && rejection.phases.includes('slot-selected')) | |
| 170 | assert.ok(!rejection.phases.includes('slot-image-written') && !rejection.phases.includes('runtime-started') && !rejection.phases.includes('runtime-input-committed')) | |
| 171 | const fullVolume = checkedVolume(volume) | |
| 172 | assert.equal(fullVolume.freeBytes, 0) | |
| 173 | await writeFile(join(directory, 'full-filesystem.json'), JSON.stringify(fullVolume, null, 2)) | |
| 174 | await archiveVolume(volume, join(directory, 'full-volume')) | |
| 175 | await unlink(join(volume, 'filler.bin')) | |
| 176 | const failed = await inspectRuntimeFirmwareSlots(reference) | |
| 177 | await writeFile(join(directory, 'failed-slots.json'), JSON.stringify(failed, null, 2)) | |
| 178 | assert.deepEqual(failed.knownGood, good.knownGood); assert.deepEqual(failed.installed, good.installed) | |
| 179 | assert.equal(failed.trial!.phase, 'selected'); assert.equal(failed.trial!.health, null) | |
| 180 | assert.equal(failed.trial!.integrity, 'mismatch'); assert.equal(failed.store.attempt!.phase, 'pending') | |
| 181 | assert.equal(failed.store.releaseFloor, 9) | |
| 182 | const version = (role: string) => failed.store.metadata.find(row => row.role === role)!.version | |
| 183 | assert.equal(version('root'), 2); assert.equal(version('targets'), 3) | |
| 184 | const partialPath = join(reference.directory, 'host-slots', failed.trial!.id, 'image.bin'), partial = await readFile(partialPath) | |
| 185 | assert.ok(partial.length > 0 && partial.length < image.length); assert.deepEqual(partial, image.subarray(0, partial.length)) | |
| 186 | const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 187 | assert.deepEqual(await readFile(previousImage), image) | |
| 188 | await archiveVolume(volume, join(directory, 'failed-volume')) | |
| 189 | await assert.rejects(abandonRuntimeFirmwareTrial({ ...reference, trialId: failed.trial!.id }), /Resolve metadata/) | |
| 190 | await writeFile(inputPath, JSON.stringify({ ...input, trialId: failed.trial!.id })) | |
| 191 | assert.equal(await worker('recover'), 'RECOVERY_COMPLETED\n') | |
| 192 | const recovered = await inspectRuntimeFirmwareSlots(reference) | |
| 193 | await writeFile(join(directory, 'recovered-slots.json'), JSON.stringify(recovered, null, 2)) | |
| 194 | assert.deepEqual(recovered.trial, failed.trial); assert.deepEqual(recovered.knownGood, good.knownGood) | |
| 195 | assert.equal(recovered.store.attempt!.phase, 'recovered'); assert.equal(recovered.store.releaseFloor, 9) | |
| 196 | assert.deepEqual(recovered.store.metadata, failed.store.metadata) | |
| 197 | assert.equal(await worker('abandon'), 'ABANDON_COMPLETED\n') | |
| 198 | const abandoned = await inspectRuntimeFirmwareSlots(reference) | |
| 199 | await writeFile(join(directory, 'abandoned-slots.json'), JSON.stringify(abandoned, null, 2)) | |
| 200 | assert.equal(abandoned.trial!.phase, 'abandoned'); assert.equal(abandoned.trial!.health, null) | |
| 201 | assert.deepEqual(abandoned.knownGood, good.knownGood) | |
| 202 | assert.equal(await worker('fresh'), 'FRESH_TRIAL_COMPLETED\n') | |
| 203 | const fresh = await inspectRuntimeFirmwareSlots(reference) | |
| 204 | await writeFile(join(directory, 'confirmed-slots.json'), JSON.stringify(fresh, null, 2)) | |
| 205 | assert.equal(fresh.knownGood!.description.releaseSequence, 9); assert.equal(fresh.knownGood!.health!.cleanProcessExit, true) | |
| 206 | assert.notEqual(fresh.knownGood!.id, failed.trial!.id) | |
| 207 | assert.notEqual(fresh.knownGood!.health!.bootId, good.knownGood!.health!.bootId) | |
| 208 | assert.notEqual(fresh.knownGood!.health!.sessionId, good.knownGood!.health!.sessionId) | |
| 209 | assert.deepEqual(await readFile(partialPath), partial); assert.deepEqual(await readFile(previousImage), image) | |
| 210 | assert.deepEqual(await readFile(join(reference.directory, 'host-slots', fresh.knownGood!.id, 'image.bin')), image) | |
| 211 | const result = { evidence: 'host-actual-enospc-v1', filesystemType: 'tmpfs', filesystemBytes: mounted.capacityBytes, freeBytesAfterFailure: fullVolume.freeBytes, sourceRevision, authorizedImageBytes: image.length, authorizedImageSha256: signed.imageHash, partialImageBytes: partial.length, partialImageSha256: digest(partial), errorCodes: rejection.errorCodes, interruptedPhase: failed.trial!.phase, interruptedAttempt: failed.store.attempt!.phase, acceptedRootVersion: version('root'), acceptedTargetsVersion: version('targets'), releaseFloor: fresh.store.releaseFloor, abandonedPhase: abandoned.trial!.phase, previousSlotId: good.knownGood!.id, failedSlotId: failed.trial!.id, confirmedSlotId: fresh.knownGood!.id, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood!.health!.bootId, physicalDeploymentAllowed: fresh.physicalDeploymentAllowed } | |
| 212 | await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) | |
| 213 | return result | |
| 214 | } | |
| 215 | finally { | |
| 216 | try { await archiveVolume(volume, join(directory, 'final-volume')) } | |
| 217 | finally { await command(['umount', '--no-mtab', volume], 'Private bounded update filesystem unmount') } | |
| 218 | } | |
| 219 | } | |
| 220 | ||
| 221 | if (import.meta.main) { | |
| 222 | try { | |
| 223 | assert.equal(process.argv.length, 4) | |
| 224 | const mode = process.argv[2], path = resolve(process.argv[3]!) | |
| 225 | if (mode === 'qualify') console.log(JSON.stringify(await qualifyHostFullVolume(path))) | |
| 226 | else if (mode === 'limited') console.log(JSON.stringify(await limited(path))) | |
| 227 | else { | |
| 228 | const input = await workerInput(path) | |
| 229 | if (mode === 'recover') { await recoverRuntimeTrustStore(input.reference); console.log('RECOVERY_COMPLETED') } | |
| 230 | else if (mode === 'abandon') { assert.ok(input.trialId); await abandonRuntimeFirmwareTrial({ ...input.reference, trialId: input.trialId }); console.log('ABANDON_COMPLETED') } | |
| 231 | else { assert.equal(mode, 'fresh'); await trialRuntimeFirmwareCandidate({ ...input.reference, repository: input.repository, project: input.project, candidateId: input.candidateId, clock }); console.log('FRESH_TRIAL_COMPLETED') } | |
| 232 | } | |
| 233 | } | |
| 234 | catch (error) { console.error('Host full-volume fixture failed', error); process.exitCode = 1 } | |
| 235 | } | |
| @@ -8,26 +8,11 @@ import { createRuntimeTrustStore, inspectRuntimeFirmwareSlots, recoverRuntimeTru | ||
| 8 | 8 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' |
| 9 | 9 | import { createHostSourceProject } from './host-project' |
| 10 | 10 | import { digest, repository, target } from './repository' |
| 11 | import { storageErrorCodes } from './storage-errors' | |
| 11 | 12 | |
| 12 | 13 | const fileLimitBytes = 1048576 |
| 13 | 14 | const clock = { source: 'independent-resource-fixture-clock', read: () => new Date('2030-01-01T00:00:00Z') } |
| 14 | 15 | |
| 15 | function errorCodes(error: unknown) { | |
| 16 | const pending = [error], seen = new Set<unknown>(), codes: string[] = [] | |
| 17 | while (pending.length && seen.size < 32) { | |
| 18 | const item = pending.shift() | |
| 19 | if (!item || typeof item !== 'object' || seen.has(item)) continue | |
| 20 | seen.add(item) | |
| 21 | const code = Object.getOwnPropertyDescriptor(item, 'code')?.value | |
| 22 | if ((typeof code === 'string' || typeof code === 'number') && String(code).length <= 64) codes.push(String(code)) | |
| 23 | const cause = Object.getOwnPropertyDescriptor(item, 'cause')?.value | |
| 24 | if (cause !== undefined) pending.push(cause) | |
| 25 | const errors = Object.getOwnPropertyDescriptor(item, 'errors')?.value | |
| 26 | if (Array.isArray(errors)) pending.push(...errors.slice(0, 32)) | |
| 27 | } | |
| 28 | return codes | |
| 29 | } | |
| 30 | ||
| 31 | 16 | async function fixtureInput(inputPath: string) { |
| 32 | 17 | assert.equal(basename(inputPath), 'worker-input.json') |
| 33 | 18 | const root = dirname(resolve(inputPath)) |
| @@ -51,7 +36,7 @@ async function limitedTrial(inputPath: string) { | ||
| 51 | 36 | assert.ok(phases.length < 512); phases.push(event.phase) |
| 52 | 37 | } }) |
| 53 | 38 | } |
| 54 | catch (error) { caught = true; codes = errorCodes(error) } | |
| 39 | catch (error) { caught = true; codes = storageErrorCodes(error) } | |
| 55 | 40 | assert.equal(caught, true, 'The actual kernel file limit must reject the trial') |
| 56 | 41 | return { errorCodes: codes, phases } |
| 57 | 42 | } |
| @@ -0,0 +1,122 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { randomBytes } from 'node:crypto' | |
| 3 | import { mkdir, readFile, writeFile } from 'node:fs/promises' | |
| 4 | import { basename, dirname, join, resolve } from 'node:path' | |
| 5 | import { continueRuntimeTrustMaintenance, inspectRuntimeTrustStore, recoverRuntimeTrustStore, refreshRuntimeTrustStore, reserveRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 6 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 7 | import { maintenanceScenario } from './maintenance-scenario' | |
| 8 | ||
| 9 | const cuts = ['maintenance-handoff-committed', 'input-committed', 'metadata-committed'] as const | |
| 10 | const clock = { source: 'synthetic-independent-maintenance-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 11 | ||
| 12 | function object(value: unknown) { assert.ok(value && typeof value === 'object' && !Array.isArray(value)); return value as Record<string, unknown> } | |
| 13 | function text(value: unknown) { assert.equal(typeof value, 'string'); return value as string } | |
| 14 | function hex(value: unknown, length: 32 | 64) { const result = text(value); assert.match(result, new RegExp(`^[a-f0-9]{${length}}$`)); return result } | |
| 15 | async function input(path: string) { | |
| 16 | const root = dirname(resolve(path)); assert.equal(basename(path), 'worker.json') | |
| 17 | assert.ok(Bun.file(path).size < 16384) | |
| 18 | const value = object(JSON.parse(await readFile(path, 'utf8'))), marker = object(await Bun.file(join(root, 'fixture.json')).json()) | |
| 19 | assert.equal(marker.kind, 'maintenance-process-cut-v1'); assert.equal(value.nonce, marker.nonce); hex(value.nonce, 32) | |
| 20 | assert.equal(value.repository, join(root, 'repository')); assert.ok(cuts.some(cut => cut === value.cut)) | |
| 21 | const reference = object(value.reference) | |
| 22 | assert.equal(reference.directory, join(root, 'store')); assert.equal(reference.enrollmentDirectory, join(root, 'enrollment')) | |
| 23 | return { cut: value.cut, repository: text(value.repository), previousAttemptId: hex(value.previousAttemptId, 32), reference: { directory: text(reference.directory), enrollmentDirectory: text(reference.enrollmentDirectory), enrollmentSha256: hex(reference.enrollmentSha256, 64), storeSha256: hex(reference.storeSha256, 64) } } | |
| 24 | } | |
| 25 | ||
| 26 | async function worker(mode: string, path: string) { | |
| 27 | const value = await input(path) | |
| 28 | if (mode === 'contend') { | |
| 29 | try { await inspectRuntimeTrustStore(value.reference); process.exitCode = 18 } | |
| 30 | catch (error) { | |
| 31 | const code = error && typeof error === 'object' ? Object.getOwnPropertyDescriptor(error, 'code')?.value : undefined | |
| 32 | console.log(JSON.stringify({ blocked: code === 'SQLITE_BUSY', code })); process.exitCode = code === 'SQLITE_BUSY' ? 17 : 19 | |
| 33 | } | |
| 34 | } | |
| 35 | else if (mode === 'recover') console.log(JSON.stringify(await recoverRuntimeTrustStore(value.reference))) | |
| 36 | else { | |
| 37 | assert.equal(mode, 'hold') | |
| 38 | await continueRuntimeTrustMaintenance({ ...value.reference, repository: value.repository, previousAttemptId: value.previousAttemptId, acknowledgeUnresolvedInputs: true, clock, async onProgress(event) { | |
| 39 | if (event.phase !== value.cut || event.ordinal !== 0) return | |
| 40 | console.log(JSON.stringify({ ready: true, phase: event.phase, ordinal: event.ordinal })) | |
| 41 | await Bun.stdin.text() | |
| 42 | } }) | |
| 43 | throw new Error('Held maintenance worker unexpectedly completed') | |
| 44 | } | |
| 45 | } | |
| 46 | ||
| 47 | /** Real SIGKILL, competing writer and fresh-process active-successor recovery. */ | |
| 48 | export async function qualifyMaintenanceInterruptions(directory: string) { | |
| 49 | directory = resolve(directory); await mkdir(directory, { mode: 0o700 }) | |
| 50 | const results = [] | |
| 51 | for (const cut of cuts) { | |
| 52 | const root = join(directory, cut); await mkdir(root, { mode: 0o700 }) | |
| 53 | const scenario = await maintenanceScenario(root) | |
| 54 | await scenario.signed.publish({ image: scenario.signed.image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 55 | const path = join(root, 'worker.json'), config = join(root, 'worker.bunfig.toml'), nonce = randomBytes(16).toString('hex') | |
| 56 | await writeFile(join(root, 'fixture.json'), JSON.stringify({ kind: 'maintenance-process-cut-v1', nonce }), { mode: 0o600 }) | |
| 57 | await writeFile(path, JSON.stringify({ nonce, cut, reference: scenario.reference, repository: scenario.source, previousAttemptId: scenario.failed.attempt!.id }), { mode: 0o600 }) | |
| 58 | await writeFile(config, 'preload=[]\n', { mode: 0o600 }) | |
| 59 | const argv = [process.execPath, '--no-env-file', `--config=${config}`, import.meta.path] | |
| 60 | const env = { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } | |
| 61 | const child = Bun.spawn([...argv, 'hold', path], { cwd: root, env, stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' }) | |
| 62 | let deadlineFired = false | |
| 63 | const timer = setTimeout(() => { deadlineFired = true; child.kill('SIGKILL') }, 15000) | |
| 64 | const reader = child.stdout.getReader(), decoder = new TextDecoder() | |
| 65 | let stdout = '', stderr: string | undefined | |
| 66 | try { | |
| 67 | while (!stdout.includes('\n')) { | |
| 68 | const part = await reader.read() | |
| 69 | if (part.done) throw new Error('Maintenance worker exited before its requested cut') | |
| 70 | stdout += decoder.decode(part.value, { stream: true }); assert.ok(stdout.length <= 4096) | |
| 71 | } | |
| 72 | assert.deepEqual(JSON.parse(stdout.trim()), { ready: true, phase: cut, ordinal: 0 }) | |
| 73 | const contender = await runBoundedHostProcess([...argv, 'contend', path], { cwd: root, env, timeoutMs: 10000, stdoutBytes: 4096, stderrBytes: 4096, label: 'Competing maintenance writer' }) | |
| 74 | await writeFile(join(root, 'contender.json'), JSON.stringify(contender, null, 2)) | |
| 75 | assert.equal(contender.exitCode, 17); assert.equal(contender.stderr, '') | |
| 76 | assert.deepEqual(JSON.parse(contender.stdout), { blocked: true, code: 'SQLITE_BUSY' }) | |
| 77 | child.kill('SIGKILL'); const exitCode = await child.exited | |
| 78 | clearTimeout(timer) | |
| 79 | assert.equal(child.signalCode, 'SIGKILL'); assert.notEqual(exitCode, 0); assert.equal(deadlineFired, false) | |
| 80 | stderr = await new Response(child.stderr).text(); assert.ok(stderr.length <= 65536); assert.equal(stderr, '') | |
| 81 | const interrupted = await inspectRuntimeTrustStore(scenario.reference) | |
| 82 | assert.equal(interrupted.attempt!.phase, 'pending') | |
| 83 | assert.equal(interrupted.maintenanceHandoffs.length, 1) | |
| 84 | assert.deepEqual(interrupted.maintenanceHandoffs[0]!.previousAttempt, scenario.failed.attempt) | |
| 85 | assert.equal(interrupted.maintenanceHandoffs[0]!.nextAttemptId, interrupted.attempt!.id) | |
| 86 | assert.equal(interrupted.releaseFloor, 9) | |
| 87 | await writeFile(join(root, 'interrupted.json'), JSON.stringify(interrupted, null, 2)) | |
| 88 | const recoveredProcess = await runBoundedHostProcess([...argv, 'recover', path], { cwd: root, env, timeoutMs: 15000, stdoutBytes: 65536, stderrBytes: 4096, label: 'Fresh maintenance recovery process' }) | |
| 89 | await writeFile(join(root, 'recovery-process.json'), JSON.stringify(recoveredProcess, null, 2)) | |
| 90 | assert.equal(recoveredProcess.exitCode, 0); assert.equal(recoveredProcess.stderr, '') | |
| 91 | const recovered = await inspectRuntimeTrustStore(scenario.reference) | |
| 92 | assert.deepEqual(JSON.parse(recoveredProcess.stdout), recovered) | |
| 93 | assert.equal(recovered.attempt!.phase, 'recovered'); assert.equal(recovered.attempt!.id, interrupted.attempt!.id) | |
| 94 | assert.deepEqual(recovered.maintenanceHandoffs[0]!.previousAttempt, scenario.failed.attempt) | |
| 95 | assert.equal(recovered.metadata.find(row => row.role === 'root')!.version, cut === 'maintenance-handoff-committed' ? 1 : 2) | |
| 96 | assert.equal(recovered.releaseFloor, 9) | |
| 97 | const fresh = await refreshRuntimeTrustStore({ ...scenario.reference, repository: scenario.source, clock }) | |
| 98 | assert.equal(fresh.attempt!.phase, 'verified'); assert.equal(fresh.releaseFloor, 9) | |
| 99 | assert.deepEqual(fresh.maintenanceHandoffs[0]!.previousAttempt, scenario.failed.attempt) | |
| 100 | const prepared = await reserveRuntimeFirmwareCandidate({ ...scenario.reference, repository: scenario.source, clock }) | |
| 101 | assert.equal(prepared.stateFormat, 4); assert.equal(prepared.candidate!.phase, 'prepared') | |
| 102 | assert.equal(prepared.candidate!.integrity, 'recorded-bytes-match'); assert.equal(prepared.candidate!.currentTrustValidated, false) | |
| 103 | results.push({ cut, pid: child.pid, exitCode, signal: child.signalCode, deadlineFired, contenderExit: contender.exitCode, previous: scenario.failed, interrupted, recovered, fresh, prepared }) | |
| 104 | await writeFile(join(directory, 'qualification.json'), JSON.stringify({ evidence: 'maintenance-host-process-cuts-v1', results, physicalDeploymentAllowed: false }, null, 2)) | |
| 105 | } | |
| 106 | finally { | |
| 107 | clearTimeout(timer) | |
| 108 | if (child.exitCode === null) child.kill('SIGKILL') | |
| 109 | await child.exited; reader.releaseLock() | |
| 110 | await writeFile(join(root, 'worker.stdout'), stdout) | |
| 111 | if (stderr === undefined) stderr = await new Response(child.stderr).text() | |
| 112 | await writeFile(join(root, 'worker.stderr'), stderr) | |
| 113 | } | |
| 114 | } | |
| 115 | return { evidence: 'maintenance-host-process-cuts-v1', cuts: results.length, physicalDeploymentAllowed: false } | |
| 116 | } | |
| 117 | ||
| 118 | if (import.meta.main) { | |
| 119 | if (process.argv.length === 4) await worker(process.argv[2]!, process.argv[3]!) | |
| 120 | else if (process.argv.length === 3) console.log(JSON.stringify(await qualifyMaintenanceInterruptions(process.argv[2]!))) | |
| 121 | else throw new Error('Use maintenance-interruptions.ts NEW_DIRECTORY') | |
| 122 | } | |
| @@ -0,0 +1,19 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { join } from 'node:path' | |
| 3 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 4 | import { createRuntimeTrustStore, inspectRuntimeTrustStore, refreshRuntimeTrustStore } from '../../../resources/functions/drone/updates/store' | |
| 5 | import { platform, repository, target } from './repository' | |
| 6 | ||
| 7 | /** Real signed release 9 followed by a deliberately wrong-key targets signature. */ | |
| 8 | export async function maintenanceScenario(root: string, clock = { source: 'independent-maintenance-clock', read: () => new Date('2030-01-01T00:00:00Z') }) { | |
| 9 | const source = join(root, 'repository'), signed = await repository(source, { custom: { releaseSequence: 9 } }) | |
| 10 | const enrollmentDirectory = join(root, 'enrollment') | |
| 11 | const enrolled = enrollRuntimeTrust({ target, platform, trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 12 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(root, 'store') }) | |
| 13 | const reference = { directory: join(root, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 14 | const good = await refreshRuntimeTrustStore({ ...reference, repository: source, clock }) | |
| 15 | await signed.publish({ image: signed.image, version: 2, custom: { releaseSequence: 9 }, badTargetsSignature: true }) | |
| 16 | await assert.rejects(refreshRuntimeTrustStore({ ...reference, repository: source, clock })) | |
| 17 | const failed = await inspectRuntimeTrustStore(reference) | |
| 18 | return { source, signed, reference, clock, good, failed } | |
| 19 | } | |
| @@ -0,0 +1,92 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { chmod, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises' | |
| 3 | import { join, resolve } from 'node:path' | |
| 4 | import { buildRuntime } from '../../../resources/functions/drone/runtime/build' | |
| 5 | import { enrollRuntimeTrust } from '../../../resources/functions/drone/updates/enrollment' | |
| 6 | import { createRuntimeTrustStore, inspectRuntimeTrustStore, refreshRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../../resources/functions/drone/updates/store' | |
| 7 | import { runBoundedHostProcess } from '../../../resources/functions/host/process' | |
| 8 | import { createHostSourceProject } from './host-project' | |
| 9 | import { digest, repository, target } from './repository' | |
| 10 | ||
| 11 | /** Copied real executable outside the checkout, signed owned host image and format-4 health. */ | |
| 12 | export async function qualifyMaintenanceCommands(directory: string, executable: string, olderExecutable?: string) { | |
| 13 | directory = resolve(directory); executable = resolve(executable) | |
| 14 | await mkdir(directory, { mode: 0o700 }) | |
| 15 | const binary = join(directory, 'drone'), binaryHash = digest(await readFile(executable)) | |
| 16 | await copyFile(executable, binary); await chmod(binary, 0o700) | |
| 17 | assert.equal(digest(await readFile(binary)), binaryHash) | |
| 18 | const { project, sourceRevision } = await createHostSourceProject(directory) | |
| 19 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 20 | const source = join(directory, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision, releaseSequence: 9 } }) | |
| 21 | const enrollmentDirectory = join(directory, 'enrollment') | |
| 22 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 23 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(directory, 'store') }) | |
| 24 | const reference = { directory: join(directory, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 25 | const clock = { source: 'operator-managed-command-fixture-clock', read: () => new Date() } | |
| 26 | const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 27 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: candidate.candidate!.id }) | |
| 28 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 }, badTargetsSignature: true }) | |
| 29 | await assert.rejects(refreshRuntimeTrustStore({ ...reference, repository: source, clock })) | |
| 30 | const failed = await inspectRuntimeTrustStore(reference) | |
| 31 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 32 | const flags = ['--enrollment', enrollmentDirectory, '--enrollment-sha256', enrolled.enrollmentSha256, '--store-sha256', created.storeSha256] | |
| 33 | const freshFlags = [...flags, '--repository', source, '--trusted-system-clock-source', clock.source] | |
| 34 | const operations: { args: string[], exitCode: number }[] = [] | |
| 35 | async function command(args: string[], expected = 0, artifact = binary) { | |
| 36 | const result = await runBoundedHostProcess([artifact, ...args], { cwd: directory, timeoutMs: 30000, stdoutBytes: 262144, stderrBytes: 65536, label: 'Copied maintenance CLI qualification' }) | |
| 37 | const ordinal = operations.length | |
| 38 | await writeFile(join(directory, `command-${ordinal}.stdout`), result.stdout) | |
| 39 | await writeFile(join(directory, `command-${ordinal}.stderr`), result.stderr) | |
| 40 | operations.push({ args, exitCode: result.exitCode }) | |
| 41 | await writeFile(join(directory, 'operations.json'), JSON.stringify(operations, null, 2)) | |
| 42 | assert.equal(result.exitCode, expected, result.stderr) | |
| 43 | if (expected === 0) { assert.equal(result.stderr, ''); return JSON.parse(result.stdout) } | |
| 44 | assert.equal(result.stdout, '') | |
| 45 | assert.equal(result.stderr, 'Runtime update store unavailable: invalid trust, clock, state or storage failure\n') | |
| 46 | return null | |
| 47 | } | |
| 48 | const maintain = ['updates:maintain', reference.directory, ...freshFlags, '--attempt', failed.attempt!.id] | |
| 49 | await command(maintain, 1) | |
| 50 | assert.deepEqual(await inspectRuntimeTrustStore(reference), failed) | |
| 51 | const verified = await command([...maintain, '--acknowledge-unresolved-inputs']) | |
| 52 | assert.equal(verified.attempt.phase, 'verified'); assert.equal(verified.releaseFloor, 9) | |
| 53 | assert.deepEqual(verified.maintenanceHandoffs[0].previousAttempt, failed.attempt) | |
| 54 | assert.deepEqual(await command(['updates:store', reference.directory, ...flags]), verified) | |
| 55 | const maintained = await command(['updates:slots', reference.directory, ...flags]) | |
| 56 | assert.equal(maintained.stateFormat, 4); assert.deepEqual(maintained.knownGood, good.knownGood) | |
| 57 | assert.deepEqual(maintained.installed, good.installed) | |
| 58 | await command([...maintain, '--acknowledge-unresolved-inputs'], 1) | |
| 59 | await command(['updates:recover', reference.directory, ...flags], 1) | |
| 60 | const prepared = await command(['updates:reserve', reference.directory, ...freshFlags]) | |
| 61 | assert.equal(prepared.stateFormat, 4); assert.equal(prepared.candidate.phase, 'prepared') | |
| 62 | const fresh = await command(['updates:trial', reference.directory, ...freshFlags, '--candidate', prepared.candidate.id, '--project', project]) | |
| 63 | assert.equal(fresh.stateFormat, 4); assert.equal(fresh.knownGood.health.cleanProcessExit, true) | |
| 64 | assert.notEqual(fresh.knownGood.id, good.knownGood!.id) | |
| 65 | assert.notEqual(fresh.knownGood.health.bootId, good.knownGood!.health!.bootId) | |
| 66 | assert.notEqual(fresh.knownGood.health.sessionId, good.knownGood!.health!.sessionId) | |
| 67 | const rollback = await command(['updates:rollback', reference.directory, ...freshFlags, '--project', project]) | |
| 68 | assert.notEqual(rollback.knownGood.health.bootId, fresh.knownGood.health.bootId) | |
| 69 | assert.equal(rollback.knownGood.health.cleanProcessExit, true) | |
| 70 | assert.equal(rollback.physicalDeploymentAllowed, false) | |
| 71 | const oldImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 72 | assert.deepEqual(await readFile(oldImage), image) | |
| 73 | let oldReader: { qualified: false } | { qualified: true, executableSha256: string, databasesUnchanged: true } = { qualified: false } | |
| 74 | if (olderExecutable) { | |
| 75 | olderExecutable = resolve(olderExecutable) | |
| 76 | const oldHash = digest(await readFile(olderExecutable)), paths = ['state.sqlite', 'writer.sqlite'].map(name => join(reference.directory, name)) | |
| 77 | const before = await Promise.all(paths.map(async path => digest(await readFile(path)))) | |
| 78 | await command(['updates:store', reference.directory, ...flags], 1, olderExecutable) | |
| 79 | assert.deepEqual(await Promise.all(paths.map(async path => digest(await readFile(path)))), before) | |
| 80 | assert.equal(digest(await readFile(olderExecutable)), oldHash) | |
| 81 | oldReader = { qualified: true, executableSha256: oldHash, databasesUnchanged: true } | |
| 82 | } | |
| 83 | assert.equal(digest(await readFile(binary)), binaryHash) | |
| 84 | const result = { evidence: 'copied-maintenance-command-v1', sourceRevision, executableSha256: binaryHash, operations: operations.length, retainedAttempt: failed.attempt, maintenanceHandoffs: rollback.store.maintenanceHandoffs, previousBootId: good.knownGood!.health!.bootId, freshBootId: fresh.knownGood.health.bootId, rollbackBootId: rollback.knownGood.health.bootId, releaseFloor: rollback.store.releaseFloor, oldReader, physicalDeploymentAllowed: rollback.physicalDeploymentAllowed } | |
| 85 | await writeFile(join(directory, 'qualification.json'), JSON.stringify(result, null, 2)) | |
| 86 | return result | |
| 87 | } | |
| 88 | ||
| 89 | if (import.meta.main) { | |
| 90 | assert.ok(process.argv.length === 4 || process.argv.length === 5) | |
| 91 | console.log(JSON.stringify(await qualifyMaintenanceCommands(process.argv[2]!, process.argv[3]!, process.argv[4]))) | |
| 92 | } | |
| @@ -59,7 +59,7 @@ export async function repository(directory: string, options: { image?: Uint8Arra | ||
| 59 | 59 | const imagePath = join(directory, 'targets/runtime', options.consistentSnapshot === false ? 'pico2-bench.bin' : `${imageHash}.pico2-bench.bin`) |
| 60 | 60 | await writeFile(imagePath, image) |
| 61 | 61 | let publishedTargetsSigner = keys.targets, publishedRootVersion = root.version |
| 62 | async function publish(update: { image: Uint8Array, version: number, custom?: Record<string, string | number | boolean>, rotateTargets?: boolean }) { | |
| 62 | async function publish(update: { image: Uint8Array, version: number, custom?: Record<string, string | number | boolean>, rotateTargets?: boolean, badTargetsSignature?: true }) { | |
| 63 | 63 | if (update.rotateTargets) { |
| 64 | 64 | if (options.rotate) throw new Error('Fixture target rotation requires its original root signer') |
| 65 | 65 | publishedTargetsSigner = signer() |
| @@ -71,7 +71,7 @@ export async function repository(directory: string, options: { image?: Uint8Arra | ||
| 71 | 71 | } |
| 72 | 72 | const bytes = Buffer.from(update.image), hash = digest(bytes), revision = { ...fields, version: update.version } |
| 73 | 73 | const nextTargets = new Metadata(new Targets({ ...revision, targets: { [target]: new TargetFile({ path: target, length: bytes.length, hashes: { sha256: hash }, unrecognizedFields: { custom: { drone: { ...custom, ...update.custom } } } }) } })) |
| 74 | nextTargets.sign(publishedTargetsSigner.sign) | |
| 74 | nextTargets.sign(update.badTargetsSignature ? signer().sign : publishedTargetsSigner.sign) | |
| 75 | 75 | const targetsBytes = Buffer.from(JSON.stringify(nextTargets.toJSON())) |
| 76 | 76 | const nextSnapshot = new Metadata(new Snapshot({ ...revision, meta: { 'targets.json': new MetaFile({ version: update.version, length: targetsBytes.length, hashes: { sha256: digest(targetsBytes) } }) } })) |
| 77 | 77 | nextSnapshot.sign(keys.snapshot.sign) |
| @@ -0,0 +1,16 @@ | ||
| 1 | /** Bounded own error descriptors only, including actual SQLite/OS aggregate causes. */ | |
| 2 | export function storageErrorCodes(error: unknown) { | |
| 3 | const pending = [error], seen = new Set<unknown>(), codes: string[] = [] | |
| 4 | while (pending.length && seen.size < 32) { | |
| 5 | const item = pending.shift() | |
| 6 | if (!item || typeof item !== 'object' || seen.has(item)) continue | |
| 7 | seen.add(item) | |
| 8 | const code = Object.getOwnPropertyDescriptor(item, 'code')?.value | |
| 9 | if ((typeof code === 'string' || typeof code === 'number') && String(code).length <= 64) codes.push(String(code)) | |
| 10 | const cause = Object.getOwnPropertyDescriptor(item, 'cause')?.value | |
| 11 | if (cause !== undefined) pending.push(cause) | |
| 12 | const errors = Object.getOwnPropertyDescriptor(item, 'errors')?.value | |
| 13 | if (Array.isArray(errors)) pending.push(...errors.slice(0, 32)) | |
| 14 | } | |
| 15 | return codes | |
| 16 | } | |
| @@ -0,0 +1,36 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdtemp, rm } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join, resolve } from 'node:path' | |
| 5 | import { qualifyHostFullVolume } from '../fixtures/updates/host-volume-full' | |
| 6 | ||
| 7 | // Actual privileged Linux namespace qualification only. A skip is not ENOSPC evidence. | |
| 8 | test.skipIf(process.env.RUNTIME_HOST_ENOSPC_TEST !== '1')('a full host filesystem preserves accepted trust and requires explicit recovery before fresh confirmation', async () => { | |
| 9 | const evidence = process.env.RUNTIME_HOST_ENOSPC_EVIDENCE | |
| 10 | const temporary = evidence ? undefined : await mkdtemp(join(tmpdir(), 'drone-host-enospc-')) | |
| 11 | const root = evidence ? resolve(evidence) : join(temporary!, 'campaign') | |
| 12 | let completed = false | |
| 13 | try { | |
| 14 | const result = await qualifyHostFullVolume(root) | |
| 15 | expect(result.filesystemType).toBe('tmpfs') | |
| 16 | expect(result.filesystemBytes).toBe(16777216) | |
| 17 | expect(result.freeBytesAfterFailure).toBe(0) | |
| 18 | expect(result.partialImageBytes).toBeGreaterThan(0) | |
| 19 | expect(result.partialImageBytes).toBeLessThan(result.authorizedImageBytes) | |
| 20 | expect(result.errorCodes).toContain('ENOSPC') | |
| 21 | expect(result.errorCodes).toContain('SQLITE_FULL') | |
| 22 | expect(result.interruptedPhase).toBe('selected') | |
| 23 | expect(result.interruptedAttempt).toBe('pending') | |
| 24 | expect(result.acceptedRootVersion).toBe(2) | |
| 25 | expect(result.acceptedTargetsVersion).toBe(3) | |
| 26 | expect(result.releaseFloor).toBe(9) | |
| 27 | expect(result.abandonedPhase).toBe('abandoned') | |
| 28 | expect(result.freshBootId).not.toBe(result.previousBootId) | |
| 29 | expect(result.physicalDeploymentAllowed).toBe(false) | |
| 30 | completed = true | |
| 31 | } | |
| 32 | finally { | |
| 33 | if (completed && temporary) await rm(temporary, { recursive: true, force: true }) | |
| 34 | else console.log(`Host full-volume campaign retained at ${root}`) | |
| 35 | } | |
| 36 | }, 180000) | |
| @@ -33,6 +33,7 @@ | ||
| 33 | 33 | "../resources/functions/drone/updates/**/*.ts", |
| 34 | 34 | "./unit/updates*.test.ts", |
| 35 | 35 | "./fixtures/updates/**/*.ts", |
| 36 | "./integration/runtime-host-storage.test.ts", | |
| 36 | 37 | "./unit/diagnostics.test.ts", |
| 37 | 38 | "./fixtures/diagnostics/**/*.ts", |
| 38 | 39 | "../resources/functions/drone/missions/**/*.ts", |
| @@ -0,0 +1,36 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { command } from '@stacksjs/testing' | |
| 3 | import { mkdtemp, rm } from 'node:fs/promises' | |
| 4 | import { tmpdir } from 'node:os' | |
| 5 | import { join } from 'node:path' | |
| 6 | import { inspectRuntimeTrustStore } from '../../resources/functions/drone/updates/store' | |
| 7 | import { maintenanceScenario } from '../fixtures/updates/maintenance-scenario' | |
| 8 | ||
| 9 | test('native maintenance requires explicit unresolved-input acknowledgement and verifies fresh metadata with retained pins', async () => { | |
| 10 | const root = await mkdtemp(join(tmpdir(), 'drone-maintenance-cli-')) | |
| 11 | let completed = false | |
| 12 | try { | |
| 13 | const item = await maintenanceScenario(root, { source: 'operator-managed-test-clock', read: () => new Date() }) | |
| 14 | await item.signed.publish({ image: item.signed.image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 15 | const flags = ['--enrollment', item.reference.enrollmentDirectory, '--enrollment-sha256', item.reference.enrollmentSha256, '--store-sha256', item.reference.storeSha256, '--repository', item.source, '--trusted-system-clock-source', 'operator-managed-test-clock', '--attempt', item.failed.attempt!.id] | |
| 16 | const noAcknowledgement = await command(['updates:maintain', item.reference.directory, ...flags]).withTimeout(20000) | |
| 17 | expect(noAcknowledgement.exitCode).toBe(1) | |
| 18 | expect(noAcknowledgement.stdout).toBe('') | |
| 19 | expect(await inspectRuntimeTrustStore(item.reference)).toEqual(item.failed) | |
| 20 | const result = await command(['updates:maintain', item.reference.directory, ...flags, '--acknowledge-unresolved-inputs']).withTimeout(20000) | |
| 21 | expect(result.exitCode).toBe(0) | |
| 22 | const observed = JSON.parse(result.stdout) | |
| 23 | expect(observed.attempt.phase).toBe('verified') | |
| 24 | expect(observed.maintenanceHandoffs[0].previousAttempt).toEqual(item.failed.attempt) | |
| 25 | expect(observed.releaseFloor).toBe(9) | |
| 26 | expect(observed.physicalDeploymentAllowed).toBe(false) | |
| 27 | const staleAttempt = await command(['updates:maintain', item.reference.directory, ...flags, '--acknowledge-unresolved-inputs']).withTimeout(20000) | |
| 28 | expect(staleAttempt.exitCode).toBe(1) | |
| 29 | expect(staleAttempt.stdout).toBe('') | |
| 30 | completed = true | |
| 31 | } | |
| 32 | finally { | |
| 33 | if (completed) await rm(root, { recursive: true, force: true }) | |
| 34 | else console.error(`Failed maintenance CLI case retained at ${root}`) | |
| 35 | } | |
| 36 | }, 120000) | |
| @@ -0,0 +1,22 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdir, mkdtemp, rm } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join, resolve } from 'node:path' | |
| 5 | import { qualifyMaintenanceInterruptions } from '../fixtures/updates/maintenance-interruptions' | |
| 6 | ||
| 7 | test('actual maintenance process cuts retain unresolved predecessors and recover only the active successor', async () => { | |
| 8 | const evidence = process.env.RUNTIME_MAINTENANCE_EVIDENCE | |
| 9 | const root = evidence ? resolve(evidence) : await mkdtemp(join(tmpdir(), 'drone-maintenance-cuts-')) | |
| 10 | if (evidence) await mkdir(root, { mode: 0o700 }) | |
| 11 | let completed = false | |
| 12 | try { | |
| 13 | const result = await qualifyMaintenanceInterruptions(join(root, 'campaign')) | |
| 14 | expect(result.cuts).toBe(3) | |
| 15 | expect(result.physicalDeploymentAllowed).toBe(false) | |
| 16 | completed = true | |
| 17 | } | |
| 18 | finally { | |
| 19 | if (completed && !evidence) await rm(root, { recursive: true, force: true }) | |
| 20 | else console.log(`Maintenance process-cut campaign retained at ${root}`) | |
| 21 | } | |
| 22 | }, 90000) | |
| @@ -0,0 +1,53 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdtemp, readFile, rm } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join } from 'node:path' | |
| 5 | import { buildRuntime } from '../../resources/functions/drone/runtime/build' | |
| 6 | import { enrollRuntimeTrust } from '../../resources/functions/drone/updates/enrollment' | |
| 7 | import { continueRuntimeTrustMaintenance, createRuntimeTrustStore, inspectRuntimeFirmwareSlots, inspectRuntimeTrustStore, refreshRuntimeTrustStore, reserveRuntimeFirmwareCandidate, trialRuntimeFirmwareCandidate } from '../../resources/functions/drone/updates/store' | |
| 8 | import { createHostSourceProject } from '../fixtures/updates/host-project' | |
| 9 | import { repository, target } from '../fixtures/updates/repository' | |
| 10 | ||
| 11 | test('format-4 maintenance preserves the known-good image and needs a fresh actual host session for new confirmation', async () => { | |
| 12 | const root = await mkdtemp(join(tmpdir(), 'drone-maintenance-host-')) | |
| 13 | let completed = false | |
| 14 | try { | |
| 15 | const { project, sourceRevision } = await createHostSourceProject(root) | |
| 16 | const built = await buildRuntime({ project }), image = await readFile(built.artifact) | |
| 17 | const source = join(root, 'repository'), signed = await repository(source, { image, custom: { platform: 'host-neutral-bench-v1', sourceRevision, releaseSequence: 9 } }) | |
| 18 | const enrollmentDirectory = join(root, 'enrollment') | |
| 19 | const enrolled = enrollRuntimeTrust({ target, platform: 'host-neutral-bench-v1', trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 20 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(root, 'store') }) | |
| 21 | const reference = { directory: join(root, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 22 | const clock = { source: 'independent-maintenance-host-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 23 | const candidate = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 24 | const good = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: candidate.candidate!.id }) | |
| 25 | const previousImage = join(reference.directory, 'host-slots', good.knownGood!.id, 'image.bin') | |
| 26 | await signed.publish({ image, version: 2, custom: { releaseSequence: 9 }, badTargetsSignature: true }) | |
| 27 | await expect(refreshRuntimeTrustStore({ ...reference, repository: source, clock })).rejects.toThrow() | |
| 28 | const failed = await inspectRuntimeTrustStore(reference) | |
| 29 | await signed.publish({ image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 30 | await continueRuntimeTrustMaintenance({ ...reference, repository: source, clock, previousAttemptId: failed.attempt!.id, acknowledgeUnresolvedInputs: true }) | |
| 31 | const maintained = await inspectRuntimeFirmwareSlots(reference) | |
| 32 | expect(maintained.stateFormat).toBe(4) | |
| 33 | expect(maintained.knownGood).toEqual(good.knownGood) | |
| 34 | expect(maintained.installed).toEqual(good.installed) | |
| 35 | expect(await readFile(previousImage)).toEqual(image) | |
| 36 | const next = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 37 | const fresh = await trialRuntimeFirmwareCandidate({ ...reference, repository: source, clock, project, candidateId: next.candidate!.id }) | |
| 38 | expect(fresh.stateFormat).toBe(4) | |
| 39 | expect(fresh.knownGood!.id).not.toBe(good.knownGood!.id) | |
| 40 | expect(fresh.knownGood!.health!.bootId).not.toBe(good.knownGood!.health!.bootId) | |
| 41 | expect(fresh.knownGood!.health!.sessionId).not.toBe(good.knownGood!.health!.sessionId) | |
| 42 | expect(fresh.knownGood!.health!.cleanProcessExit).toBe(true) | |
| 43 | expect(fresh.store.releaseFloor).toBe(9) | |
| 44 | expect(fresh.store.maintenanceHandoffs[0]!.previousAttempt).toEqual(failed.attempt!) | |
| 45 | expect(await readFile(previousImage)).toEqual(image) | |
| 46 | expect(fresh.physicalDeploymentAllowed).toBe(false) | |
| 47 | completed = true | |
| 48 | } | |
| 49 | finally { | |
| 50 | if (completed) await rm(root, { recursive: true, force: true }) | |
| 51 | else console.error(`Failed maintenance host case retained at ${root}`) | |
| 52 | } | |
| 53 | }, 120000) | |
| @@ -0,0 +1,127 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdtemp, readFile, rm } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join } from 'node:path' | |
| 5 | import { enrollRuntimeTrust } from '../../resources/functions/drone/updates/enrollment' | |
| 6 | import { continueRuntimeTrustMaintenance, createRuntimeTrustStore, inspectRuntimeFirmwareCandidates, inspectRuntimeFirmwareSlots, inspectRuntimeTrustStore, recoverRuntimeTrustStore, refreshRuntimeTrustStore, reserveRuntimeFirmwareCandidate, retireRuntimeFirmwareCandidate } from '../../resources/functions/drone/updates/store' | |
| 7 | import { platform, repository, target } from '../fixtures/updates/repository' | |
| 8 | import { maintenanceScenario } from '../fixtures/updates/maintenance-scenario' | |
| 9 | ||
| 10 | // Existing public coordinator seam: retain an unresolved rejection while a fully | |
| 11 | // authenticated fresh successor succeeds. This does not repair a lost trust root. | |
| 12 | test('explicit maintenance retains rejected input and security floors while fresh signed key rotation completes', async () => { | |
| 13 | const root = await mkdtemp(join(tmpdir(), 'drone-update-maintenance-')) | |
| 14 | let completed = false | |
| 15 | try { | |
| 16 | const source = join(root, 'repository'), signed = await repository(source, { custom: { releaseSequence: 9 } }) | |
| 17 | const enrollmentDirectory = join(root, 'enrollment') | |
| 18 | const enrolled = enrollRuntimeTrust({ target, platform, trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 19 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(root, 'store') }) | |
| 20 | const reference = { directory: join(root, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 21 | const clock = { source: 'independent-maintenance-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 22 | const good = await refreshRuntimeTrustStore({ ...reference, repository: source, clock }) | |
| 23 | expect(good.releaseFloor).toBe(9) | |
| 24 | await signed.publish({ image: signed.image, version: 2, custom: { releaseSequence: 9 }, badTargetsSignature: true }) | |
| 25 | await expect(refreshRuntimeTrustStore({ ...reference, repository: source, clock })).rejects.toThrow() | |
| 26 | const failed = await inspectRuntimeTrustStore(reference) | |
| 27 | expect(failed.attempt!.phase).toBe('maintenance') | |
| 28 | expect(failed.attempt!.inputRecords).toBeGreaterThan(0) | |
| 29 | await expect(recoverRuntimeTrustStore(reference)).rejects.toThrow() | |
| 30 | expect(await inspectRuntimeTrustStore(reference)).toEqual(failed) | |
| 31 | await signed.publish({ image: signed.image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 32 | const fresh = await continueRuntimeTrustMaintenance({ ...reference, repository: source, clock, previousAttemptId: failed.attempt!.id, acknowledgeUnresolvedInputs: true }) | |
| 33 | expect(fresh.attempt!.phase).toBe('verified') | |
| 34 | expect(fresh.attempt!.id).not.toBe(failed.attempt!.id) | |
| 35 | expect(fresh.maintenanceHandoffs).toEqual([{ previousAttempt: failed.attempt!, nextAttemptId: fresh.attempt!.id, generation: failed.generation }]) | |
| 36 | expect(fresh.releaseFloor).toBe(9) | |
| 37 | expect(fresh.storeSha256).toBe(reference.storeSha256) | |
| 38 | expect(fresh.enrollmentSha256).toBe(reference.enrollmentSha256) | |
| 39 | expect(fresh.clockFloorMs).toBe(failed.clockFloorMs) | |
| 40 | expect(fresh.retainedMetadataRecords).toBeGreaterThan(failed.retainedMetadataRecords) | |
| 41 | expect(fresh.metadata.map(row => [row.role, row.version])).toEqual([['root', 2], ['snapshot', 3], ['targets', 3], ['timestamp', 3]]) | |
| 42 | expect(fresh.physicalDeploymentAllowed).toBe(false) | |
| 43 | const slots = await inspectRuntimeFirmwareSlots(reference) | |
| 44 | expect(slots.stateFormat).toBe(4) | |
| 45 | expect(slots.knownGood).toBeNull() | |
| 46 | expect(slots.installed).toBeNull() | |
| 47 | expect(await inspectRuntimeTrustStore(reference)).toEqual(fresh) | |
| 48 | completed = true | |
| 49 | } | |
| 50 | finally { | |
| 51 | if (completed) await rm(root, { recursive: true, force: true }) | |
| 52 | else console.error(`Failed maintenance case retained at ${root}`) | |
| 53 | } | |
| 54 | }, 30000) | |
| 55 | ||
| 56 | test('format-2 maintenance preserves the prepared candidate until explicit retirement and a new reservation', async () => { | |
| 57 | const root = await mkdtemp(join(tmpdir(), 'drone-maintenance-candidate-')) | |
| 58 | let completed = false | |
| 59 | try { | |
| 60 | const source = join(root, 'repository'), signed = await repository(source, { custom: { releaseSequence: 9 } }) | |
| 61 | const enrollmentDirectory = join(root, 'enrollment') | |
| 62 | const enrolled = enrollRuntimeTrust({ target, platform, trustedRootSha256: signed.rootSha256, minimumReleaseSequence: 7, trustedRoot: signed.rootPath, output: enrollmentDirectory }) | |
| 63 | const created = await createRuntimeTrustStore({ enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, output: join(root, 'store') }) | |
| 64 | const reference = { directory: join(root, 'store'), enrollmentDirectory, enrollmentSha256: enrolled.enrollmentSha256, storeSha256: created.storeSha256 } | |
| 65 | const clock = { source: 'independent-candidate-maintenance-clock', read: () => new Date('2030-01-01T00:00:00Z') } | |
| 66 | const prepared = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 67 | expect(prepared.stateFormat).toBe(2) | |
| 68 | await signed.publish({ image: signed.image, version: 2, custom: { releaseSequence: 9 }, badTargetsSignature: true }) | |
| 69 | await expect(refreshRuntimeTrustStore({ ...reference, repository: source, clock })).rejects.toThrow() | |
| 70 | const failed = await inspectRuntimeTrustStore(reference) | |
| 71 | await signed.publish({ image: signed.image, version: 3, custom: { releaseSequence: 9 }, rotateTargets: true }) | |
| 72 | await continueRuntimeTrustMaintenance({ ...reference, repository: source, clock, previousAttemptId: failed.attempt!.id, acknowledgeUnresolvedInputs: true }) | |
| 73 | const retained = await inspectRuntimeFirmwareCandidates(reference) | |
| 74 | expect(retained.stateFormat).toBe(4) | |
| 75 | expect(retained.candidate).toEqual(prepared.candidate) | |
| 76 | expect(retained.store.storeSha256).toBe(reference.storeSha256) | |
| 77 | expect(retained.store.enrollmentSha256).toBe(reference.enrollmentSha256) | |
| 78 | expect((await inspectRuntimeFirmwareSlots(reference)).knownGood).toBeNull() | |
| 79 | await expect(reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock })).rejects.toThrow('active candidate') | |
| 80 | await retireRuntimeFirmwareCandidate({ ...reference, candidateId: prepared.candidate!.id }) | |
| 81 | const next = await reserveRuntimeFirmwareCandidate({ ...reference, repository: source, clock }) | |
| 82 | expect(next.candidate!.id).not.toBe(prepared.candidate!.id) | |
| 83 | expect(next.candidate!.description.rootVersion).toBe(2) | |
| 84 | expect(next.store.releaseFloor).toBe(9) | |
| 85 | expect(await readFile(join(prepared.candidate!.directory, 'image.bin'))).toEqual(signed.image) | |
| 86 | completed = true | |
| 87 | } | |
| 88 | finally { | |
| 89 | if (completed) await rm(root, { recursive: true, force: true }) | |
| 90 | else console.error(`Failed format-2 maintenance case retained at ${root}`) | |
| 91 | } | |
| 92 | }, 30000) | |
| 93 | ||
| 94 | test('a rejected maintenance successor keeps the original history and blocks ordinary refresh until explicit recovery or continuation', async () => { | |
| 95 | const root = await mkdtemp(join(tmpdir(), 'drone-maintenance-rejection-')) | |
| 96 | let completed = false | |
| 97 | try { | |
| 98 | const item = await maintenanceScenario(root) | |
| 99 | const options = { ...item.reference, repository: item.source, clock: item.clock, previousAttemptId: item.failed.attempt!.id, acknowledgeUnresolvedInputs: true as const } | |
| 100 | await item.signed.publish({ image: item.signed.image, version: 3, custom: { releaseSequence: 8 }, rotateTargets: true }) | |
| 101 | await expect(continueRuntimeTrustMaintenance(options)).rejects.toThrow('below the trusted minimum') | |
| 102 | const rejected = await inspectRuntimeTrustStore(item.reference) | |
| 103 | expect(rejected.releaseFloor).toBe(9) | |
| 104 | expect(rejected.attempt!.phase).toBe('maintenance') | |
| 105 | expect(rejected.attempt!.id).not.toBe(item.failed.attempt!.id) | |
| 106 | expect(rejected.maintenanceHandoffs[0]!.previousAttempt).toEqual(item.failed.attempt!) | |
| 107 | await expect(refreshRuntimeTrustStore({ ...item.reference, repository: item.source, clock: item.clock })).rejects.toThrow('requires recovery') | |
| 108 | await expect(continueRuntimeTrustMaintenance(options)).rejects.toThrow('Exact unresolved') | |
| 109 | await expect(continueRuntimeTrustMaintenance({ ...options, previousAttemptId: rejected.attempt!.id, clock: { source: 'reversed-maintenance-clock', read: () => new Date('2029-01-01T00:00:00Z') } })).rejects.toThrow('reversed') | |
| 110 | expect(await inspectRuntimeTrustStore(item.reference)).toEqual(rejected) | |
| 111 | await item.signed.publish({ image: item.signed.image, version: 4, custom: { releaseSequence: 9 } }) | |
| 112 | const verified = await continueRuntimeTrustMaintenance({ ...options, previousAttemptId: rejected.attempt!.id }) | |
| 113 | expect(verified.maintenanceHandoffs).toHaveLength(2) | |
| 114 | expect(verified.maintenanceHandoffs[0]!.previousAttempt).toEqual(item.failed.attempt!) | |
| 115 | expect(verified.maintenanceHandoffs[1]!.previousAttempt).toEqual(rejected.attempt!) | |
| 116 | expect(verified.maintenanceHandoffs[1]!.nextAttemptId).toBe(verified.attempt!.id) | |
| 117 | expect(verified.attempt!.phase).toBe('verified') | |
| 118 | expect(verified.releaseFloor).toBe(9) | |
| 119 | expect(verified.physicalDeploymentAllowed).toBe(false) | |
| 120 | expect((await refreshRuntimeTrustStore({ ...item.reference, repository: item.source, clock: item.clock })).attempt!.phase).toBe('verified') | |
| 121 | completed = true | |
| 122 | } | |
| 123 | finally { | |
| 124 | if (completed) await rm(root, { recursive: true, force: true }) | |
| 125 | else console.error(`Failed maintenance rejection retained at ${root}`) | |
| 126 | } | |
| 127 | }, 30000) | |