fix: hydrate durable registry snapshots before serving writes

packages/registry/src/build-status-hydration.test.tsadded+58-0

Changes to packages/registry/src/build-status-hydration.test.ts

		@@ -0,0 +1,58 @@
	1	+import type { S3Client } from './storage/aws-client'
	2	+import { expect, it } from 'bun:test'
	3	+import { S3RequestError } from './storage/aws-client'
	4	+import { BuildStatusStore } from './storage/build-status'
	5	+
	6	+for (const failedKey of ['build-status/status.json', 'build-status/coverage.json']) {
	7	+ it(`preserves both snapshots when ${failedKey} cannot hydrate, then permits a healthy retry`, async () => {
	8	+ const objects = new Map<string, Buffer>()
	9	+ let unavailable = false
	10	+ let writes = 0
	11	+ const client = {
	12	+ async getObjectBuffer(_bucket: string, key: string) {
	13	+ if (unavailable && key === failedKey)
	14	+ throw new S3RequestError('GET', 503, 'ServiceUnavailable')
	15	+ const value = objects.get(key)
	16	+ if (!value) throw new S3RequestError('GET', 404, 'NoSuchKey')
	17	+ return value
	18	+ },
	19	+ async putObject(input: { key: string, body: string | Buffer }) {
	20	+ writes++
	21	+ objects.set(input.key, Buffer.from(input.body))
	22	+ },
	23	+ async list() {
	24	+ return [{ Key: 'binaries/bun.sh/1.2.3/linux-arm64/bun.sh-1.2.3.tar.gz', LastModified: '2026-10-18T00:00:00Z' }]
	25	+ },
	26	+ } as unknown as S3Client
	27	+ const original = new BuildStatusStore(client, 'registry')
	28	+ await original.load()
	29	+ await original.refreshCoverage(true)
	30	+ original.record({ domain: 'bun.sh', version: '1.2.3', platform: 'linux-arm64', state: 'built' })
	31	+ await original.flush()
	32	+ const status = Buffer.from(objects.get('build-status/status.json')!)
	33	+ const coverage = Buffer.from(objects.get('build-status/coverage.json')!)
	34	+ const initialWrites = writes
	35	+
	36	+ unavailable = true
	37	+ const restarted = new BuildStatusStore(client, 'registry')
	38	+ await expect(restarted.load()).rejects.toThrow('503')
	39	+ if (failedKey.endsWith('coverage.json')) {
	40	+ restarted.record({ domain: 'git-scm.org', version: '2.53.0', platform: 'linux-arm64', state: 'built' })
	41	+ await expect(restarted.flush()).rejects.toThrow()
	42	+ }
	43	+ else {
	44	+ await expect(restarted.refreshCoverage(true)).rejects.toThrow()
	45	+ }
	46	+ expect(writes).toBe(initialWrites)
	47	+ expect(objects.get('build-status/status.json')).toEqual(status)
	48	+ expect(objects.get('build-status/coverage.json')).toEqual(coverage)
	49	+
	50	+ unavailable = false
	51	+ await restarted.load()
	52	+ restarted.record({ domain: 'curl.se', version: '8.22.0', platform: 'linux-arm64', state: 'built' })
	53	+ await restarted.flush()
	54	+ const recovered = JSON.parse(objects.get('build-status/status.json')!.toString())
	55	+ expect(recovered.recent.map((event: { domain: string }) => event.domain)).toContain('bun.sh')
	56	+ expect(recovered.recent.map((event: { domain: string }) => event.domain)).toContain('curl.se')
	57	+ })
	58	+}

packages/registry/src/build-status.test.tsmodified+5-4

Changes to packages/registry/src/build-status.test.ts

		@@ -1,3 +1,4 @@
	1	+import { S3RequestError } from './storage/aws-client'
1	2	import { describe, expect, it } from 'bun:test'
2	3	import { BuildStatusStore } from './storage/build-status'
3	4
		⋯3 linesShow the 3 lines above this hunk@@ -7,7 +8,7 @@ function mockBucket() {
7	8	const client = {
8	9	async getObjectBuffer(_bucket: string, key: string): Promise<Buffer> {
9	10	const value = objects.get(key)
10		- if (!value) throw new Error('S3 GET failed: 404 Not Found')
	11	+ if (!value) throw new S3RequestError('GET', 404, 'NoSuchKey')
11	12	return value
12	13	},
13	14	async putObject(input: { key: string, body: Buffer | string }): Promise<void> {
		⋯60 linesShow the 60 lines above this hunk@@ -74,7 +75,7 @@ describe('BuildStatusStore partial coverage', () => {
74	75	const client = {
75	76	async getObjectBuffer(_bucket: string, key: string): Promise<Buffer> {
76	77	const value = objects.get(key)
77		- if (!value) throw new Error('S3 GET failed: 404 Not Found')
	78	+ if (!value) throw new S3RequestError('GET', 404, 'NoSuchKey')
78	79	return value
79	80	},
80	81	async putObject(input: { key: string, body: Buffer | string }): Promise<void> {
		⋯54 linesShow the 54 lines above this hunk@@ -135,7 +136,7 @@ describe('BuildStatusStore partial coverage', () => {
135	136	const client = {
136	137	async getObjectBuffer(_bucket: string, key: string): Promise<Buffer> {
137	138	const value = objects.get(key)
138		- if (!value) throw new Error('S3 GET failed: 404 Not Found')
	139	+ if (!value) throw new S3RequestError('GET', 404, 'NoSuchKey')
139	140	return value
140	141	},
141	142	async putObject(input: { key: string, body: Buffer | string }): Promise<void> {
		⋯54 linesShow the 54 lines above this hunk@@ -196,7 +197,7 @@ describe('BuildStatusStore retired platforms', () => {
196	197	const client = {
197	198	async getObjectBuffer(_bucket: string, key: string): Promise<Buffer> {
198	199	const value = objects.get(key)
199		- if (!value) throw new Error('S3 GET failed: 404 Not Found')
	200	+ if (!value) throw new S3RequestError('GET', 404, 'NoSuchKey')
200	201	return value
201	202	},
202	203	async putObject(input: { key: string, body: Buffer | string }): Promise<void> {

packages/registry/src/object-analytics.test.tsmodified+2-1

Changes to packages/registry/src/object-analytics.test.ts

		⋯7 linesShow the 7 lines above this hunk@@ -8,6 +8,7 @@
8	8	* starts empty.
9	9	*/
10	10
	11	+import { S3RequestError } from './storage/aws-client'
11	12	import { describe, expect, it } from 'bun:test'
12	13	import { ObjectAnalytics } from './storage/object-analytics'
13	14
		⋯3 linesShow the 3 lines above this hunk@@ -17,7 +18,7 @@ function mockBucket() {
17	18	async getObjectBuffer(_bucket: string, key: string): Promise<Buffer> {
18	19	const v = store.get(key)
19	20	if (!v)
20		- throw new Error('S3 GET failed: 404 Not Found')
	21	+ throw new S3RequestError('GET', 404, 'NoSuchKey')
21	22	return v
22	23	},
23	24	async putObject(o: { bucket: string, key: string, body: Buffer | string }): Promise<void> {

packages/registry/src/object-metadata.test.tsmodified+2-1

Changes to packages/registry/src/object-metadata.test.ts

		⋯6 linesShow the 6 lines above this hunk@@ -7,6 +7,7 @@
7	7	* used to hold), and that a missing snapshot starts empty rather than throwing.
8	8	*/
9	9
	10	+import { S3RequestError } from './storage/aws-client'
10	11	import type { CommitPublish, PackageMetadata } from './types'
11	12	import { describe, expect, it } from 'bun:test'
12	13	import { ObjectMetadataStorage } from './storage/object-metadata'
		⋯5 linesShow the 5 lines above this hunk@@ -18,7 +19,7 @@ function mockBucket() {
18	19	async getObjectBuffer(_bucket: string, key: string): Promise<Buffer> {
19	20	const v = store.get(key)
20	21	if (!v)
21		- throw new Error('S3 GET failed: 404 Not Found')
	22	+ throw new S3RequestError('GET', 404, 'NoSuchKey')
22	23	return v
23	24	},
24	25	async putObject(o: { bucket: string, key: string, body: Buffer | string }): Promise<void> {

packages/registry/src/object-package-storage.test.tsmodified+2-1

Changes to packages/registry/src/object-package-storage.test.ts

		@@ -1,3 +1,4 @@
	1	+import { S3RequestError } from './storage/aws-client'
1	2	import { describe, expect, test } from 'bun:test'
2	3	import { ObjectPhpStorage } from './php'
3	4	import { ObjectZigStorage } from './zig'
		⋯10 linesShow the 10 lines above this hunk@@ -14,7 +15,7 @@ class MockS3 {
14	15	async getObjectBuffer(bucket: string, key: string): Promise<Buffer> {
15	16	const v = this.store.get(`${bucket}/${key}`)
16	17	if (!v)
17		- throw new Error('404 Not Found')
	18	+ throw new S3RequestError('GET', 404, 'NoSuchKey')
18	19	return v
19	20	}
20	21

packages/registry/src/object-snapshot.test.tsadded+83-0

Changes to packages/registry/src/object-snapshot.test.ts

		@@ -0,0 +1,83 @@
	1	+import type { S3Client } from './storage/aws-client'
	2	+import { describe, expect, it } from 'bun:test'
	3	+import { S3RequestError } from './storage/aws-client'
	4	+import { ObjectSnapshot } from './storage/object-snapshot'
	5	+import { ObjectMetadataStorage } from './storage/object-metadata'
	6	+import { ObjectAnalytics } from './storage/object-analytics'
	7	+
	8	+function bucket(body: string, failure?: Error) {
	9	+ let reads = 0
	10	+ let writes = 0
	11	+ const client = {
	12	+ async getObjectBuffer() {
	13	+ reads++
	14	+ if (failure) throw failure
	15	+ return Buffer.from(body)
	16	+ },
	17	+ async putObject() { writes++ },
	18	+ } as unknown as S3Client
	19	+ return { client, reads: () => reads, writes: () => writes }
	20	+}
	21	+
	22	+describe('snapshot hydration safety', () => {
	23	+ for (const failure of [
	24	+ new S3RequestError('GET', 503, 'ServiceUnavailable'),
	25	+ new S3RequestError('GET', 403, 'AccessDenied'),
	26	+ new S3RequestError('GET', 404, 'NoSuchBucket'),
	27	+ new Error('transport unavailable'),
	28	+ new Error('adapter failed: 404 Not Found'),
	29	+ ]) {
	30	+ it(`rejects ${failure.message} without overwriting remote state`, async () => {
	31	+ const remote = bucket('{"existing":true}', failure)
	32	+ const snapshot = new ObjectSnapshot(remote.client, 'b', 'snapshot.json', () => ({}))
	33	+ await expect(snapshot.load()).rejects.toBe(failure)
	34	+ await expect(snapshot.flush()).rejects.toThrow()
	35	+ expect(remote.writes()).toBe(0)
	36	+ })
	37	+ }
	38	+
	39	+ it('cannot flush before the initial read completes', async () => {
	40	+ const remote = bucket('{}')
	41	+ const snapshot = new ObjectSnapshot(remote.client, 'b', 'snapshot.json', () => ({}))
	42	+ await expect(snapshot.flush()).rejects.toThrow()
	43	+ expect(remote.reads()).toBe(0)
	44	+ expect(remote.writes()).toBe(0)
	45	+ await snapshot.load()
	46	+ await snapshot.flush()
	47	+ expect(remote.writes()).toBe(1)
	48	+ })
	49	+
	50	+ it('permits a first save only after confirmed object absence', async () => {
	51	+ const remote = bucket('', new S3RequestError('GET', 404, 'NoSuchKey'))
	52	+ const snapshot = new ObjectSnapshot(remote.client, 'b', 'snapshot.json', () => ({}))
	53	+ expect(await snapshot.load()).toBeNull()
	54	+ await snapshot.flush()
	55	+ expect(remote.writes()).toBe(1)
	56	+ })
	57	+
	58	+ it('rejects corrupt snapshots without replacing them', async () => {
	59	+ for (const body of ['', '{broken']) {
	60	+ const remote = bucket(body)
	61	+ const snapshot = new ObjectSnapshot(remote.client, 'b', 'snapshot.json', () => ({}))
	62	+ await expect(snapshot.load()).rejects.toThrow()
	63	+ await expect(snapshot.flush()).rejects.toThrow()
	64	+ expect(remote.writes()).toBe(0)
	65	+ }
	66	+ })
	67	+
	68	+ it('metadata readiness rejects an outage and flush cannot overwrite the index', async () => {
	69	+ const remote = bucket('{}', new S3RequestError('GET', 503, 'ServiceUnavailable'))
	70	+ const storage = new ObjectMetadataStorage(remote.client, 'b')
	71	+ await expect(storage.ready()).rejects.toThrow('503')
	72	+ await expect(storage.flush()).rejects.toThrow()
	73	+ expect(remote.writes()).toBe(0)
	74	+ })
	75	+
	76	+ it('analytics readiness rejects an outage and flush cannot overwrite counters', async () => {
	77	+ const remote = bucket('{}', new S3RequestError('GET', 503, 'ServiceUnavailable'))
	78	+ const storage = new ObjectAnalytics(remote.client, 'b')
	79	+ await expect(storage.ready()).rejects.toThrow('503')
	80	+ await expect(storage.flush()).rejects.toThrow()
	81	+ expect(remote.writes()).toBe(0)
	82	+ })
	83	+})

packages/registry/src/server.tsmodified+10-1

Changes to packages/registry/src/server.ts

		⋯6,845 linesShow the 6,845 lines above this hunk@@ -6846,7 +6846,16 @@ if (import.meta.main) {
6846	6846	// Ensure auth storage uses the same DynamoDB table as the registry
6847	6847	const authStorage = createAuthStorage(dynamoTable, awsRegion)
6848	6848
6849		- const { start } = createServer(registry, port, analytics, undefined, undefined, undefined, authStorage)
	6849	+ const zigStorage = createZigStorage()
	6850	+ const phpStorage = createPhpStorage()
	6851	+ // Hydrate durable state before accepting any request that can mutate it.
	6852	+ // Only a confirmed missing object can initialize an empty registry.
	6853	+ await Promise.all([registry.metadata, analytics, authStorage, zigStorage, phpStorage].map(async (store) => {
	6854	+ if ('ready' in store && typeof store.ready === 'function')
	6855	+ await store.ready()
	6856	+ }))
	6857	+
	6858	+ const { start } = createServer(registry, port, analytics, zigStorage, undefined, phpStorage, authStorage)
6850	6859	start()
6851	6860
6852	6861	console.log('\nEnvironment:')

packages/registry/src/storage/build-status.tsmodified+15-2

Changes to packages/registry/src/storage/build-status.ts

		⋯178 linesShow the 178 lines above this hunk@@ -179,6 +179,7 @@ export class BuildStatusStore {
179	179	private unavailable = new Map<string, UnavailableVersion>()
180	180	private snapshot: ObjectSnapshot
181	181	private coverageSnapshot: ObjectSnapshot
	182	+ private hydrated = false
182	183
183	184	// Derived coverage cache (domain -> version -> set of platforms) + freshness.
184	185	private coverage = new Map<string, Map<string, Set<string>>>()
		⋯18 linesShow the 18 lines above this hunk@@ -203,8 +204,18 @@ export class BuildStatusStore {
203	204	private logs = new Map<string, BuildLogLine[]>()
204	205
205	206	constructor(private s3: S3Client, private bucket: string) {
206		- this.snapshot = new ObjectSnapshot(s3, bucket, 'build-status/status.json', () => this.captureState())
207		- this.coverageSnapshot = new ObjectSnapshot(s3, bucket, 'build-status/coverage.json', () => this.captureCoverage())
	207	+ this.snapshot = new ObjectSnapshot(s3, bucket, 'build-status/status.json', () => {
	208	+ this.assertHydrated()
	209	+ return this.captureState()
	210	+ })
	211	+ this.coverageSnapshot = new ObjectSnapshot(s3, bucket, 'build-status/coverage.json', () => {
	212	+ this.assertHydrated()
	213	+ return this.captureCoverage()
	214	+ })
	215	+ }
	216	+
	217	+ private assertHydrated(): void {
	218	+ if (!this.hydrated) throw new Error('Build status snapshots have not been applied')
208	219	}
209	220
210	221	/**
		⋯101 linesShow the 101 lines above this hunk@@ -312,6 +323,7 @@ export class BuildStatusStore {
312	323	}
313	324
314	325	async load(): Promise<void> {
	326	+ this.hydrated = false
315	327	const [state, coverage] = await Promise.all([
316	328	this.snapshot.load() as Promise<PersistedState | null>,
317	329	this.coverageSnapshot.load() as Promise<PersistedCoverage | null>,
		⋯26 linesShow the 26 lines above this hunk@@ -344,6 +356,7 @@ export class BuildStatusStore {
344	356	if (event.state === 'built' && event.version)
345	357	this.mergeCoverage(event.domain, event.version, event.platform, new Date(event.ts).toISOString())
346	358	}
	359	+ this.hydrated = true
347	360	}
348	361
349	362	private key(e: { domain: string, version: string, platform: string }): string {

packages/registry/src/storage/object-analytics.tsmodified+8-7

Changes to packages/registry/src/storage/object-analytics.ts

		@@ -1,4 +1,5 @@
1	1	import type { S3Client } from './aws-client'
	2	+import { isS3ObjectNotFound } from './aws-client'
2	3	import { InMemoryAnalytics } from '../analytics'
3	4
4	5	/**
		⋯11 linesShow the 11 lines above this hunk@@ -16,6 +17,7 @@ import { InMemoryAnalytics } from '../analytics'
16	17	* are debounced (counters accumulate in memory between flushes).
17	18	*/
18	19	export class ObjectAnalytics extends InMemoryAnalytics {
	20	+ private hydrated = false
19	21	private s3: S3Client
20	22	private bucket: string
21	23	private key: string
		⋯17 linesShow the 17 lines above this hunk@@ -39,16 +41,12 @@ export class ObjectAnalytics extends InMemoryAnalytics {
39	41	private async load(): Promise<void> {
40	42	try {
41	43	const buf = await this.s3.getObjectBuffer(this.bucket, this.key)
42		- if (!buf || buf.byteLength === 0)
43		- return
44	44	this.applyAnalyticsState(JSON.parse(buf.toString('utf-8')))
	45	+ this.hydrated = true
45	46	}
46	47	catch (err) {
47		- const msg = (err as Error).message || ''
48		- // A missing object on first run is expected (404/403); start empty otherwise.
49		- if (!/\b40[34]\b/.test(msg)) {
50		- console.error(`ObjectAnalytics: failed to load ${this.bucket}/${this.key}: ${msg}. Starting empty.`)
51		- }
	48	+ if (!isS3ObjectNotFound(err)) throw err
	49	+ this.hydrated = true
52	50	}
53	51	}
54	52
		⋯11 linesShow the 11 lines above this hunk@@ -66,6 +64,8 @@ export class ObjectAnalytics extends InMemoryAnalytics {
66	64	}
67	65
68	66	private async save(): Promise<void> {
	67	+ if (!this.hydrated)
	68	+ throw new Error(`Snapshot not hydrated: ${this.key}`)
69	69	while (this.savePromise) await this.savePromise
70	70	this.savePromise = this.doSave()
71	71	try {
		⋯16 linesShow the 16 lines above this hunk@@ -88,6 +88,7 @@ export class ObjectAnalytics extends InMemoryAnalytics {
88	88
89	89	/** Flush any pending save immediately (e.g. before shutdown). */
90	90	async flush(): Promise<void> {
	91	+ await this.loaded
91	92	if (this.saveTimeout) {
92	93	clearTimeout(this.saveTimeout)
93	94	this.saveTimeout = null

packages/registry/src/storage/object-metadata.tsmodified+8-9

Changes to packages/registry/src/storage/object-metadata.ts

		@@ -1,4 +1,5 @@
1	1	import type { S3Client } from './aws-client'
	2	+import { isS3ObjectNotFound } from './aws-client'
2	3	import { InMemoryMetadataStorage } from './metadata'
3	4
4	5	/**
		⋯10 linesShow the 10 lines above this hunk@@ -15,6 +16,7 @@ import { InMemoryMetadataStorage } from './metadata'
15	16	* snapshot from the bucket.
16	17	*/
17	18	export class ObjectMetadataStorage extends InMemoryMetadataStorage {
	19	+ private hydrated = false
18	20	private s3: S3Client
19	21	private bucket: string
20	22	private key: string
		⋯17 linesShow the 17 lines above this hunk@@ -38,19 +40,13 @@ export class ObjectMetadataStorage extends InMemoryMetadataStorage {
38	40	private async load(): Promise<void> {
39	41	try {
40	42	const buf = await this.s3.getObjectBuffer(this.bucket, this.key)
41		- if (!buf || buf.byteLength === 0)
42		- return
43	43	const data = JSON.parse(buf.toString('utf-8'))
44	44	this.applyState(data)
	45	+ this.hydrated = true
45	46	}
46	47	catch (err) {
47		- // A missing object on first run is expected (404/403). Anything else we
48		- // log loudly but still start empty rather than block the server boot —
49		- // and we do NOT overwrite the remote object until the next real save.
50		- const msg = (err as Error).message || ''
51		- if (!/\b40[34]\b/.test(msg)) {
52		- console.error(`ObjectMetadataStorage: failed to load ${this.bucket}/${this.key}: ${msg}. Starting empty.`)
53		- }
	48	+ if (!isS3ObjectNotFound(err)) throw err
	49	+ this.hydrated = true
54	50	}
55	51	}
56	52
		⋯8 linesShow the 8 lines above this hunk@@ -65,6 +61,8 @@ export class ObjectMetadataStorage extends InMemoryMetadataStorage {
65	61	}
66	62
67	63	private async save(): Promise<void> {
	64	+ if (!this.hydrated)
	65	+ throw new Error(`Snapshot not hydrated: ${this.key}`)
68	66	// Serialize saves so a slow upload can't be overtaken by a newer one.
69	67	while (this.savePromise) await this.savePromise
70	68	this.savePromise = this._doSave()
		⋯17 linesShow the 17 lines above this hunk@@ -88,6 +86,7 @@ export class ObjectMetadataStorage extends InMemoryMetadataStorage {
88	86
89	87	/** Flush any pending save immediately (e.g. before shutdown). */
90	88	async flush(): Promise<void> {
	89	+ await this.loaded
91	90	if (this.saveTimeout) {
92	91	clearTimeout(this.saveTimeout)
93	92	this.saveTimeout = null

packages/registry/src/storage/object-snapshot.tsmodified+9-9

Changes to packages/registry/src/storage/object-snapshot.ts

		@@ -1,4 +1,5 @@
1	1	import type { S3Client } from './aws-client'
	2	+import { isS3ObjectNotFound } from './aws-client'
2	3
3	4	/**
4	5	* Reusable debounced JSON-snapshot persistence to an S3-compatible bucket.
		⋯8 linesShow the 8 lines above this hunk@@ -13,6 +14,7 @@ import type { S3Client } from './aws-client'
13	14	* the bucket under their own keys by the owning store.
14	15	*/
15	16	export class ObjectSnapshot {
	17	+ private hydrated = false
16	18	private saveTimeout: ReturnType<typeof setTimeout> | null = null
17	19	private savePromise: Promise<void> | null = null
18	20
		⋯8 linesShow the 8 lines above this hunk@@ -27,17 +29,13 @@ export class ObjectSnapshot {
27	29	async load(): Promise<unknown | null> {
28	30	try {
29	31	const buf = await this.s3.getObjectBuffer(this.bucket, this.key)
30		- if (!buf || buf.byteLength === 0)
31		- return null
32		- return JSON.parse(buf.toString('utf-8'))
	32	+ const state = JSON.parse(buf.toString('utf-8'))
	33	+ this.hydrated = true
	34	+ return state
33	35	}
34	36	catch (err) {
35		- // A missing object on first run is expected (404/403). Anything else we
36		- // surface but still start empty rather than block boot — and we do NOT
37		- // overwrite the remote object until the next real save.
38		- const msg = (err as Error).message || ''
39		- if (!/\b40[34]\b/.test(msg))
40		- console.error(`ObjectSnapshot: failed to load ${this.bucket}/${this.key}: ${msg}. Starting empty.`)
	37	+ if (!isS3ObjectNotFound(err)) throw err
	38	+ this.hydrated = true
41	39	return null
42	40	}
43	41	}
		⋯9 linesShow the 9 lines above this hunk@@ -53,6 +51,8 @@ export class ObjectSnapshot {
53	51	}
54	52
55	53	private async save(): Promise<void> {
	54	+ if (!this.hydrated)
	55	+ throw new Error(`Snapshot not hydrated: ${this.key}`)
56	56	// Serialize saves so a slow upload can't be overtaken by a newer one.
57	57	while (this.savePromise) await this.savePromise
58	58	this.savePromise = this.s3.putObject({