ReviewOS

also looking at this

craft-native/craft

Gate releases on shipped artifact findings

#316
Merged glennmichael123 wants to merge codex/gate-shipped-artifacts into main
11 files +454 -25

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

scripts/npm-packages.test.tsmodified+21-1
Changes to scripts/npm-packages.test.ts
@@ -1,5 +1,9 @@
11import { expect, test } from 'bun:test'
2import { manifestTargets, verifyArchiveTargets } from './npm-packages'
2import { createHash } from 'node:crypto'
3import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
4import { tmpdir } from 'node:os'
5import { join } from 'node:path'
6import { manifestTargets, verifyArchiveTargets, verifyNpmPackages } from './npm-packages'
37
48test('rejects absent built entries even when the manifest advertises them', () => {
59 const pkg = { name: '@craft-native/react', main: 'dist/index.js', exports: { '.': { import: './dist/index.mjs', types: './dist/index.d.ts' } } }
@@ -31,3 +35,19 @@ test('fails closed for targets that escape the package or need wildcard handling
3135 for (const target of ['../outside.js', '/outside.js', './dist/../outside.js', '.\\outside.js', './dist/*.js'])
3236 expect(() => manifestTargets({ name: 'fixture', exports: target })).toThrow()
3337})
38
39test('exports every verified npm archive with its exact digest', async () => {
40 const root = mkdtempSync(join(tmpdir(), 'craft-npm-archive-test-'))
41 const output = join(root, 'archives')
42 try {
43 await verifyNpmPackages(false, output)
44 const manifest = JSON.parse(readFileSync(join(output, 'manifest.json'), 'utf8')) as { packages: { name: string, file: string, sha256: string }[] }
45 expect(manifest.packages).toHaveLength(8)
46 for (const archive of manifest.packages) {
47 const path = join(output, archive.file)
48 expect(existsSync(path)).toBe(true)
49 expect(createHash('sha256').update(readFileSync(path)).digest('hex')).toBe(archive.sha256)
50 }
51 }
52 finally { rmSync(root, { recursive: true, force: true }) }
53}, 120_000)
scripts/npm-packages.tsmodified+27-6
Changes to scripts/npm-packages.ts
@@ -1,9 +1,11 @@
1import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
1import { createHash } from 'node:crypto'
2import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
23import { tmpdir } from 'node:os'
3import { dirname, join, resolve } from 'node:path'
4import { dirname, join, resolve, sep } from 'node:path'
45
56type Manifest = {
67 name: string
8 version: string
79 private?: boolean
810 scripts?: Record<string, string>
911 peerDependencies?: Record<string, string>
@@ -76,10 +78,18 @@ function importSpecifiers(pkg: Manifest): string[] {
7678 return [pkg.name]
7779}
7880
79export async function verifyNpmPackages(buildOnly = false): Promise<void> {
81export async function verifyNpmPackages(buildOnly = false, archiveDir?: string): Promise<void> {
8082 const packages = publicPackages()
8183 if (packages.length === 0)
8284 throw new Error('No public npm workspaces found')
85 if (buildOnly && archiveDir) throw new Error('Cannot export archives in build-only mode')
86 const output = archiveDir ? resolve(archiveDir) : undefined
87 if (output) {
88 if (output === root || output.startsWith(`${root}${sep}`))
89 throw new Error('Archive output must be outside the repository')
90 mkdirSync(output, { recursive: true })
91 if (readdirSync(output).length) throw new Error('Archive output must be empty')
92 }
8393 const temp = mkdtempSync(join(tmpdir(), 'craft-npm-verify-'))
8494 try {
8595 for (const { dir, pkg } of packages) {
@@ -95,12 +105,17 @@ export async function verifyNpmPackages(buildOnly = false): Promise<void> {
95105 return
96106
97107 const specifiers: string[] = []
108 const archives: { name: string, version: string, file: string, sha256: string }[] = []
98109 for (const [index, { dir, pkg }] of packages.entries()) {
99 const archive = join(temp, `${index}.tgz`)
110 const file = `${index}.tgz`
111 const archive = join(output ?? temp, file)
100112 run([process.execPath, 'pm', 'pack', '--filename', archive], dir)
101113 const entries = run(['tar', '-tzf', archive], temp).trim().split('\n')
102114 const packed: Manifest = JSON.parse(run(['tar', '-xOzf', archive, 'package/package.json'], temp))
103115 verifyArchiveTargets(packed, entries)
116 if (packed.name !== pkg.name || packed.version !== pkg.version)
117 throw new Error(`${pkg.name}: packed identity differs from the source manifest`)
118 archives.push({ name: packed.name, version: packed.version, file, sha256: createHash('sha256').update(readFileSync(archive)).digest('hex') })
104119 const installed = join(temp, 'node_modules', pkg.name)
105120 mkdirSync(installed, { recursive: true })
106121 run(['tar', '-xzf', archive, '--strip-components=1', '-C', installed], temp)
@@ -133,6 +148,8 @@ export async function verifyNpmPackages(buildOnly = false): Promise<void> {
133148 '--target', 'esnext', '--skipLibCheck', '--ignoreConfig',
134149 '--typeRoots', join(root, 'node_modules/@types'),
135150 ], temp)
151 if (output)
152 writeFileSync(join(output, 'manifest.json'), `${JSON.stringify({ packages: archives }, null, 2)}\n`)
136153 console.log(`Verified ${packages.length} npm archives and ${specifiers.length} installed import/type entry points.`)
137154 }
138155 finally {
@@ -140,5 +157,9 @@ export async function verifyNpmPackages(buildOnly = false): Promise<void> {
140157 }
141158}
142159
143if (import.meta.main)
144 await verifyNpmPackages(Bun.argv.includes('--build-only'))
160if (import.meta.main) {
161 const index = Bun.argv.indexOf('--archive-dir')
162 if (index >= 0 && (!Bun.argv[index + 1] || Bun.argv[index + 1]!.startsWith('--')))
163 throw new Error('--archive-dir requires an output path')
164 await verifyNpmPackages(Bun.argv.includes('--build-only'), index >= 0 ? Bun.argv[index + 1] : undefined)
165}
scripts/publish-npm-archives.tsadded+42-0
Changes to scripts/publish-npm-archives.ts
@@ -0,0 +1,42 @@
1import { join, resolve } from 'node:path'
2import { readArchiveManifest } from './scan-release-artifacts'
3
4type Command = (args: string[]) => { exitCode: number | null, stdout: string, stderr: string }
5
6const npmCommand: Command = args => {
7 const result = Bun.spawnSync(['npm', ...args], { stdout: 'pipe', stderr: 'pipe' })
8 return { exitCode: result.exitCode, stdout: result.stdout.toString(), stderr: result.stderr.toString() }
9}
10
11export function publishNpmArchives(directory: string, command: Command = npmCommand, dryRun = false): void {
12 const root = resolve(directory)
13 const archives = readArchiveManifest(root)
14 for (const archive of archives) {
15 const path = join(root, archive.file)
16 if (!dryRun) {
17 const lookup = command(['view', `${archive.name}@${archive.version}`, 'version', '--json'])
18 if (lookup.exitCode === 0) {
19 if (JSON.parse(lookup.stdout) !== archive.version)
20 throw new Error(`${archive.name}: npm returned an unexpected published version`)
21 console.log(`${archive.name}@${archive.version} already published; skipping`)
22 continue
23 }
24 if (!lookup.stderr.includes('E404'))
25 throw new Error(`${archive.name}: npm version lookup failed: ${lookup.stderr}`)
26 }
27 // A failed or changed archive must never be published after the scan.
28 readArchiveManifest(root)
29 const args = ['publish', path, '--access', 'public', '--ignore-scripts', '--provenance']
30 if (dryRun) args.push('--dry-run')
31 const publish = command(args)
32 if (publish.exitCode !== 0)
33 throw new Error(`${archive.name}: npm publish failed: ${publish.stderr}`)
34 console.log(`${dryRun ? 'Dry-run checked' : 'Published'} ${archive.name}@${archive.version}`)
35 }
36}
37
38if (import.meta.main) {
39 const directory = Bun.argv[2]
40 if (!directory) throw new Error('Usage: publish-npm-archives.ts <archive-dir> [--dry-run]')
41 publishNpmArchives(directory, npmCommand, Bun.argv.includes('--dry-run'))
42}
scripts/scan-release-artifacts.test.tsadded+88-0
Changes to scripts/scan-release-artifacts.test.ts
@@ -0,0 +1,88 @@
1import { afterEach, expect, test } from 'bun:test'
2import { createHash } from 'node:crypto'
3import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
4import { tmpdir } from 'node:os'
5import { join } from 'node:path'
6import { publishNpmArchives } from './publish-npm-archives'
7import { expectedNpmPackages, nativeInventory, readArchiveManifest, summarizeBunAudit } from './scan-release-artifacts'
8
9const roots: string[] = []
10afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) })
11const fixture = () => {
12 const root = mkdtempSync(join(tmpdir(), 'craft-release-artifacts-test-'))
13 roots.push(root)
14 return root
15}
16
17function archives(root: string) {
18 const packages = expectedNpmPackages().map((name, index) => {
19 const file = `${index}.tgz`
20 const bytes = `${name}@1.2.3`
21 writeFileSync(join(root, file), bytes)
22 return { name, version: '1.2.3', file, sha256: createHash('sha256').update(bytes).digest('hex') }
23 })
24 writeFileSync(join(root, 'manifest.json'), JSON.stringify({ packages }))
25 return packages
26}
27
28test('release manifest requires every public archive and unchanged bytes', () => {
29 const root = fixture()
30 const packages = archives(root)
31 expect(readArchiveManifest(root)).toHaveLength(packages.length)
32 writeFileSync(join(root, packages[0]!.file), 'changed')
33 expect(() => readArchiveManifest(root)).toThrow('hash mismatch')
34 writeFileSync(join(root, packages[0]!.file), `${packages[0]!.name}@1.2.3`)
35 packages[0]!.file = '../escape.tgz'
36 writeFileSync(join(root, 'manifest.json'), JSON.stringify({ packages }))
37 expect(() => readArchiveManifest(root)).toThrow('Malformed')
38 packages.shift()
39 writeFileSync(join(root, 'manifest.json'), JSON.stringify({ packages }))
40 expect(() => readArchiveManifest(root)).toThrow('does not cover every')
41})
42
43test('Bun audit parser fails closed and only High or Critical blocks', () => {
44 expect(summarizeBunAudit({})).toEqual({ Critical: 0, High: 0 })
45 expect(summarizeBunAudit({ svelte: [{ severity: 'moderate' }], unsafe: [{ severity: 'high' }, { severity: 'critical' }] }))
46 .toEqual({ Critical: 1, High: 1 })
47 for (const invalid of [null, [], { package: null }, { package: [{}] }, { package: [{ severity: 'unknown' }] }])
48 expect(() => summarizeBunAudit(invalid)).toThrow()
49})
50
51function binary(root: string, path: string, magic: string) {
52 const absolute = join(root, path)
53 mkdirSync(join(absolute, '..'), { recursive: true })
54 const bytes = Buffer.alloc(2048)
55 Buffer.from(magic, 'hex').copy(bytes)
56 writeFileSync(absolute, bytes)
57}
58
59test('native inventory binds exact binaries and Zig runtime version', () => {
60 const root = fixture()
61 binary(root, 'bin/craft', '7f454c46')
62 binary(root, 'cross/windows-x64/craft.exe', '4d5a')
63 const inventory = nativeInventory(root, 'linux', '1.2.3', '0.17.0-dev.1963')
64 expect(inventory.components.map(component => component.name)).toEqual(['craft-linux-x64', 'craft-windows-x64', 'zig-std'])
65 expect(inventory.components[0]?.hashes).toEqual([{ alg: 'SHA-256', content: createHash('sha256').update(readFileSync(join(root, 'bin/craft'))).digest('hex') }])
66 writeFileSync(join(root, 'cross/windows-x64/craft.exe'), 'not a binary')
67 expect(() => nativeInventory(root, 'linux', '1.2.3', '0.17.0-dev.1963')).toThrow('missing or empty')
68 binary(root, 'cross/windows-x64/craft.exe', '7f454c46')
69 expect(() => nativeInventory(root, 'linux', '1.2.3', '0.17.0-dev.1963')).toThrow('unexpected executable format')
70})
71
72test('publisher uses only hash-verified archives and skips an existing version', () => {
73 const root = fixture()
74 const packages = archives(root)
75 const calls: string[][] = []
76 publishNpmArchives(root, (args) => {
77 calls.push(args)
78 if (args[0] === 'view')
79 return args[1] === `${packages[0]!.name}@1.2.3`
80 ? { exitCode: 0, stdout: '"1.2.3"', stderr: '' }
81 : { exitCode: 1, stdout: '', stderr: 'E404' }
82 return { exitCode: 0, stdout: '', stderr: '' }
83 })
84 expect(calls.filter(args => args[0] === 'publish')).toHaveLength(packages.length - 1)
85 expect(calls.find(args => args[0] === 'publish')).toContain('--ignore-scripts')
86 expect(calls.find(args => args[0] === 'publish')).toContain('--provenance')
87 expect(calls.some(args => args[0] === 'publish' && args[1] === join(root, packages[0]!.file))).toBe(false)
88})
scripts/scan-release-artifacts.tsadded+170-0
Changes to scripts/scan-release-artifacts.ts
@@ -0,0 +1,170 @@
1import { createHash } from 'node:crypto'
2import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
3import { tmpdir } from 'node:os'
4import { join, resolve } from 'node:path'
5import { enforceScanPolicy, scanSbom } from './scan-sbom'
6
7type Archive = { name: string, version: string, file: string, sha256: string }
8type Component = { name: string, version?: string, 'bom-ref'?: string, [key: string]: unknown }
9type Inventory = { bomFormat: string, components: Component[], [key: string]: unknown }
10
11const root = resolve(import.meta.dir, '..')
12const sha256 = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex')
13
14function run(args: string[], cwd = root) {
15 const result = Bun.spawnSync(args, { cwd, stdout: 'pipe', stderr: 'pipe' })
16 if (result.exitCode !== 0)
17 throw new Error(`${args[0]} failed (${result.exitCode}): ${result.stderr.toString()}`)
18 return result.stdout.toString()
19}
20
21export function expectedNpmPackages(): string[] {
22 const names: string[] = []
23 for (const path of new Bun.Glob('packages/*/package.json').scanSync({ cwd: root })) {
24 const pkg = JSON.parse(readFileSync(join(root, path), 'utf8')) as { name?: string, private?: boolean }
25 if (!pkg.private) {
26 if (!pkg.name) throw new Error(`${path}: public package has no name`)
27 names.push(pkg.name)
28 }
29 }
30 return names.sort()
31}
32
33export function readArchiveManifest(directory: string, expected: string[] = expectedNpmPackages()): Archive[] {
34 const manifest = JSON.parse(readFileSync(join(directory, 'manifest.json'), 'utf8')) as { packages?: Archive[] }
35 if (!Array.isArray(manifest.packages) || manifest.packages.length !== expected.length)
36 throw new Error('Release archive manifest does not cover every public npm package')
37 const names = new Set<string>()
38 const files = new Set<string>()
39 for (const archive of manifest.packages) {
40 if (!archive || typeof archive.name !== 'string' || !archive.name || typeof archive.version !== 'string' || !archive.version
41 || typeof archive.file !== 'string' || !/^\d+\.tgz$/.test(archive.file)
42 || typeof archive.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(archive.sha256))
43 throw new Error('Malformed release archive manifest')
44 if (names.has(archive.name) || files.has(archive.file)) throw new Error('Duplicate release archive identity')
45 names.add(archive.name)
46 files.add(archive.file)
47 const path = join(directory, archive.file)
48 if (!existsSync(path) || !statSync(path).isFile() || sha256(path) !== archive.sha256)
49 throw new Error(`${archive.name}: release archive hash mismatch`)
50 }
51 if (expected.some(name => !names.has(name)))
52 throw new Error('Release archive manifest does not cover every public npm package')
53 return manifest.packages
54}
55
56export function summarizeBunAudit(report: unknown): { Critical: number, High: number } {
57 if (!report || typeof report !== 'object' || Array.isArray(report)) throw new Error('Invalid Bun audit report')
58 const counts = { Critical: 0, High: 0 }
59 for (const advisories of Object.values(report)) {
60 if (!Array.isArray(advisories)) throw new Error('Invalid Bun audit advisories')
61 for (const advisory of advisories) {
62 const severity = advisory?.severity
63 if (!['critical', 'high', 'moderate', 'low', 'info'].includes(severity))
64 throw new Error('Invalid Bun audit severity')
65 if (severity === 'critical') counts.Critical++
66 if (severity === 'high') counts.High++
67 }
68 }
69 return counts
70}
71
72function scanNpm(directory: string, output: string, syft: string, grype: string) {
73 const archives = readArchiveManifest(directory)
74 const consumer = mkdtempSync(join(tmpdir(), 'craft-release-consumer-'))
75 try {
76 const dependencies = Object.fromEntries(archives.map(({ name, file }) => [name, `file:${join(resolve(directory), file)}`]))
77 writeFileSync(join(consumer, 'package.json'), `${JSON.stringify({ private: true, dependencies }, null, 2)}\n`)
78 run([process.execPath, 'install', '--ignore-scripts', '--save', '--save-text-lockfile'], consumer)
79 const audit = Bun.spawnSync([process.execPath, 'audit', '--production', '--json'], { cwd: consumer, stdout: 'pipe', stderr: 'pipe' })
80 const auditText = audit.stdout.toString()
81 let auditCounts: ReturnType<typeof summarizeBunAudit>
82 try {
83 const report = JSON.parse(auditText)
84 auditCounts = summarizeBunAudit(report)
85 if (audit.exitCode !== 0 && Object.keys(report).length === 0)
86 throw new Error('Bun audit failed without reporting any advisories')
87 }
88 catch { throw new Error(`Bun production audit did not return a valid report: ${audit.stderr.toString()}`) }
89 mkdirSync(output, { recursive: true })
90 writeFileSync(join(output, 'bun-audit.json'), auditText)
91
92 const sbom = join(output, 'npm.cyclonedx.json')
93 run([syft, join(consumer, 'node_modules'), '--select-catalogers', '+javascript-package-cataloger', '-o', `cyclonedx-json=${sbom}`])
94 const inventory = JSON.parse(readFileSync(sbom, 'utf8')) as Inventory
95 if (inventory.bomFormat !== 'CycloneDX' || !Array.isArray(inventory.components))
96 throw new Error('Invalid npm release inventory')
97 for (const archive of archives) {
98 if (!inventory.components.some(component => component.name === archive.name && component.version === archive.version))
99 throw new Error(`${archive.name}: package missing from installed release inventory`)
100 }
101 const summary = scanSbom(sbom, join(output, 'grype'), grype)
102 enforceScanPolicy(summary.counts, true)
103 if (auditCounts.Critical || auditCounts.High)
104 throw new Error('High or Critical findings in npm production dependencies block this release')
105 readArchiveManifest(directory)
106 console.log(`Scanned ${archives.length} packed npm packages and their resolved production dependencies`)
107 }
108 finally { rmSync(consumer, { recursive: true, force: true }) }
109}
110
111export function nativeInventory(binaryRoot: string, platform: 'macos' | 'linux', version: string, zigVersion: string, discovered: Component[][] = []): Inventory {
112 const entries = platform === 'macos'
113 ? [{ path: 'bin/craft', name: 'craft-darwin-arm64', magic: ['cffaedfe', 'feedfacf'] }, { path: 'cross/darwin-x64/craft', name: 'craft-darwin-x64', magic: ['cffaedfe', 'feedfacf'] }]
114 : [{ path: 'bin/craft', name: 'craft-linux-x64', magic: ['7f454c46'] }, { path: 'cross/windows-x64/craft.exe', name: 'craft-windows-x64', magic: ['4d5a'] }]
115 const binaries: Component[] = entries.map(({ path, name, magic }) => {
116 const absolute = join(binaryRoot, path)
117 if (!existsSync(absolute) || !statSync(absolute).isFile() || statSync(absolute).size < 1024)
118 throw new Error(`${name}: missing or empty release binary`)
119 const bytes = readFileSync(absolute)
120 if (!magic.some(prefix => bytes.subarray(0, prefix.length / 2).toString('hex') === prefix))
121 throw new Error(`${name}: unexpected executable format`)
122 return { 'bom-ref': `urn:craft:binary:${name}`, type: 'application', name, version,
123 hashes: [{ alg: 'SHA-256', content: createHash('sha256').update(bytes).digest('hex') }],
124 properties: [{ name: 'craft:artifact-path', value: path }],
125 }
126 })
127 const zig = { 'bom-ref': 'urn:craft:zig-std', type: 'library', name: 'zig-std', version: zigVersion }
128 return {
129 bomFormat: 'CycloneDX', specVersion: '1.6', version: 1,
130 metadata: { component: { type: 'application', name: 'craft-native-release', version } },
131 components: [...binaries, zig, ...discovered.flat()],
132 dependencies: binaries.map(binary => ({ ref: binary['bom-ref'], dependsOn: [zig['bom-ref']] })),
133 }
134}
135
136function scanNative(binaryRoot: string, platform: 'macos' | 'linux', output: string, syft: string, grype: string) {
137 mkdirSync(output, { recursive: true })
138 const { version } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')) as { version: string }
139 const zigVersion = Bun.JSONC.parse(readFileSync(join(root, 'pantry.jsonc'), 'utf8')).dependencies['ziglang.org'] as string
140 const paths = platform === 'macos' ? ['bin/craft', 'cross/darwin-x64/craft'] : ['bin/craft', 'cross/windows-x64/craft.exe']
141 const discovered = paths.map((path, index) => {
142 const report = join(output, `native-${index}.cyclonedx.json`)
143 run([syft, join(binaryRoot, path), '-o', `cyclonedx-json=${report}`])
144 const inventory = JSON.parse(readFileSync(report, 'utf8')) as Inventory
145 if (inventory.bomFormat !== 'CycloneDX' || (inventory.components !== undefined && !Array.isArray(inventory.components)))
146 throw new Error(`${path}: invalid binary scanner inventory`)
147 return inventory.components ?? []
148 })
149 const inventory = nativeInventory(binaryRoot, platform, version, zigVersion, discovered)
150 const sbom = join(output, 'native.cyclonedx.json')
151 writeFileSync(sbom, `${JSON.stringify(inventory, null, 2)}\n`)
152 const summary = scanSbom(sbom, join(output, 'grype'), grype)
153 enforceScanPolicy(summary.counts, true)
154 const after = nativeInventory(binaryRoot, platform, version, zigVersion)
155 for (const [index, binary] of after.components.slice(0, 2).entries()) {
156 if (JSON.stringify(binary.hashes) !== JSON.stringify(inventory.components[index]!.hashes))
157 throw new Error(`${binary.name}: release binary changed after scanning`)
158 }
159 console.log(`Scanned ${platform} release binaries and declared Zig runtime dependencies`)
160}
161
162if (import.meta.main) {
163 const [mode, input, output] = Bun.argv.slice(2)
164 if (!mode || !input || !output) throw new Error('Usage: scan-release-artifacts.ts <npm|macos|linux> <input> <output>')
165 const syft = process.env.CRAFT_SYFT ?? 'syft'
166 const grype = process.env.CRAFT_GRYPE ?? 'grype'
167 if (mode === 'npm') scanNpm(resolve(input), resolve(output), syft, grype)
168 else if (mode === 'macos' || mode === 'linux') scanNative(resolve(input), mode, resolve(output), syft, grype)
169 else throw new Error(`Unknown release artifact mode: ${mode}`)
170}