ReviewOS

also looking at this

pantry-pm/pantry

fix(installer): resolve versions against live sources when one is unreachable

#255
Merged glennmichael123 wants to merge fix/242-version-source-fallback into main
6 files +400 -44
packages/ts-pantry/src/installer.tsmodified+189-42
Changes to packages/ts-pantry/src/installer.ts
@@ -53,6 +53,17 @@ export interface InstallOptions {
5353 quiet?: boolean
5454 /** Receive bounded transport-retry diagnostics. */
5555 onRetry?: (message: string) => void
56 /**
57 * Receive diagnostics that change which version is installed — currently a
58 * version source that could not be consulted.
59 *
60 * Separate from `quiet`, which suppresses progress output: a caller that
61 * silences progress (the GitHub Action passes `quiet: true` for exactly
62 * that) still needs to hear that a pin resolved off a stale snapshot,
63 * because CI is where that happens. Unset falls back to `console.warn`,
64 * which `quiet` does suppress.
65 */
66 onWarn?: (message: string) => void
5667 /**
5768 * Mirror this package's binaries to a stable user-level directory so they
5869 * appear on PATH after a one-time `pantry shell-init`. When `true` the
@@ -395,7 +406,7 @@ export async function resolvePackageVersion(
395406 domain: string,
396407 version: string,
397408 platform: Platform = detectPlatform(),
398 options: Pick<InstallOptions, 'onRetry'> = {},
409 options: Pick<InstallOptions, 'onRetry' | 'quiet' | 'onWarn'> = {},
399410): Promise<string> {
400411 if (domain === 'git-scm.org' && platform.os === 'windows') {
401412 if (['latest', '*', ''].includes(version))
@@ -411,7 +422,10 @@ export async function resolvePackageVersion(
411422
412423 // Resolve semver constraints (^, ~, >=, etc.) to concrete versions
413424 if (/^[\^~>=<]/.test(version)) {
414 version = await resolveVersionConstraint(domain, version, platform)
425 version = await resolveInstallerVersionConstraint(domain, version, platform, {
426 quiet: options.quiet,
427 onWarn: options.onWarn,
428 })
415429 }
416430 else if (version === 'latest' || version === '*' || !version) {
417431 version = await resolveLatestVersion(domain, platform)
@@ -687,9 +701,16 @@ export async function resolveLatestVersion(domain: string, platform: Platform =
687701 const resp = await fetchJSON('https://api.github.com/repos/oven-sh/bun/releases/latest').catch(() => null)
688702 const tag = (resp as { tag_name?: string } | null)?.tag_name?.replace(/^bun-v/, '') || ''
689703 if (tag) return tag
704 // The registry mirrors bun's releases per platform, so it answers "latest"
705 // correctly during a rate-limited hour. It sits ahead of the bundled
706 // catalog because the catalog is frozen at whichever release this pantry
707 // build shipped with — `latest` resolving to a months-old version is a
708 // wrong answer that looks exactly like a right one.
709 const mirrored = (await registryVersions(domain, platform).catch(() => [])).sort(compareVersions)[0]
710 if (mirrored) return mirrored
690711 const fallback = await latestFromPackageMetadata(domain)
691712 if (fallback) return fallback
692 throw new Error('Failed to resolve latest bun.sh version (GitHub API unreachable, no bundled metadata)')
713 throw new Error('Failed to resolve latest bun.sh version (GitHub API unreachable, registry publishes none, no bundled metadata)')
693714 }
694715 if (domain === 'ziglang.org') {
695716 const versions = await registryVersions(domain, platform)
@@ -702,6 +723,8 @@ export async function resolveLatestVersion(domain: string, platform: Platform =
702723 const lts = versions.find(v => v.lts)
703724 const v = (lts?.version || versions[0]?.version || '').replace(/^v/, '')
704725 if (v) return v
726 const mirrored = (await registryVersions(domain, platform).catch(() => [])).sort(compareVersions)[0]
727 if (mirrored) return mirrored
705728 const fallback = await latestFromPackageMetadata(domain)
706729 if (fallback) return fallback
707730 throw new Error('Failed to resolve latest nodejs.org version')
@@ -1056,27 +1079,118 @@ export function satisfiesInstallerConstraint(version: string, constraint: Instal
10561079 return false
10571080}
10581081
1082// ── Version sources ──
1083
1084/** What one version source was asked, and what it answered. */
1085export interface VersionSourceOutcome {
1086 /** Short identifier for diagnostics, e.g. `api.github.com/…/releases`. */
1087 source: string
1088 /** The versions it returned; `[]` when it errored. */
1089 versions: readonly string[]
1090 /** Why it could not be consulted, when it could not be. */
1091 error?: string
1092}
1093
1094/**
1095 * Whether a secondary source is worth consulting after `primary` answered.
1096 *
1097 * The errored case is the one that matters: `[]` from a 403 is
1098 * indistinguishable from `[]` from "upstream published nothing", and reading
1099 * the first as the second is what turned a satisfiable pin into a hard
1100 * "no version satisfies" failure.
1101 */
1102export function shouldConsultFallbackSource(primary: VersionSourceOutcome): boolean {
1103 return Boolean(primary.error) || primary.versions.length === 0
1104}
1105
1106/** Run one version source, recording its failure instead of throwing it. */
1107async function versionSource(source: string, load: () => Promise<string[]>): Promise<VersionSourceOutcome> {
1108 try {
1109 return { source, versions: await load() }
1110 }
1111 catch (error) {
1112 return { source, versions: [], error: errorMessage(error) }
1113 }
1114}
1115
1116function registrySourceName(domain: string, platform: Platform): string {
1117 return `${registryBase()}/binaries/${domain} (${registryPlatformKey(platform)})`
1118}
1119
1120/** Every version any source returned, de-duplicated and newest-first. */
1121export function mergeVersionOutcomes(outcomes: readonly VersionSourceOutcome[]): string[] {
1122 return [...new Set(outcomes.flatMap(outcome => [...outcome.versions]))].sort(compareInstallerVersionsDesc)
1123}
1124
1125/** One line per source: what it was, and what it answered. */
1126export function describeVersionSources(outcomes: readonly VersionSourceOutcome[]): string {
1127 return outcomes.map((outcome) => {
1128 if (outcome.error) return ` - ${outcome.source}: unavailable (${outcome.error})`
1129 const newest = [...outcome.versions].sort(compareInstallerVersionsDesc)[0]
1130 const count = `${outcome.versions.length} version${outcome.versions.length === 1 ? '' : 's'}`
1131 return ` - ${outcome.source}: ${count}${newest ? `, newest ${newest}` : ''}`
1132 }).join('\n')
1133}
1134
10591135/**
1060 * Live version list for the domains that publish one, newest-first.
1136 * Why `constraint` could not be met, naming every source that was consulted
1137 * and every one that could not be.
1138 *
1139 * The message this replaces read "checked 143 versions, newest 1.4.0" — a
1140 * count that matched no registry endpoint and named no source, so the single
1141 * fact that explained the failure (the live lookup had 403'd, leaving the
1142 * bundled snapshot to answer alone) was the fact it omitted.
1143 */
1144export function unsatisfiableConstraintMessage(
1145 domain: string,
1146 constraint: string,
1147 outcomes: readonly VersionSourceOutcome[],
1148): string {
1149 const degraded = outcomes.filter(outcome => outcome.error)
1150 return [
1151 `No version of ${domain} satisfies "${constraint}".`,
1152 ...(degraded.length > 0
1153 ? [`${degraded.length} of ${outcomes.length} version sources could not be consulted, so this list is incomplete — the pin may well be satisfiable.`]
1154 : []),
1155 'Version sources consulted:',
1156 describeVersionSources(outcomes),
1157 ].join('\n')
1158}
1159
1160/**
1161 * Live version lists for the domains that publish one, as per-source outcomes.
10611162 *
10621163 * The bundled catalog in `packages/index.js` is a point-in-time snapshot: it
10631164 * goes stale the moment upstream cuts a release (bun.sh tops out at 1.3.0
10641165 * there while 1.3.14 is current), so a constraint like `^1.3.14` matched
10651166 * nothing and we silently installed "latest" instead. Consulting upstream
10661167 * keeps a correct `deps.yaml` pin from depending on when this package was
1067 * last published. Returns [] for domains with no live source — the bundled
1068 * list is then the only answer.
1168 * last published.
1169 *
1170 * Each domain gets a fallback source behind its origin, because returning the
1171 * outcomes rather than a bare list is only half the fix: a source that cannot
1172 * be reached has to be *replaced*, not just reported.
10691173 */
1070async function fetchLiveVersions(domain: string, platform: Platform): Promise<string[]> {
1174async function fetchLiveVersions(domain: string, platform: Platform): Promise<VersionSourceOutcome[]> {
10711175 if (isBunDomain(domain)) {
1072 const resp = await fetchJSON('https://api.github.com/repos/oven-sh/bun/releases?per_page=100').catch(() => null)
1073 const releases = (resp as Array<{ tag_name?: string }> | null) || []
1074 return releases.map(r => (r.tag_name || '').replace(/^bun-v/, '')).filter(Boolean)
1176 const upstream = await versionSource('api.github.com/repos/oven-sh/bun/releases', async () => {
1177 const resp = await fetchJSON('https://api.github.com/repos/oven-sh/bun/releases?per_page=100') as Array<{ tag_name?: string }> | null
1178 return (resp || []).map(r => (r.tag_name || '').replace(/^bun-v/, '')).filter(Boolean)
1179 })
1180 // Unauthenticated api.github.com is 60 req/hr per IP and CI runners share
1181 // egress addresses, so a 403 here is routine rather than exceptional. The
1182 // registry mirrors the same releases per platform, which is what keeps a
1183 // rate-limited hour from resolving off the bundled snapshot instead.
1184 if (!shouldConsultFallbackSource(upstream)) return [upstream]
1185 return [upstream, await versionSource(registrySourceName(domain, platform), () => registryVersions(domain, platform))]
10751186 }
10761187 if (domain === 'nodejs.org') {
1077 const resp = await fetchJSON('https://nodejs.org/dist/index.json').catch(() => null)
1078 const versions = (resp as Array<{ version?: string }> | null) || []
1079 return versions.map(v => (v.version || '').replace(/^v/, '')).filter(Boolean)
1188 const upstream = await versionSource('nodejs.org/dist/index.json', async () => {
1189 const resp = await fetchJSON('https://nodejs.org/dist/index.json') as Array<{ version?: string }> | null
1190 return (resp || []).map(v => (v.version || '').replace(/^v/, '')).filter(Boolean)
1191 })
1192 if (!shouldConsultFallbackSource(upstream)) return [upstream]
1193 return [upstream, await versionSource(registrySourceName(domain, platform), () => registryVersions(domain, platform))]
10801194 }
10811195 if (domain === 'ziglang.org') {
10821196 // The installer downloads zig from ziglang.org itself, and ziglang.org
@@ -1086,15 +1200,17 @@ async function fetchLiveVersions(domain: string, platform: Platform): Promise<st
10861200 // job that refreshes the mirror could not install zig to run, so the
10871201 // stale pin could never heal itself. Origin first; the registry is the
10881202 // fallback for when the origin index is unreachable.
1089 const index = await fetchJSON('https://ziglang.org/download/index.json').catch(() => null) as ZigDownloadIndex | null
1090 const origin = zigOriginVersions(index, platform)
1091 if (origin.length > 0) return origin
1092 return await registryVersions(domain, platform)
1203 const origin = await versionSource('ziglang.org/download/index.json', async () => {
1204 const index = await fetchJSON('https://ziglang.org/download/index.json') as ZigDownloadIndex | null
1205 return zigOriginVersions(index, platform)
1206 })
1207 if (!shouldConsultFallbackSource(origin)) return [origin]
1208 return [origin, await versionSource(registrySourceName(domain, platform), () => registryVersions(domain, platform))]
10931209 }
10941210 // For a registry-published package the registry *is* the live list, so a
10951211 // `^0.0.37`-style constraint resolves against what can actually be installed
10961212 // rather than against nothing.
1097 return await registryVersions(domain, platform)
1213 return [await versionSource(registrySourceName(domain, platform), () => registryVersions(domain, platform))]
10981214}
10991215
11001216export type ZigDownloadIndex = Record<string, Record<string, unknown> & { version?: string }>
@@ -1119,44 +1235,67 @@ export function resolveInstallerConstraintFromCandidates(constraint: string, can
11191235 return [...new Set(candidates)].sort(compareInstallerVersionsDesc).find(version => satisfiesInstallerConstraint(version, parsed)) || null
11201236}
11211237
1122/**
1123 * Resolve a semver constraint (^1.0.0, ~1.2.0, >=1.0.0, etc.) to the best matching concrete version.
1124 */
1125async function resolveVersionConstraint(domain: string, constraint: string, platform: Platform): Promise<string> {
1126 if (!parseInstallerConstraint(constraint)) return resolveLatestVersion(domain, platform)
1127
1128 // Get available versions from package metadata
1129 let bundledVersions: readonly string[]
1238/** Versions `packages/index.js` was built with, for `domain`. Point-in-time. */
1239async function bundledCatalogVersions(domain: string): Promise<string[]> {
11301240 try {
11311241 const pkgKey = domain.replace(/[^a-z0-9]/gi, '').toLowerCase()
11321242 const { packages } = await import('./packages/index.js').catch(() => import('./index.js'))
1133 const pkg = (packages as any)[pkgKey]
1134 bundledVersions = pkg?.versions || []
1243 const pkg = (packages as unknown as Record<string, { versions?: readonly string[] }>)[pkgKey]
1244 return [...(pkg?.versions || [])]
11351245 }
11361246 catch {
1137 bundledVersions = []
1247 return []
11381248 }
1249}
1250
1251/**
1252 * Resolve a semver constraint (^1.0.0, ~1.2.0, >=1.0.0, etc.) to the best matching concrete version.
1253 *
1254 * Named apart from `dependency-resolver`'s same-purpose export because the
1255 * package index re-exports both modules with `export *`.
1256 */
1257export async function resolveInstallerVersionConstraint(
1258 domain: string,
1259 constraint: string,
1260 platform: Platform,
1261 options: Pick<InstallOptions, 'quiet' | 'onWarn'> = {},
1262): Promise<string> {
1263 if (!parseInstallerConstraint(constraint)) return resolveLatestVersion(domain, platform)
11391264
11401265 // Union of upstream + bundled, sorted so the first match is the highest one.
11411266 // Merging rather than preferring either list keeps resolution working when
11421267 // upstream is unreachable (bundled-only) and when it's ahead (live-only).
1143 const live = await fetchLiveVersions(domain, platform)
1144 // The Pantry registry is authoritative for Zig binaries. Bundled recipe
1268 //
1269 // The Pantry registry is authoritative for Zig binaries: bundled recipe
11451270 // versions do not prove that a particular target artifact was published.
1146 const candidates = domain === 'ziglang.org'
1147 ? live.sort(compareInstallerVersionsDesc)
1148 : [...new Set([...live, ...bundledVersions])].sort(compareInstallerVersionsDesc)
1149 const resolved = resolveInstallerConstraintFromCandidates(constraint, candidates)
1150 if (resolved) return resolved
1271 const live = await fetchLiveVersions(domain, platform)
1272 const outcomes: VersionSourceOutcome[] = domain === 'ziglang.org'
1273 ? live
1274 : [...live, { source: 'bundled catalog (this pantry build)', versions: await bundledCatalogVersions(domain) }]
1275
1276 const resolved = resolveInstallerConstraintFromCandidates(constraint, mergeVersionOutcomes(outcomes))
1277 const degraded = outcomes.filter(outcome => outcome.error)
1278
1279 if (resolved) {
1280 // Resolving while a source is down means the answer came from whatever
1281 // was left — possibly the bundled snapshot, which can be releases behind.
1282 // Say so: silently installing an older version than upstream has is the
1283 // half of this bug that never announces itself as a failure.
1284 if (degraded.length > 0) {
1285 const warn = options.onWarn || (options.quiet ? undefined : console.warn)
1286 warn?.(
1287 `${domain}: resolved "${constraint}" to ${resolved} with ${degraded.length} of `
1288 + `${outcomes.length} version sources unavailable; a newer match may exist upstream\n`
1289 + describeVersionSources(outcomes),
1290 )
1291 }
1292 return resolved
1293 }
11511294
11521295 // Nothing upstream or bundled satisfies the pin. Installing "latest" here
11531296 // would hand back a version the caller explicitly did not ask for, which is
11541297 // worse than stopping: a `^1.2` pin quietly became 1.3.x.
1155 throw new Error(
1156 `No version of ${domain} satisfies "${constraint}" `
1157 + `(checked ${candidates.length} version${candidates.length === 1 ? '' : 's'}`
1158 + `${candidates.length > 0 ? `, newest ${candidates[0]}` : ''}).`,
1159 )
1298 throw new Error(unsatisfiableConstraintMessage(domain, constraint, outcomes))
11601299}
11611300
11621301// ── Helpers ──
@@ -1411,8 +1550,16 @@ function fetchJSONOnce(url: string, maxRedirects: number, deadline = Date.now()
14111550 }
14121551 // Rebuild credentials on every redirect. A path, query, subdomain, HTTP
14131552 // endpoint or non-default port cannot acquire the GitHub API credential.
1553 //
1554 // PANTRY_GITHUB_TOKEN comes first so a caller can authenticate these reads
1555 // without setting GITHUB_TOKEN — which is NOT present on an Actions runner
1556 // unless the workflow passes it through, and which other code paths read
1557 // for their own purposes (the action's release step among them). The
1558 // Setup Pantry action sets it from its `github-token` input, whose default
1559 // is the job's own token; unauthenticated reads share a 60 req/hr quota
1560 // across the whole runner pool's IP range.
14141561 if (target.origin === 'https://api.github.com' && !target.username && !target.password) {
1415 const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN
1562 const token = process.env.PANTRY_GITHUB_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN
14161563 if (token) headers.Authorization = `Bearer ${token}`
14171564 }
14181565 let response: http.IncomingMessage | undefined