ReviewOS

also looking at this

stacks/rpx

[DO NOT MERGE] diag: #2267 CLI-hang discriminator

#2269
Closed chris wants to merge diag/2267-cli-hang into main
5 files +370 -6

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

packages/rpx/test/zz-2267-diagnostic.test.tsadded+198-0
Changes to packages/rpx/test/zz-2267-diagnostic.test.ts
@@ -0,0 +1,198 @@
1/**
2 * TEMPORARY — diagnostic for #2267, not for merge.
3 *
4 * On GitHub's Linux runners (and only there), a `bun bin/cli.ts --help` spawned
5 * asynchronously from a bun test process that has the rpx graph loaded produces
6 * zero bytes and never exits, while `spawnSync` of the same CLI from a file with
7 * no rpx imports passes in ~100ms. Nothing reproduces it in an oven/bun
8 * container. This file runs one variant after another in the failing parent
9 * shape (it imports `../src/start` and `../src/gateway` on purpose), so a single
10 * CI run says which knob matters, and dumps the stuck child's /proc state
11 * before killing it. Every line is prefixed `[2267-diag]` so it is greppable in
12 * the CI log. The test itself always passes; the log is the result.
13 */
14import { describe, expect, it } from 'bun:test'
15import { spawn, spawnSync } from 'node:child_process'
16import * as fs from 'node:fs'
17import * as os from 'node:os'
18import * as path from 'node:path'
19import * as process from 'node:process'
20import { readGatewayFragments } from '../src/gateway'
21import * as Start from '../src/start'
22
23const CLI = path.join(import.meta.dir, '..', 'bin', 'cli.ts')
24const BUDGET_MS = 12_000
25const P = '[2267-diag]'
26
27function say(...parts: unknown[]): void {
28 console.log(P, ...parts)
29}
30
31interface Result {
32 name: string
33 outcome: string
34 ms: number
35 bytes: number
36 head: string
37 diag: string
38}
39
40type Stdio = Array<'ignore' | 'pipe' | 'inherit'>
41
42function sh(cmd: string[]): string {
43 try {
44 const r = Bun.spawnSync(cmd, { stdout: 'pipe', stderr: 'pipe' })
45 const dec = new TextDecoder()
46 return `${dec.decode(r.stdout)}${dec.decode(r.stderr)}`.trim()
47 }
48 catch (e) {
49 return `ERR ${e}`
50 }
51}
52
53function readMaybe(p: string): string {
54 try {
55 return fs.readFileSync(p, 'utf8').trim()
56 }
57 catch {
58 return ''
59 }
60}
61
62/** Everything /proc will tell us about a child that has gone quiet. */
63function procDump(pid: number): string {
64 if (process.platform !== 'linux')
65 return '(no /proc on this platform)'
66 const lines: string[] = []
67 const status = readMaybe(`/proc/${pid}/status`)
68 .split('\n')
69 .filter(l => /^(?:State|Threads|VmRSS|SigBlk|SigIgn|SigCgt|PPid)/.test(l))
70 .join(' | ')
71 lines.push(`status: ${status}`)
72 lines.push(`wchan: ${readMaybe(`/proc/${pid}/wchan`)}`)
73 lines.push(`stack: ${readMaybe(`/proc/${pid}/stack`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/stack`])}`)
74 lines.push(`threads: ${sh(['sh', '-c', `for t in /proc/${pid}/task/*; do echo "$(basename "$t"):$(cat "$t/wchan" 2>/dev/null || sudo -n cat "$t/wchan" 2>/dev/null)"; done | tr '\\n' ' '`])}`)
75 lines.push(`thread stacks: ${sh(['sh', '-c', `for t in /proc/${pid}/task/*; do echo "== $(basename "$t")"; (cat "$t/stack" 2>/dev/null || sudo -n cat "$t/stack" 2>/dev/null) | head -8; done | tr '\\n' ' '`])}`)
76 lines.push(`fds: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`)
77 lines.push(`cwd: ${sh(['readlink', `/proc/${pid}/cwd`])}`)
78 lines.push(`cmdline: ${readMaybe(`/proc/${pid}/cmdline`).replace(/\0/g, ' ')}`)
79 lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`)
80 lines.push(`io: ${readMaybe(`/proc/${pid}/io`).replace(/\n/g, ' ')}`)
81 return lines.join('\n ')
82}
83
84function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }): Promise<Result> {
85 return new Promise((resolve) => {
86 const t0 = performance.now()
87 let out = ''
88 let settled = false
89 const child = spawn(process.execPath, argv, {
90 stdio: opts.stdio,
91 env: { ...process.env, ...(opts.env ?? {}) },
92 cwd: opts.cwd,
93 })
94 child.stdout?.on('data', (c: Buffer) => { out += c.toString() })
95 child.stderr?.on('data', (c: Buffer) => { out += c.toString() })
96 if (opts.stdio[0] === 'pipe')
97 child.stdin?.end()
98 const finish = (outcome: string, diag = ''): void => {
99 if (settled)
100 return
101 settled = true
102 resolve({ name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag })
103 }
104 child.on('error', e => finish(`spawn-error ${e.message}`))
105 child.on('exit', (code, sig) => finish(`exit code=${code} signal=${sig}`))
106 setTimeout(() => {
107 if (settled)
108 return
109 const diag = child.pid ? procDump(child.pid) : '(no pid)'
110 try {
111 child.kill('SIGKILL')
112 }
113 catch {}
114 finish(`HUNG >${BUDGET_MS}ms`, diag)
115 }, BUDGET_MS)
116 })
117}
118
119async function viaBunSpawn(name: string, argv: string[]): Promise<Result> {
120 const t0 = performance.now()
121 const proc = Bun.spawn([process.execPath, ...argv], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: process.env })
122 let out = ''
123 const dec = new TextDecoder()
124 const pump = async (s: ReadableStream<Uint8Array> | null | undefined): Promise<void> => {
125 if (!s)
126 return
127 for await (const c of s) out += dec.decode(c)
128 }
129 const done = Promise.all([pump(proc.stdout), pump(proc.stderr), proc.exited]).then(([, , code]) => `exit code=${code}`)
130 const timer = new Promise<string>(r => setTimeout(() => r('TIMEOUT'), BUDGET_MS))
131 const raced = await Promise.race([done, timer])
132 let diag = ''
133 if (raced === 'TIMEOUT') {
134 diag = procDump(proc.pid)
135 try {
136 proc.kill(9)
137 }
138 catch {}
139 }
140 return { name, outcome: raced === 'TIMEOUT' ? `HUNG >${BUDGET_MS}ms` : raced, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag }
141}
142
143function viaSpawnSync(name: string, argv: string[]): Result {
144 const t0 = performance.now()
145 const r = spawnSync(process.execPath, argv, { encoding: 'utf8', timeout: BUDGET_MS })
146 const out = `${r.stdout ?? ''}${r.stderr ?? ''}`
147 const outcome = r.error
148 ? `error ${r.error.message}`
149 : (r.signal ? `HUNG >${BUDGET_MS}ms (killed ${r.signal})` : `exit code=${r.status}`)
150 return { name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag: '' }
151}
152
153describe('2267 diagnostic', () => {
154 it('records how each spawn variant of the CLI behaves here', async () => {
155 say('platform', process.platform, process.arch, 'bun', Bun.version, Bun.revision)
156 say('execPath', process.execPath, '->', fs.realpathSync(process.execPath))
157 say('cwd', process.cwd(), '| tmpdir', os.tmpdir(), '| cpus', os.cpus().length, '| uid', typeof process.getuid === 'function' ? process.getuid() : 'n/a')
158 say('env TMPDIR=', process.env.TMPDIR, '| RUNNER_TEMP=', process.env.RUNNER_TEMP, '| BUN_RUNTIME_TRANSPILER_CACHE_PATH=', process.env.BUN_RUNTIME_TRANSPILER_CACHE_PATH, '| HOME=', process.env.HOME)
159 say('bunfig.toml in cwd:', fs.existsSync(path.join(process.cwd(), 'bunfig.toml')))
160 say('limits:', readMaybe('/proc/self/limits').split('\n').filter(l => /open files|processes/.test(l)).join(' | ') || 'n/a')
161 say('sudo -n:', sh(['sudo', '-n', 'true']) === '' ? 'ok' : 'unavailable')
162 say('parent graph loaded:', typeof Start.startProxies, typeof readGatewayFragments)
163
164 const results: Result[] = []
165 const base: Stdio = ['ignore', 'pipe', 'pipe']
166
167 // G: is it bun-at-all, or the CLI graph? Cheapest discriminator first.
168 results.push(await viaNodeSpawn('G bun --version | node spawn ignore/pipe/pipe', ['--version'], { stdio: base }))
169 // A: the exact gateway.test.ts shape. Expected to hang on GitHub Linux.
170 results.push(await viaNodeSpawn('A cli --help | node spawn ignore/pipe/pipe (= gateway.test.ts baseline)', [CLI, '--help'], { stdio: base }))
171 // B: transpiler-cache hypothesis.
172 results.push(await viaNodeSpawn('B A + BUN_RUNTIME_TRANSPILER_CACHE_PATH=0', [CLI, '--help'], { stdio: base, env: { BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } }))
173 // C: stdin-ignore hypothesis.
174 results.push(await viaNodeSpawn('C A + stdin pipe (ended) instead of ignore', [CLI, '--help'], { stdio: ['pipe', 'pipe', 'pipe'] }))
175 // F: cwd hypothesis (repo-root bunfig.toml, project files).
176 results.push(await viaNodeSpawn('F A + cwd=tmpdir', [CLI, '--help'], { stdio: base, cwd: os.tmpdir() }))
177 // I: bypass the pipes entirely; if help appears in the CI log between the
178 // markers, the child runs and the fault is in pipe plumbing.
179 say('--- I begin (inherit stdout/stderr; any help text below is from the child) ---')
180 results.push(await viaNodeSpawn('I A + stdout/stderr inherit', [CLI, '--help'], { stdio: ['ignore', 'inherit', 'inherit'] }))
181 say('--- I end ---')
182 // D: is it the node:child_process shim specifically?
183 results.push(await viaBunSpawn('D cli --help | Bun.spawn', [CLI, '--help']))
184 // E: cli.test.ts's call shape, but from a parent that HAS the graph.
185 results.push(viaSpawnSync('E cli --help | spawnSync (= cli.test.ts shape, parent has graph)', [CLI, '--help']))
186
187 say('===== RESULTS =====')
188 for (const r of results) {
189 say(r.name)
190 say(` -> ${r.outcome} in ${r.ms}ms, ${r.bytes} bytes${r.head ? `, head: ${r.head}` : ''}`)
191 if (r.diag)
192 say(` diag:\n ${r.diag}`)
193 }
194 say('===== END =====')
195
196 expect(results).toHaveLength(8)
197 }, 200_000)
198})