ReviewOS

also looking at this

stacks/rpx

[DO NOT MERGE] diag: #2267 CLI-hang discriminator

#2269
Closed chris wants to merge diag/2267-cli-hang into main
5 files +370 -6
bunfig.tomlmodified+4-0
Changes to bunfig.toml
@@ -1,3 +1,7 @@
11[install]
22registry = { url = "https://registry.npmjs.org/", token = "$BUN_AUTH_TOKEN" }
33linker = "hoisted"
4
5# TEMPORARY — #2267 canary, diag branch only.
6[test]
7preload = ["./packages/rpx/test/helpers/canary.preload.ts"]
packages/rpx/test/gateway.test.tsmodified+34-6
Changes to packages/rpx/test/gateway.test.ts
@@ -22,6 +22,7 @@ import * as path from 'node:path'
2222import * as process from 'node:process'
2323import { mergeGatewayFragments, readGatewayFragments, resolveGatewayOptions, startGateway } from '../src/gateway'
2424import * as Start from '../src/start'
25import { envDump, jsFields, parentDump, procDump, runSweep, say } from './helpers/spawn-diag'
2526
2627const CERTS_DIR = '/etc/rpx/certs'
2728
@@ -353,9 +354,16 @@ async function freePort(): Promise<number> {
353354 return port
354355}
355356
356const CLI_STARTS = process.platform !== 'linux'
357
358describe.skipIf(!CLI_STARTS)('rpx gateway (CLI, end to end)', () => {
357/**
358 * #2267 diagnostic, in situ. The Linux skip is lifted on this branch so the
359 * baseline runs exactly where the hang was recorded — this file is position 23
360 * of 48 in bun's deterministic order, and the same shape passes from a
361 * standalone file at position 38. On a hang the child's and this parent's
362 * /proc state is dumped before the kill, then the variant sweep runs from the
363 * same spot. The long-lived case stays skipped on Linux: it is not the
364 * discriminator and would only add 45s.
365 */
366describe('rpx gateway (CLI, end to end)', () => {
359367 it('starts and prints its help', async () => {
360368 const cli = path.join(import.meta.dir, '..', 'bin', 'cli.ts')
361369 const child = spawn(process.execPath, [cli, '--help'], { stdio: ['ignore', 'pipe', 'pipe'] })
@@ -368,13 +376,33 @@ describe.skipIf(!CLI_STARTS)('rpx gateway (CLI, end to end)', () => {
368376 new Promise<string>(resolve => setTimeout(() => resolve('still running after 45s'), 45_000)),
369377 ])
370378
371 if (child.exitCode === null && child.signalCode === null)
372 child.kill('SIGKILL')
379 // Trigger on the race, not on exitCode/signalCode: last run the dud
380 // ChildProcess reported a non-null one at 45s with no exit event and no
381 // bytes, which took the wrong branch and skipped the dump.
382 if (exit.startsWith('still running')) {
383 say('IN-SITU BASELINE HUNG at gateway.test.ts')
384 say(` ${jsFields(child)}`)
385 say(` child:\n ${child.pid ? procDump(child.pid) : '(no pid)'}`)
386 say(` parent:\n ${parentDump()}`)
387 try {
388 child.kill('SIGKILL')
389 }
390 catch {}
391 }
392 else {
393 say('IN-SITU BASELINE OK:', exit, `${output.length} bytes`)
394 }
373395
374396 expect(output, `rpx --help produced nothing (${exit})`).toContain('gateway')
375397 }, 60_000)
376398
377 it('serves a fragment\'s route over plain HTTP with --no-https on the given port', async () => {
399 it('[2267-diag] in-situ spawn variant sweep', async () => {
400 envDump()
401 const results = await runSweep(path.join(import.meta.dir, '..', 'bin', 'cli.ts'), 'in-situ')
402 expect(results).toHaveLength(8)
403 }, 200_000)
404
405 it.skipIf(process.platform === 'linux')('serves a fragment\'s route over plain HTTP with --no-https on the given port', async () => {
378406 const upstream = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: req => new Response(`gateway-upstream ${req.headers.get('x-forwarded-host')}`) })
379407 const httpPort = await freePort()
380408 // Two routes so the fragment looks like a real box (and the shared
packages/rpx/test/helpers/canary.preload.tsadded+69-0
Changes to packages/rpx/test/helpers/canary.preload.ts
@@ -0,0 +1,69 @@
1/**
2 * TEMPORARY — #2267 canary. Not for merge.
3 *
4 * Loaded through `[test] preload` in the root bunfig.toml on the diag branch,
5 * so this `beforeEach` runs inside every test file. `Bun.main` names the file
6 * currently under test, so the canary fires once per file, before that file's
7 * first test: it spawns `bun --version` through `node:child_process.spawn` —
8 * the path that returns a dud ChildProcess (pid 12345, no process) at
9 * gateway.test.ts's position — with a short budget, and logs whether a real
10 * child ran. The first file whose canary is a DUD names the file BEFORE it as
11 * the one that poisons the spawn path; the first OK after a DUD names the file
12 * that clears it. The parent's pipe/thread state is dumped at both edges.
13 *
14 * Every line is prefixed `[2267-canary]`.
15 */
16import { beforeEach } from 'bun:test'
17import { spawn } from 'node:child_process'
18import * as process from 'node:process'
19import { parentDump } from './spawn-diag'
20
21const P = '[2267-canary]'
22const BUDGET_MS = 3_000
23
24let lastFile = ''
25let fileIndex = 0
26let wasDud = false
27
28function canary(): Promise<string> {
29 return new Promise((resolve) => {
30 const t0 = performance.now()
31 let out = ''
32 let done = false
33 const child = spawn(process.execPath, ['--version'], { stdio: ['ignore', 'pipe', 'pipe'] })
34 child.stdout?.on('data', (c: Buffer) => { out += c.toString() })
35 const finish = (verdict: string): void => {
36 if (done)
37 return
38 done = true
39 resolve(`${verdict} pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} ${Math.round(performance.now() - t0)}ms`)
40 }
41 child.on('error', e => finish(`ERROR(${e.message})`))
42 child.on('exit', code => finish(out.trim() ? `OK(${out.trim()})` : `EXIT-NO-OUTPUT(code=${code})`))
43 setTimeout(() => {
44 if (done)
45 return
46 try {
47 child.kill('SIGKILL')
48 }
49 catch {}
50 finish('DUD')
51 }, BUDGET_MS)
52 })
53}
54
55beforeEach(async () => {
56 const file = Bun.main.split('/').pop() ?? Bun.main
57 if (file === lastFile)
58 return
59 lastFile = file
60 fileIndex++
61 const result = await canary()
62 const isDud = result.startsWith('DUD')
63 console.log(P, `file#${fileIndex}`, file, '->', result)
64 if (isDud !== wasDud) {
65 console.log(P, isDud ? 'FIRST DUD — state right after the poisoning file:' : 'RECOVERED — state right after the clearing file:')
66 console.log(P, ` ${parentDump()}`)
67 wasDud = isDud
68 }
69})
packages/rpx/test/helpers/spawn-diag.tsadded+236-0
Changes to packages/rpx/test/helpers/spawn-diag.ts
@@ -0,0 +1,236 @@
1/**
2 * TEMPORARY — shared helpers for the #2267 diagnostics. Not for merge.
3 *
4 * Not a test file (no `.test.` in the name), so `bun test` never runs it
5 * directly; `gateway.test.ts` and `zz-2267-diagnostic.test.ts` import it.
6 * Every line printed is prefixed `[2267-diag]` so it is greppable in CI logs.
7 */
8import type { ChildProcess } from 'node:child_process'
9import { spawn, spawnSync } from 'node:child_process'
10import * as fs from 'node:fs'
11import * as os from 'node:os'
12import * as process from 'node:process'
13
14export const BUDGET_MS = 12_000
15const P = '[2267-diag]'
16
17export function say(...parts: unknown[]): void {
18 console.log(P, ...parts)
19}
20
21export interface Result {
22 name: string
23 outcome: string
24 ms: number
25 bytes: number
26 head: string
27 diag: string
28}
29
30export type Stdio = Array<'ignore' | 'pipe' | 'inherit'>
31
32export function sh(cmd: string[]): string {
33 try {
34 const r = Bun.spawnSync(cmd, { stdout: 'pipe', stderr: 'pipe' })
35 const dec = new TextDecoder()
36 return `${dec.decode(r.stdout)}${dec.decode(r.stderr)}`.trim()
37 }
38 catch (e) {
39 return `ERR ${e}`
40 }
41}
42
43export function readMaybe(p: string): string {
44 try {
45 return fs.readFileSync(p, 'utf8').trim()
46 }
47 catch {
48 return ''
49 }
50}
51
52/** One line per thread of `pid`: `tid:<first lines of /proc/pid/task/tid/file>`. */
53function perTask(pid: number, file: string): string {
54 return sh(['sh', '-c', `for t in /proc/${pid}/task/*; do printf '%s:%s ' "$(basename "$t")" "$( (cat "$t/${file}" 2>/dev/null || sudo -n cat "$t/${file}" 2>/dev/null) | head -6 | tr '\\n' '|')"; done`])
55}
56
57/** Everything /proc will tell us about a child that has gone quiet. */
58export function procDump(pid: number): string {
59 if (process.platform !== 'linux')
60 return '(no /proc on this platform)'
61 const lines: string[] = []
62 const status = readMaybe(`/proc/${pid}/status`)
63 .split('\n')
64 .filter(l => /^(?:State|PPid|Threads|VmRSS|SigBlk|SigIgn|SigCgt)/.test(l))
65 .join(' | ')
66 lines.push(`status: ${status}`)
67 lines.push(`wchan: ${readMaybe(`/proc/${pid}/wchan`)} | syscall: ${readMaybe(`/proc/${pid}/syscall`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/syscall`])}`)
68 lines.push(`stack: ${(readMaybe(`/proc/${pid}/stack`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/stack`])).replace(/\n/g, ' | ')}`)
69 lines.push(`task wchan: ${perTask(pid, 'wchan')}`)
70 lines.push(`task syscall: ${perTask(pid, 'syscall')}`)
71 lines.push(`task stack: ${perTask(pid, 'stack')}`)
72 lines.push(`fds: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`)
73 lines.push(`cwd: ${sh(['readlink', `/proc/${pid}/cwd`])} | cmdline: ${readMaybe(`/proc/${pid}/cmdline`).replace(/\0/g, ' ')}`)
74 lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`)
75 lines.push(`io: ${readMaybe(`/proc/${pid}/io`).replace(/\n/g, ' ')}`)
76 return lines.join('\n ')
77}
78
79/**
80 * The test process's own state at the moment a child went quiet. Uses the
81 * numeric pid throughout: `/proc/self` inside a shelled-out command would name
82 * the shell, not us.
83 */
84export function parentDump(): string {
85 if (process.platform !== 'linux')
86 return '(no /proc on this platform)'
87 const pid = process.pid
88 const lines: string[] = []
89 const status = readMaybe(`/proc/${pid}/status`)
90 .split('\n')
91 .filter(l => /^(?:Threads|VmRSS|FDSize)/.test(l))
92 .join(' | ')
93 lines.push(`pid ${pid} | ${status}`)
94 lines.push(`fd count: ${sh(['sh', '-c', `ls /proc/${pid}/fd | wc -l`])} | by kind: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $11}' | sed -E 's/:.*//; s/\\/.*//' | sort | uniq -c | tr '\\n' ' '`])}`)
95 lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`)
96 lines.push(`task wchan: ${perTask(pid, 'wchan')}`)
97 lines.push(`task syscall: ${perTask(pid, 'syscall')}`)
98 lines.push(`task stack: ${perTask(pid, 'stack')}`)
99 lines.push(`fd table: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`)
100 // For every pipe this process holds, which OTHER processes hold the same
101 // pipe — i.e. who is on the far end of a read one of our threads is stuck in.
102 lines.push(`pipe peers: ${sh(['sh', '-c', `
103for l in $(ls -l /proc/${pid}/fd 2>/dev/null | awk '$11 ~ /^pipe:/ {print $11}' | sort -u); do
104 e=$(printf '%s' "$l" | sed 's/\\[/\\\\[/; s/\\]/\\\\]/')
105 printf '%s<=' "$l"
106 sudo -n find /proc/[0-9]*/fd -lname "$e" 2>/dev/null | awk -F/ -v me=${pid} '$3 != me {print $3}' | sort -u | while read p; do
107 printf '%s(%s) ' "$p" "$(tr '\\0' ' ' < /proc/$p/cmdline 2>/dev/null | head -c 70)"
108 done
109 printf '; '
110done`])}`)
111 lines.push(`listening sockets: ${sh(['sh', '-c', '(ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null) | tail -n +2 | wc -l'])}`)
112 return lines.join('\n ')
113}
114
115export function envDump(): void {
116 say('platform', process.platform, process.arch, 'bun', Bun.version, Bun.revision)
117 say('execPath', process.execPath, '->', fs.realpathSync(process.execPath))
118 say('cwd', process.cwd(), '| tmpdir', os.tmpdir(), '| cpus', os.cpus().length, '| uid', typeof process.getuid === 'function' ? process.getuid() : 'n/a', '| pid', process.pid)
119 say('env TMPDIR=', process.env.TMPDIR, '| RUNNER_TEMP=', process.env.RUNNER_TEMP, '| BUN_RUNTIME_TRANSPILER_CACHE_PATH=', process.env.BUN_RUNTIME_TRANSPILER_CACHE_PATH, '| HOME=', process.env.HOME)
120 say('limits:', readMaybe('/proc/self/limits').split('\n').filter(l => /open files|processes/.test(l)).join(' | ') || 'n/a')
121 say('sudo -n:', sh(['sudo', '-n', 'true']) === '' ? 'ok' : 'unavailable')
122}
123
124function shape(out: string): string {
125 return out.slice(0, 100).replace(/\n/g, '⏎')
126}
127
128/** What the JS ChildProcess object itself claims about a child that went quiet. */
129export function jsFields(child: ChildProcess): string {
130 return `js: pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} killed=${child.killed} connected=${child.connected} spawnfile=${child.spawnfile} stdin=${child.stdin === null ? 'null' : typeof child.stdin} stdout=${child.stdout === null ? 'null' : typeof child.stdout}`
131}
132
133export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }, budget = BUDGET_MS): Promise<Result> {
134 return new Promise((resolve) => {
135 const t0 = performance.now()
136 let out = ''
137 let settled = false
138 const child = spawn(process.execPath, argv, {
139 stdio: opts.stdio,
140 env: { ...process.env, ...(opts.env ?? {}) },
141 cwd: opts.cwd,
142 })
143 child.stdout?.on('data', (c: Buffer) => { out += c.toString() })
144 child.stderr?.on('data', (c: Buffer) => { out += c.toString() })
145 if (opts.stdio[0] === 'pipe')
146 child.stdin?.end()
147 const finish = (outcome: string, diag = ''): void => {
148 if (settled)
149 return
150 settled = true
151 resolve({ name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag })
152 }
153 child.on('error', e => finish(`spawn-error ${e.message}`))
154 child.on('exit', (code, sig) => finish(`exit code=${code} signal=${sig}`))
155 setTimeout(() => {
156 if (settled)
157 return
158 const diag = `${jsFields(child)}\n ${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}`
159 try {
160 child.kill('SIGKILL')
161 }
162 catch {}
163 finish(`HUNG >${budget}ms`, diag)
164 }, budget)
165 })
166}
167
168export async function viaBunSpawn(name: string, argv: string[], budget = BUDGET_MS): Promise<Result> {
169 const t0 = performance.now()
170 const proc = Bun.spawn([process.execPath, ...argv], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: process.env })
171 let out = ''
172 const dec = new TextDecoder()
173 const pump = async (s: ReadableStream<Uint8Array> | null | undefined): Promise<void> => {
174 if (!s)
175 return
176 for await (const c of s) out += dec.decode(c)
177 }
178 const done = Promise.all([pump(proc.stdout), pump(proc.stderr), proc.exited]).then(([, , code]) => `exit code=${code}`)
179 const timer = new Promise<string>(r => setTimeout(() => r('TIMEOUT'), budget))
180 const raced = await Promise.race([done, timer])
181 let diag = ''
182 if (raced === 'TIMEOUT') {
183 diag = `${procDump(proc.pid)}\n PARENT: ${parentDump()}`
184 try {
185 proc.kill(9)
186 }
187 catch {}
188 }
189 return { name, outcome: raced === 'TIMEOUT' ? `HUNG >${budget}ms` : raced, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag }
190}
191
192export function viaSpawnSync(name: string, argv: string[], budget = BUDGET_MS): Result {
193 const t0 = performance.now()
194 const r = spawnSync(process.execPath, argv, { encoding: 'utf8', timeout: budget })
195 const out = `${r.stdout ?? ''}${r.stderr ?? ''}`
196 const outcome = r.error
197 ? `error ${r.error.message}`
198 : (r.signal ? `HUNG >${budget}ms (killed ${r.signal})` : `exit code=${r.status}`)
199 return { name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag: '' }
200}
201
202/** The eight variants, sequentially, followed by a printed report. */
203export async function runSweep(cli: string, where: string): Promise<Result[]> {
204 const results: Result[] = []
205 const base: Stdio = ['ignore', 'pipe', 'pipe']
206
207 // G: is it bun-at-all, or the CLI graph? Cheapest discriminator first.
208 results.push(await viaNodeSpawn('G bun --version | node spawn ignore/pipe/pipe', ['--version'], { stdio: base }))
209 // A: the exact gateway.test.ts shape.
210 results.push(await viaNodeSpawn('A cli --help | node spawn ignore/pipe/pipe (= gateway.test.ts baseline)', [cli, '--help'], { stdio: base }))
211 // B: transpiler-cache hypothesis.
212 results.push(await viaNodeSpawn('B A + BUN_RUNTIME_TRANSPILER_CACHE_PATH=0', [cli, '--help'], { stdio: base, env: { BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } }))
213 // C: stdin-ignore hypothesis.
214 results.push(await viaNodeSpawn('C A + stdin pipe (ended) instead of ignore', [cli, '--help'], { stdio: ['pipe', 'pipe', 'pipe'] }))
215 // F: cwd hypothesis (repo-root bunfig.toml, project files).
216 results.push(await viaNodeSpawn('F A + cwd=tmpdir', [cli, '--help'], { stdio: base, cwd: os.tmpdir() }))
217 // I: bypass the pipes entirely; help text between the markers means the
218 // child runs and the fault is in pipe plumbing.
219 say(`--- I begin [${where}] (inherit stdout/stderr; any help text below is from the child) ---`)
220 results.push(await viaNodeSpawn('I A + stdout/stderr inherit', [cli, '--help'], { stdio: ['ignore', 'inherit', 'inherit'] }))
221 say(`--- I end [${where}] ---`)
222 // D: is it the node:child_process shim specifically?
223 results.push(await viaBunSpawn('D cli --help | Bun.spawn', [cli, '--help']))
224 // E: cli.test.ts's call shape, but from a parent that HAS the graph.
225 results.push(viaSpawnSync('E cli --help | spawnSync (= cli.test.ts shape, parent has graph)', [cli, '--help']))
226
227 say(`===== RESULTS [${where}] =====`)
228 for (const r of results) {
229 say(r.name)
230 say(` -> ${r.outcome} in ${r.ms}ms, ${r.bytes} bytes${r.head ? `, head: ${r.head}` : ''}`)
231 if (r.diag)
232 say(` diag:\n ${r.diag}`)
233 }
234 say(`===== END [${where}] =====`)
235 return results
236}
packages/rpx/test/zz-2267-diagnostic.test.tsadded+27-0
Changes to packages/rpx/test/zz-2267-diagnostic.test.ts
@@ -0,0 +1,27 @@
1/**
2 * TEMPORARY — #2267 diagnostic, standalone file. Not for merge.
3 *
4 * `bun test` orders files deterministically but not alphabetically; this one
5 * lands at position 38 of 48, fifteen files after `gateway.test.ts` (23), so it
6 * answers "does the failing shape hang from a parent with the graph loaded, at
7 * a LATER point in the suite?" The in-situ copy of the same sweep inside
8 * `gateway.test.ts` answers it at the position where the hang was recorded.
9 * Both import the rpx graph on purpose. The test always passes; the log is the
10 * result, every line prefixed `[2267-diag]`.
11 */
12import { describe, expect, it } from 'bun:test'
13import * as path from 'node:path'
14import { readGatewayFragments } from '../src/gateway'
15import * as Start from '../src/start'
16import { envDump, runSweep, say } from './helpers/spawn-diag'
17
18const CLI = path.join(import.meta.dir, '..', 'bin', 'cli.ts')
19
20describe('2267 diagnostic (standalone file)', () => {
21 it('records how each spawn variant of the CLI behaves here', async () => {
22 envDump()
23 say('parent graph loaded:', typeof Start.startProxies, typeof readGatewayFragments)
24 const results = await runSweep(CLI, 'standalone')
25 expect(results).toHaveLength(8)
26 }, 200_000)
27})