ReviewOS

also looking at this

stacks/rpx

[DO NOT MERGE] diag: #2267 CLI-hang discriminator

#2269
Closed chris wants to merge diag/2267-cli-hang into main
5 files +370 -6

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

bunfig.tomlmodified+4-0
Changes to bunfig.toml
@@ -1,3 +1,7 @@
11[install]
22registry = { url = "https://registry.npmjs.org/", token = "$BUN_AUTH_TOKEN" }
33linker = "hoisted"
4
5# TEMPORARY — #2267 canary, diag branch only.
6[test]
7preload = ["./packages/rpx/test/helpers/canary.preload.ts"]
packages/rpx/test/gateway.test.tsmodified+11-4
Changes to packages/rpx/test/gateway.test.ts
@@ -22,7 +22,7 @@ import * as path from 'node:path'
2222import * as process from 'node:process'
2323import { mergeGatewayFragments, readGatewayFragments, resolveGatewayOptions, startGateway } from '../src/gateway'
2424import * as Start from '../src/start'
25import { envDump, parentDump, procDump, runSweep, say } from './helpers/spawn-diag'
25import { envDump, jsFields, parentDump, procDump, runSweep, say } from './helpers/spawn-diag'
2626
2727const CERTS_DIR = '/etc/rpx/certs'
2828
@@ -376,11 +376,18 @@ describe('rpx gateway (CLI, end to end)', () => {
376376 new Promise<string>(resolve => setTimeout(() => resolve('still running after 45s'), 45_000)),
377377 ])
378378
379 if (child.exitCode === null && child.signalCode === null) {
380 say('IN-SITU BASELINE HUNG at gateway.test.ts; child pid', child.pid)
379 // Trigger on the race, not on exitCode/signalCode: last run the dud
380 // ChildProcess reported a non-null one at 45s with no exit event and no
381 // bytes, which took the wrong branch and skipped the dump.
382 if (exit.startsWith('still running')) {
383 say('IN-SITU BASELINE HUNG at gateway.test.ts')
384 say(` ${jsFields(child)}`)
381385 say(` child:\n ${child.pid ? procDump(child.pid) : '(no pid)'}`)
382386 say(` parent:\n ${parentDump()}`)
383 child.kill('SIGKILL')
387 try {
388 child.kill('SIGKILL')
389 }
390 catch {}
384391 }
385392 else {
386393 say('IN-SITU BASELINE OK:', exit, `${output.length} bytes`)
packages/rpx/test/helpers/canary.preload.tsadded+69-0
Changes to packages/rpx/test/helpers/canary.preload.ts
@@ -0,0 +1,69 @@
1/**
2 * TEMPORARY — #2267 canary. Not for merge.
3 *
4 * Loaded through `[test] preload` in the root bunfig.toml on the diag branch,
5 * so this `beforeEach` runs inside every test file. `Bun.main` names the file
6 * currently under test, so the canary fires once per file, before that file's
7 * first test: it spawns `bun --version` through `node:child_process.spawn` —
8 * the path that returns a dud ChildProcess (pid 12345, no process) at
9 * gateway.test.ts's position — with a short budget, and logs whether a real
10 * child ran. The first file whose canary is a DUD names the file BEFORE it as
11 * the one that poisons the spawn path; the first OK after a DUD names the file
12 * that clears it. The parent's pipe/thread state is dumped at both edges.
13 *
14 * Every line is prefixed `[2267-canary]`.
15 */
16import { beforeEach } from 'bun:test'
17import { spawn } from 'node:child_process'
18import * as process from 'node:process'
19import { parentDump } from './spawn-diag'
20
21const P = '[2267-canary]'
22const BUDGET_MS = 3_000
23
24let lastFile = ''
25let fileIndex = 0
26let wasDud = false
27
28function canary(): Promise<string> {
29 return new Promise((resolve) => {
30 const t0 = performance.now()
31 let out = ''
32 let done = false
33 const child = spawn(process.execPath, ['--version'], { stdio: ['ignore', 'pipe', 'pipe'] })
34 child.stdout?.on('data', (c: Buffer) => { out += c.toString() })
35 const finish = (verdict: string): void => {
36 if (done)
37 return
38 done = true
39 resolve(`${verdict} pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} ${Math.round(performance.now() - t0)}ms`)
40 }
41 child.on('error', e => finish(`ERROR(${e.message})`))
42 child.on('exit', code => finish(out.trim() ? `OK(${out.trim()})` : `EXIT-NO-OUTPUT(code=${code})`))
43 setTimeout(() => {
44 if (done)
45 return
46 try {
47 child.kill('SIGKILL')
48 }
49 catch {}
50 finish('DUD')
51 }, BUDGET_MS)
52 })
53}
54
55beforeEach(async () => {
56 const file = Bun.main.split('/').pop() ?? Bun.main
57 if (file === lastFile)
58 return
59 lastFile = file
60 fileIndex++
61 const result = await canary()
62 const isDud = result.startsWith('DUD')
63 console.log(P, `file#${fileIndex}`, file, '->', result)
64 if (isDud !== wasDud) {
65 console.log(P, isDud ? 'FIRST DUD — state right after the poisoning file:' : 'RECOVERED — state right after the clearing file:')
66 console.log(P, ` ${parentDump()}`)
67 wasDud = isDud
68 }
69})
packages/rpx/test/helpers/spawn-diag.tsmodified+20-1
Changes to packages/rpx/test/helpers/spawn-diag.ts
@@ -5,6 +5,7 @@
55 * directly; `gateway.test.ts` and `zz-2267-diagnostic.test.ts` import it.
66 * Every line printed is prefixed `[2267-diag]` so it is greppable in CI logs.
77 */
8import type { ChildProcess } from 'node:child_process'
89import { spawn, spawnSync } from 'node:child_process'
910import * as fs from 'node:fs'
1011import * as os from 'node:os'
@@ -94,6 +95,19 @@ export function parentDump(): string {
9495 lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`)
9596 lines.push(`task wchan: ${perTask(pid, 'wchan')}`)
9697 lines.push(`task syscall: ${perTask(pid, 'syscall')}`)
98 lines.push(`task stack: ${perTask(pid, 'stack')}`)
99 lines.push(`fd table: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`)
100 // For every pipe this process holds, which OTHER processes hold the same
101 // pipe — i.e. who is on the far end of a read one of our threads is stuck in.
102 lines.push(`pipe peers: ${sh(['sh', '-c', `
103for l in $(ls -l /proc/${pid}/fd 2>/dev/null | awk '$11 ~ /^pipe:/ {print $11}' | sort -u); do
104 e=$(printf '%s' "$l" | sed 's/\\[/\\\\[/; s/\\]/\\\\]/')
105 printf '%s<=' "$l"
106 sudo -n find /proc/[0-9]*/fd -lname "$e" 2>/dev/null | awk -F/ -v me=${pid} '$3 != me {print $3}' | sort -u | while read p; do
107 printf '%s(%s) ' "$p" "$(tr '\\0' ' ' < /proc/$p/cmdline 2>/dev/null | head -c 70)"
108 done
109 printf '; '
110done`])}`)
97111 lines.push(`listening sockets: ${sh(['sh', '-c', '(ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null) | tail -n +2 | wc -l'])}`)
98112 return lines.join('\n ')
99113}
@@ -111,6 +125,11 @@ function shape(out: string): string {
111125 return out.slice(0, 100).replace(/\n/g, '⏎')
112126}
113127
128/** What the JS ChildProcess object itself claims about a child that went quiet. */
129export function jsFields(child: ChildProcess): string {
130 return `js: pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} killed=${child.killed} connected=${child.connected} spawnfile=${child.spawnfile} stdin=${child.stdin === null ? 'null' : typeof child.stdin} stdout=${child.stdout === null ? 'null' : typeof child.stdout}`
131}
132
114133export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }, budget = BUDGET_MS): Promise<Result> {
115134 return new Promise((resolve) => {
116135 const t0 = performance.now()
@@ -136,7 +155,7 @@ export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio,
136155 setTimeout(() => {
137156 if (settled)
138157 return
139 const diag = `${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}`
158 const diag = `${jsFields(child)}\n ${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}`
140159 try {
141160 child.kill('SIGKILL')
142161 }