also looking at this
[DO NOT MERGE] diag: #2267 CLI-hang discriminator
#2269
5 files
+370
-6
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -22,7 +22,7 @@ import * as path from 'node:path' | ||
| 22 | 22 | import * as process from 'node:process' |
| 23 | 23 | import { mergeGatewayFragments, readGatewayFragments, resolveGatewayOptions, startGateway } from '../src/gateway' |
| 24 | 24 | import * as Start from '../src/start' |
| 25 | import { envDump, parentDump, procDump, runSweep, say } from './helpers/spawn-diag' | |
| 25 | import { envDump, jsFields, parentDump, procDump, runSweep, say } from './helpers/spawn-diag' | |
| 26 | 26 | |
| 27 | 27 | const CERTS_DIR = '/etc/rpx/certs' |
| 28 | 28 | |
| @@ -376,11 +376,18 @@ describe('rpx gateway (CLI, end to end)', () => { | ||
| 376 | 376 | new Promise<string>(resolve => setTimeout(() => resolve('still running after 45s'), 45_000)), |
| 377 | 377 | ]) |
| 378 | 378 | |
| 379 | if (child.exitCode === null && child.signalCode === null) { | |
| 380 | say('IN-SITU BASELINE HUNG at gateway.test.ts; child pid', child.pid) | |
| 379 | // Trigger on the race, not on exitCode/signalCode: last run the dud | |
| 380 | // ChildProcess reported a non-null one at 45s with no exit event and no | |
| 381 | // bytes, which took the wrong branch and skipped the dump. | |
| 382 | if (exit.startsWith('still running')) { | |
| 383 | say('IN-SITU BASELINE HUNG at gateway.test.ts') | |
| 384 | say(` ${jsFields(child)}`) | |
| 381 | 385 | say(` child:\n ${child.pid ? procDump(child.pid) : '(no pid)'}`) |
| 382 | 386 | say(` parent:\n ${parentDump()}`) |
| 383 | child.kill('SIGKILL') | |
| 387 | try { | |
| 388 | child.kill('SIGKILL') | |
| 389 | } | |
| 390 | catch {} | |
| 384 | 391 | } |
| 385 | 392 | else { |
| 386 | 393 | say('IN-SITU BASELINE OK:', exit, `${output.length} bytes`) |
| @@ -0,0 +1,69 @@ | ||
| 1 | /** | |
| 2 | * TEMPORARY — #2267 canary. Not for merge. | |
| 3 | * | |
| 4 | * Loaded through `[test] preload` in the root bunfig.toml on the diag branch, | |
| 5 | * so this `beforeEach` runs inside every test file. `Bun.main` names the file | |
| 6 | * currently under test, so the canary fires once per file, before that file's | |
| 7 | * first test: it spawns `bun --version` through `node:child_process.spawn` — | |
| 8 | * the path that returns a dud ChildProcess (pid 12345, no process) at | |
| 9 | * gateway.test.ts's position — with a short budget, and logs whether a real | |
| 10 | * child ran. The first file whose canary is a DUD names the file BEFORE it as | |
| 11 | * the one that poisons the spawn path; the first OK after a DUD names the file | |
| 12 | * that clears it. The parent's pipe/thread state is dumped at both edges. | |
| 13 | * | |
| 14 | * Every line is prefixed `[2267-canary]`. | |
| 15 | */ | |
| 16 | import { beforeEach } from 'bun:test' | |
| 17 | import { spawn } from 'node:child_process' | |
| 18 | import * as process from 'node:process' | |
| 19 | import { parentDump } from './spawn-diag' | |
| 20 | ||
| 21 | const P = '[2267-canary]' | |
| 22 | const BUDGET_MS = 3_000 | |
| 23 | ||
| 24 | let lastFile = '' | |
| 25 | let fileIndex = 0 | |
| 26 | let wasDud = false | |
| 27 | ||
| 28 | function canary(): Promise<string> { | |
| 29 | return new Promise((resolve) => { | |
| 30 | const t0 = performance.now() | |
| 31 | let out = '' | |
| 32 | let done = false | |
| 33 | const child = spawn(process.execPath, ['--version'], { stdio: ['ignore', 'pipe', 'pipe'] }) | |
| 34 | child.stdout?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 35 | const finish = (verdict: string): void => { | |
| 36 | if (done) | |
| 37 | return | |
| 38 | done = true | |
| 39 | resolve(`${verdict} pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} ${Math.round(performance.now() - t0)}ms`) | |
| 40 | } | |
| 41 | child.on('error', e => finish(`ERROR(${e.message})`)) | |
| 42 | child.on('exit', code => finish(out.trim() ? `OK(${out.trim()})` : `EXIT-NO-OUTPUT(code=${code})`)) | |
| 43 | setTimeout(() => { | |
| 44 | if (done) | |
| 45 | return | |
| 46 | try { | |
| 47 | child.kill('SIGKILL') | |
| 48 | } | |
| 49 | catch {} | |
| 50 | finish('DUD') | |
| 51 | }, BUDGET_MS) | |
| 52 | }) | |
| 53 | } | |
| 54 | ||
| 55 | beforeEach(async () => { | |
| 56 | const file = Bun.main.split('/').pop() ?? Bun.main | |
| 57 | if (file === lastFile) | |
| 58 | return | |
| 59 | lastFile = file | |
| 60 | fileIndex++ | |
| 61 | const result = await canary() | |
| 62 | const isDud = result.startsWith('DUD') | |
| 63 | console.log(P, `file#${fileIndex}`, file, '->', result) | |
| 64 | if (isDud !== wasDud) { | |
| 65 | console.log(P, isDud ? 'FIRST DUD — state right after the poisoning file:' : 'RECOVERED — state right after the clearing file:') | |
| 66 | console.log(P, ` ${parentDump()}`) | |
| 67 | wasDud = isDud | |
| 68 | } | |
| 69 | }) | |
| @@ -5,6 +5,7 @@ | ||
| 5 | 5 | * directly; `gateway.test.ts` and `zz-2267-diagnostic.test.ts` import it. |
| 6 | 6 | * Every line printed is prefixed `[2267-diag]` so it is greppable in CI logs. |
| 7 | 7 | */ |
| 8 | import type { ChildProcess } from 'node:child_process' | |
| 8 | 9 | import { spawn, spawnSync } from 'node:child_process' |
| 9 | 10 | import * as fs from 'node:fs' |
| 10 | 11 | import * as os from 'node:os' |
| @@ -94,6 +95,19 @@ export function parentDump(): string { | ||
| 94 | 95 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) |
| 95 | 96 | lines.push(`task wchan: ${perTask(pid, 'wchan')}`) |
| 96 | 97 | lines.push(`task syscall: ${perTask(pid, 'syscall')}`) |
| 98 | lines.push(`task stack: ${perTask(pid, 'stack')}`) | |
| 99 | lines.push(`fd table: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`) | |
| 100 | // For every pipe this process holds, which OTHER processes hold the same | |
| 101 | // pipe — i.e. who is on the far end of a read one of our threads is stuck in. | |
| 102 | lines.push(`pipe peers: ${sh(['sh', '-c', ` | |
| 103 | for l in $(ls -l /proc/${pid}/fd 2>/dev/null | awk '$11 ~ /^pipe:/ {print $11}' | sort -u); do | |
| 104 | e=$(printf '%s' "$l" | sed 's/\\[/\\\\[/; s/\\]/\\\\]/') | |
| 105 | printf '%s<=' "$l" | |
| 106 | sudo -n find /proc/[0-9]*/fd -lname "$e" 2>/dev/null | awk -F/ -v me=${pid} '$3 != me {print $3}' | sort -u | while read p; do | |
| 107 | printf '%s(%s) ' "$p" "$(tr '\\0' ' ' < /proc/$p/cmdline 2>/dev/null | head -c 70)" | |
| 108 | done | |
| 109 | printf '; ' | |
| 110 | done`])}`) | |
| 97 | 111 | lines.push(`listening sockets: ${sh(['sh', '-c', '(ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null) | tail -n +2 | wc -l'])}`) |
| 98 | 112 | return lines.join('\n ') |
| 99 | 113 | } |
| @@ -111,6 +125,11 @@ function shape(out: string): string { | ||
| 111 | 125 | return out.slice(0, 100).replace(/\n/g, '⏎') |
| 112 | 126 | } |
| 113 | 127 | |
| 128 | /** What the JS ChildProcess object itself claims about a child that went quiet. */ | |
| 129 | export function jsFields(child: ChildProcess): string { | |
| 130 | return `js: pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} killed=${child.killed} connected=${child.connected} spawnfile=${child.spawnfile} stdin=${child.stdin === null ? 'null' : typeof child.stdin} stdout=${child.stdout === null ? 'null' : typeof child.stdout}` | |
| 131 | } | |
| 132 | ||
| 114 | 133 | export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }, budget = BUDGET_MS): Promise<Result> { |
| 115 | 134 | return new Promise((resolve) => { |
| 116 | 135 | const t0 = performance.now() |
| @@ -136,7 +155,7 @@ export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, | ||
| 136 | 155 | setTimeout(() => { |
| 137 | 156 | if (settled) |
| 138 | 157 | return |
| 139 | const diag = `${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}` | |
| 158 | const diag = `${jsFields(child)}\n ${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}` | |
| 140 | 159 | try { |
| 141 | 160 | child.kill('SIGKILL') |
| 142 | 161 | } |