also looking at this
[DO NOT MERGE] diag: #2267 CLI-hang discriminator
#2269
5 files
+370
-6
| @@ -0,0 +1,236 @@ | ||
| 1 | /** | |
| 2 | * TEMPORARY — shared helpers for the #2267 diagnostics. Not for merge. | |
| 3 | * | |
| 4 | * Not a test file (no `.test.` in the name), so `bun test` never runs it | |
| 5 | * directly; `gateway.test.ts` and `zz-2267-diagnostic.test.ts` import it. | |
| 6 | * Every line printed is prefixed `[2267-diag]` so it is greppable in CI logs. | |
| 7 | */ | |
| 8 | import type { ChildProcess } from 'node:child_process' | |
| 9 | import { spawn, spawnSync } from 'node:child_process' | |
| 10 | import * as fs from 'node:fs' | |
| 11 | import * as os from 'node:os' | |
| 12 | import * as process from 'node:process' | |
| 13 | ||
| 14 | export const BUDGET_MS = 12_000 | |
| 15 | const P = '[2267-diag]' | |
| 16 | ||
| 17 | export function say(...parts: unknown[]): void { | |
| 18 | console.log(P, ...parts) | |
| 19 | } | |
| 20 | ||
| 21 | export interface Result { | |
| 22 | name: string | |
| 23 | outcome: string | |
| 24 | ms: number | |
| 25 | bytes: number | |
| 26 | head: string | |
| 27 | diag: string | |
| 28 | } | |
| 29 | ||
| 30 | export type Stdio = Array<'ignore' | 'pipe' | 'inherit'> | |
| 31 | ||
| 32 | export function sh(cmd: string[]): string { | |
| 33 | try { | |
| 34 | const r = Bun.spawnSync(cmd, { stdout: 'pipe', stderr: 'pipe' }) | |
| 35 | const dec = new TextDecoder() | |
| 36 | return `${dec.decode(r.stdout)}${dec.decode(r.stderr)}`.trim() | |
| 37 | } | |
| 38 | catch (e) { | |
| 39 | return `ERR ${e}` | |
| 40 | } | |
| 41 | } | |
| 42 | ||
| 43 | export function readMaybe(p: string): string { | |
| 44 | try { | |
| 45 | return fs.readFileSync(p, 'utf8').trim() | |
| 46 | } | |
| 47 | catch { | |
| 48 | return '' | |
| 49 | } | |
| 50 | } | |
| 51 | ||
| 52 | /** One line per thread of `pid`: `tid:<first lines of /proc/pid/task/tid/file>`. */ | |
| 53 | function perTask(pid: number, file: string): string { | |
| 54 | return sh(['sh', '-c', `for t in /proc/${pid}/task/*; do printf '%s:%s ' "$(basename "$t")" "$( (cat "$t/${file}" 2>/dev/null || sudo -n cat "$t/${file}" 2>/dev/null) | head -6 | tr '\\n' '|')"; done`]) | |
| 55 | } | |
| 56 | ||
| 57 | /** Everything /proc will tell us about a child that has gone quiet. */ | |
| 58 | export function procDump(pid: number): string { | |
| 59 | if (process.platform !== 'linux') | |
| 60 | return '(no /proc on this platform)' | |
| 61 | const lines: string[] = [] | |
| 62 | const status = readMaybe(`/proc/${pid}/status`) | |
| 63 | .split('\n') | |
| 64 | .filter(l => /^(?:State|PPid|Threads|VmRSS|SigBlk|SigIgn|SigCgt)/.test(l)) | |
| 65 | .join(' | ') | |
| 66 | lines.push(`status: ${status}`) | |
| 67 | lines.push(`wchan: ${readMaybe(`/proc/${pid}/wchan`)} | syscall: ${readMaybe(`/proc/${pid}/syscall`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/syscall`])}`) | |
| 68 | lines.push(`stack: ${(readMaybe(`/proc/${pid}/stack`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/stack`])).replace(/\n/g, ' | ')}`) | |
| 69 | lines.push(`task wchan: ${perTask(pid, 'wchan')}`) | |
| 70 | lines.push(`task syscall: ${perTask(pid, 'syscall')}`) | |
| 71 | lines.push(`task stack: ${perTask(pid, 'stack')}`) | |
| 72 | lines.push(`fds: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`) | |
| 73 | lines.push(`cwd: ${sh(['readlink', `/proc/${pid}/cwd`])} | cmdline: ${readMaybe(`/proc/${pid}/cmdline`).replace(/\0/g, ' ')}`) | |
| 74 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) | |
| 75 | lines.push(`io: ${readMaybe(`/proc/${pid}/io`).replace(/\n/g, ' ')}`) | |
| 76 | return lines.join('\n ') | |
| 77 | } | |
| 78 | ||
| 79 | /** | |
| 80 | * The test process's own state at the moment a child went quiet. Uses the | |
| 81 | * numeric pid throughout: `/proc/self` inside a shelled-out command would name | |
| 82 | * the shell, not us. | |
| 83 | */ | |
| 84 | export function parentDump(): string { | |
| 85 | if (process.platform !== 'linux') | |
| 86 | return '(no /proc on this platform)' | |
| 87 | const pid = process.pid | |
| 88 | const lines: string[] = [] | |
| 89 | const status = readMaybe(`/proc/${pid}/status`) | |
| 90 | .split('\n') | |
| 91 | .filter(l => /^(?:Threads|VmRSS|FDSize)/.test(l)) | |
| 92 | .join(' | ') | |
| 93 | lines.push(`pid ${pid} | ${status}`) | |
| 94 | lines.push(`fd count: ${sh(['sh', '-c', `ls /proc/${pid}/fd | wc -l`])} | by kind: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $11}' | sed -E 's/:.*//; s/\\/.*//' | sort | uniq -c | tr '\\n' ' '`])}`) | |
| 95 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) | |
| 96 | lines.push(`task wchan: ${perTask(pid, 'wchan')}`) | |
| 97 | lines.push(`task syscall: ${perTask(pid, 'syscall')}`) | |
| 98 | lines.push(`task stack: ${perTask(pid, 'stack')}`) | |
| 99 | lines.push(`fd table: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`) | |
| 100 | // For every pipe this process holds, which OTHER processes hold the same | |
| 101 | // pipe — i.e. who is on the far end of a read one of our threads is stuck in. | |
| 102 | lines.push(`pipe peers: ${sh(['sh', '-c', ` | |
| 103 | for l in $(ls -l /proc/${pid}/fd 2>/dev/null | awk '$11 ~ /^pipe:/ {print $11}' | sort -u); do | |
| 104 | e=$(printf '%s' "$l" | sed 's/\\[/\\\\[/; s/\\]/\\\\]/') | |
| 105 | printf '%s<=' "$l" | |
| 106 | sudo -n find /proc/[0-9]*/fd -lname "$e" 2>/dev/null | awk -F/ -v me=${pid} '$3 != me {print $3}' | sort -u | while read p; do | |
| 107 | printf '%s(%s) ' "$p" "$(tr '\\0' ' ' < /proc/$p/cmdline 2>/dev/null | head -c 70)" | |
| 108 | done | |
| 109 | printf '; ' | |
| 110 | done`])}`) | |
| 111 | lines.push(`listening sockets: ${sh(['sh', '-c', '(ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null) | tail -n +2 | wc -l'])}`) | |
| 112 | return lines.join('\n ') | |
| 113 | } | |
| 114 | ||
| 115 | export function envDump(): void { | |
| 116 | say('platform', process.platform, process.arch, 'bun', Bun.version, Bun.revision) | |
| 117 | say('execPath', process.execPath, '->', fs.realpathSync(process.execPath)) | |
| 118 | say('cwd', process.cwd(), '| tmpdir', os.tmpdir(), '| cpus', os.cpus().length, '| uid', typeof process.getuid === 'function' ? process.getuid() : 'n/a', '| pid', process.pid) | |
| 119 | say('env TMPDIR=', process.env.TMPDIR, '| RUNNER_TEMP=', process.env.RUNNER_TEMP, '| BUN_RUNTIME_TRANSPILER_CACHE_PATH=', process.env.BUN_RUNTIME_TRANSPILER_CACHE_PATH, '| HOME=', process.env.HOME) | |
| 120 | say('limits:', readMaybe('/proc/self/limits').split('\n').filter(l => /open files|processes/.test(l)).join(' | ') || 'n/a') | |
| 121 | say('sudo -n:', sh(['sudo', '-n', 'true']) === '' ? 'ok' : 'unavailable') | |
| 122 | } | |
| 123 | ||
| 124 | function shape(out: string): string { | |
| 125 | return out.slice(0, 100).replace(/\n/g, '⏎') | |
| 126 | } | |
| 127 | ||
| 128 | /** What the JS ChildProcess object itself claims about a child that went quiet. */ | |
| 129 | export function jsFields(child: ChildProcess): string { | |
| 130 | return `js: pid=${child.pid} exitCode=${child.exitCode} signalCode=${child.signalCode} killed=${child.killed} connected=${child.connected} spawnfile=${child.spawnfile} stdin=${child.stdin === null ? 'null' : typeof child.stdin} stdout=${child.stdout === null ? 'null' : typeof child.stdout}` | |
| 131 | } | |
| 132 | ||
| 133 | export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }, budget = BUDGET_MS): Promise<Result> { | |
| 134 | return new Promise((resolve) => { | |
| 135 | const t0 = performance.now() | |
| 136 | let out = '' | |
| 137 | let settled = false | |
| 138 | const child = spawn(process.execPath, argv, { | |
| 139 | stdio: opts.stdio, | |
| 140 | env: { ...process.env, ...(opts.env ?? {}) }, | |
| 141 | cwd: opts.cwd, | |
| 142 | }) | |
| 143 | child.stdout?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 144 | child.stderr?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 145 | if (opts.stdio[0] === 'pipe') | |
| 146 | child.stdin?.end() | |
| 147 | const finish = (outcome: string, diag = ''): void => { | |
| 148 | if (settled) | |
| 149 | return | |
| 150 | settled = true | |
| 151 | resolve({ name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag }) | |
| 152 | } | |
| 153 | child.on('error', e => finish(`spawn-error ${e.message}`)) | |
| 154 | child.on('exit', (code, sig) => finish(`exit code=${code} signal=${sig}`)) | |
| 155 | setTimeout(() => { | |
| 156 | if (settled) | |
| 157 | return | |
| 158 | const diag = `${jsFields(child)}\n ${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}` | |
| 159 | try { | |
| 160 | child.kill('SIGKILL') | |
| 161 | } | |
| 162 | catch {} | |
| 163 | finish(`HUNG >${budget}ms`, diag) | |
| 164 | }, budget) | |
| 165 | }) | |
| 166 | } | |
| 167 | ||
| 168 | export async function viaBunSpawn(name: string, argv: string[], budget = BUDGET_MS): Promise<Result> { | |
| 169 | const t0 = performance.now() | |
| 170 | const proc = Bun.spawn([process.execPath, ...argv], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: process.env }) | |
| 171 | let out = '' | |
| 172 | const dec = new TextDecoder() | |
| 173 | const pump = async (s: ReadableStream<Uint8Array> | null | undefined): Promise<void> => { | |
| 174 | if (!s) | |
| 175 | return | |
| 176 | for await (const c of s) out += dec.decode(c) | |
| 177 | } | |
| 178 | const done = Promise.all([pump(proc.stdout), pump(proc.stderr), proc.exited]).then(([, , code]) => `exit code=${code}`) | |
| 179 | const timer = new Promise<string>(r => setTimeout(() => r('TIMEOUT'), budget)) | |
| 180 | const raced = await Promise.race([done, timer]) | |
| 181 | let diag = '' | |
| 182 | if (raced === 'TIMEOUT') { | |
| 183 | diag = `${procDump(proc.pid)}\n PARENT: ${parentDump()}` | |
| 184 | try { | |
| 185 | proc.kill(9) | |
| 186 | } | |
| 187 | catch {} | |
| 188 | } | |
| 189 | return { name, outcome: raced === 'TIMEOUT' ? `HUNG >${budget}ms` : raced, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag } | |
| 190 | } | |
| 191 | ||
| 192 | export function viaSpawnSync(name: string, argv: string[], budget = BUDGET_MS): Result { | |
| 193 | const t0 = performance.now() | |
| 194 | const r = spawnSync(process.execPath, argv, { encoding: 'utf8', timeout: budget }) | |
| 195 | const out = `${r.stdout ?? ''}${r.stderr ?? ''}` | |
| 196 | const outcome = r.error | |
| 197 | ? `error ${r.error.message}` | |
| 198 | : (r.signal ? `HUNG >${budget}ms (killed ${r.signal})` : `exit code=${r.status}`) | |
| 199 | return { name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag: '' } | |
| 200 | } | |
| 201 | ||
| 202 | /** The eight variants, sequentially, followed by a printed report. */ | |
| 203 | export async function runSweep(cli: string, where: string): Promise<Result[]> { | |
| 204 | const results: Result[] = [] | |
| 205 | const base: Stdio = ['ignore', 'pipe', 'pipe'] | |
| 206 | ||
| 207 | // G: is it bun-at-all, or the CLI graph? Cheapest discriminator first. | |
| 208 | results.push(await viaNodeSpawn('G bun --version | node spawn ignore/pipe/pipe', ['--version'], { stdio: base })) | |
| 209 | // A: the exact gateway.test.ts shape. | |
| 210 | results.push(await viaNodeSpawn('A cli --help | node spawn ignore/pipe/pipe (= gateway.test.ts baseline)', [cli, '--help'], { stdio: base })) | |
| 211 | // B: transpiler-cache hypothesis. | |
| 212 | results.push(await viaNodeSpawn('B A + BUN_RUNTIME_TRANSPILER_CACHE_PATH=0', [cli, '--help'], { stdio: base, env: { BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } })) | |
| 213 | // C: stdin-ignore hypothesis. | |
| 214 | results.push(await viaNodeSpawn('C A + stdin pipe (ended) instead of ignore', [cli, '--help'], { stdio: ['pipe', 'pipe', 'pipe'] })) | |
| 215 | // F: cwd hypothesis (repo-root bunfig.toml, project files). | |
| 216 | results.push(await viaNodeSpawn('F A + cwd=tmpdir', [cli, '--help'], { stdio: base, cwd: os.tmpdir() })) | |
| 217 | // I: bypass the pipes entirely; help text between the markers means the | |
| 218 | // child runs and the fault is in pipe plumbing. | |
| 219 | say(`--- I begin [${where}] (inherit stdout/stderr; any help text below is from the child) ---`) | |
| 220 | results.push(await viaNodeSpawn('I A + stdout/stderr inherit', [cli, '--help'], { stdio: ['ignore', 'inherit', 'inherit'] })) | |
| 221 | say(`--- I end [${where}] ---`) | |
| 222 | // D: is it the node:child_process shim specifically? | |
| 223 | results.push(await viaBunSpawn('D cli --help | Bun.spawn', [cli, '--help'])) | |
| 224 | // E: cli.test.ts's call shape, but from a parent that HAS the graph. | |
| 225 | results.push(viaSpawnSync('E cli --help | spawnSync (= cli.test.ts shape, parent has graph)', [cli, '--help'])) | |
| 226 | ||
| 227 | say(`===== RESULTS [${where}] =====`) | |
| 228 | for (const r of results) { | |
| 229 | say(r.name) | |
| 230 | say(` -> ${r.outcome} in ${r.ms}ms, ${r.bytes} bytes${r.head ? `, head: ${r.head}` : ''}`) | |
| 231 | if (r.diag) | |
| 232 | say(` diag:\n ${r.diag}`) | |
| 233 | } | |
| 234 | say(`===== END [${where}] =====`) | |
| 235 | return results | |
| 236 | } | |