also looking at this
[DO NOT MERGE] diag: #2267 CLI-hang discriminator
#2269
5 files
+370
-6
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -0,0 +1,198 @@ | ||
| 1 | /** | |
| 2 | * TEMPORARY — diagnostic for #2267, not for merge. | |
| 3 | * | |
| 4 | * On GitHub's Linux runners (and only there), a `bun bin/cli.ts --help` spawned | |
| 5 | * asynchronously from a bun test process that has the rpx graph loaded produces | |
| 6 | * zero bytes and never exits, while `spawnSync` of the same CLI from a file with | |
| 7 | * no rpx imports passes in ~100ms. Nothing reproduces it in an oven/bun | |
| 8 | * container. This file runs one variant after another in the failing parent | |
| 9 | * shape (it imports `../src/start` and `../src/gateway` on purpose), so a single | |
| 10 | * CI run says which knob matters, and dumps the stuck child's /proc state | |
| 11 | * before killing it. Every line is prefixed `[2267-diag]` so it is greppable in | |
| 12 | * the CI log. The test itself always passes; the log is the result. | |
| 13 | */ | |
| 14 | import { describe, expect, it } from 'bun:test' | |
| 15 | import { spawn, spawnSync } from 'node:child_process' | |
| 16 | import * as fs from 'node:fs' | |
| 17 | import * as os from 'node:os' | |
| 18 | import * as path from 'node:path' | |
| 19 | import * as process from 'node:process' | |
| 20 | import { readGatewayFragments } from '../src/gateway' | |
| 21 | import * as Start from '../src/start' | |
| 22 | ||
| 23 | const CLI = path.join(import.meta.dir, '..', 'bin', 'cli.ts') | |
| 24 | const BUDGET_MS = 12_000 | |
| 25 | const P = '[2267-diag]' | |
| 26 | ||
| 27 | function say(...parts: unknown[]): void { | |
| 28 | console.log(P, ...parts) | |
| 29 | } | |
| 30 | ||
| 31 | interface Result { | |
| 32 | name: string | |
| 33 | outcome: string | |
| 34 | ms: number | |
| 35 | bytes: number | |
| 36 | head: string | |
| 37 | diag: string | |
| 38 | } | |
| 39 | ||
| 40 | type Stdio = Array<'ignore' | 'pipe' | 'inherit'> | |
| 41 | ||
| 42 | function sh(cmd: string[]): string { | |
| 43 | try { | |
| 44 | const r = Bun.spawnSync(cmd, { stdout: 'pipe', stderr: 'pipe' }) | |
| 45 | const dec = new TextDecoder() | |
| 46 | return `${dec.decode(r.stdout)}${dec.decode(r.stderr)}`.trim() | |
| 47 | } | |
| 48 | catch (e) { | |
| 49 | return `ERR ${e}` | |
| 50 | } | |
| 51 | } | |
| 52 | ||
| 53 | function readMaybe(p: string): string { | |
| 54 | try { | |
| 55 | return fs.readFileSync(p, 'utf8').trim() | |
| 56 | } | |
| 57 | catch { | |
| 58 | return '' | |
| 59 | } | |
| 60 | } | |
| 61 | ||
| 62 | /** Everything /proc will tell us about a child that has gone quiet. */ | |
| 63 | function procDump(pid: number): string { | |
| 64 | if (process.platform !== 'linux') | |
| 65 | return '(no /proc on this platform)' | |
| 66 | const lines: string[] = [] | |
| 67 | const status = readMaybe(`/proc/${pid}/status`) | |
| 68 | .split('\n') | |
| 69 | .filter(l => /^(?:State|Threads|VmRSS|SigBlk|SigIgn|SigCgt|PPid)/.test(l)) | |
| 70 | .join(' | ') | |
| 71 | lines.push(`status: ${status}`) | |
| 72 | lines.push(`wchan: ${readMaybe(`/proc/${pid}/wchan`)}`) | |
| 73 | lines.push(`stack: ${readMaybe(`/proc/${pid}/stack`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/stack`])}`) | |
| 74 | lines.push(`threads: ${sh(['sh', '-c', `for t in /proc/${pid}/task/*; do echo "$(basename "$t"):$(cat "$t/wchan" 2>/dev/null || sudo -n cat "$t/wchan" 2>/dev/null)"; done | tr '\\n' ' '`])}`) | |
| 75 | lines.push(`thread stacks: ${sh(['sh', '-c', `for t in /proc/${pid}/task/*; do echo "== $(basename "$t")"; (cat "$t/stack" 2>/dev/null || sudo -n cat "$t/stack" 2>/dev/null) | head -8; done | tr '\\n' ' '`])}`) | |
| 76 | lines.push(`fds: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`) | |
| 77 | lines.push(`cwd: ${sh(['readlink', `/proc/${pid}/cwd`])}`) | |
| 78 | lines.push(`cmdline: ${readMaybe(`/proc/${pid}/cmdline`).replace(/\0/g, ' ')}`) | |
| 79 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) | |
| 80 | lines.push(`io: ${readMaybe(`/proc/${pid}/io`).replace(/\n/g, ' ')}`) | |
| 81 | return lines.join('\n ') | |
| 82 | } | |
| 83 | ||
| 84 | function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }): Promise<Result> { | |
| 85 | return new Promise((resolve) => { | |
| 86 | const t0 = performance.now() | |
| 87 | let out = '' | |
| 88 | let settled = false | |
| 89 | const child = spawn(process.execPath, argv, { | |
| 90 | stdio: opts.stdio, | |
| 91 | env: { ...process.env, ...(opts.env ?? {}) }, | |
| 92 | cwd: opts.cwd, | |
| 93 | }) | |
| 94 | child.stdout?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 95 | child.stderr?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 96 | if (opts.stdio[0] === 'pipe') | |
| 97 | child.stdin?.end() | |
| 98 | const finish = (outcome: string, diag = ''): void => { | |
| 99 | if (settled) | |
| 100 | return | |
| 101 | settled = true | |
| 102 | resolve({ name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag }) | |
| 103 | } | |
| 104 | child.on('error', e => finish(`spawn-error ${e.message}`)) | |
| 105 | child.on('exit', (code, sig) => finish(`exit code=${code} signal=${sig}`)) | |
| 106 | setTimeout(() => { | |
| 107 | if (settled) | |
| 108 | return | |
| 109 | const diag = child.pid ? procDump(child.pid) : '(no pid)' | |
| 110 | try { | |
| 111 | child.kill('SIGKILL') | |
| 112 | } | |
| 113 | catch {} | |
| 114 | finish(`HUNG >${BUDGET_MS}ms`, diag) | |
| 115 | }, BUDGET_MS) | |
| 116 | }) | |
| 117 | } | |
| 118 | ||
| 119 | async function viaBunSpawn(name: string, argv: string[]): Promise<Result> { | |
| 120 | const t0 = performance.now() | |
| 121 | const proc = Bun.spawn([process.execPath, ...argv], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: process.env }) | |
| 122 | let out = '' | |
| 123 | const dec = new TextDecoder() | |
| 124 | const pump = async (s: ReadableStream<Uint8Array> | null | undefined): Promise<void> => { | |
| 125 | if (!s) | |
| 126 | return | |
| 127 | for await (const c of s) out += dec.decode(c) | |
| 128 | } | |
| 129 | const done = Promise.all([pump(proc.stdout), pump(proc.stderr), proc.exited]).then(([, , code]) => `exit code=${code}`) | |
| 130 | const timer = new Promise<string>(r => setTimeout(() => r('TIMEOUT'), BUDGET_MS)) | |
| 131 | const raced = await Promise.race([done, timer]) | |
| 132 | let diag = '' | |
| 133 | if (raced === 'TIMEOUT') { | |
| 134 | diag = procDump(proc.pid) | |
| 135 | try { | |
| 136 | proc.kill(9) | |
| 137 | } | |
| 138 | catch {} | |
| 139 | } | |
| 140 | return { name, outcome: raced === 'TIMEOUT' ? `HUNG >${BUDGET_MS}ms` : raced, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag } | |
| 141 | } | |
| 142 | ||
| 143 | function viaSpawnSync(name: string, argv: string[]): Result { | |
| 144 | const t0 = performance.now() | |
| 145 | const r = spawnSync(process.execPath, argv, { encoding: 'utf8', timeout: BUDGET_MS }) | |
| 146 | const out = `${r.stdout ?? ''}${r.stderr ?? ''}` | |
| 147 | const outcome = r.error | |
| 148 | ? `error ${r.error.message}` | |
| 149 | : (r.signal ? `HUNG >${BUDGET_MS}ms (killed ${r.signal})` : `exit code=${r.status}`) | |
| 150 | return { name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag: '' } | |
| 151 | } | |
| 152 | ||
| 153 | describe('2267 diagnostic', () => { | |
| 154 | it('records how each spawn variant of the CLI behaves here', async () => { | |
| 155 | say('platform', process.platform, process.arch, 'bun', Bun.version, Bun.revision) | |
| 156 | say('execPath', process.execPath, '->', fs.realpathSync(process.execPath)) | |
| 157 | say('cwd', process.cwd(), '| tmpdir', os.tmpdir(), '| cpus', os.cpus().length, '| uid', typeof process.getuid === 'function' ? process.getuid() : 'n/a') | |
| 158 | say('env TMPDIR=', process.env.TMPDIR, '| RUNNER_TEMP=', process.env.RUNNER_TEMP, '| BUN_RUNTIME_TRANSPILER_CACHE_PATH=', process.env.BUN_RUNTIME_TRANSPILER_CACHE_PATH, '| HOME=', process.env.HOME) | |
| 159 | say('bunfig.toml in cwd:', fs.existsSync(path.join(process.cwd(), 'bunfig.toml'))) | |
| 160 | say('limits:', readMaybe('/proc/self/limits').split('\n').filter(l => /open files|processes/.test(l)).join(' | ') || 'n/a') | |
| 161 | say('sudo -n:', sh(['sudo', '-n', 'true']) === '' ? 'ok' : 'unavailable') | |
| 162 | say('parent graph loaded:', typeof Start.startProxies, typeof readGatewayFragments) | |
| 163 | ||
| 164 | const results: Result[] = [] | |
| 165 | const base: Stdio = ['ignore', 'pipe', 'pipe'] | |
| 166 | ||
| 167 | // G: is it bun-at-all, or the CLI graph? Cheapest discriminator first. | |
| 168 | results.push(await viaNodeSpawn('G bun --version | node spawn ignore/pipe/pipe', ['--version'], { stdio: base })) | |
| 169 | // A: the exact gateway.test.ts shape. Expected to hang on GitHub Linux. | |
| 170 | results.push(await viaNodeSpawn('A cli --help | node spawn ignore/pipe/pipe (= gateway.test.ts baseline)', [CLI, '--help'], { stdio: base })) | |
| 171 | // B: transpiler-cache hypothesis. | |
| 172 | results.push(await viaNodeSpawn('B A + BUN_RUNTIME_TRANSPILER_CACHE_PATH=0', [CLI, '--help'], { stdio: base, env: { BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } })) | |
| 173 | // C: stdin-ignore hypothesis. | |
| 174 | results.push(await viaNodeSpawn('C A + stdin pipe (ended) instead of ignore', [CLI, '--help'], { stdio: ['pipe', 'pipe', 'pipe'] })) | |
| 175 | // F: cwd hypothesis (repo-root bunfig.toml, project files). | |
| 176 | results.push(await viaNodeSpawn('F A + cwd=tmpdir', [CLI, '--help'], { stdio: base, cwd: os.tmpdir() })) | |
| 177 | // I: bypass the pipes entirely; if help appears in the CI log between the | |
| 178 | // markers, the child runs and the fault is in pipe plumbing. | |
| 179 | say('--- I begin (inherit stdout/stderr; any help text below is from the child) ---') | |
| 180 | results.push(await viaNodeSpawn('I A + stdout/stderr inherit', [CLI, '--help'], { stdio: ['ignore', 'inherit', 'inherit'] })) | |
| 181 | say('--- I end ---') | |
| 182 | // D: is it the node:child_process shim specifically? | |
| 183 | results.push(await viaBunSpawn('D cli --help | Bun.spawn', [CLI, '--help'])) | |
| 184 | // E: cli.test.ts's call shape, but from a parent that HAS the graph. | |
| 185 | results.push(viaSpawnSync('E cli --help | spawnSync (= cli.test.ts shape, parent has graph)', [CLI, '--help'])) | |
| 186 | ||
| 187 | say('===== RESULTS =====') | |
| 188 | for (const r of results) { | |
| 189 | say(r.name) | |
| 190 | say(` -> ${r.outcome} in ${r.ms}ms, ${r.bytes} bytes${r.head ? `, head: ${r.head}` : ''}`) | |
| 191 | if (r.diag) | |
| 192 | say(` diag:\n ${r.diag}`) | |
| 193 | } | |
| 194 | say('===== END =====') | |
| 195 | ||
| 196 | expect(results).toHaveLength(8) | |
| 197 | }, 200_000) | |
| 198 | }) | |