also looking at this
[DO NOT MERGE] diag: #2267 CLI-hang discriminator
#2269
5 files
+370
-6
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -22,6 +22,7 @@ import * as path from 'node:path' | ||
| 22 | 22 | import * as process from 'node:process' |
| 23 | 23 | import { mergeGatewayFragments, readGatewayFragments, resolveGatewayOptions, startGateway } from '../src/gateway' |
| 24 | 24 | import * as Start from '../src/start' |
| 25 | import { envDump, parentDump, procDump, runSweep, say } from './helpers/spawn-diag' | |
| 25 | 26 | |
| 26 | 27 | const CERTS_DIR = '/etc/rpx/certs' |
| 27 | 28 | |
| @@ -353,9 +354,16 @@ async function freePort(): Promise<number> { | ||
| 353 | 354 | return port |
| 354 | 355 | } |
| 355 | 356 | |
| 356 | const CLI_STARTS = process.platform !== 'linux' | |
| 357 | ||
| 358 | describe.skipIf(!CLI_STARTS)('rpx gateway (CLI, end to end)', () => { | |
| 357 | /** | |
| 358 | * #2267 diagnostic, in situ. The Linux skip is lifted on this branch so the | |
| 359 | * baseline runs exactly where the hang was recorded — this file is position 23 | |
| 360 | * of 48 in bun's deterministic order, and the same shape passes from a | |
| 361 | * standalone file at position 38. On a hang the child's and this parent's | |
| 362 | * /proc state is dumped before the kill, then the variant sweep runs from the | |
| 363 | * same spot. The long-lived case stays skipped on Linux: it is not the | |
| 364 | * discriminator and would only add 45s. | |
| 365 | */ | |
| 366 | describe('rpx gateway (CLI, end to end)', () => { | |
| 359 | 367 | it('starts and prints its help', async () => { |
| 360 | 368 | const cli = path.join(import.meta.dir, '..', 'bin', 'cli.ts') |
| 361 | 369 | const child = spawn(process.execPath, [cli, '--help'], { stdio: ['ignore', 'pipe', 'pipe'] }) |
| @@ -368,13 +376,26 @@ describe.skipIf(!CLI_STARTS)('rpx gateway (CLI, end to end)', () => { | ||
| 368 | 376 | new Promise<string>(resolve => setTimeout(() => resolve('still running after 45s'), 45_000)), |
| 369 | 377 | ]) |
| 370 | 378 | |
| 371 | if (child.exitCode === null && child.signalCode === null) | |
| 379 | if (child.exitCode === null && child.signalCode === null) { | |
| 380 | say('IN-SITU BASELINE HUNG at gateway.test.ts; child pid', child.pid) | |
| 381 | say(` child:\n ${child.pid ? procDump(child.pid) : '(no pid)'}`) | |
| 382 | say(` parent:\n ${parentDump()}`) | |
| 372 | 383 | child.kill('SIGKILL') |
| 384 | } | |
| 385 | else { | |
| 386 | say('IN-SITU BASELINE OK:', exit, `${output.length} bytes`) | |
| 387 | } | |
| 373 | 388 | |
| 374 | 389 | expect(output, `rpx --help produced nothing (${exit})`).toContain('gateway') |
| 375 | 390 | }, 60_000) |
| 376 | 391 | |
| 377 | it('serves a fragment\'s route over plain HTTP with --no-https on the given port', async () => { | |
| 392 | it('[2267-diag] in-situ spawn variant sweep', async () => { | |
| 393 | envDump() | |
| 394 | const results = await runSweep(path.join(import.meta.dir, '..', 'bin', 'cli.ts'), 'in-situ') | |
| 395 | expect(results).toHaveLength(8) | |
| 396 | }, 200_000) | |
| 397 | ||
| 398 | it.skipIf(process.platform === 'linux')('serves a fragment\'s route over plain HTTP with --no-https on the given port', async () => { | |
| 378 | 399 | const upstream = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: req => new Response(`gateway-upstream ${req.headers.get('x-forwarded-host')}`) }) |
| 379 | 400 | const httpPort = await freePort() |
| 380 | 401 | // Two routes so the fragment looks like a real box (and the shared |
| @@ -0,0 +1,217 @@ | ||
| 1 | /** | |
| 2 | * TEMPORARY — shared helpers for the #2267 diagnostics. Not for merge. | |
| 3 | * | |
| 4 | * Not a test file (no `.test.` in the name), so `bun test` never runs it | |
| 5 | * directly; `gateway.test.ts` and `zz-2267-diagnostic.test.ts` import it. | |
| 6 | * Every line printed is prefixed `[2267-diag]` so it is greppable in CI logs. | |
| 7 | */ | |
| 8 | import { spawn, spawnSync } from 'node:child_process' | |
| 9 | import * as fs from 'node:fs' | |
| 10 | import * as os from 'node:os' | |
| 11 | import * as process from 'node:process' | |
| 12 | ||
| 13 | export const BUDGET_MS = 12_000 | |
| 14 | const P = '[2267-diag]' | |
| 15 | ||
| 16 | export function say(...parts: unknown[]): void { | |
| 17 | console.log(P, ...parts) | |
| 18 | } | |
| 19 | ||
| 20 | export interface Result { | |
| 21 | name: string | |
| 22 | outcome: string | |
| 23 | ms: number | |
| 24 | bytes: number | |
| 25 | head: string | |
| 26 | diag: string | |
| 27 | } | |
| 28 | ||
| 29 | export type Stdio = Array<'ignore' | 'pipe' | 'inherit'> | |
| 30 | ||
| 31 | export function sh(cmd: string[]): string { | |
| 32 | try { | |
| 33 | const r = Bun.spawnSync(cmd, { stdout: 'pipe', stderr: 'pipe' }) | |
| 34 | const dec = new TextDecoder() | |
| 35 | return `${dec.decode(r.stdout)}${dec.decode(r.stderr)}`.trim() | |
| 36 | } | |
| 37 | catch (e) { | |
| 38 | return `ERR ${e}` | |
| 39 | } | |
| 40 | } | |
| 41 | ||
| 42 | export function readMaybe(p: string): string { | |
| 43 | try { | |
| 44 | return fs.readFileSync(p, 'utf8').trim() | |
| 45 | } | |
| 46 | catch { | |
| 47 | return '' | |
| 48 | } | |
| 49 | } | |
| 50 | ||
| 51 | /** One line per thread of `pid`: `tid:<first lines of /proc/pid/task/tid/file>`. */ | |
| 52 | function perTask(pid: number, file: string): string { | |
| 53 | return sh(['sh', '-c', `for t in /proc/${pid}/task/*; do printf '%s:%s ' "$(basename "$t")" "$( (cat "$t/${file}" 2>/dev/null || sudo -n cat "$t/${file}" 2>/dev/null) | head -6 | tr '\\n' '|')"; done`]) | |
| 54 | } | |
| 55 | ||
| 56 | /** Everything /proc will tell us about a child that has gone quiet. */ | |
| 57 | export function procDump(pid: number): string { | |
| 58 | if (process.platform !== 'linux') | |
| 59 | return '(no /proc on this platform)' | |
| 60 | const lines: string[] = [] | |
| 61 | const status = readMaybe(`/proc/${pid}/status`) | |
| 62 | .split('\n') | |
| 63 | .filter(l => /^(?:State|PPid|Threads|VmRSS|SigBlk|SigIgn|SigCgt)/.test(l)) | |
| 64 | .join(' | ') | |
| 65 | lines.push(`status: ${status}`) | |
| 66 | lines.push(`wchan: ${readMaybe(`/proc/${pid}/wchan`)} | syscall: ${readMaybe(`/proc/${pid}/syscall`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/syscall`])}`) | |
| 67 | lines.push(`stack: ${(readMaybe(`/proc/${pid}/stack`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/stack`])).replace(/\n/g, ' | ')}`) | |
| 68 | lines.push(`task wchan: ${perTask(pid, 'wchan')}`) | |
| 69 | lines.push(`task syscall: ${perTask(pid, 'syscall')}`) | |
| 70 | lines.push(`task stack: ${perTask(pid, 'stack')}`) | |
| 71 | lines.push(`fds: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`) | |
| 72 | lines.push(`cwd: ${sh(['readlink', `/proc/${pid}/cwd`])} | cmdline: ${readMaybe(`/proc/${pid}/cmdline`).replace(/\0/g, ' ')}`) | |
| 73 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) | |
| 74 | lines.push(`io: ${readMaybe(`/proc/${pid}/io`).replace(/\n/g, ' ')}`) | |
| 75 | return lines.join('\n ') | |
| 76 | } | |
| 77 | ||
| 78 | /** | |
| 79 | * The test process's own state at the moment a child went quiet. Uses the | |
| 80 | * numeric pid throughout: `/proc/self` inside a shelled-out command would name | |
| 81 | * the shell, not us. | |
| 82 | */ | |
| 83 | export function parentDump(): string { | |
| 84 | if (process.platform !== 'linux') | |
| 85 | return '(no /proc on this platform)' | |
| 86 | const pid = process.pid | |
| 87 | const lines: string[] = [] | |
| 88 | const status = readMaybe(`/proc/${pid}/status`) | |
| 89 | .split('\n') | |
| 90 | .filter(l => /^(?:Threads|VmRSS|FDSize)/.test(l)) | |
| 91 | .join(' | ') | |
| 92 | lines.push(`pid ${pid} | ${status}`) | |
| 93 | lines.push(`fd count: ${sh(['sh', '-c', `ls /proc/${pid}/fd | wc -l`])} | by kind: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $11}' | sed -E 's/:.*//; s/\\/.*//' | sort | uniq -c | tr '\\n' ' '`])}`) | |
| 94 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) | |
| 95 | lines.push(`task wchan: ${perTask(pid, 'wchan')}`) | |
| 96 | lines.push(`task syscall: ${perTask(pid, 'syscall')}`) | |
| 97 | lines.push(`listening sockets: ${sh(['sh', '-c', '(ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null) | tail -n +2 | wc -l'])}`) | |
| 98 | return lines.join('\n ') | |
| 99 | } | |
| 100 | ||
| 101 | export function envDump(): void { | |
| 102 | say('platform', process.platform, process.arch, 'bun', Bun.version, Bun.revision) | |
| 103 | say('execPath', process.execPath, '->', fs.realpathSync(process.execPath)) | |
| 104 | say('cwd', process.cwd(), '| tmpdir', os.tmpdir(), '| cpus', os.cpus().length, '| uid', typeof process.getuid === 'function' ? process.getuid() : 'n/a', '| pid', process.pid) | |
| 105 | say('env TMPDIR=', process.env.TMPDIR, '| RUNNER_TEMP=', process.env.RUNNER_TEMP, '| BUN_RUNTIME_TRANSPILER_CACHE_PATH=', process.env.BUN_RUNTIME_TRANSPILER_CACHE_PATH, '| HOME=', process.env.HOME) | |
| 106 | say('limits:', readMaybe('/proc/self/limits').split('\n').filter(l => /open files|processes/.test(l)).join(' | ') || 'n/a') | |
| 107 | say('sudo -n:', sh(['sudo', '-n', 'true']) === '' ? 'ok' : 'unavailable') | |
| 108 | } | |
| 109 | ||
| 110 | function shape(out: string): string { | |
| 111 | return out.slice(0, 100).replace(/\n/g, '⏎') | |
| 112 | } | |
| 113 | ||
| 114 | export function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }, budget = BUDGET_MS): Promise<Result> { | |
| 115 | return new Promise((resolve) => { | |
| 116 | const t0 = performance.now() | |
| 117 | let out = '' | |
| 118 | let settled = false | |
| 119 | const child = spawn(process.execPath, argv, { | |
| 120 | stdio: opts.stdio, | |
| 121 | env: { ...process.env, ...(opts.env ?? {}) }, | |
| 122 | cwd: opts.cwd, | |
| 123 | }) | |
| 124 | child.stdout?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 125 | child.stderr?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 126 | if (opts.stdio[0] === 'pipe') | |
| 127 | child.stdin?.end() | |
| 128 | const finish = (outcome: string, diag = ''): void => { | |
| 129 | if (settled) | |
| 130 | return | |
| 131 | settled = true | |
| 132 | resolve({ name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag }) | |
| 133 | } | |
| 134 | child.on('error', e => finish(`spawn-error ${e.message}`)) | |
| 135 | child.on('exit', (code, sig) => finish(`exit code=${code} signal=${sig}`)) | |
| 136 | setTimeout(() => { | |
| 137 | if (settled) | |
| 138 | return | |
| 139 | const diag = `${child.pid ? procDump(child.pid) : '(no pid)'}\n PARENT: ${parentDump()}` | |
| 140 | try { | |
| 141 | child.kill('SIGKILL') | |
| 142 | } | |
| 143 | catch {} | |
| 144 | finish(`HUNG >${budget}ms`, diag) | |
| 145 | }, budget) | |
| 146 | }) | |
| 147 | } | |
| 148 | ||
| 149 | export async function viaBunSpawn(name: string, argv: string[], budget = BUDGET_MS): Promise<Result> { | |
| 150 | const t0 = performance.now() | |
| 151 | const proc = Bun.spawn([process.execPath, ...argv], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: process.env }) | |
| 152 | let out = '' | |
| 153 | const dec = new TextDecoder() | |
| 154 | const pump = async (s: ReadableStream<Uint8Array> | null | undefined): Promise<void> => { | |
| 155 | if (!s) | |
| 156 | return | |
| 157 | for await (const c of s) out += dec.decode(c) | |
| 158 | } | |
| 159 | const done = Promise.all([pump(proc.stdout), pump(proc.stderr), proc.exited]).then(([, , code]) => `exit code=${code}`) | |
| 160 | const timer = new Promise<string>(r => setTimeout(() => r('TIMEOUT'), budget)) | |
| 161 | const raced = await Promise.race([done, timer]) | |
| 162 | let diag = '' | |
| 163 | if (raced === 'TIMEOUT') { | |
| 164 | diag = `${procDump(proc.pid)}\n PARENT: ${parentDump()}` | |
| 165 | try { | |
| 166 | proc.kill(9) | |
| 167 | } | |
| 168 | catch {} | |
| 169 | } | |
| 170 | return { name, outcome: raced === 'TIMEOUT' ? `HUNG >${budget}ms` : raced, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag } | |
| 171 | } | |
| 172 | ||
| 173 | export function viaSpawnSync(name: string, argv: string[], budget = BUDGET_MS): Result { | |
| 174 | const t0 = performance.now() | |
| 175 | const r = spawnSync(process.execPath, argv, { encoding: 'utf8', timeout: budget }) | |
| 176 | const out = `${r.stdout ?? ''}${r.stderr ?? ''}` | |
| 177 | const outcome = r.error | |
| 178 | ? `error ${r.error.message}` | |
| 179 | : (r.signal ? `HUNG >${budget}ms (killed ${r.signal})` : `exit code=${r.status}`) | |
| 180 | return { name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: shape(out), diag: '' } | |
| 181 | } | |
| 182 | ||
| 183 | /** The eight variants, sequentially, followed by a printed report. */ | |
| 184 | export async function runSweep(cli: string, where: string): Promise<Result[]> { | |
| 185 | const results: Result[] = [] | |
| 186 | const base: Stdio = ['ignore', 'pipe', 'pipe'] | |
| 187 | ||
| 188 | // G: is it bun-at-all, or the CLI graph? Cheapest discriminator first. | |
| 189 | results.push(await viaNodeSpawn('G bun --version | node spawn ignore/pipe/pipe', ['--version'], { stdio: base })) | |
| 190 | // A: the exact gateway.test.ts shape. | |
| 191 | results.push(await viaNodeSpawn('A cli --help | node spawn ignore/pipe/pipe (= gateway.test.ts baseline)', [cli, '--help'], { stdio: base })) | |
| 192 | // B: transpiler-cache hypothesis. | |
| 193 | results.push(await viaNodeSpawn('B A + BUN_RUNTIME_TRANSPILER_CACHE_PATH=0', [cli, '--help'], { stdio: base, env: { BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } })) | |
| 194 | // C: stdin-ignore hypothesis. | |
| 195 | results.push(await viaNodeSpawn('C A + stdin pipe (ended) instead of ignore', [cli, '--help'], { stdio: ['pipe', 'pipe', 'pipe'] })) | |
| 196 | // F: cwd hypothesis (repo-root bunfig.toml, project files). | |
| 197 | results.push(await viaNodeSpawn('F A + cwd=tmpdir', [cli, '--help'], { stdio: base, cwd: os.tmpdir() })) | |
| 198 | // I: bypass the pipes entirely; help text between the markers means the | |
| 199 | // child runs and the fault is in pipe plumbing. | |
| 200 | say(`--- I begin [${where}] (inherit stdout/stderr; any help text below is from the child) ---`) | |
| 201 | results.push(await viaNodeSpawn('I A + stdout/stderr inherit', [cli, '--help'], { stdio: ['ignore', 'inherit', 'inherit'] })) | |
| 202 | say(`--- I end [${where}] ---`) | |
| 203 | // D: is it the node:child_process shim specifically? | |
| 204 | results.push(await viaBunSpawn('D cli --help | Bun.spawn', [cli, '--help'])) | |
| 205 | // E: cli.test.ts's call shape, but from a parent that HAS the graph. | |
| 206 | results.push(viaSpawnSync('E cli --help | spawnSync (= cli.test.ts shape, parent has graph)', [cli, '--help'])) | |
| 207 | ||
| 208 | say(`===== RESULTS [${where}] =====`) | |
| 209 | for (const r of results) { | |
| 210 | say(r.name) | |
| 211 | say(` -> ${r.outcome} in ${r.ms}ms, ${r.bytes} bytes${r.head ? `, head: ${r.head}` : ''}`) | |
| 212 | if (r.diag) | |
| 213 | say(` diag:\n ${r.diag}`) | |
| 214 | } | |
| 215 | say(`===== END [${where}] =====`) | |
| 216 | return results | |
| 217 | } | |
| @@ -1,198 +1,27 @@ | ||
| 1 | 1 | /** |
| 2 | * TEMPORARY — diagnostic for #2267, not for merge. | |
| 2 | * TEMPORARY — #2267 diagnostic, standalone file. Not for merge. | |
| 3 | 3 | * |
| 4 | * On GitHub's Linux runners (and only there), a `bun bin/cli.ts --help` spawned | |
| 5 | * asynchronously from a bun test process that has the rpx graph loaded produces | |
| 6 | * zero bytes and never exits, while `spawnSync` of the same CLI from a file with | |
| 7 | * no rpx imports passes in ~100ms. Nothing reproduces it in an oven/bun | |
| 8 | * container. This file runs one variant after another in the failing parent | |
| 9 | * shape (it imports `../src/start` and `../src/gateway` on purpose), so a single | |
| 10 | * CI run says which knob matters, and dumps the stuck child's /proc state | |
| 11 | * before killing it. Every line is prefixed `[2267-diag]` so it is greppable in | |
| 12 | * the CI log. The test itself always passes; the log is the result. | |
| 4 | * `bun test` orders files deterministically but not alphabetically; this one | |
| 5 | * lands at position 38 of 48, fifteen files after `gateway.test.ts` (23), so it | |
| 6 | * answers "does the failing shape hang from a parent with the graph loaded, at | |
| 7 | * a LATER point in the suite?" The in-situ copy of the same sweep inside | |
| 8 | * `gateway.test.ts` answers it at the position where the hang was recorded. | |
| 9 | * Both import the rpx graph on purpose. The test always passes; the log is the | |
| 10 | * result, every line prefixed `[2267-diag]`. | |
| 13 | 11 | */ |
| 14 | 12 | import { describe, expect, it } from 'bun:test' |
| 15 | import { spawn, spawnSync } from 'node:child_process' | |
| 16 | import * as fs from 'node:fs' | |
| 17 | import * as os from 'node:os' | |
| 18 | 13 | import * as path from 'node:path' |
| 19 | import * as process from 'node:process' | |
| 20 | 14 | import { readGatewayFragments } from '../src/gateway' |
| 21 | 15 | import * as Start from '../src/start' |
| 16 | import { envDump, runSweep, say } from './helpers/spawn-diag' | |
| 22 | 17 | |
| 23 | 18 | const CLI = path.join(import.meta.dir, '..', 'bin', 'cli.ts') |
| 24 | const BUDGET_MS = 12_000 | |
| 25 | const P = '[2267-diag]' | |
| 26 | 19 | |
| 27 | function say(...parts: unknown[]): void { | |
| 28 | console.log(P, ...parts) | |
| 29 | } | |
| 30 | ||
| 31 | interface Result { | |
| 32 | name: string | |
| 33 | outcome: string | |
| 34 | ms: number | |
| 35 | bytes: number | |
| 36 | head: string | |
| 37 | diag: string | |
| 38 | } | |
| 39 | ||
| 40 | type Stdio = Array<'ignore' | 'pipe' | 'inherit'> | |
| 41 | ||
| 42 | function sh(cmd: string[]): string { | |
| 43 | try { | |
| 44 | const r = Bun.spawnSync(cmd, { stdout: 'pipe', stderr: 'pipe' }) | |
| 45 | const dec = new TextDecoder() | |
| 46 | return `${dec.decode(r.stdout)}${dec.decode(r.stderr)}`.trim() | |
| 47 | } | |
| 48 | catch (e) { | |
| 49 | return `ERR ${e}` | |
| 50 | } | |
| 51 | } | |
| 52 | ||
| 53 | function readMaybe(p: string): string { | |
| 54 | try { | |
| 55 | return fs.readFileSync(p, 'utf8').trim() | |
| 56 | } | |
| 57 | catch { | |
| 58 | return '' | |
| 59 | } | |
| 60 | } | |
| 61 | ||
| 62 | /** Everything /proc will tell us about a child that has gone quiet. */ | |
| 63 | function procDump(pid: number): string { | |
| 64 | if (process.platform !== 'linux') | |
| 65 | return '(no /proc on this platform)' | |
| 66 | const lines: string[] = [] | |
| 67 | const status = readMaybe(`/proc/${pid}/status`) | |
| 68 | .split('\n') | |
| 69 | .filter(l => /^(?:State|Threads|VmRSS|SigBlk|SigIgn|SigCgt|PPid)/.test(l)) | |
| 70 | .join(' | ') | |
| 71 | lines.push(`status: ${status}`) | |
| 72 | lines.push(`wchan: ${readMaybe(`/proc/${pid}/wchan`)}`) | |
| 73 | lines.push(`stack: ${readMaybe(`/proc/${pid}/stack`) || sh(['sudo', '-n', 'cat', `/proc/${pid}/stack`])}`) | |
| 74 | lines.push(`threads: ${sh(['sh', '-c', `for t in /proc/${pid}/task/*; do echo "$(basename "$t"):$(cat "$t/wchan" 2>/dev/null || sudo -n cat "$t/wchan" 2>/dev/null)"; done | tr '\\n' ' '`])}`) | |
| 75 | lines.push(`thread stacks: ${sh(['sh', '-c', `for t in /proc/${pid}/task/*; do echo "== $(basename "$t")"; (cat "$t/stack" 2>/dev/null || sudo -n cat "$t/stack" 2>/dev/null) | head -8; done | tr '\\n' ' '`])}`) | |
| 76 | lines.push(`fds: ${sh(['sh', '-c', `ls -l /proc/${pid}/fd 2>/dev/null | tail -n +2 | awk '{print $9"->"$11}' | tr '\\n' ' '`])}`) | |
| 77 | lines.push(`cwd: ${sh(['readlink', `/proc/${pid}/cwd`])}`) | |
| 78 | lines.push(`cmdline: ${readMaybe(`/proc/${pid}/cmdline`).replace(/\0/g, ' ')}`) | |
| 79 | lines.push(`children: ${sh(['sh', '-c', `ps -o pid,stat,wchan:30,etimes,cmd --ppid ${pid} 2>/dev/null | tail -n +2 | tr '\\n' ';'`])}`) | |
| 80 | lines.push(`io: ${readMaybe(`/proc/${pid}/io`).replace(/\n/g, ' ')}`) | |
| 81 | return lines.join('\n ') | |
| 82 | } | |
| 83 | ||
| 84 | function viaNodeSpawn(name: string, argv: string[], opts: { stdio: Stdio, env?: Record<string, string>, cwd?: string }): Promise<Result> { | |
| 85 | return new Promise((resolve) => { | |
| 86 | const t0 = performance.now() | |
| 87 | let out = '' | |
| 88 | let settled = false | |
| 89 | const child = spawn(process.execPath, argv, { | |
| 90 | stdio: opts.stdio, | |
| 91 | env: { ...process.env, ...(opts.env ?? {}) }, | |
| 92 | cwd: opts.cwd, | |
| 93 | }) | |
| 94 | child.stdout?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 95 | child.stderr?.on('data', (c: Buffer) => { out += c.toString() }) | |
| 96 | if (opts.stdio[0] === 'pipe') | |
| 97 | child.stdin?.end() | |
| 98 | const finish = (outcome: string, diag = ''): void => { | |
| 99 | if (settled) | |
| 100 | return | |
| 101 | settled = true | |
| 102 | resolve({ name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag }) | |
| 103 | } | |
| 104 | child.on('error', e => finish(`spawn-error ${e.message}`)) | |
| 105 | child.on('exit', (code, sig) => finish(`exit code=${code} signal=${sig}`)) | |
| 106 | setTimeout(() => { | |
| 107 | if (settled) | |
| 108 | return | |
| 109 | const diag = child.pid ? procDump(child.pid) : '(no pid)' | |
| 110 | try { | |
| 111 | child.kill('SIGKILL') | |
| 112 | } | |
| 113 | catch {} | |
| 114 | finish(`HUNG >${BUDGET_MS}ms`, diag) | |
| 115 | }, BUDGET_MS) | |
| 116 | }) | |
| 117 | } | |
| 118 | ||
| 119 | async function viaBunSpawn(name: string, argv: string[]): Promise<Result> { | |
| 120 | const t0 = performance.now() | |
| 121 | const proc = Bun.spawn([process.execPath, ...argv], { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: process.env }) | |
| 122 | let out = '' | |
| 123 | const dec = new TextDecoder() | |
| 124 | const pump = async (s: ReadableStream<Uint8Array> | null | undefined): Promise<void> => { | |
| 125 | if (!s) | |
| 126 | return | |
| 127 | for await (const c of s) out += dec.decode(c) | |
| 128 | } | |
| 129 | const done = Promise.all([pump(proc.stdout), pump(proc.stderr), proc.exited]).then(([, , code]) => `exit code=${code}`) | |
| 130 | const timer = new Promise<string>(r => setTimeout(() => r('TIMEOUT'), BUDGET_MS)) | |
| 131 | const raced = await Promise.race([done, timer]) | |
| 132 | let diag = '' | |
| 133 | if (raced === 'TIMEOUT') { | |
| 134 | diag = procDump(proc.pid) | |
| 135 | try { | |
| 136 | proc.kill(9) | |
| 137 | } | |
| 138 | catch {} | |
| 139 | } | |
| 140 | return { name, outcome: raced === 'TIMEOUT' ? `HUNG >${BUDGET_MS}ms` : raced, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag } | |
| 141 | } | |
| 142 | ||
| 143 | function viaSpawnSync(name: string, argv: string[]): Result { | |
| 144 | const t0 = performance.now() | |
| 145 | const r = spawnSync(process.execPath, argv, { encoding: 'utf8', timeout: BUDGET_MS }) | |
| 146 | const out = `${r.stdout ?? ''}${r.stderr ?? ''}` | |
| 147 | const outcome = r.error | |
| 148 | ? `error ${r.error.message}` | |
| 149 | : (r.signal ? `HUNG >${BUDGET_MS}ms (killed ${r.signal})` : `exit code=${r.status}`) | |
| 150 | return { name, outcome, ms: Math.round(performance.now() - t0), bytes: out.length, head: out.slice(0, 100).replace(/\n/g, '⏎'), diag: '' } | |
| 151 | } | |
| 152 | ||
| 153 | describe('2267 diagnostic', () => { | |
| 20 | describe('2267 diagnostic (standalone file)', () => { | |
| 154 | 21 | it('records how each spawn variant of the CLI behaves here', async () => { |
| 155 | say('platform', process.platform, process.arch, 'bun', Bun.version, Bun.revision) | |
| 156 | say('execPath', process.execPath, '->', fs.realpathSync(process.execPath)) | |
| 157 | say('cwd', process.cwd(), '| tmpdir', os.tmpdir(), '| cpus', os.cpus().length, '| uid', typeof process.getuid === 'function' ? process.getuid() : 'n/a') | |
| 158 | say('env TMPDIR=', process.env.TMPDIR, '| RUNNER_TEMP=', process.env.RUNNER_TEMP, '| BUN_RUNTIME_TRANSPILER_CACHE_PATH=', process.env.BUN_RUNTIME_TRANSPILER_CACHE_PATH, '| HOME=', process.env.HOME) | |
| 159 | say('bunfig.toml in cwd:', fs.existsSync(path.join(process.cwd(), 'bunfig.toml'))) | |
| 160 | say('limits:', readMaybe('/proc/self/limits').split('\n').filter(l => /open files|processes/.test(l)).join(' | ') || 'n/a') | |
| 161 | say('sudo -n:', sh(['sudo', '-n', 'true']) === '' ? 'ok' : 'unavailable') | |
| 22 | envDump() | |
| 162 | 23 | say('parent graph loaded:', typeof Start.startProxies, typeof readGatewayFragments) |
| 163 | ||
| 164 | const results: Result[] = [] | |
| 165 | const base: Stdio = ['ignore', 'pipe', 'pipe'] | |
| 166 | ||
| 167 | // G: is it bun-at-all, or the CLI graph? Cheapest discriminator first. | |
| 168 | results.push(await viaNodeSpawn('G bun --version | node spawn ignore/pipe/pipe', ['--version'], { stdio: base })) | |
| 169 | // A: the exact gateway.test.ts shape. Expected to hang on GitHub Linux. | |
| 170 | results.push(await viaNodeSpawn('A cli --help | node spawn ignore/pipe/pipe (= gateway.test.ts baseline)', [CLI, '--help'], { stdio: base })) | |
| 171 | // B: transpiler-cache hypothesis. | |
| 172 | results.push(await viaNodeSpawn('B A + BUN_RUNTIME_TRANSPILER_CACHE_PATH=0', [CLI, '--help'], { stdio: base, env: { BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } })) | |
| 173 | // C: stdin-ignore hypothesis. | |
| 174 | results.push(await viaNodeSpawn('C A + stdin pipe (ended) instead of ignore', [CLI, '--help'], { stdio: ['pipe', 'pipe', 'pipe'] })) | |
| 175 | // F: cwd hypothesis (repo-root bunfig.toml, project files). | |
| 176 | results.push(await viaNodeSpawn('F A + cwd=tmpdir', [CLI, '--help'], { stdio: base, cwd: os.tmpdir() })) | |
| 177 | // I: bypass the pipes entirely; if help appears in the CI log between the | |
| 178 | // markers, the child runs and the fault is in pipe plumbing. | |
| 179 | say('--- I begin (inherit stdout/stderr; any help text below is from the child) ---') | |
| 180 | results.push(await viaNodeSpawn('I A + stdout/stderr inherit', [CLI, '--help'], { stdio: ['ignore', 'inherit', 'inherit'] })) | |
| 181 | say('--- I end ---') | |
| 182 | // D: is it the node:child_process shim specifically? | |
| 183 | results.push(await viaBunSpawn('D cli --help | Bun.spawn', [CLI, '--help'])) | |
| 184 | // E: cli.test.ts's call shape, but from a parent that HAS the graph. | |
| 185 | results.push(viaSpawnSync('E cli --help | spawnSync (= cli.test.ts shape, parent has graph)', [CLI, '--help'])) | |
| 186 | ||
| 187 | say('===== RESULTS =====') | |
| 188 | for (const r of results) { | |
| 189 | say(r.name) | |
| 190 | say(` -> ${r.outcome} in ${r.ms}ms, ${r.bytes} bytes${r.head ? `, head: ${r.head}` : ''}`) | |
| 191 | if (r.diag) | |
| 192 | say(` diag:\n ${r.diag}`) | |
| 193 | } | |
| 194 | say('===== END =====') | |
| 195 | ||
| 24 | const results = await runSweep(CLI, 'standalone') | |
| 196 | 25 | expect(results).toHaveLength(8) |
| 197 | 26 | }, 200_000) |
| 198 | 27 | }) |