also looking at this
test: label native browser transport failures
#66
7 files
+177
-11
| @@ -0,0 +1,38 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { writeFile } from 'node:fs/promises' | |
| 3 | import { basename, join } from 'node:path' | |
| 4 | ||
| 5 | // Controlled external HTTP/WebSocket peer, not a Chrome implementation. | |
| 6 | const root = process.cwd(), mode = process.env.BROWSER_PEER_MODE | |
| 7 | assert(basename(root).startsWith('drone-incident-access-browser-operation-')) | |
| 8 | assert(['version-fetch', 'version-json', 'target-fetch', 'target-json', 'browser-socket', 'page-socket', 'control-fetch', 'control-json', 'control-ok', 'ok'].includes(mode!)) | |
| 9 | const profile = Bun.argv.find(value => value.startsWith('--user-data-dir='))?.slice('--user-data-dir='.length) | |
| 10 | assert.equal(profile, join(root, 'browser-profile')) | |
| 11 | const stalledBody = () => new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode('{')) } }), { headers: { 'content-type': 'application/json' } }) | |
| 12 | const server = Bun.serve({ | |
| 13 | hostname: '127.0.0.1', port: 0, | |
| 14 | fetch(request, owner): Response | Promise<Response> | undefined { | |
| 15 | const path = new URL(request.url).pathname | |
| 16 | if (path === '/browser' || path === '/page') { | |
| 17 | if ((path === '/browser' && mode === 'browser-socket') || (path === '/page' && mode === 'page-socket')) return new Promise<Response>(() => {}) | |
| 18 | assert(owner.upgrade(request)); return | |
| 19 | } | |
| 20 | if (path === '/__fixture/status') { | |
| 21 | assert.equal(request.headers.get('x-fixture-control'), 'synthetic-peer-control') | |
| 22 | if (mode === 'control-fetch') return new Promise<Response>(() => {}) | |
| 23 | if (mode === 'control-json') return stalledBody() | |
| 24 | return Response.json({ ok: true }) | |
| 25 | } | |
| 26 | if (path === '/json/version' || path === '/json/new') { | |
| 27 | const operation = path === '/json/version' ? 'version' : 'target' | |
| 28 | if (mode === `${operation}-fetch`) return new Promise<Response>(() => {}) | |
| 29 | if (mode === `${operation}-json`) return stalledBody() | |
| 30 | return Response.json({ webSocketDebuggerUrl: `ws://127.0.0.1:${server.port}/${operation === 'version' ? 'browser' : 'page'}` }) | |
| 31 | } | |
| 32 | return new Response(null, { status: 404 }) | |
| 33 | }, | |
| 34 | websocket: { message() {} }, | |
| 35 | }) | |
| 36 | process.on('SIGTERM', () => { void server.stop(true).then(() => process.exit(0)) }) | |
| 37 | await writeFile(join(root, 'peer.pid'), String(process.pid), { mode: 0o600, flag: 'wx' }) | |
| 38 | await writeFile(join(profile!, 'DevToolsActivePort'), `${server.port}\n/browser\n`, { mode: 0o600, flag: 'wx' }) | |
| @@ -0,0 +1,41 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import { closeSync, openSync, writeSync } from 'node:fs' | |
| 3 | import { readFile } from 'node:fs/promises' | |
| 4 | import { basename, join } from 'node:path' | |
| 5 | import { startWorkspaceBrowser } from '../../helpers/workspace-browser' | |
| 6 | import { requestWorkspaceControl } from '../../helpers/workspace-control' | |
| 7 | ||
| 8 | const root = process.cwd(), mode = process.env.BROWSER_PEER_MODE | |
| 9 | assert(basename(root).startsWith('drone-incident-access-browser-operation-')) | |
| 10 | const unwritable = process.env.BROWSER_OPERATION_UNWRITABLE_DIAGNOSTIC === 'true' | |
| 11 | if (unwritable) { | |
| 12 | closeSync(2) | |
| 13 | assert.equal(openSync('/dev/null', 'r'), 2, 'Keep descriptor two reserved and read-only') | |
| 14 | assert.throws(() => writeSync(2, 'probe'), { code: 'EBADF' }) | |
| 15 | } | |
| 16 | let browser: Awaited<ReturnType<typeof startWorkspaceBrowser>> | undefined | |
| 17 | let caught: unknown | |
| 18 | const began = performance.now() | |
| 19 | try { | |
| 20 | browser = await startWorkspaceBrowser(root) | |
| 21 | if (mode?.startsWith('target-') || mode === 'page-socket') await browser.page('http://fixture.invalid/incidents') | |
| 22 | if (mode?.startsWith('control-')) { | |
| 23 | const port = (await readFile(join(root, 'browser-profile/DevToolsActivePort'), 'utf8')).split('\n')[0] | |
| 24 | const result = await requestWorkspaceControl({ url: `http://127.0.0.1:${port}`, control: 'synthetic-peer-control' }, 'status') | |
| 25 | assert.deepEqual(result, { ok: true }) | |
| 26 | } | |
| 27 | } | |
| 28 | catch (error) { caught = error } | |
| 29 | finally { await browser?.close() } | |
| 30 | if (unwritable) assert.throws(() => writeSync(2, 'probe'), { code: 'EBADF' }) | |
| 31 | if (mode === 'ok' || mode === 'control-ok') assert.equal(caught, undefined) | |
| 32 | else { | |
| 33 | assert(caught instanceof Error) | |
| 34 | if (mode?.endsWith('-socket')) { assert.equal(caught.name, 'Error'); assert.match(caught.message, /^Browser socket /) } | |
| 35 | else assert(['TimeoutError', 'AbortError'].includes(caught.name), 'Preserve the native abort exception instead of replacing it') | |
| 36 | assert(performance.now() - began >= 5000, 'Exercise the original five-second native fetch timeout') | |
| 37 | } | |
| 38 | const peer = Number(await readFile(join(root, 'peer.pid'), 'utf8')) | |
| 39 | assert(Number.isSafeInteger(peer) && peer > 1) | |
| 40 | assert.throws(() => process.kill(peer, 0), 'Owned peer must actually exit before files are removed') | |
| 41 | console.log(JSON.stringify({ oracle: 'browser-operation-peer-result', mode, nativeError: caught instanceof Error ? caught.name : null, elapsedMs: performance.now() - began, peerReaped: true })) | |
| @@ -0,0 +1,25 @@ | ||
| 1 | import { writeSync } from 'node:fs' | |
| 2 | ||
| 3 | export type BrowserControlAction = 'status' | 'release' | 'verify' | 'read-only' | 'revoke' | |
| 4 | type BrowserOperation = 'version-fetch' | 'version-json' | 'browser-socket' | 'target-fetch' | 'target-json' | 'page-socket' | `control-${BrowserControlAction}-${'fetch' | 'json'}` | |
| 5 | ||
| 6 | /** Observe an owned transport boundary without replacing its result, error or timeout. */ | |
| 7 | export async function observeBrowserOperation<T>(operation: BrowserOperation, work: () => Promise<T>): Promise<T> { | |
| 8 | const began = performance.now() | |
| 9 | try { return await work() } | |
| 10 | catch (error) { | |
| 11 | // Reporting is best effort. A closed diagnostic pipe must preserve the native error. | |
| 12 | try { | |
| 13 | const phase = operation.startsWith('version-') || operation === 'browser-socket' ? 'browser-start' : operation.startsWith('control-') ? 'workspace-control' : 'page-open' | |
| 14 | const nativeError = error instanceof Error && ['TimeoutError', 'AbortError', 'SyntaxError'].includes(error.name) ? error.name : 'other' | |
| 15 | const bytes = new TextEncoder().encode(`${JSON.stringify({ oracle: 'browser-operation-failure', phase, operation, nativeError, elapsedMs: performance.now() - began })}\n`) | |
| 16 | const report = (descriptor: number) => { | |
| 17 | try { return writeSync(descriptor, bytes) === bytes.byteLength } | |
| 18 | catch { return false } | |
| 19 | } | |
| 20 | if (!report(2)) report(1) | |
| 21 | } | |
| 22 | catch {} | |
| 23 | throw error | |
| 24 | } | |
| 25 | } | |
| @@ -2,17 +2,15 @@ import assert from 'node:assert/strict' | ||
| 2 | 2 | import { mkdir, readFile, writeFile } from 'node:fs/promises' |
| 3 | 3 | import { basename, join } from 'node:path' |
| 4 | 4 | import { startWorkspaceBrowser } from './workspace-browser' |
| 5 | import type { BrowserControlAction } from './browser-operation' | |
| 6 | import { requestWorkspaceControl } from './workspace-control' | |
| 5 | 7 | |
| 6 | 8 | // Controller is isolated from native Auth/SQLite and reads owner-only fixture credentials. |
| 7 | 9 | const root = process.cwd() |
| 8 | 10 | assert(basename(root).startsWith('drone-incident-access-')) |
| 9 | 11 | const data = JSON.parse(await readFile(join(root, 'credentials.json'), 'utf8')) as { url: string, key: string, sourceDigest: string, senderToken: string, recipientToken: string, recipientId: number, control: string, capture: { id: string, width: number }, pixel: number } |
| 10 | 12 | assert(/^http:\/\/127\.0\.0\.1:\d+$/.test(data.url)) |
| 11 | const control = async (action: string): Promise<any> => { | |
| 12 | const response = await fetch(`${data.url}/__fixture/${action}`, { method: 'GET', headers: { 'x-fixture-control': data.control }, signal: AbortSignal.timeout(5000) }) | |
| 13 | assert(response.ok, `Private fixture control must succeed: ${action}, ${response.status}`) | |
| 14 | return response.json() | |
| 15 | } | |
| 13 | const control = async (action: BrowserControlAction): Promise<any> => requestWorkspaceControl(data, action) | |
| 16 | 14 | const browser = await startWorkspaceBrowser(root) |
| 17 | 15 | const evidence = process.env.WORKSPACE_BROWSER_EVIDENCE ?? join(root, 'evidence') |
| 18 | 16 | await mkdir(evidence, { recursive: true }) |
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | import assert from 'node:assert/strict' |
| 2 | 2 | import { mkdir, readFile } from 'node:fs/promises' |
| 3 | 3 | import { join } from 'node:path' |
| 4 | import { observeBrowserOperation } from './browser-operation' | |
| 4 | 5 | |
| 5 | 6 | /** Dependency-free CDP, with a fresh profile. Credentials never enter command arguments. */ |
| 6 | 7 | async function openSocket(url: string): Promise<WebSocket> { |
| @@ -26,22 +27,24 @@ export async function startWorkspaceBrowser(root: string) { | ||
| 26 | 27 | if (port) break |
| 27 | 28 | await Bun.sleep(50) |
| 28 | 29 | } |
| 29 | if (!port) { if (child.exitCode === null) child.kill(); await Promise.race([child.exited, Bun.sleep(2000)]); throw new Error('Fresh browser must start') } | |
| 30 | if (!port) { closing = true; if (child.exitCode === null) child.kill(); await Promise.race([child.exited, Bun.sleep(2000)]); throw new Error('Fresh browser must start') } | |
| 30 | 31 | // Keep a browser-level session, as well as page sessions, for the owned lifecycle. |
| 31 | 32 | let browserSocket: WebSocket |
| 32 | 33 | try { |
| 33 | const version = await (await fetch(`http://127.0.0.1:${port}/json/version`, { signal: AbortSignal.timeout(5000) })).json() as { webSocketDebuggerUrl: string } | |
| 34 | browserSocket = await openSocket(version.webSocketDebuggerUrl) | |
| 34 | const response = await observeBrowserOperation('version-fetch', () => fetch(`http://127.0.0.1:${port}/json/version`, { signal: AbortSignal.timeout(5000) })) | |
| 35 | const version = await observeBrowserOperation('version-json', () => response.json()) as { webSocketDebuggerUrl: string } | |
| 36 | browserSocket = await observeBrowserOperation('browser-socket', () => openSocket(version.webSocketDebuggerUrl)) | |
| 35 | 37 | } |
| 36 | catch (error) { if (child.exitCode === null) child.kill(); await Promise.race([child.exited, Bun.sleep(2000)]); throw error } | |
| 38 | catch (error) { closing = true; if (child.exitCode === null) child.kill(); await Promise.race([child.exited, Bun.sleep(2000)]); throw error } | |
| 37 | 39 | let browserSequence = 0 |
| 38 | 40 | browserSocket.onmessage = () => {} |
| 39 | 41 | const keepAlive = setInterval(() => { if (browserSocket.readyState === WebSocket.OPEN) browserSocket.send(JSON.stringify({ id: ++browserSequence, method: 'Browser.getVersion' })) }, 5000) |
| 40 | 42 | const pages: Array<{ close: () => void }> = [] |
| 41 | 43 | return { |
| 42 | 44 | async page(url: string, javascript = true) { |
| 43 | const target = await (await fetch(`http://127.0.0.1:${port}/json/new?about:blank`, { method: 'PUT', signal: AbortSignal.timeout(5000) })).json() as { webSocketDebuggerUrl: string } | |
| 44 | const socket = await openSocket(target.webSocketDebuggerUrl) | |
| 45 | const response = await observeBrowserOperation('target-fetch', () => fetch(`http://127.0.0.1:${port}/json/new?about:blank`, { method: 'PUT', signal: AbortSignal.timeout(5000) })) | |
| 46 | const target = await observeBrowserOperation('target-json', () => response.json()) as { webSocketDebuggerUrl: string } | |
| 47 | const socket = await observeBrowserOperation('page-socket', () => openSocket(target.webSocketDebuggerUrl)) | |
| 45 | 48 | let sequence = 0, phase = 'initial page' |
| 46 | 49 | const waiting = new Map<number, { resolve: (value: any) => void, reject: (error: Error) => void }>() |
| 47 | 50 | const errors: string[] = [] |
| @@ -0,0 +1,10 @@ | ||
| 1 | import assert from 'node:assert/strict' | |
| 2 | import type { BrowserControlAction } from './browser-operation' | |
| 3 | import { observeBrowserOperation } from './browser-operation' | |
| 4 | ||
| 5 | /** The driver's private fixture control transport, with no credential logging. */ | |
| 6 | export async function requestWorkspaceControl(context: { url: string, control: string }, action: BrowserControlAction): Promise<unknown> { | |
| 7 | const response = await observeBrowserOperation(`control-${action}-fetch`, () => fetch(`${context.url}/__fixture/${action}`, { method: 'GET', headers: { 'x-fixture-control': context.control }, signal: AbortSignal.timeout(5000) })) | |
| 8 | assert(response.ok, `Private fixture control must succeed: ${action}, ${response.status}`) | |
| 9 | return observeBrowserOperation(`control-${action}-json`, () => response.json()) | |
| 10 | } | |
| @@ -0,0 +1,51 @@ | ||
| 1 | import { expect, test } from 'bun:test' | |
| 2 | import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' | |
| 3 | import { tmpdir } from 'node:os' | |
| 4 | import { join, resolve } from 'node:path' | |
| 5 | import { spawnWorkspaceFixture } from '../helpers/workspace-process' | |
| 6 | ||
| 7 | for (const mode of ['version-fetch', 'version-json', 'target-fetch', 'target-json', 'browser-socket', 'page-socket', 'control-fetch', 'control-json', 'control-ok', 'ok', 'unwritable-diagnostic']) test(`native browser operation ${mode}`, async () => { | |
| 8 | const evidence = process.env.BROWSER_OPERATION_EVIDENCE | |
| 9 | if (evidence) await mkdir(resolve(evidence), { recursive: true, mode: 0o700 }) | |
| 10 | const root = await mkdtemp(join(evidence ? resolve(evidence) : tmpdir(), 'drone-incident-access-browser-operation-')) | |
| 11 | let owned: ReturnType<typeof spawnWorkspaceFixture> | undefined | |
| 12 | let passed = false | |
| 13 | try { | |
| 14 | const config = join(root, 'bunfig.toml'), launcher = join(root, 'browser-peer') | |
| 15 | await writeFile(config, 'preload=[]\n[install]\nauto="disable"\n', { mode: 0o600 }) | |
| 16 | await writeFile(launcher, '#!/bin/sh\nexec "$BROWSER_PEER_BUN" --no-install "--config=$BROWSER_PEER_CONFIG" --no-env-file "$BROWSER_PEER_SOURCE" "$@"\n', { mode: 0o700 }) | |
| 17 | const fixture = `${import.meta.dir}/../fixtures/public-safety-access` | |
| 18 | const peerMode = mode === 'unwritable-diagnostic' ? 'version-fetch' : mode | |
| 19 | owned = spawnWorkspaceFixture([process.execPath, '--no-install', `--config=${config}`, '--no-env-file', `${fixture}/browser-operation-probe.ts`], root, { BROWSE_BROWSER: launcher, BROWSER_PEER_BUN: process.execPath, BROWSER_PEER_CONFIG: config, BROWSER_PEER_SOURCE: `${fixture}/browser-operation-peer.ts`, BROWSER_PEER_MODE: peerMode, BROWSER_OPERATION_UNWRITABLE_DIAGNOSTIC: mode === 'unwritable-diagnostic' ? 'true' : 'false' }) | |
| 20 | const current = owned, watchdog = setTimeout(() => { void current.stop().catch(() => {}) }, 12000) | |
| 21 | try { | |
| 22 | const [code, stdout, stderr] = await Promise.all([owned.child.exited, new Response(owned.child.stdout as ReadableStream).text(), new Response(owned.child.stderr as ReadableStream).text()]) | |
| 23 | await writeFile(join(root, 'fixture.stdout'), stdout, { mode: 0o600 }); await writeFile(join(root, 'fixture.stderr'), stderr, { mode: 0o600 }) | |
| 24 | for (const line of stdout.split('\n')) if (line.startsWith('{"oracle":')) console.log(line) | |
| 25 | expect(code, `${stdout}\n${stderr}`).toBe(0) | |
| 26 | expect(stdout).toContain('"peerReaped":true') | |
| 27 | expect(stdout).not.toContain('QA isolated Chrome exited unexpectedly') | |
| 28 | if (mode === 'ok' || mode === 'control-ok' || mode === 'unwritable-diagnostic') expect(stderr).toBe('') | |
| 29 | if (mode !== 'ok' && mode !== 'control-ok') { | |
| 30 | const stream = mode === 'unwritable-diagnostic' ? stdout : stderr | |
| 31 | const lines = stream.split('\n').filter(line => line.startsWith('{"oracle":"browser-operation-failure"')); expect(lines).toHaveLength(1) | |
| 32 | const observation = JSON.parse(lines[0]!) | |
| 33 | expect(observation.oracle).toBe('browser-operation-failure') | |
| 34 | expect(observation.operation).toBe(peerMode.startsWith('control-') ? peerMode.replace('control-', 'control-status-') : peerMode) | |
| 35 | expect(observation.phase).toBe(peerMode.startsWith('control-') ? 'workspace-control' : peerMode.startsWith('version') || peerMode === 'browser-socket' ? 'browser-start' : 'page-open') | |
| 36 | expect(peerMode.endsWith('-socket') ? ['other'] : ['TimeoutError', 'AbortError']).toContain(observation.nativeError) | |
| 37 | expect(observation.elapsedMs).toBeGreaterThanOrEqual(4900) | |
| 38 | expect(stream).not.toContain(root) | |
| 39 | expect(stream).not.toContain('http:') | |
| 40 | expect(stream).not.toContain('synthetic-peer-control') | |
| 41 | } | |
| 42 | passed = true | |
| 43 | } | |
| 44 | finally { clearTimeout(watchdog) } | |
| 45 | } | |
| 46 | finally { | |
| 47 | await owned?.stop() | |
| 48 | if (passed) await rm(root, { recursive: true, force: true }) | |
| 49 | else console.error('BROWSER_OPERATION_FIXTURE_RETAINED', root) | |
| 50 | } | |
| 51 | }, 18000) | |