also looking at this
feat: add scoped cargo receiving mission tasks
#102
21 files
+1186
-15
| @@ -0,0 +1,167 @@ | ||
| 1 | import { createHash } from 'node:crypto' | |
| 2 | import { lstat, open, readFile, readlink, readdir } from 'node:fs/promises' | |
| 3 | import type { FileHandle } from 'node:fs/promises' | |
| 4 | import { join } from 'node:path' | |
| 5 | import { browserFileErrorCode } from './browser-launch-evidence' | |
| 6 | ||
| 7 | const processLimit = 64, threadLimit = 32, executableLimit = 4, executableByteLimit = 512 * 1024 * 1024 | |
| 8 | async function text(path: string, signal: AbortSignal) { | |
| 9 | if (signal.aborted) return { value: null, code: 'ABORT_ERR' } | |
| 10 | try { | |
| 11 | const value = await readFile(path, { encoding: 'utf8', signal }) | |
| 12 | return value.length <= 32768 ? { value, code: null } : { value: null, code: 'BYTE_LIMIT' } | |
| 13 | } | |
| 14 | catch (error) { return { value: null, code: browserFileErrorCode(error) } } | |
| 15 | } | |
| 16 | const decimal = (value: string | undefined) => value && /^\d{1,20}$/.test(value) ? value : null | |
| 17 | const integer = (value: string | undefined) => { const n = decimal(value); return n !== null && Number.isSafeInteger(Number(n)) ? Number(n) : null } | |
| 18 | async function processState(pid: number, signal: AbortSignal) { | |
| 19 | const [raw, status, wait, children] = await Promise.all([text(`/proc/${pid}/stat`, signal), text(`/proc/${pid}/status`, signal), text(`/proc/${pid}/wchan`, signal), text(`/proc/${pid}/task/${pid}/children`, signal)]) | |
| 20 | signal.throwIfAborted() | |
| 21 | const end = raw.value?.lastIndexOf(') ') ?? -1 | |
| 22 | // tcomm may contain spaces/parentheses and is deliberately discarded. | |
| 23 | const fields = end >= 0 ? raw.value!.slice(end + 2).trim().split(/\s+/) : [] | |
| 24 | const valid = fields.length >= 22 && /^[RSDZTtXxKWPI]$/.test(fields[0]!) && decimal(fields[19]) !== null | |
| 25 | const rss = status.value?.match(/^VmRSS:\s+(\d+)\s+kB$/m)?.[1] | |
| 26 | const ids = children.value?.trim().split(/\s+/).map(integer).filter((n): n is number => n !== null && n > 0) ?? [] | |
| 27 | return { | |
| 28 | record: { pid, code: valid ? null : raw.code ?? 'FORMAT', state: valid ? fields[0]! : null, | |
| 29 | parentPid: valid ? integer(fields[1]) : null, group: valid ? integer(fields[2]) : null, session: valid ? integer(fields[3]) : null, | |
| 30 | startTicks: valid ? decimal(fields[19]) : null, userTicks: valid ? decimal(fields[11]) : null, systemTicks: valid ? decimal(fields[12]) : null, | |
| 31 | threads: valid ? integer(fields[17]) : null, rssKiB: integer(rss), statusCode: status.code, | |
| 32 | wait: wait.value !== null && /^[A-Za-z0-9_.:]{1,80}$/.test(wait.value.trim()) ? wait.value.trim() : null, waitCode: wait.code, childrenCode: children.code }, | |
| 33 | children: ids.slice(0, processLimit), childrenCapped: ids.length > processLimit, | |
| 34 | } | |
| 35 | } | |
| 36 | async function counters(path: string, allowed: string[], signal: AbortSignal) { | |
| 37 | const result = await text(path, signal), values: Record<string, string> = {} | |
| 38 | signal.throwIfAborted() | |
| 39 | for (const line of result.value?.split('\n') ?? []) { | |
| 40 | const [key, value] = line.trim().split(/\s+/) | |
| 41 | if (key && allowed.includes(key) && decimal(value) !== null) values[key] = value! | |
| 42 | } | |
| 43 | return { values, code: result.code } | |
| 44 | } | |
| 45 | async function scalar(path: string, signal: AbortSignal) { | |
| 46 | const result = await text(path, signal), value = result.value?.trim() | |
| 47 | signal.throwIfAborted() | |
| 48 | return { value: value === 'max' || decimal(value) !== null ? value! : null, code: result.code ?? (value === 'max' || decimal(value) !== null ? null : 'FORMAT') } | |
| 49 | } | |
| 50 | async function pressure(resource: string, signal: AbortSignal) { | |
| 51 | const result = await text(`/proc/pressure/${resource}`, signal), values: Record<string, { avg10: number, avg60: number, avg300: number, totalMicroseconds: string }> = {} | |
| 52 | for (const line of result.value?.split('\n') ?? []) { | |
| 53 | const match = /^(some|full) avg10=(\d{1,3}(?:\.\d{1,6})?) avg60=(\d{1,3}(?:\.\d{1,6})?) avg300=(\d{1,3}(?:\.\d{1,6})?) total=(\d{1,20})$/.exec(line) | |
| 54 | if (match) values[match[1]!] = { avg10: Number(match[2]), avg60: Number(match[3]), avg300: Number(match[4]), totalMicroseconds: match[5]! } | |
| 55 | } | |
| 56 | return { values, code: result.code } | |
| 57 | } | |
| 58 | async function resources(signal: AbortSignal) { | |
| 59 | signal.throwIfAborted() | |
| 60 | const membership = await text('/proc/self/cgroup', signal) | |
| 61 | signal.throwIfAborted() | |
| 62 | const path = membership.value?.split('\n').find(line => line.startsWith('0::'))?.slice(3) | |
| 63 | const base = path?.startsWith('/') && !path.split('/').includes('..') ? join('/sys/fs/cgroup', path) : null | |
| 64 | const [cpuPressure, memoryPressure, ioPressure] = await Promise.all(['cpu', 'memory', 'io'].map(name => pressure(name, signal))) | |
| 65 | signal.throwIfAborted() | |
| 66 | const cgroup = base ? { | |
| 67 | memoryCurrent: await scalar(join(base, 'memory.current'), signal), memoryMax: await scalar(join(base, 'memory.max'), signal), | |
| 68 | memoryEvents: await counters(join(base, 'memory.events'), ['low', 'high', 'max', 'oom', 'oom_kill', 'oom_group_kill', 'sock_throttled'], signal), | |
| 69 | cpu: await counters(join(base, 'cpu.stat'), ['usage_usec', 'user_usec', 'system_usec', 'nr_periods', 'nr_throttled', 'throttled_usec', 'nr_bursts', 'burst_usec'], signal), | |
| 70 | pidsCurrent: await scalar(join(base, 'pids.current'), signal), pidsMax: await scalar(join(base, 'pids.max'), signal), | |
| 71 | } : null | |
| 72 | return { scope: 'observer cgroup, potentially shared with other runner processes', pressureScope: 'system pressure, shared with other host workloads', membershipCode: membership.code ?? (base ? null : 'NOT_V2'), cgroup, cpuPressure, memoryPressure, ioPressure } | |
| 73 | } | |
| 74 | ||
| 75 | type Executable = { path: string, device: string, inode: string, byteLength: number, observedPids: number[], openedAtMs: number, sha256: string | null, hashedAfterDiscovery: boolean, code: string | null, closeCode: string | null } | |
| 76 | /** No argv, environment, comm, stack or memory dump is exported. */ | |
| 77 | export function createLinuxStartupObservation(pid: number, profile: string, began: number) { | |
| 78 | const tracked = new Map<number, string>(), pinned = new Map<string, { handle: FileHandle, mtimeNs: bigint, ctimeNs: bigint, evidence: Executable }>() | |
| 79 | const identityErrors: Array<{ pid: number, code: string }> = [] | |
| 80 | let handlesSealed = false, executableLimitReached = false | |
| 81 | async function executable(id: number, signal: AbortSignal) { | |
| 82 | if (handlesSealed || signal.aborted) return | |
| 83 | let handle: FileHandle | undefined | |
| 84 | try { | |
| 85 | handle = await open(`/proc/${id}/exe`, 'r') | |
| 86 | if (handlesSealed || signal.aborted) return | |
| 87 | const stat = await handle.stat({ bigint: true }), key = `${stat.dev}:${stat.ino}` | |
| 88 | if (handlesSealed || signal.aborted) return | |
| 89 | const prior = pinned.get(key) | |
| 90 | if (prior) { if (!prior.evidence.observedPids.includes(id)) prior.evidence.observedPids.push(id); return } | |
| 91 | if (pinned.size >= executableLimit) { executableLimitReached = true; return } | |
| 92 | const path = await readlink(`/proc/self/fd/${handle.fd}`) | |
| 93 | if (handlesSealed || signal.aborted) return | |
| 94 | if (!stat.isFile() || stat.size <= 0n || stat.size > BigInt(executableByteLimit) || path.length > 4096) { if (identityErrors.length < 8) identityErrors.push({ pid: id, code: 'IDENTITY_LIMIT' }); return } | |
| 95 | pinned.set(key, { handle, mtimeNs: stat.mtimeNs, ctimeNs: stat.ctimeNs, evidence: { path, device: String(stat.dev), inode: String(stat.ino), byteLength: Number(stat.size), observedPids: [id], openedAtMs: performance.now() - began, sha256: null, hashedAfterDiscovery: false, code: null, closeCode: null } }) | |
| 96 | handle = undefined | |
| 97 | } | |
| 98 | catch (error) { if (identityErrors.length < 8) identityErrors.push({ pid: id, code: browserFileErrorCode(error) }) } | |
| 99 | finally { | |
| 100 | if (handle) { | |
| 101 | try { await handle.close() } | |
| 102 | catch (closeError) { if (identityErrors.length < 8) identityErrors.push({ pid: id, code: browserFileErrorCode(closeError) }) } | |
| 103 | } | |
| 104 | } | |
| 105 | } | |
| 106 | return { | |
| 107 | async snapshot(signal: AbortSignal) { | |
| 108 | signal.throwIfAborted() | |
| 109 | const root = await processState(pid, signal), descendants: Array<typeof root.record & { reused: boolean }> = [] | |
| 110 | const check = (id: number, start: string | null) => { | |
| 111 | if (!start) return false | |
| 112 | const prior = tracked.get(id) | |
| 113 | if (prior && prior !== start) return false | |
| 114 | if (tracked.size < processLimit) tracked.set(id, start) | |
| 115 | return true | |
| 116 | } | |
| 117 | const rootMatches = check(pid, root.record.startTicks) | |
| 118 | if (rootMatches) await executable(pid, signal) | |
| 119 | const queue = [...new Set([...(rootMatches ? root.children : []), ...tracked.keys()].filter(id => id !== pid))] | |
| 120 | const seen = new Set<number>([pid]); let processLimitReached = root.childrenCapped | |
| 121 | while (queue.length && seen.size < processLimit && !signal.aborted) { | |
| 122 | const id = queue.shift()! | |
| 123 | if (seen.has(id)) continue | |
| 124 | seen.add(id) | |
| 125 | const state = await processState(id, signal), matches = check(id, state.record.startTicks) | |
| 126 | descendants.push({ ...state.record, reused: state.record.startTicks !== null && !matches }) | |
| 127 | if (matches) { await executable(id, signal); queue.push(...state.children.filter(n => !seen.has(n))); processLimitReached ||= state.childrenCapped } | |
| 128 | } | |
| 129 | processLimitReached ||= queue.length > 0 || tracked.size === processLimit | |
| 130 | signal.throwIfAborted() | |
| 131 | let threadCode: string | null = null | |
| 132 | const allThreads = await readdir(`/proc/${pid}/task`).catch(error => { threadCode = browserFileErrorCode(error); return [] }) | |
| 133 | signal.throwIfAborted() | |
| 134 | const threads = await Promise.all(allThreads.filter(id => /^\d+$/.test(id)).slice(0, threadLimit).map(async id => { | |
| 135 | const wait = await text(`/proc/${pid}/task/${id}/wchan`, signal) | |
| 136 | return { tid: Number(id), wait: wait.value !== null && /^[A-Za-z0-9_.:]{1,80}$/.test(wait.value.trim()) ? wait.value.trim() : null, code: wait.code } | |
| 137 | })) | |
| 138 | signal.throwIfAborted() | |
| 139 | const discoveryFile = await lstat(join(profile, 'DevToolsActivePort')).then(value => ({ present: true, regular: value.isFile(), bytes: value.size, code: null })).catch(error => ({ present: false, regular: null, bytes: null, code: browserFileErrorCode(error) })) | |
| 140 | signal.throwIfAborted() | |
| 141 | return { status: 'observed' as const, root: { ...root.record, reused: root.record.startTicks !== null && !rootMatches }, descendants, processLimitReached, | |
| 142 | childScope: 'main-task children and previously observed descendants; incomplete during fork/exit/reparenting', threads, threadCode, threadDetailsCapped: allThreads.length > threadLimit, discoveryFile, resources: await resources(signal) } | |
| 143 | }, | |
| 144 | async settleExecutables(shouldHash: boolean) { | |
| 145 | handlesSealed = true | |
| 146 | const abort = new AbortController(), timer = setTimeout(() => abort.abort(), 2000) | |
| 147 | try { | |
| 148 | for (const { handle, evidence, mtimeNs, ctimeNs } of pinned.values()) { | |
| 149 | try { | |
| 150 | if (!shouldHash) { evidence.code = 'OBSERVATION_CANCELED'; continue } | |
| 151 | if (abort.signal.aborted) { evidence.code = 'HASH_BUDGET'; continue } | |
| 152 | const hash = createHash('sha256'); let bytes = 0 | |
| 153 | for await (const chunk of handle.createReadStream({ autoClose: false, start: 0, end: evidence.byteLength - 1, signal: abort.signal })) { hash.update(chunk); bytes += chunk.length } | |
| 154 | const after = await handle.stat({ bigint: true }) | |
| 155 | if (abort.signal.aborted) { evidence.code = 'HASH_BUDGET'; continue } | |
| 156 | if (bytes !== evidence.byteLength || after.size !== BigInt(bytes) || after.mtimeNs !== mtimeNs || after.ctimeNs !== ctimeNs) { evidence.code = 'EXECUTABLE_CHANGED'; continue } | |
| 157 | evidence.sha256 = hash.digest('hex'); evidence.hashedAfterDiscovery = true | |
| 158 | } | |
| 159 | catch (error) { evidence.code = browserFileErrorCode(error) } | |
| 160 | finally { await handle.close().catch(error => { evidence.closeCode = browserFileErrorCode(error) }) } | |
| 161 | } | |
| 162 | } | |
| 163 | finally { clearTimeout(timer) } | |
| 164 | }, | |
| 165 | evidence() { return { executables: [...pinned.values()].map(value => structuredClone(value.evidence)), identityErrors: structuredClone(identityErrors), executableLimitReached } }, | |
| 166 | } | |
| 167 | } | |