ReviewOS

also looking at this

stacks/drone

test: observe native execution callback verification cost

#86
Merged chris wants to merge codex/observe-execution-verification into main
8 files +363 -18
tests/helpers/execution-browser-driver.tsmodified+64-4
Changes to tests/helpers/execution-browser-driver.ts
@@ -121,11 +121,25 @@ try {
121121 await first.fill('#execution-scenario', 'nominal'); await first.fill('#execution-seconds', '100')
122122 await first.click('Start synthetic run')
123123 mark('real nominal100 completion')
124 await first.wait("document.querySelector('[data-execution-phase]')?.textContent.trim() === 'completed' && !!document.querySelector('[data-execution-completion]')", 'Actual Worker recording must complete', 40000)
124 // Consume the completed fields from the same authorized DOM frame that satisfies readiness.
125 // A later evaluation could run after native renewal has correctly hidden private content.
126 const completedView = `(() => {
127 const status = document.querySelector('[data-workspace-status]')?.textContent.trim();
128 const phase = document.querySelector('[data-execution-phase]')?.textContent.trim();
129 const digest = document.querySelector('[data-execution-digest]')?.textContent.trim();
130 const execution = document.querySelector('[data-execution-id]')?.textContent.trim();
131 const originalSelected = document.querySelector('[data-workspace-source]')?.textContent.trim() === ${JSON.stringify(data.sourceDigest)};
132 return status === 'current' && phase === 'completed' && !!document.querySelector('[data-execution-completion]') && /^[a-f0-9]{64}$/.test(digest || '') && !!execution && originalSelected ? { digest, execution } : false;
133 })()`
134 const completed = await first.wait<{ digest: string, execution: string }>(completedView, 'Actual Worker recording must complete', 40000)
125135 await control('before-admission')
126 const digest = await first.evaluate<string>("document.querySelector('[data-execution-digest]')?.textContent.trim()")
127 assert(/^[a-f0-9]{64}$/.test(digest))
128 assert.equal(await first.evaluate("document.querySelector('[data-workspace-source]')?.textContent.trim()"), data.sourceDigest, 'Completion cannot change selected imagery')
136 await control('hold-completion-read-renewal')
137 await first.wait("!document.querySelector('[data-execution-completion]')", 'Native completion-read renewal clears private content')
138 await control('release-workspace')
139 const restored = await first.wait<typeof completed>(completedView, 'Fresh authorization must restore the same completed result')
140 assert.deepEqual(restored, completed)
141 const { digest } = completed
142 await writeFile(join(evidence, 'completion-read-renewal.json'), `${JSON.stringify({ privateContentCleared: true, sameCompletedResultRestored: true, readinessFieldsReadAtomically: true })}\n`, { mode: 0o600, flag: 'wx' })
129143 await first.evaluate("document.querySelector('[data-execution-panel]').scrollIntoView()")
130144 await first.screenshot(join(evidence, 'execution-completed-desktop.png'))
131145 await first.send('Emulation.setDeviceMetricsOverride', { width: 375, height: 900, deviceScaleFactor: 1, mobile: true })
@@ -146,8 +160,32 @@ try {
146160 observer.observe(document.body, { childList: true, characterData: true, subtree: true });
147161 sample(); globalThis.executionSourceProbe = { probe, stop: () => observer.disconnect() };
148162 })()`)
163 await control('arm-pre-admission-renewal')
164 // Force the ordinary refresh ordering that can also occur from the native poll.
165 // The held read must leave the admission action available until this explicit click.
166 await first.click('Refresh current records')
167 let preAdmissionHeld = false
168 for (let attempt = 0; attempt < 100 && !preAdmissionHeld; attempt++) { preAdmissionHeld = (await control('status')).preAdmissionHeldReads > 0; if (!preAdmissionHeld) await Bun.sleep(10) }
169 assert(preAdmissionHeld, 'An ordinary native refresh must reach the read barrier before admission')
170 assert.equal((await control('status')).preAdmissionRenewed, false, 'Holding an ordinary refresh never conceals authorization')
171 await first.click('Admit completed source')
172 await first.wait("document.querySelector('[data-workspace-status]')?.textContent.trim() === 'current' && document.querySelector('[data-execution-phase]')?.textContent.trim() === 'completed' && document.querySelector('[data-execution-admit]')?.disabled === true", 'The explicit admission decision must be pending while native reads are held')
173 await control('renew-pre-admission-held-read')
174 await first.wait("!document.querySelector('[data-workspace-source]') && !document.querySelector('[data-execution-completion]')", 'Native renewal clears private content during the admission read')
175 await control('release-workspace')
176 assert.deepEqual(await first.wait<typeof completed>(completedView, 'Fresh authorization restores completed execution after the canceled admission'), completed)
177 const canceledDecision = await control('status')
178 assert.equal(canceledDecision.preAdmissionRenewed, true)
179 assert.equal(canceledDecision.preAdmissionHoldExpired, false, 'The controlled native renewal must complete within the original read bound')
180 assert.equal(canceledDecision.admissionRequests, 0, 'Lost availability cancels the earlier decision before any admission POST')
181 assert.equal(await first.evaluate("!!document.querySelector('[data-execution-source-exists]')"), false)
182 await control('before-admission')
183 mark('explicit source admission after fresh authorization')
149184 await first.click('Admit completed source')
150185 await first.wait("!!document.querySelector('[data-execution-source-exists]')", 'Explicit admission must refresh actual source catalog', 15000)
186 const acceptedDecision = await control('status')
187 assert.equal(acceptedDecision.admissionRequests, 1, 'Only the new explicit decision submits one admission POST')
188 await writeFile(join(evidence, 'pre-admission-renewal.json'), `${JSON.stringify({ ordinaryRefreshHeldBeforeClick: preAdmissionHeld, pendingDecisionObservedBeforeRenewal: true, nativeRenewalObserved: canceledDecision.preAdmissionRenewed, postsAfterCanceledDecision: canceledDecision.admissionRequests, postsAfterFreshExplicitDecision: acceptedDecision.admissionRequests, originalSelectionRestored: true })}\n`, { mode: 0o600, flag: 'wx' })
151189 await control('verify-admitted')
152190 mark('fresh workspace after admission renewal')
153191 assert.equal(await first.evaluate("!!document.querySelector('[data-workspace-source]') || !!document.querySelector('[data-execution-completion]')"), false, 'Native renewal must clear private panes before fresh authorization')
@@ -176,6 +214,28 @@ try {
176214 await control('revoke')
177215 await first.wait("document.querySelector('[data-workspace-status]')?.textContent.trim() === 'refused'", 'Revocation must clear private execution content', 16000)
178216 assert.equal(await first.evaluate("!!document.querySelector('[data-execution-id]') || document.body.textContent.includes(" + JSON.stringify(digest) + ")"), false)
217 const callbackTimings = await control('worker-callback-timings')
218 assert.equal(callbackTimings.overflow, false)
219 assert.equal(callbackTimings.unavailable, false)
220 assert.equal(callbackTimings.observations.length, 2, 'Both genuine core Workers must have bounded callback observations')
221 const nominalCallbacks = callbackTimings.observations[1]
222 assert.equal(nominalCallbacks.ordinal, 2)
223 assert.equal(nominalCallbacks.inCallback, false)
224 assert.equal(nominalCallbacks.capped, false)
225 assert(nominalCallbacks.costs.event.calls > 0)
226 assert.equal(nominalCallbacks.costs.result.calls, 1)
227 assert.equal(nominalCallbacks.costs.finished.calls, 1)
228 for (const observation of callbackTimings.observations) {
229 assert.equal(observation.inCallback, false)
230 assert.equal(observation.capped, false)
231 for (const kind of ['event', 'result', 'finished', 'other']) {
232 const bucket = observation.costs[kind]
233 assert(Number.isSafeInteger(bucket.calls) && bucket.calls >= 0 && bucket.calls <= 1000000)
234 assert(Number.isFinite(bucket.totalMs) && bucket.totalMs >= 0 && bucket.totalMs <= Number.MAX_SAFE_INTEGER)
235 assert(Number.isFinite(bucket.maxMs) && bucket.maxMs >= 0 && bucket.maxMs <= bucket.totalMs)
236 }
237 }
238 await writeFile(join(evidence, 'worker-callback-timings.json'), `${JSON.stringify(callbackTimings)}\n`, { mode: 0o600, flag: 'wx' })
179239 console.log('PASS actual execution browser: native live-before-return/GET renewal/stop/completion/explicit admission/read-only/revocation')
180240}
181241catch (error) {