also looking at this
test(mobile): run the generated apps on a simulator and an emulator
#199
26 files
+3009
-293
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -0,0 +1,254 @@ | ||
| 1 | <!DOCTYPE html> | |
| 2 | <html lang="en"> | |
| 3 | <head> | |
| 4 | <meta charset="UTF-8"> | |
| 5 | <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| 6 | <title>Craft mobile E2E</title> | |
| 7 | <style> | |
| 8 | body { font: 14px -apple-system, Roboto, sans-serif; background: #0b1712; color: #e6f2ec; padding: 16px; } | |
| 9 | li { margin: 4px 0; font-family: ui-monospace, monospace; } | |
| 10 | .pass { color: #7ee08a; } | |
| 11 | .fail { color: #ff8a80; } | |
| 12 | </style> | |
| 13 | </head> | |
| 14 | <body> | |
| 15 | <h1 id="headline">Craft mobile E2E</h1> | |
| 16 | <ul id="results"></ul> | |
| 17 | <script> | |
| 18 | // The page drives itself and reports through the bridge's own `log` action, | |
| 19 | // which is the one channel both runtimes already forward off-device: iOS | |
| 20 | // prints "[Craft Web] <message>" to the process stderr that | |
| 21 | // `simctl launch --console-pty` captures, and Android writes it to logcat | |
| 22 | // under the CraftBridge tag. Nothing else in the generated app reaches a | |
| 23 | // host-side reader, so the harness would otherwise be watching a screenshot. | |
| 24 | // | |
| 25 | // Every line is CRAFT-E2E followed by one JSON object. The harness reads | |
| 26 | // three kinds: `plan` (once, up front), `case` (once per case), and `done`. | |
| 27 | // Announcing the plan before running anything is what lets the harness tell | |
| 28 | // "this case failed" apart from "this case never ran" — a suite that dies | |
| 29 | // halfway through is a failure, not a pass with fewer cases. | |
| 30 | (function () { | |
| 31 | var NONCE = '__CRAFT_E2E_NONCE__' | |
| 32 | // Which platform the harness built this leg for. Substituted in from | |
| 33 | // outside, because the page comparing window.craft.platform against a | |
| 34 | // value it read out of window.craft.platform is a tautology - it was one, | |
| 35 | // and bridge.ready passed unconditionally until this arrived. | |
| 36 | var EXPECTED_PLATFORM = '__CRAFT_E2E_PLATFORM__' | |
| 37 | var CASE_TIMEOUT_MS = 8000 | |
| 38 | ||
| 39 | function emit(payload) { | |
| 40 | var line = 'CRAFT-E2E ' + JSON.stringify(payload) | |
| 41 | // console.log first: on Android it reaches logcat under `chromium` even | |
| 42 | // when the bridge never arrived, which is the case the harness most | |
| 43 | // needs evidence for. | |
| 44 | try { console.log(line) } catch (ignored) {} | |
| 45 | try { if (window.craft && window.craft.log) window.craft.log(line) } catch (ignored) {} | |
| 46 | } | |
| 47 | ||
| 48 | function show(name, ok, detail) { | |
| 49 | var li = document.createElement('li') | |
| 50 | li.className = ok ? 'pass' : 'fail' | |
| 51 | li.textContent = (ok ? 'PASS ' : 'FAIL ') + name + (detail ? ' — ' + detail : '') | |
| 52 | document.getElementById('results').appendChild(li) | |
| 53 | } | |
| 54 | ||
| 55 | // Rejections arrive in three shapes across the two runtimes: an Error from | |
| 56 | // iOS's _rejectCallback, a bare string from Android's named reject globals | |
| 57 | // (promise-runtime.ts parks `window._craftPushReject` and Kotlin calls it | |
| 58 | // with a quoted string), and an object with `code`/`message` from iOS's | |
| 59 | // capability gate. Normalise once here so a case can assert on a code | |
| 60 | // without caring which runtime produced it. | |
| 61 | function describeError(error) { | |
| 62 | if (error === null || error === undefined) return { code: '', message: '' } | |
| 63 | if (typeof error === 'string') return { code: '', message: error } | |
| 64 | return { code: String(error.code || ''), message: String(error.message || error) } | |
| 65 | } | |
| 66 | ||
| 67 | function withTimeout(run) { | |
| 68 | return new Promise(function (resolve) { | |
| 69 | var settled = false | |
| 70 | var timer = setTimeout(function () { | |
| 71 | if (settled) return | |
| 72 | settled = true | |
| 73 | resolve({ state: 'timeout' }) | |
| 74 | }, CASE_TIMEOUT_MS) | |
| 75 | ||
| 76 | function finish(value) { | |
| 77 | if (settled) return | |
| 78 | settled = true | |
| 79 | clearTimeout(timer) | |
| 80 | resolve(value) | |
| 81 | } | |
| 82 | ||
| 83 | try { | |
| 84 | Promise.resolve(run()).then( | |
| 85 | function (value) { finish({ state: 'resolved', value: value }) }, | |
| 86 | function (error) { finish({ state: 'rejected', error: describeError(error) }) } | |
| 87 | ) | |
| 88 | } | |
| 89 | catch (thrown) { | |
| 90 | finish({ state: 'rejected', error: describeError(thrown) }) | |
| 91 | } | |
| 92 | }) | |
| 93 | } | |
| 94 | ||
| 95 | // A case passes only by returning nothing. Returning a string is how it | |
| 96 | // explains what it saw instead, and that string lands in the report — so a | |
| 97 | // failure names the actual value rather than just the assertion that broke. | |
| 98 | var CASES = [ | |
| 99 | { | |
| 100 | name: 'bridge.ready', | |
| 101 | platforms: ['ios', 'android'], | |
| 102 | run: function () { | |
| 103 | if (!window.craft) return 'window.craft is absent' | |
| 104 | if (typeof window.craft.log !== 'function') return 'window.craft has no log function' | |
| 105 | if (window.craft.platform !== EXPECTED_PLATFORM) | |
| 106 | return 'platform is ' + window.craft.platform + ', expected ' + EXPECTED_PLATFORM | |
| 107 | } | |
| 108 | }, | |
| 109 | { | |
| 110 | // Ground truth the page cannot invent: `isSimulator` comes from Swift's | |
| 111 | // `#if targetEnvironment(simulator)`, decided by the compiler, and | |
| 112 | // `isEmulator` from the device's own Build properties. A stubbed bridge | |
| 113 | // that echoed plausible JSON would have to guess it right. | |
| 114 | name: 'deviceInfo.isSimulator', | |
| 115 | platforms: ['ios'], | |
| 116 | run: function () { | |
| 117 | return withTimeout(function () { return window.craft.getDeviceInfo() }).then(function (outcome) { | |
| 118 | if (outcome.state !== 'resolved') return 'getDeviceInfo ' + outcome.state | |
| 119 | if (!outcome.value) return 'getDeviceInfo resolved with nothing' | |
| 120 | if (outcome.value.platform !== 'ios') return 'platform is ' + outcome.value.platform | |
| 121 | if (outcome.value.isSimulator !== true) return 'isSimulator is ' + outcome.value.isSimulator | |
| 122 | }) | |
| 123 | } | |
| 124 | }, | |
| 125 | { | |
| 126 | name: 'deviceInfo.isEmulator', | |
| 127 | platforms: ['android'], | |
| 128 | run: function () { | |
| 129 | return withTimeout(function () { return window.craft.getDeviceInfo() }).then(function (outcome) { | |
| 130 | if (outcome.state !== 'resolved') return 'getDeviceInfo ' + outcome.state | |
| 131 | if (!outcome.value) return 'getDeviceInfo resolved with nothing' | |
| 132 | ||
| 133 | // Reported before anything is asserted, so a failing assertion | |
| 134 | // still leaves the values behind. The first run of this case said | |
| 135 | // only "isEmulator is false" and the log did not say what the | |
| 136 | // device had called itself, which is the one thing needed to fix | |
| 137 | // it. | |
| 138 | emit({ event: 'observed', name: 'sdkVersion', value: outcome.value.sdkVersion }) | |
| 139 | emit({ event: 'observed', name: 'model', value: outcome.value.model }) | |
| 140 | emit({ event: 'observed', name: 'manufacturer', value: outcome.value.manufacturer }) | |
| 141 | emit({ event: 'observed', name: 'device', value: outcome.value.device }) | |
| 142 | ||
| 143 | if (outcome.value.platform !== 'android') return 'platform is ' + outcome.value.platform | |
| 144 | if (!outcome.value.sdkVersion) return 'sdkVersion is ' + outcome.value.sdkVersion | |
| 145 | if (outcome.value.isEmulator !== true) | |
| 146 | return 'isEmulator is ' + outcome.value.isEmulator + ' on model ' + outcome.value.model | |
| 147 | }) | |
| 148 | } | |
| 149 | }, | |
| 150 | { | |
| 151 | // The success path. On iOS the harness then reads the device pasteboard | |
| 152 | // from the host with `simctl pbpaste`, so the case asserts the value | |
| 153 | // reached real system clipboard state rather than a variable the bridge | |
| 154 | // handed back. Android has no equally cheap host-side read, so its | |
| 155 | // external check is the sdkVersion cross-reference below instead. | |
| 156 | name: 'clipboard.roundTrip', | |
| 157 | platforms: ['ios', 'android'], | |
| 158 | run: function () { | |
| 159 | return withTimeout(function () { return window.craft.clipboard.write(NONCE) }).then(function (written) { | |
| 160 | if (written.state !== 'resolved') return 'clipboard.write ' + written.state + ' ' + (written.error ? written.error.message : '') | |
| 161 | return withTimeout(function () { return window.craft.clipboard.read() }).then(function (read) { | |
| 162 | if (read.state !== 'resolved') return 'clipboard.read ' + read.state + ' ' + (read.error ? read.error.message : '') | |
| 163 | if (read.value !== NONCE) return 'read back ' + JSON.stringify(read.value) | |
| 164 | }) | |
| 165 | }) | |
| 166 | } | |
| 167 | }, | |
| 168 | { | |
| 169 | // The failure path. Geolocation is off in the harness config, so the | |
| 170 | // native side must answer — with a code that names the configuration — | |
| 171 | // rather than leave the promise pending. A dropped callback here is | |
| 172 | // indistinguishable from a slow one to the page, which is exactly why | |
| 173 | // the case asserts the rejection and not merely "did not resolve". | |
| 174 | name: 'geolocation.disabled.rejects', | |
| 175 | platforms: ['ios'], | |
| 176 | run: function () { | |
| 177 | return withTimeout(function () { return window.craft.getCurrentPosition() }).then(function (outcome) { | |
| 178 | if (outcome.state === 'timeout') return 'getCurrentPosition never settled' | |
| 179 | if (outcome.state === 'resolved') return 'getCurrentPosition resolved on a disabled capability' | |
| 180 | if (outcome.error.code !== 'CAPABILITY_DISABLED') return 'code is ' + JSON.stringify(outcome.error.code) | |
| 181 | }) | |
| 182 | } | |
| 183 | }, | |
| 184 | { | |
| 185 | // Android's equivalent gate. With push disabled the generator emits a | |
| 186 | // literal reject call rather than a Firebase block, so the rejection | |
| 187 | // string is fixed at generation time and is safe to match on. | |
| 188 | name: 'push.disabled.rejects', | |
| 189 | platforms: ['android'], | |
| 190 | run: function () { | |
| 191 | return withTimeout(function () { return window.craft.registerPush() }).then(function (outcome) { | |
| 192 | if (outcome.state === 'timeout') return 'registerPush never settled' | |
| 193 | if (outcome.state === 'resolved') return 'registerPush resolved while push is disabled' | |
| 194 | if (outcome.error.message.indexOf('Push notifications are disabled') === -1) | |
| 195 | return 'message is ' + JSON.stringify(outcome.error.message) | |
| 196 | }) | |
| 197 | } | |
| 198 | } | |
| 199 | ] | |
| 200 | ||
| 201 | function waitForBridge(attempt) { | |
| 202 | if (window.craft && window.craft.getDeviceInfo) return Promise.resolve(true) | |
| 203 | if (attempt >= 150) return Promise.resolve(false) | |
| 204 | return new Promise(function (resolve) { | |
| 205 | setTimeout(function () { resolve(waitForBridge(attempt + 1)) }, 100) | |
| 206 | }) | |
| 207 | } | |
| 208 | ||
| 209 | waitForBridge(0).then(function (ready) { | |
| 210 | if (!ready) { | |
| 211 | emit({ event: 'fatal', reason: 'window.craft never appeared' }) | |
| 212 | document.getElementById('headline').textContent = 'FATAL: no bridge' | |
| 213 | return | |
| 214 | } | |
| 215 | ||
| 216 | var platform = window.craft.platform | |
| 217 | var selected = CASES.filter(function (entry) { return entry.platforms.indexOf(platform) !== -1 }) | |
| 218 | emit({ event: 'plan', platform: platform, run: NONCE, cases: selected.map(function (entry) { return entry.name }) }) | |
| 219 | ||
| 220 | var passed = 0 | |
| 221 | var failed = 0 | |
| 222 | ||
| 223 | // Sequential, not Promise.all. Android's promise runtime is single-flight | |
| 224 | // per channel and rejects a second in-flight request outright; running | |
| 225 | // the suite in parallel would fail cases for colliding with each other. | |
| 226 | function step(index) { | |
| 227 | if (index >= selected.length) { | |
| 228 | emit({ event: 'done', passed: passed, failed: failed }) | |
| 229 | document.getElementById('headline').textContent = failed ? 'FAILED' : 'PASSED' | |
| 230 | return | |
| 231 | } | |
| 232 | ||
| 233 | var entry = selected[index] | |
| 234 | Promise.resolve() | |
| 235 | .then(function () { return entry.run() }) | |
| 236 | .then( | |
| 237 | function (detail) { return detail ? String(detail) : '' }, | |
| 238 | function (thrown) { return 'threw: ' + describeError(thrown).message } | |
| 239 | ) | |
| 240 | .then(function (detail) { | |
| 241 | if (detail) failed++ | |
| 242 | else passed++ | |
| 243 | emit({ event: 'case', name: entry.name, status: detail ? 'fail' : 'pass', detail: detail }) | |
| 244 | show(entry.name, !detail, detail) | |
| 245 | step(index + 1) | |
| 246 | }) | |
| 247 | } | |
| 248 | ||
| 249 | step(0) | |
| 250 | }) | |
| 251 | })() | |
| 252 | </script> | |
| 253 | </body> | |
| 254 | </html> | |
| @@ -0,0 +1,278 @@ | ||
| 1 | import { describe, expect, it } from 'bun:test' | |
| 2 | import { evaluateRun, hasTerminated, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, ZIG_REFUSED_ACTION, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigRefusals } from './protocol' | |
| 3 | ||
| 4 | const ESC = String.fromCharCode(27) | |
| 5 | ||
| 6 | /** A transcript shaped like the one a real iOS run produces. */ | |
| 7 | const RUN = 'craft-e2e-ios-shim-1234' | |
| 8 | ||
| 9 | function iosTranscript(overrides: { cases?: string[], omit?: string[], fail?: Record<string, string>, done?: boolean, run?: string } = {}): string { | |
| 10 | const planned = overrides.cases ?? REQUIRED_CASES.ios | |
| 11 | const omit = overrides.omit ?? [] | |
| 12 | const fail = overrides.fail ?? {} | |
| 13 | const reported = planned.filter(name => !omit.includes(name)) | |
| 14 | ||
| 15 | const lines = [ | |
| 16 | 'Some unrelated system chatter from the device', | |
| 17 | `[Craft Web] CRAFT-E2E ${JSON.stringify({ event: 'plan', platform: 'ios', run: overrides.run ?? RUN, cases: planned })}`, | |
| 18 | ...reported.map(name => `[Craft Web] CRAFT-E2E ${JSON.stringify({ | |
| 19 | event: 'case', | |
| 20 | name, | |
| 21 | status: fail[name] ? 'fail' : 'pass', | |
| 22 | detail: fail[name] ?? '', | |
| 23 | })}`), | |
| 24 | ] | |
| 25 | ||
| 26 | if (overrides.done !== false) { | |
| 27 | const failed = reported.filter(name => fail[name]).length | |
| 28 | lines.push(`[Craft Web] CRAFT-E2E ${JSON.stringify({ event: 'done', passed: reported.length - failed, failed })}`) | |
| 29 | } | |
| 30 | ||
| 31 | return `${lines.join('\n')}\n` | |
| 32 | } | |
| 33 | ||
| 34 | describe('required cases', () => { | |
| 35 | it('keeps a success path and a rejection path on every platform', () => { | |
| 36 | expect(requiredCaseProblems()).toEqual([]) | |
| 37 | }) | |
| 38 | ||
| 39 | it('names the same bridge.ready case on both platforms', () => { | |
| 40 | expect(REQUIRED_CASES.ios).toContain('bridge.ready') | |
| 41 | expect(REQUIRED_CASES.android).toContain('bridge.ready') | |
| 42 | }) | |
| 43 | }) | |
| 44 | ||
| 45 | describe('parseDriverOutput', () => { | |
| 46 | it('finds events inside noisy, colour-coded device logs', () => { | |
| 47 | const text = [ | |
| 48 | 'unrelated', | |
| 49 | `${ESC}[33m[Craft Web] CRAFT-E2E {"event":"plan","platform":"ios","cases":["a"]}${ESC}[0m`, | |
| 50 | 'more unrelated', | |
| 51 | ].join('\n') | |
| 52 | ||
| 53 | const { events, malformed } = parseDriverOutput(text) | |
| 54 | expect(malformed).toEqual([]) | |
| 55 | expect(events).toEqual([{ event: 'plan', platform: 'ios', cases: ['a'] }]) | |
| 56 | }) | |
| 57 | ||
| 58 | it('reports a garbled event rather than skipping it', () => { | |
| 59 | const { events, malformed } = parseDriverOutput('CRAFT-E2E {"event":"plan"\n') | |
| 60 | expect(events).toEqual([]) | |
| 61 | expect(malformed).toEqual(['{"event":"plan"']) | |
| 62 | }) | |
| 63 | ||
| 64 | it('tolerates carriage returns from the console pty', () => { | |
| 65 | const { events } = parseDriverOutput('CRAFT-E2E {"event":"done","passed":1,"failed":0}\r\n') | |
| 66 | expect(events).toEqual([{ event: 'done', passed: 1, failed: 0 }]) | |
| 67 | }) | |
| 68 | ||
| 69 | // The exact shape the first Android run produced. Chromium wraps a page's | |
| 70 | // console.log before it reaches logcat, and taking the rest of the line | |
| 71 | // turned every event into an unparseable one - six of them, which failed a | |
| 72 | // run whose bridge had in fact worked. | |
| 73 | it('reads an event out of a Chromium console wrapper', () => { | |
| 74 | const line = '09-15 13:49:12.345 1234 1234 I chromium: [INFO:CONSOLE(39)] "CRAFT-E2E ' | |
| 75 | + '{"event":"case","name":"clipboard.roundTrip","status":"pass","detail":""}", ' | |
| 76 | + 'source: https://appassets.androidplatform.net/ (39)' | |
| 77 | ||
| 78 | const { events, malformed } = parseDriverOutput(line) | |
| 79 | expect(malformed).toEqual([]) | |
| 80 | expect(events).toEqual([{ event: 'case', name: 'clipboard.roundTrip', status: 'pass', detail: '' }]) | |
| 81 | }) | |
| 82 | ||
| 83 | it('keeps one copy when the page reports through both channels', () => { | |
| 84 | const payload = '{"event":"case","name":"bridge.ready","status":"pass","detail":""}' | |
| 85 | const { events } = parseDriverOutput([ | |
| 86 | `I chromium: [INFO:CONSOLE(39)] "CRAFT-E2E ${payload}", source: https://appassets.androidplatform.net/ (39)`, | |
| 87 | `D CraftBridge: CRAFT-E2E ${payload}`, | |
| 88 | ].join('\n')) | |
| 89 | ||
| 90 | expect(events).toEqual([{ event: 'case', name: 'bridge.ready', status: 'pass', detail: '' }]) | |
| 91 | }) | |
| 92 | ||
| 93 | it('does not stop at a brace inside a string', () => { | |
| 94 | const { events, malformed } = parseDriverOutput( | |
| 95 | 'CRAFT-E2E {"event":"case","name":"a","status":"fail","detail":"read back \\"{oops}\\""}', | |
| 96 | ) | |
| 97 | expect(malformed).toEqual([]) | |
| 98 | expect(events).toEqual([{ event: 'case', name: 'a', status: 'fail', detail: 'read back "{oops}"' }]) | |
| 99 | }) | |
| 100 | ||
| 101 | it('still reports a genuinely truncated event', () => { | |
| 102 | const { events, malformed } = parseDriverOutput('CRAFT-E2E {"event":"plan","cases":["a"') | |
| 103 | expect(events).toEqual([]) | |
| 104 | expect(malformed).toEqual(['{"event":"plan","cases":["a"']) | |
| 105 | }) | |
| 106 | }) | |
| 107 | ||
| 108 | describe('evaluateRun', () => { | |
| 109 | it('passes a complete run', () => { | |
| 110 | const verdict = evaluateRun('ios', iosTranscript()) | |
| 111 | expect(verdict.failures).toEqual([]) | |
| 112 | expect(verdict.ok).toBe(true) | |
| 113 | expect(verdict.passed).toEqual(REQUIRED_CASES.ios) | |
| 114 | }) | |
| 115 | ||
| 116 | // The failure the workflow this replaces actually shipped: nothing ran, and | |
| 117 | // nothing said so. | |
| 118 | it('fails an empty log instead of finding nothing to complain about', () => { | |
| 119 | const verdict = evaluateRun('ios', '') | |
| 120 | expect(verdict.ok).toBe(false) | |
| 121 | expect(verdict.failures).toEqual(['the test page never announced a plan; it did not reach the native bridge']) | |
| 122 | }) | |
| 123 | ||
| 124 | it('fails a log with device chatter but no driver output', () => { | |
| 125 | const verdict = evaluateRun('ios', 'Booting\nAssertion failed somewhere unrelated\n') | |
| 126 | expect(verdict.ok).toBe(false) | |
| 127 | }) | |
| 128 | ||
| 129 | it('fails when a planned case never reports', () => { | |
| 130 | const verdict = evaluateRun('ios', iosTranscript({ omit: ['clipboard.roundTrip'], done: false })) | |
| 131 | expect(verdict.ok).toBe(false) | |
| 132 | expect(verdict.failures).toContain('case clipboard.roundTrip was planned but never reported; the suite stopped early') | |
| 133 | expect(verdict.failures).toContain('the suite never reported done') | |
| 134 | }) | |
| 135 | ||
| 136 | it('fails when the suite is hollowed out to the easy cases', () => { | |
| 137 | const verdict = evaluateRun('ios', iosTranscript({ cases: ['bridge.ready'] })) | |
| 138 | expect(verdict.ok).toBe(false) | |
| 139 | expect(verdict.failures).toContain('required case geolocation.disabled.rejects is not in the suite the page ran') | |
| 140 | expect(verdict.failures).toContain('required case clipboard.roundTrip is not in the suite the page ran') | |
| 141 | }) | |
| 142 | ||
| 143 | it('carries the page detail into the failure so the reason is the value seen', () => { | |
| 144 | const verdict = evaluateRun('ios', iosTranscript({ fail: { 'clipboard.roundTrip': 'read back ""' } })) | |
| 145 | expect(verdict.ok).toBe(false) | |
| 146 | expect(verdict.failures).toContain('case clipboard.roundTrip failed: read back ""') | |
| 147 | expect(verdict.failed).toEqual(['clipboard.roundTrip']) | |
| 148 | }) | |
| 149 | ||
| 150 | it('fails when the tally disagrees with the transcript', () => { | |
| 151 | const text = [ | |
| 152 | 'CRAFT-E2E {"event":"plan","platform":"ios","cases":["bridge.ready","deviceInfo.isSimulator","clipboard.roundTrip","geolocation.disabled.rejects"]}', | |
| 153 | ...REQUIRED_CASES.ios.map(name => `CRAFT-E2E {"event":"case","name":"${name}","status":"pass","detail":""}`), | |
| 154 | 'CRAFT-E2E {"event":"done","passed":99,"failed":0}', | |
| 155 | ].join('\n') | |
| 156 | ||
| 157 | const verdict = evaluateRun('ios', text) | |
| 158 | expect(verdict.ok).toBe(false) | |
| 159 | expect(verdict.failures).toContain('done says 99 passed / 0 failed, transcript shows 4 / 0') | |
| 160 | }) | |
| 161 | ||
| 162 | it('fails when the app is not the platform the leg was built for', () => { | |
| 163 | const text = iosTranscript().replace('"platform":"ios"', '"platform":"android"') | |
| 164 | const verdict = evaluateRun('ios', text) | |
| 165 | expect(verdict.ok).toBe(false) | |
| 166 | expect(verdict.failures).toContain('the app reports platform android, expected ios') | |
| 167 | }) | |
| 168 | ||
| 169 | // Android reads its transcript out of a ring buffer that survives an | |
| 170 | // uninstall, so a leg whose app printed nothing could otherwise be evaluated | |
| 171 | // against the previous run's output. | |
| 172 | it('refuses a transcript left over from an earlier run', () => { | |
| 173 | const verdict = evaluateRun('ios', iosTranscript({ run: 'craft-e2e-ios-shim-OLD' }), RUN) | |
| 174 | expect(verdict.ok).toBe(false) | |
| 175 | expect(verdict.failures).toContain( | |
| 176 | `the transcript is from run "craft-e2e-ios-shim-OLD", not "${RUN}"; this is stale output, not this run's`, | |
| 177 | ) | |
| 178 | }) | |
| 179 | ||
| 180 | it('accepts the transcript when the run matches', () => { | |
| 181 | expect(evaluateRun('ios', iosTranscript(), RUN).ok).toBe(true) | |
| 182 | }) | |
| 183 | ||
| 184 | it('does not check the run when none was given', () => { | |
| 185 | expect(evaluateRun('ios', iosTranscript({ run: 'anything' })).ok).toBe(true) | |
| 186 | }) | |
| 187 | ||
| 188 | // "Planned, and did report" is not the same as "passed". A third status | |
| 189 | // satisfies both of the other checks while leaving the case unasserted. | |
| 190 | it('refuses a status that is neither pass nor fail', () => { | |
| 191 | const text = [ | |
| 192 | `CRAFT-E2E ${JSON.stringify({ event: 'plan', platform: 'ios', run: RUN, cases: REQUIRED_CASES.ios })}`, | |
| 193 | ...REQUIRED_CASES.ios.map((name, index) => `CRAFT-E2E ${JSON.stringify({ | |
| 194 | event: 'case', | |
| 195 | name, | |
| 196 | status: index === 0 ? 'skipped' : 'pass', | |
| 197 | detail: '', | |
| 198 | })}`), | |
| 199 | `CRAFT-E2E ${JSON.stringify({ event: 'done', passed: 3, failed: 0 })}`, | |
| 200 | ].join('\n') | |
| 201 | ||
| 202 | const verdict = evaluateRun('ios', text, RUN) | |
| 203 | expect(verdict.ok).toBe(false) | |
| 204 | expect(verdict.failures).toContain('case bridge.ready reported status "skipped", which is neither pass nor fail') | |
| 205 | }) | |
| 206 | ||
| 207 | it('reports the driver fatal when the bridge never appeared', () => { | |
| 208 | const verdict = evaluateRun('ios', 'CRAFT-E2E {"event":"fatal","reason":"window.craft never appeared"}\n') | |
| 209 | expect(verdict.ok).toBe(false) | |
| 210 | expect(verdict.failures).toContain('driver reported a fatal condition: window.craft never appeared') | |
| 211 | }) | |
| 212 | ||
| 213 | it('evaluates an android transcript against the android list', () => { | |
| 214 | const lines = [ | |
| 215 | `CRAFT-E2E ${JSON.stringify({ event: 'plan', platform: 'android', run: RUN, cases: REQUIRED_CASES.android })}`, | |
| 216 | ...REQUIRED_CASES.android.map(name => `CRAFT-E2E ${JSON.stringify({ event: 'case', name, status: 'pass', detail: '' })}`), | |
| 217 | `CRAFT-E2E ${JSON.stringify({ event: 'done', passed: REQUIRED_CASES.android.length, failed: 0 })}`, | |
| 218 | ].join('\n') | |
| 219 | ||
| 220 | expect(evaluateRun('android', lines).ok).toBe(true) | |
| 221 | // The same transcript must not satisfy the iOS leg. | |
| 222 | expect(evaluateRun('ios', lines).ok).toBe(false) | |
| 223 | }) | |
| 224 | }) | |
| 225 | ||
| 226 | describe('hasTerminated', () => { | |
| 227 | it('waits for the whole event, not the first bytes of it', () => { | |
| 228 | // The harness kills the app the moment this is true, so a substring match | |
| 229 | // would cut the line it was waiting for and report it as malformed. | |
| 230 | expect(hasTerminated('[Craft Web] CRAFT-E2E {"event":"done","pa')).toBe(false) | |
| 231 | expect(hasTerminated('[Craft Web] CRAFT-E2E {"event":"done","passed":4,"failed":0}')).toBe(true) | |
| 232 | }) | |
| 233 | ||
| 234 | it('treats a fatal as terminal too', () => { | |
| 235 | expect(hasTerminated('CRAFT-E2E {"event":"fatal","reason":"window.craft never appeared"}')).toBe(true) | |
| 236 | }) | |
| 237 | ||
| 238 | it('is false while cases are still arriving', () => { | |
| 239 | expect(hasTerminated('CRAFT-E2E {"event":"case","name":"a","status":"pass"}')).toBe(false) | |
| 240 | }) | |
| 241 | }) | |
| 242 | ||
| 243 | describe('zig attribution', () => { | |
| 244 | it('names the actions the Zig dispatcher saw', () => { | |
| 245 | const text = [ | |
| 246 | 'info: craft-bridge dispatch t=mobile a=getDeviceInfo i=3', | |
| 247 | `${ESC}[32minfo: craft-bridge dispatch t=mobile a=clipboardWrite i=4${ESC}[0m`, | |
| 248 | 'info: something else entirely', | |
| 249 | ].join('\n') | |
| 250 | ||
| 251 | expect(zigDispatchedActions(text)).toEqual(['clipboardWrite', 'getDeviceInfo']) | |
| 252 | }) | |
| 253 | ||
| 254 | // The reason this counts actions and not lines: every report the page makes | |
| 255 | // goes through craft.log, and log is itself dispatched. A bare count is | |
| 256 | // satisfied by the reporting traffic while Swift serves everything tested. | |
| 257 | it('is not satisfied by the page reporting through craft.log', () => { | |
| 258 | const logOnly = Array.from({ length: 12 }, () => 'info: craft-bridge dispatch t=mobile a=log i=null').join('\n') | |
| 259 | const dispatched = zigDispatchedActions(logOnly) | |
| 260 | ||
| 261 | expect(dispatched).toEqual(['log']) | |
| 262 | expect(ZIG_TESTED_ACTIONS.filter(action => !dispatched.includes(action))).toEqual(ZIG_TESTED_ACTIONS) | |
| 263 | }) | |
| 264 | ||
| 265 | it('is empty for a run the platform shim served alone', () => { | |
| 266 | expect(zigDispatchedActions(iosTranscript())).toEqual([]) | |
| 267 | expect(zigRefusals(iosTranscript())).toEqual([]) | |
| 268 | }) | |
| 269 | ||
| 270 | it('reads the refusal Zig writes through its own capability gate', () => { | |
| 271 | const text = 'info: ios: refusing getCurrentPosition; enableGeolocation is not enabled in craft.config.json' | |
| 272 | expect(zigRefusals(text)).toEqual([ZIG_REFUSED_ACTION]) | |
| 273 | }) | |
| 274 | ||
| 275 | it('expects a refusal for an action the suite actually exercises', () => { | |
| 276 | expect(ZIG_TESTED_ACTIONS).toContain(ZIG_REFUSED_ACTION) | |
| 277 | }) | |
| 278 | }) | |
| @@ -0,0 +1,359 @@ | ||
| 1 | /** | |
| 2 | * The wire between the test page and the harness, and the rules for reading it. | |
| 3 | * | |
| 4 | * Kept separate from the two runners because this is the part that can be | |
| 5 | * tested without a simulator: `scripts/mobile-e2e/protocol.test.ts` feeds it | |
| 6 | * transcripts a device would have produced, including the ones that matter | |
| 7 | * most - a suite that died halfway, a case that never ran, a run with no | |
| 8 | * output at all. The runners stay untested; the reading of their output does | |
| 9 | * not. Same split as scripts/native-lifecycle-plan.ts and its test. | |
| 10 | */ | |
| 11 | ||
| 12 | /** Emitted once, before any case runs, so a truncated run is detectable. */ | |
| 13 | export interface PlanEvent { | |
| 14 | event: 'plan' | |
| 15 | platform: string | |
| 16 | cases: string[] | |
| 17 | /** This run's nonce, so a stale transcript cannot be mistaken for this one. */ | |
| 18 | run?: string | |
| 19 | } | |
| 20 | ||
| 21 | export interface CaseEvent { | |
| 22 | event: 'case' | |
| 23 | name: string | |
| 24 | status: 'pass' | 'fail' | |
| 25 | /** Empty on pass; on failure, what the page actually saw. */ | |
| 26 | detail?: string | |
| 27 | } | |
| 28 | ||
| 29 | export interface DoneEvent { | |
| 30 | event: 'done' | |
| 31 | passed: number | |
| 32 | failed: number | |
| 33 | } | |
| 34 | ||
| 35 | export interface FatalEvent { | |
| 36 | event: 'fatal' | |
| 37 | reason: string | |
| 38 | } | |
| 39 | ||
| 40 | export interface ObservedEvent { | |
| 41 | event: 'observed' | |
| 42 | name: string | |
| 43 | value: unknown | |
| 44 | } | |
| 45 | ||
| 46 | export type DriverEvent = PlanEvent | CaseEvent | DoneEvent | FatalEvent | ObservedEvent | |
| 47 | ||
| 48 | export type MobilePlatform = 'ios' | 'android' | |
| 49 | ||
| 50 | /** | |
| 51 | * The cases each platform must run, named here and nowhere else. | |
| 52 | * | |
| 53 | * The page decides what a case *asserts*; this decides which ones have to | |
| 54 | * exist. Both halves are needed: without the page there is nothing to assert, | |
| 55 | * and without this list a suite can be hollowed out to one trivially passing | |
| 56 | * case and still report green - which is the failure mode the workflow this | |
| 57 | * replaces actually had, for months, with `if: hashFiles(...)` guards that | |
| 58 | * skipped every real step. | |
| 59 | * | |
| 60 | * Each platform's list must contain at least one success path and one | |
| 61 | * rejection path; `requiredCaseProblems` enforces that rather than trusting | |
| 62 | * whoever edits it next. | |
| 63 | */ | |
| 64 | export const REQUIRED_CASES: Record<MobilePlatform, string[]> = { | |
| 65 | ios: [ | |
| 66 | 'bridge.ready', | |
| 67 | 'deviceInfo.isSimulator', | |
| 68 | 'clipboard.roundTrip', | |
| 69 | 'geolocation.disabled.rejects', | |
| 70 | ], | |
| 71 | android: [ | |
| 72 | 'bridge.ready', | |
| 73 | 'deviceInfo.isEmulator', | |
| 74 | 'clipboard.roundTrip', | |
| 75 | 'push.disabled.rejects', | |
| 76 | ], | |
| 77 | } | |
| 78 | ||
| 79 | /** Cases whose whole point is that the native side *refuses*. */ | |
| 80 | const REJECTION_SUFFIX = '.rejects' | |
| 81 | ||
| 82 | /** | |
| 83 | * Guard the list above against being watered down. | |
| 84 | * | |
| 85 | * A platform that lost its rejection case would still look like a passing | |
| 86 | * suite - every remaining case is a success path, and success paths are the | |
| 87 | * easy half. Run from the protocol test. | |
| 88 | */ | |
| 89 | export function requiredCaseProblems(): string[] { | |
| 90 | const problems: string[] = [] | |
| 91 | ||
| 92 | for (const [platform, names] of Object.entries(REQUIRED_CASES)) { | |
| 93 | if (!names.some(name => name.endsWith(REJECTION_SUFFIX))) | |
| 94 | problems.push(`${platform} has no ${REJECTION_SUFFIX} case; a suite of success paths only proves half the bridge`) | |
| 95 | ||
| 96 | if (!names.some(name => !name.endsWith(REJECTION_SUFFIX) && name !== 'bridge.ready')) | |
| 97 | problems.push(`${platform} has no success case beyond bridge.ready`) | |
| 98 | ||
| 99 | const duplicated = names.filter((name, index) => names.indexOf(name) !== index) | |
| 100 | if (duplicated.length) | |
| 101 | problems.push(`${platform} lists ${duplicated.join(', ')} more than once`) | |
| 102 | } | |
| 103 | ||
| 104 | return problems | |
| 105 | } | |
| 106 | ||
| 107 | const MARKER = 'CRAFT-E2E ' | |
| 108 | ||
| 109 | /** | |
| 110 | * Colour codes the iOS console adds, stripped before matching. | |
| 111 | * | |
| 112 | * Built from a char code rather than written as an escape so this file stays | |
| 113 | * free of literal control characters. | |
| 114 | */ | |
| 115 | const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g') | |
| 116 | ||
| 117 | /** | |
| 118 | * The first complete JSON object in `text`, or null. | |
| 119 | * | |
| 120 | * Taking the rest of the line would be enough on iOS, where the console shows | |
| 121 | * exactly what the app printed. It is not enough on Android: the page's | |
| 122 | * `console.log` reaches logcat through Chromium, which wraps it - | |
| 123 | * | |
| 124 | * [INFO:CONSOLE(1)] "CRAFT-E2E {...}", source: https://appassets... (1) | |
| 125 | * | |
| 126 | * - so the rest of the line carries a trailing quote and a source reference. | |
| 127 | * Scanning to the matching brace reads the event out of either shape, while | |
| 128 | * still rejecting a line that was genuinely truncated. | |
| 129 | */ | |
| 130 | function firstJsonObject(text: string): string | null { | |
| 131 | const start = text.indexOf('{') | |
| 132 | if (start === -1) return null | |
| 133 | ||
| 134 | let depth = 0 | |
| 135 | let inString = false | |
| 136 | let escaped = false | |
| 137 | ||
| 138 | for (let index = start; index < text.length; index++) { | |
| 139 | const character = text[index]! | |
| 140 | ||
| 141 | if (escaped) { escaped = false; continue } | |
| 142 | if (character === '\\' && inString) { escaped = true; continue } | |
| 143 | if (character === '"') { inString = !inString; continue } | |
| 144 | if (inString) continue | |
| 145 | ||
| 146 | if (character === '{') depth++ | |
| 147 | else if (character === '}') { | |
| 148 | depth-- | |
| 149 | if (depth === 0) return text.slice(start, index + 1) | |
| 150 | } | |
| 151 | } | |
| 152 | ||
| 153 | return null | |
| 154 | } | |
| 155 | ||
| 156 | /** | |
| 157 | * Pull the driver's events out of a device log. | |
| 158 | * | |
| 159 | * The log is not ours: on iOS it is the app's whole stderr as | |
| 160 | * `simctl launch --console-pty` saw it, on Android a logcat slice, both | |
| 161 | * carrying unrelated system chatter and, on iOS, ANSI colouring. So lines are | |
| 162 | * found by marker rather than by position, colour codes are stripped first, | |
| 163 | * and a marker line with no readable event is reported rather than skipped - | |
| 164 | * a garbled event is evidence of a problem, not something to shrug at. | |
| 165 | * | |
| 166 | * The page emits each event twice on purpose, through `console.log` and | |
| 167 | * through the bridge's own `log` action, so that a run whose bridge never | |
| 168 | * arrived can still say so. Only one of those reaches the log on iOS; both do | |
| 169 | * on Android. Duplicates are dropped here rather than being left for | |
| 170 | * `evaluateRun` to trip over. | |
| 171 | */ | |
| 172 | export function parseDriverOutput(text: string): { events: DriverEvent[], malformed: string[] } { | |
| 173 | const events: DriverEvent[] = [] | |
| 174 | const malformed: string[] = [] | |
| 175 | const seen = new Set<string>() | |
| 176 | ||
| 177 | for (const rawLine of text.split('\n')) { | |
| 178 | const line = rawLine.replace(ANSI, '').replace(/\r$/, '') | |
| 179 | const at = line.indexOf(MARKER) | |
| 180 | if (at === -1) continue | |
| 181 | ||
| 182 | const rest = line.slice(at + MARKER.length).trim() | |
| 183 | const payload = firstJsonObject(rest) | |
| 184 | if (payload === null) { | |
| 185 | malformed.push(rest) | |
| 186 | continue | |
| 187 | } | |
| 188 | ||
| 189 | let parsed: DriverEvent | |
| 190 | try { | |
| 191 | parsed = JSON.parse(payload) as DriverEvent | |
| 192 | } | |
| 193 | catch { | |
| 194 | malformed.push(payload) | |
| 195 | continue | |
| 196 | } | |
| 197 | ||
| 198 | if (!parsed || typeof parsed !== 'object' || typeof parsed.event !== 'string') { | |
| 199 | malformed.push(payload) | |
| 200 | continue | |
| 201 | } | |
| 202 | ||
| 203 | if (seen.has(payload)) continue | |
| 204 | seen.add(payload) | |
| 205 | events.push(parsed) | |
| 206 | } | |
| 207 | ||
| 208 | return { events, malformed } | |
| 209 | } | |
| 210 | ||
| 211 | /** | |
| 212 | * Has the suite finished, as a complete event rather than a substring? | |
| 213 | * | |
| 214 | * The callers kill the app the moment this is true, so matching `"event":"done"` | |
| 215 | * in a half-written line would truncate the very event being waited for, and | |
| 216 | * the truncation would then be reported as a malformed event - a passing run | |
| 217 | * failing on a race. Requiring the object to parse means the line is whole. | |
| 218 | */ | |
| 219 | export function hasTerminated(text: string): boolean { | |
| 220 | return parseDriverOutput(text).events.some(event => event.event === 'done' || event.event === 'fatal') | |
| 221 | } | |
| 222 | ||
| 223 | export interface RunVerdict { | |
| 224 | ok: boolean | |
| 225 | /** One line per reason the run is not a pass. Empty when ok. */ | |
| 226 | failures: string[] | |
| 227 | planned: string[] | |
| 228 | passed: string[] | |
| 229 | failed: string[] | |
| 230 | } | |
| 231 | ||
| 232 | /** | |
| 233 | * Decide whether a captured run is a pass. | |
| 234 | * | |
| 235 | * Silence is the thing being guarded against. A run that produced no output | |
| 236 | * at all, or announced a plan and then died, or ran every case it planned but | |
| 237 | * planned only one - each of those reads as "nothing went wrong" to a naive | |
| 238 | * reader, and each is a failure here. | |
| 239 | */ | |
| 240 | export function evaluateRun(platform: MobilePlatform, text: string, expectedRun?: string): RunVerdict { | |
| 241 | const { events, malformed } = parseDriverOutput(text) | |
| 242 | const failures: string[] = [] | |
| 243 | ||
| 244 | for (const line of malformed) | |
| 245 | failures.push(`driver emitted an unparseable event: ${line}`) | |
| 246 | ||
| 247 | const fatal = events.find((event): event is FatalEvent => event.event === 'fatal') | |
| 248 | if (fatal) failures.push(`driver reported a fatal condition: ${fatal.reason}`) | |
| 249 | ||
| 250 | const plan = events.find((event): event is PlanEvent => event.event === 'plan') | |
| 251 | const cases = events.filter((event): event is CaseEvent => event.event === 'case') | |
| 252 | const done = events.find((event): event is DoneEvent => event.event === 'done') | |
| 253 | ||
| 254 | const planned = plan?.cases ?? [] | |
| 255 | const passed = cases.filter(entry => entry.status === 'pass').map(entry => entry.name) | |
| 256 | const failed = cases.filter(entry => entry.status === 'fail').map(entry => entry.name) | |
| 257 | ||
| 258 | if (!plan) { | |
| 259 | // No plan means the page never reached the bridge - the app crashed, the | |
| 260 | // page never loaded, or the bridge refused it. Everything below would be | |
| 261 | // vacuously true, so say this and stop adding noise. | |
| 262 | failures.push('the test page never announced a plan; it did not reach the native bridge') | |
| 263 | return { ok: false, failures, planned, passed, failed } | |
| 264 | } | |
| 265 | ||
| 266 | if (plan.platform !== platform) | |
| 267 | failures.push(`the app reports platform ${plan.platform}, expected ${platform}`) | |
| 268 | ||
| 269 | // Android reads its transcript out of the device's logcat ring buffer, which | |
| 270 | // survives an uninstall and is cleared only on a best-effort basis. Without | |
| 271 | // this, a leg whose app printed nothing at all could be evaluated against — | |
| 272 | // and pass on — the previous run's output still sitting in the buffer. | |
| 273 | if (expectedRun !== undefined && plan.run !== expectedRun) | |
| 274 | failures.push(`the transcript is from run ${JSON.stringify(plan.run)}, not ${JSON.stringify(expectedRun)}; this is stale output, not this run's`) | |
| 275 | ||
| 276 | for (const required of REQUIRED_CASES[platform]) { | |
| 277 | if (!planned.includes(required)) | |
| 278 | failures.push(`required case ${required} is not in the suite the page ran`) | |
| 279 | } | |
| 280 | ||
| 281 | for (const name of planned) { | |
| 282 | if (!cases.some(entry => entry.name === name)) | |
| 283 | failures.push(`case ${name} was planned but never reported; the suite stopped early`) | |
| 284 | } | |
| 285 | ||
| 286 | for (const entry of cases) { | |
| 287 | if (entry.status === 'fail') | |
| 288 | failures.push(`case ${entry.name} failed: ${entry.detail || 'no detail'}`) | |
| 289 | // Neither a pass nor a fail is not a third outcome to be tolerated. A case | |
| 290 | // reporting anything else satisfies "was planned and did report" while | |
| 291 | // never appearing in `passed`, which would leave it unasserted inside a | |
| 292 | // green run - the precise shape of failure this harness exists to remove. | |
| 293 | else if (entry.status !== 'pass') | |
| 294 | failures.push(`case ${entry.name} reported status ${JSON.stringify(entry.status)}, which is neither pass nor fail`) | |
| 295 | } | |
| 296 | ||
| 297 | if (!done) | |
| 298 | failures.push('the suite never reported done') | |
| 299 | else if (done.failed !== failed.length || done.passed !== passed.length) | |
| 300 | failures.push(`done says ${done.passed} passed / ${done.failed} failed, transcript shows ${passed.length} / ${failed.length}`) | |
| 301 | ||
| 302 | return { ok: failures.length === 0, failures, planned, passed, failed } | |
| 303 | } | |
| 304 | ||
| 305 | /** | |
| 306 | * The actions the Zig dispatcher saw, by name. | |
| 307 | * | |
| 308 | * Zig and the platform shim answer the same actions with the same shapes - | |
| 309 | * that is the point of the hand-off design - so the only way to tell them | |
| 310 | * apart from outside is what Zig's own logger writes | |
| 311 | * (`packages/zig/src/ios_dispatch.zig`). With no archive linked, `dlsym` | |
| 312 | * misses and none of this is emitted at all, which is what separates the two | |
| 313 | * iOS legs. | |
| 314 | * | |
| 315 | * By name rather than by count, because the page routes every report through | |
| 316 | * `craft.log`, and `log` is itself a dispatched action: a bare count is | |
| 317 | * satisfied by the reporting traffic alone and would pass while Swift served | |
| 318 | * every capability under test. | |
| 319 | */ | |
| 320 | export function zigDispatchedActions(text: string): string[] { | |
| 321 | const plain = text.replace(ANSI, '') | |
| 322 | const seen = new Set<string>() | |
| 323 | for (const match of plain.matchAll(/craft-bridge dispatch t=\w+ a=(\w+)/g)) seen.add(match[1]!) | |
| 324 | return [...seen].sort() | |
| 325 | } | |
| 326 | ||
| 327 | /** | |
| 328 | * The actions Zig refused through its own capability gate. | |
| 329 | * | |
| 330 | * A dispatch line proves Zig was *offered* an action - it is logged on entry, | |
| 331 | * before the decision. This proves Zig *answered* one: `ios_config`'s gate | |
| 332 | * writes the line, and the rejection the page then sees came off Zig's error | |
| 333 | * route rather than Swift's. | |
| 334 | */ | |
| 335 | export function zigRefusals(text: string): string[] { | |
| 336 | const plain = text.replace(ANSI, '') | |
| 337 | const seen = new Set<string>() | |
| 338 | for (const match of plain.matchAll(/ios: refusing (\w+);/g)) seen.add(match[1]!) | |
| 339 | return [...seen].sort() | |
| 340 | } | |
| 341 | ||
| 342 | /** | |
| 343 | * The capability actions the page exercises, which the runtime leg expects to | |
| 344 | * see Zig dispatch. | |
| 345 | * | |
| 346 | * Kept in step with the cases in driver.html by hand. Derived from the page it | |
| 347 | * would have to be parsed out of HTML; named here it is at least checkable, | |
| 348 | * and a case removed from the page fails the leg loudly rather than quietly | |
| 349 | * reducing what the assertion covers. | |
| 350 | */ | |
| 351 | export const ZIG_TESTED_ACTIONS = [ | |
| 352 | 'clipboardRead', | |
| 353 | 'clipboardWrite', | |
| 354 | 'getCurrentPosition', | |
| 355 | 'getDeviceInfo', | |
| 356 | ] | |
| 357 | ||
| 358 | /** The one action the suite configures off, so Zig must refuse it by name. */ | |
| 359 | export const ZIG_REFUSED_ACTION = 'getCurrentPosition' | |
| @@ -0,0 +1,126 @@ | ||
| 1 | import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs' | |
| 2 | import { join } from 'node:path' | |
| 3 | ||
| 4 | const HERE = import.meta.dir | |
| 5 | ||
| 6 | export interface CommandOptions { | |
| 7 | cwd?: string | |
| 8 | /** Append the command line and its output here, for the evidence upload. */ | |
| 9 | logPath?: string | |
| 10 | /** Record the failure and carry on, for teardown steps whose failure is not the story. */ | |
| 11 | allowFailure?: boolean | |
| 12 | stdin?: string | |
| 13 | /** | |
| 14 | * Write raw stdout bytes here instead of returning them as text. | |
| 15 | * | |
| 16 | * `adb exec-out screencap -p` emits a PNG; decoding that as UTF-8 to hand it | |
| 17 | * back as a string is how a screenshot becomes an unopenable file. | |
| 18 | */ | |
| 19 | outPath?: string | |
| 20 | } | |
| 21 | ||
| 22 | export interface CommandResult { | |
| 23 | exitCode: number | |
| 24 | stdout: string | |
| 25 | stderr: string | |
| 26 | } | |
| 27 | ||
| 28 | /** | |
| 29 | * Run a command, keep its output, and fail loudly by default. | |
| 30 | * | |
| 31 | * Everything a leg does to a device goes through here so that the evidence | |
| 32 | * upload contains the actual command lines. A failing `xcodebuild` whose log | |
| 33 | * is three screens of Swift diagnostics is useless as an exception message and | |
| 34 | * invaluable as a file. | |
| 35 | */ | |
| 36 | export async function command(argv: string[], options: CommandOptions = {}): Promise<CommandResult> { | |
| 37 | const spawned = Bun.spawn(argv, { | |
| 38 | cwd: options.cwd, | |
| 39 | stdin: options.stdin === undefined ? 'ignore' : new TextEncoder().encode(options.stdin), | |
| 40 | stdout: 'pipe', | |
| 41 | stderr: 'pipe', | |
| 42 | }) | |
| 43 | ||
| 44 | const [raw, stderr] = await Promise.all([ | |
| 45 | new Response(spawned.stdout).arrayBuffer(), | |
| 46 | new Response(spawned.stderr).text(), | |
| 47 | ]) | |
| 48 | const exitCode = await spawned.exited | |
| 49 | ||
| 50 | if (options.outPath) writeFileSync(options.outPath, new Uint8Array(raw)) | |
| 51 | const stdout = options.outPath ? '' : new TextDecoder().decode(raw) | |
| 52 | ||
| 53 | if (options.logPath) { | |
| 54 | appendFileSync(options.logPath, [ | |
| 55 | `$ ${argv.join(' ')}`, | |
| 56 | options.cwd ? `# cwd ${options.cwd}` : '', | |
| 57 | stdout, | |
| 58 | stderr, | |
| 59 | `# exit ${exitCode}`, | |
| 60 | '', | |
| 61 | ].filter(Boolean).join('\n')) | |
| 62 | } | |
| 63 | ||
| 64 | if (exitCode !== 0 && !options.allowFailure) { | |
| 65 | const tail = `${stdout}\n${stderr}`.trim().split('\n').slice(-25).join('\n') | |
| 66 | throw new Error(`${argv.join(' ')} exited with ${exitCode}\n${tail}`) | |
| 67 | } | |
| 68 | ||
| 69 | return { exitCode, stdout, stderr } | |
| 70 | } | |
| 71 | ||
| 72 | /** | |
| 73 | * Wait for a file to say what we are waiting for. | |
| 74 | * | |
| 75 | * `refresh` exists because the two platforms deliver output differently: iOS | |
| 76 | * streams the app's stderr straight into the file, so there is nothing to do | |
| 77 | * between polls, while Android has to be asked - `adb logcat -d` dumps and | |
| 78 | * exits. Streaming logcat instead would mean killing it at exactly the right | |
| 79 | * moment, and the dump costs nothing. | |
| 80 | * | |
| 81 | * Returns false on the deadline rather than throwing, because a run that | |
| 82 | * produced no terminating event still has a log worth keeping and a screenshot | |
| 83 | * worth taking - and the caller does both before it decides anything. Throwing | |
| 84 | * here would skip the evidence for the failure that most needs it. | |
| 85 | */ | |
| 86 | export async function waitForFile( | |
| 87 | path: string, | |
| 88 | timeoutMs: number, | |
| 89 | done: (text: string) => boolean, | |
| 90 | refresh?: () => Promise<void>, | |
| 91 | ): Promise<boolean> { | |
| 92 | const deadline = Date.now() + timeoutMs | |
| 93 | ||
| 94 | while (Date.now() < deadline) { | |
| 95 | if (refresh) await refresh() | |
| 96 | if (existsSync(path)) { | |
| 97 | const text = readFileSync(path, 'utf8') | |
| 98 | if (done(text)) return true | |
| 99 | } | |
| 100 | await Bun.sleep(refresh ? 1000 : 250) | |
| 101 | } | |
| 102 | ||
| 103 | return false | |
| 104 | } | |
| 105 | ||
| 106 | /** | |
| 107 | * The test page, with this run's nonce baked in. | |
| 108 | * | |
| 109 | * One page serves both platforms: `window.craft` is injected by each runtime | |
| 110 | * with the same shape, so the suite that runs on a simulator is the same file | |
| 111 | * that runs on an emulator, and a case can only diverge where it says it does. | |
| 112 | */ | |
| 113 | export function driverPage(nonce: string, platform: string): string { | |
| 114 | const template = readFileSync(join(HERE, 'driver.html'), 'utf8') | |
| 115 | ||
| 116 | // Both placeholders checked, because a silently unsubstituted one turns its | |
| 117 | // case into an assertion about a constant the page chose for itself. | |
| 118 | for (const placeholder of ['__CRAFT_E2E_NONCE__', '__CRAFT_E2E_PLATFORM__']) { | |
| 119 | if (!template.includes(placeholder)) | |
| 120 | throw new Error(`driver.html no longer carries the ${placeholder} placeholder`) | |
| 121 | } | |
| 122 | ||
| 123 | return template | |
| 124 | .replaceAll('__CRAFT_E2E_NONCE__', nonce) | |
| 125 | .replaceAll('__CRAFT_E2E_PLATFORM__', platform) | |
| 126 | } | |
| @@ -0,0 +1,28 @@ | ||
| 1 | export interface RunnerOptions { | |
| 2 | /** Repository root, used to shorten evidence paths in the report. */ | |
| 3 | root: string | |
| 4 | /** artifacts/mobile-e2e/<label>, created before any leg runs. */ | |
| 5 | evidenceDir: string | |
| 6 | /** Scratch space for generated projects; kept under the evidence dir in CI. */ | |
| 7 | workDir: string | |
| 8 | /** How long one app may run before its leg is called a failure. */ | |
| 9 | timeoutMs: number | |
| 10 | /** Distinguishes this run's clipboard nonce from a stale one. */ | |
| 11 | runId: string | |
| 12 | /** Directory holding the Zig iOS simulator archives. */ | |
| 13 | iosRuntimeDir: string | null | |
| 14 | } | |
| 15 | ||
| 16 | export interface LegOutcome { | |
| 17 | name: string | |
| 18 | status: 'passed' | 'failed' | |
| 19 | /** Empty on a pass. Each entry is one reason, in the words of what was seen. */ | |
| 20 | failures: string[] | |
| 21 | planned: string[] | |
| 22 | passed: string[] | |
| 23 | failed: string[] | |
| 24 | /** Which actions reached the Zig dispatcher, i.e. which side answered. */ | |
| 25 | zigActions: string[] | |
| 26 | /** Repo-relative path to this leg's evidence. */ | |
| 27 | evidence: string | |
| 28 | } | |