ReviewOS

also looking at this

craft-native/craft

test(mobile): run the generated apps on a simulator and an emulator

#199
Merged glennmichael123 wants to merge fix/mobile-e2e-real-assertions into main
26 files +3009 -293

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

.gitignoremodified+1-0
Changes to .gitignore
@@ -31,6 +31,7 @@ release/
3131# Test outputs
3232test-app/
3333artifacts/native-lifecycle/
34artifacts/mobile-e2e/
3435
3536# Logs
3637*.log
scripts/mobile-e2e/driver.htmladded+254-0
Changes to scripts/mobile-e2e/driver.html
@@ -0,0 +1,254 @@
1<!DOCTYPE html>
2<html lang="en">
3<head>
4 <meta charset="UTF-8">
5 <meta name="viewport" content="width=device-width, initial-scale=1.0">
6 <title>Craft mobile E2E</title>
7 <style>
8 body { font: 14px -apple-system, Roboto, sans-serif; background: #0b1712; color: #e6f2ec; padding: 16px; }
9 li { margin: 4px 0; font-family: ui-monospace, monospace; }
10 .pass { color: #7ee08a; }
11 .fail { color: #ff8a80; }
12 </style>
13</head>
14<body>
15 <h1 id="headline">Craft mobile E2E</h1>
16 <ul id="results"></ul>
17 <script>
18 // The page drives itself and reports through the bridge's own `log` action,
19 // which is the one channel both runtimes already forward off-device: iOS
20 // prints "[Craft Web] <message>" to the process stderr that
21 // `simctl launch --console-pty` captures, and Android writes it to logcat
22 // under the CraftBridge tag. Nothing else in the generated app reaches a
23 // host-side reader, so the harness would otherwise be watching a screenshot.
24 //
25 // Every line is CRAFT-E2E followed by one JSON object. The harness reads
26 // three kinds: `plan` (once, up front), `case` (once per case), and `done`.
27 // Announcing the plan before running anything is what lets the harness tell
28 // "this case failed" apart from "this case never ran" — a suite that dies
29 // halfway through is a failure, not a pass with fewer cases.
30 (function () {
31 var NONCE = '__CRAFT_E2E_NONCE__'
32 // Which platform the harness built this leg for. Substituted in from
33 // outside, because the page comparing window.craft.platform against a
34 // value it read out of window.craft.platform is a tautology - it was one,
35 // and bridge.ready passed unconditionally until this arrived.
36 var EXPECTED_PLATFORM = '__CRAFT_E2E_PLATFORM__'
37 var CASE_TIMEOUT_MS = 8000
38
39 function emit(payload) {
40 var line = 'CRAFT-E2E ' + JSON.stringify(payload)
41 // console.log first: on Android it reaches logcat under `chromium` even
42 // when the bridge never arrived, which is the case the harness most
43 // needs evidence for.
44 try { console.log(line) } catch (ignored) {}
45 try { if (window.craft && window.craft.log) window.craft.log(line) } catch (ignored) {}
46 }
47
48 function show(name, ok, detail) {
49 var li = document.createElement('li')
50 li.className = ok ? 'pass' : 'fail'
51 li.textContent = (ok ? 'PASS ' : 'FAIL ') + name + (detail ? ' — ' + detail : '')
52 document.getElementById('results').appendChild(li)
53 }
54
55 // Rejections arrive in three shapes across the two runtimes: an Error from
56 // iOS's _rejectCallback, a bare string from Android's named reject globals
57 // (promise-runtime.ts parks `window._craftPushReject` and Kotlin calls it
58 // with a quoted string), and an object with `code`/`message` from iOS's
59 // capability gate. Normalise once here so a case can assert on a code
60 // without caring which runtime produced it.
61 function describeError(error) {
62 if (error === null || error === undefined) return { code: '', message: '' }
63 if (typeof error === 'string') return { code: '', message: error }
64 return { code: String(error.code || ''), message: String(error.message || error) }
65 }
66
67 function withTimeout(run) {
68 return new Promise(function (resolve) {
69 var settled = false
70 var timer = setTimeout(function () {
71 if (settled) return
72 settled = true
73 resolve({ state: 'timeout' })
74 }, CASE_TIMEOUT_MS)
75
76 function finish(value) {
77 if (settled) return
78 settled = true
79 clearTimeout(timer)
80 resolve(value)
81 }
82
83 try {
84 Promise.resolve(run()).then(
85 function (value) { finish({ state: 'resolved', value: value }) },
86 function (error) { finish({ state: 'rejected', error: describeError(error) }) }
87 )
88 }
89 catch (thrown) {
90 finish({ state: 'rejected', error: describeError(thrown) })
91 }
92 })
93 }
94
95 // A case passes only by returning nothing. Returning a string is how it
96 // explains what it saw instead, and that string lands in the report — so a
97 // failure names the actual value rather than just the assertion that broke.
98 var CASES = [
99 {
100 name: 'bridge.ready',
101 platforms: ['ios', 'android'],
102 run: function () {
103 if (!window.craft) return 'window.craft is absent'
104 if (typeof window.craft.log !== 'function') return 'window.craft has no log function'
105 if (window.craft.platform !== EXPECTED_PLATFORM)
106 return 'platform is ' + window.craft.platform + ', expected ' + EXPECTED_PLATFORM
107 }
108 },
109 {
110 // Ground truth the page cannot invent: `isSimulator` comes from Swift's
111 // `#if targetEnvironment(simulator)`, decided by the compiler, and
112 // `isEmulator` from the device's own Build properties. A stubbed bridge
113 // that echoed plausible JSON would have to guess it right.
114 name: 'deviceInfo.isSimulator',
115 platforms: ['ios'],
116 run: function () {
117 return withTimeout(function () { return window.craft.getDeviceInfo() }).then(function (outcome) {
118 if (outcome.state !== 'resolved') return 'getDeviceInfo ' + outcome.state
119 if (!outcome.value) return 'getDeviceInfo resolved with nothing'
120 if (outcome.value.platform !== 'ios') return 'platform is ' + outcome.value.platform
121 if (outcome.value.isSimulator !== true) return 'isSimulator is ' + outcome.value.isSimulator
122 })
123 }
124 },
125 {
126 name: 'deviceInfo.isEmulator',
127 platforms: ['android'],
128 run: function () {
129 return withTimeout(function () { return window.craft.getDeviceInfo() }).then(function (outcome) {
130 if (outcome.state !== 'resolved') return 'getDeviceInfo ' + outcome.state
131 if (!outcome.value) return 'getDeviceInfo resolved with nothing'
132
133 // Reported before anything is asserted, so a failing assertion
134 // still leaves the values behind. The first run of this case said
135 // only "isEmulator is false" and the log did not say what the
136 // device had called itself, which is the one thing needed to fix
137 // it.
138 emit({ event: 'observed', name: 'sdkVersion', value: outcome.value.sdkVersion })
139 emit({ event: 'observed', name: 'model', value: outcome.value.model })
140 emit({ event: 'observed', name: 'manufacturer', value: outcome.value.manufacturer })
141 emit({ event: 'observed', name: 'device', value: outcome.value.device })
142
143 if (outcome.value.platform !== 'android') return 'platform is ' + outcome.value.platform
144 if (!outcome.value.sdkVersion) return 'sdkVersion is ' + outcome.value.sdkVersion
145 if (outcome.value.isEmulator !== true)
146 return 'isEmulator is ' + outcome.value.isEmulator + ' on model ' + outcome.value.model
147 })
148 }
149 },
150 {
151 // The success path. On iOS the harness then reads the device pasteboard
152 // from the host with `simctl pbpaste`, so the case asserts the value
153 // reached real system clipboard state rather than a variable the bridge
154 // handed back. Android has no equally cheap host-side read, so its
155 // external check is the sdkVersion cross-reference below instead.
156 name: 'clipboard.roundTrip',
157 platforms: ['ios', 'android'],
158 run: function () {
159 return withTimeout(function () { return window.craft.clipboard.write(NONCE) }).then(function (written) {
160 if (written.state !== 'resolved') return 'clipboard.write ' + written.state + ' ' + (written.error ? written.error.message : '')
161 return withTimeout(function () { return window.craft.clipboard.read() }).then(function (read) {
162 if (read.state !== 'resolved') return 'clipboard.read ' + read.state + ' ' + (read.error ? read.error.message : '')
163 if (read.value !== NONCE) return 'read back ' + JSON.stringify(read.value)
164 })
165 })
166 }
167 },
168 {
169 // The failure path. Geolocation is off in the harness config, so the
170 // native side must answer — with a code that names the configuration —
171 // rather than leave the promise pending. A dropped callback here is
172 // indistinguishable from a slow one to the page, which is exactly why
173 // the case asserts the rejection and not merely "did not resolve".
174 name: 'geolocation.disabled.rejects',
175 platforms: ['ios'],
176 run: function () {
177 return withTimeout(function () { return window.craft.getCurrentPosition() }).then(function (outcome) {
178 if (outcome.state === 'timeout') return 'getCurrentPosition never settled'
179 if (outcome.state === 'resolved') return 'getCurrentPosition resolved on a disabled capability'
180 if (outcome.error.code !== 'CAPABILITY_DISABLED') return 'code is ' + JSON.stringify(outcome.error.code)
181 })
182 }
183 },
184 {
185 // Android's equivalent gate. With push disabled the generator emits a
186 // literal reject call rather than a Firebase block, so the rejection
187 // string is fixed at generation time and is safe to match on.
188 name: 'push.disabled.rejects',
189 platforms: ['android'],
190 run: function () {
191 return withTimeout(function () { return window.craft.registerPush() }).then(function (outcome) {
192 if (outcome.state === 'timeout') return 'registerPush never settled'
193 if (outcome.state === 'resolved') return 'registerPush resolved while push is disabled'
194 if (outcome.error.message.indexOf('Push notifications are disabled') === -1)
195 return 'message is ' + JSON.stringify(outcome.error.message)
196 })
197 }
198 }
199 ]
200
201 function waitForBridge(attempt) {
202 if (window.craft && window.craft.getDeviceInfo) return Promise.resolve(true)
203 if (attempt >= 150) return Promise.resolve(false)
204 return new Promise(function (resolve) {
205 setTimeout(function () { resolve(waitForBridge(attempt + 1)) }, 100)
206 })
207 }
208
209 waitForBridge(0).then(function (ready) {
210 if (!ready) {
211 emit({ event: 'fatal', reason: 'window.craft never appeared' })
212 document.getElementById('headline').textContent = 'FATAL: no bridge'
213 return
214 }
215
216 var platform = window.craft.platform
217 var selected = CASES.filter(function (entry) { return entry.platforms.indexOf(platform) !== -1 })
218 emit({ event: 'plan', platform: platform, run: NONCE, cases: selected.map(function (entry) { return entry.name }) })
219
220 var passed = 0
221 var failed = 0
222
223 // Sequential, not Promise.all. Android's promise runtime is single-flight
224 // per channel and rejects a second in-flight request outright; running
225 // the suite in parallel would fail cases for colliding with each other.
226 function step(index) {
227 if (index >= selected.length) {
228 emit({ event: 'done', passed: passed, failed: failed })
229 document.getElementById('headline').textContent = failed ? 'FAILED' : 'PASSED'
230 return
231 }
232
233 var entry = selected[index]
234 Promise.resolve()
235 .then(function () { return entry.run() })
236 .then(
237 function (detail) { return detail ? String(detail) : '' },
238 function (thrown) { return 'threw: ' + describeError(thrown).message }
239 )
240 .then(function (detail) {
241 if (detail) failed++
242 else passed++
243 emit({ event: 'case', name: entry.name, status: detail ? 'fail' : 'pass', detail: detail })
244 show(entry.name, !detail, detail)
245 step(index + 1)
246 })
247 }
248
249 step(0)
250 })
251 })()
252 </script>
253</body>
254</html>
scripts/mobile-e2e/protocol.test.tsadded+278-0
Changes to scripts/mobile-e2e/protocol.test.ts
@@ -0,0 +1,278 @@
1import { describe, expect, it } from 'bun:test'
2import { evaluateRun, hasTerminated, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, ZIG_REFUSED_ACTION, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigRefusals } from './protocol'
3
4const ESC = String.fromCharCode(27)
5
6/** A transcript shaped like the one a real iOS run produces. */
7const RUN = 'craft-e2e-ios-shim-1234'
8
9function iosTranscript(overrides: { cases?: string[], omit?: string[], fail?: Record<string, string>, done?: boolean, run?: string } = {}): string {
10 const planned = overrides.cases ?? REQUIRED_CASES.ios
11 const omit = overrides.omit ?? []
12 const fail = overrides.fail ?? {}
13 const reported = planned.filter(name => !omit.includes(name))
14
15 const lines = [
16 'Some unrelated system chatter from the device',
17 `[Craft Web] CRAFT-E2E ${JSON.stringify({ event: 'plan', platform: 'ios', run: overrides.run ?? RUN, cases: planned })}`,
18 ...reported.map(name => `[Craft Web] CRAFT-E2E ${JSON.stringify({
19 event: 'case',
20 name,
21 status: fail[name] ? 'fail' : 'pass',
22 detail: fail[name] ?? '',
23 })}`),
24 ]
25
26 if (overrides.done !== false) {
27 const failed = reported.filter(name => fail[name]).length
28 lines.push(`[Craft Web] CRAFT-E2E ${JSON.stringify({ event: 'done', passed: reported.length - failed, failed })}`)
29 }
30
31 return `${lines.join('\n')}\n`
32}
33
34describe('required cases', () => {
35 it('keeps a success path and a rejection path on every platform', () => {
36 expect(requiredCaseProblems()).toEqual([])
37 })
38
39 it('names the same bridge.ready case on both platforms', () => {
40 expect(REQUIRED_CASES.ios).toContain('bridge.ready')
41 expect(REQUIRED_CASES.android).toContain('bridge.ready')
42 })
43})
44
45describe('parseDriverOutput', () => {
46 it('finds events inside noisy, colour-coded device logs', () => {
47 const text = [
48 'unrelated',
49 `${ESC}[33m[Craft Web] CRAFT-E2E {"event":"plan","platform":"ios","cases":["a"]}${ESC}[0m`,
50 'more unrelated',
51 ].join('\n')
52
53 const { events, malformed } = parseDriverOutput(text)
54 expect(malformed).toEqual([])
55 expect(events).toEqual([{ event: 'plan', platform: 'ios', cases: ['a'] }])
56 })
57
58 it('reports a garbled event rather than skipping it', () => {
59 const { events, malformed } = parseDriverOutput('CRAFT-E2E {"event":"plan"\n')
60 expect(events).toEqual([])
61 expect(malformed).toEqual(['{"event":"plan"'])
62 })
63
64 it('tolerates carriage returns from the console pty', () => {
65 const { events } = parseDriverOutput('CRAFT-E2E {"event":"done","passed":1,"failed":0}\r\n')
66 expect(events).toEqual([{ event: 'done', passed: 1, failed: 0 }])
67 })
68
69 // The exact shape the first Android run produced. Chromium wraps a page's
70 // console.log before it reaches logcat, and taking the rest of the line
71 // turned every event into an unparseable one - six of them, which failed a
72 // run whose bridge had in fact worked.
73 it('reads an event out of a Chromium console wrapper', () => {
74 const line = '09-15 13:49:12.345 1234 1234 I chromium: [INFO:CONSOLE(39)] "CRAFT-E2E '
75 + '{"event":"case","name":"clipboard.roundTrip","status":"pass","detail":""}", '
76 + 'source: https://appassets.androidplatform.net/ (39)'
77
78 const { events, malformed } = parseDriverOutput(line)
79 expect(malformed).toEqual([])
80 expect(events).toEqual([{ event: 'case', name: 'clipboard.roundTrip', status: 'pass', detail: '' }])
81 })
82
83 it('keeps one copy when the page reports through both channels', () => {
84 const payload = '{"event":"case","name":"bridge.ready","status":"pass","detail":""}'
85 const { events } = parseDriverOutput([
86 `I chromium: [INFO:CONSOLE(39)] "CRAFT-E2E ${payload}", source: https://appassets.androidplatform.net/ (39)`,
87 `D CraftBridge: CRAFT-E2E ${payload}`,
88 ].join('\n'))
89
90 expect(events).toEqual([{ event: 'case', name: 'bridge.ready', status: 'pass', detail: '' }])
91 })
92
93 it('does not stop at a brace inside a string', () => {
94 const { events, malformed } = parseDriverOutput(
95 'CRAFT-E2E {"event":"case","name":"a","status":"fail","detail":"read back \\"{oops}\\""}',
96 )
97 expect(malformed).toEqual([])
98 expect(events).toEqual([{ event: 'case', name: 'a', status: 'fail', detail: 'read back "{oops}"' }])
99 })
100
101 it('still reports a genuinely truncated event', () => {
102 const { events, malformed } = parseDriverOutput('CRAFT-E2E {"event":"plan","cases":["a"')
103 expect(events).toEqual([])
104 expect(malformed).toEqual(['{"event":"plan","cases":["a"'])
105 })
106})
107
108describe('evaluateRun', () => {
109 it('passes a complete run', () => {
110 const verdict = evaluateRun('ios', iosTranscript())
111 expect(verdict.failures).toEqual([])
112 expect(verdict.ok).toBe(true)
113 expect(verdict.passed).toEqual(REQUIRED_CASES.ios)
114 })
115
116 // The failure the workflow this replaces actually shipped: nothing ran, and
117 // nothing said so.
118 it('fails an empty log instead of finding nothing to complain about', () => {
119 const verdict = evaluateRun('ios', '')
120 expect(verdict.ok).toBe(false)
121 expect(verdict.failures).toEqual(['the test page never announced a plan; it did not reach the native bridge'])
122 })
123
124 it('fails a log with device chatter but no driver output', () => {
125 const verdict = evaluateRun('ios', 'Booting\nAssertion failed somewhere unrelated\n')
126 expect(verdict.ok).toBe(false)
127 })
128
129 it('fails when a planned case never reports', () => {
130 const verdict = evaluateRun('ios', iosTranscript({ omit: ['clipboard.roundTrip'], done: false }))
131 expect(verdict.ok).toBe(false)
132 expect(verdict.failures).toContain('case clipboard.roundTrip was planned but never reported; the suite stopped early')
133 expect(verdict.failures).toContain('the suite never reported done')
134 })
135
136 it('fails when the suite is hollowed out to the easy cases', () => {
137 const verdict = evaluateRun('ios', iosTranscript({ cases: ['bridge.ready'] }))
138 expect(verdict.ok).toBe(false)
139 expect(verdict.failures).toContain('required case geolocation.disabled.rejects is not in the suite the page ran')
140 expect(verdict.failures).toContain('required case clipboard.roundTrip is not in the suite the page ran')
141 })
142
143 it('carries the page detail into the failure so the reason is the value seen', () => {
144 const verdict = evaluateRun('ios', iosTranscript({ fail: { 'clipboard.roundTrip': 'read back ""' } }))
145 expect(verdict.ok).toBe(false)
146 expect(verdict.failures).toContain('case clipboard.roundTrip failed: read back ""')
147 expect(verdict.failed).toEqual(['clipboard.roundTrip'])
148 })
149
150 it('fails when the tally disagrees with the transcript', () => {
151 const text = [
152 'CRAFT-E2E {"event":"plan","platform":"ios","cases":["bridge.ready","deviceInfo.isSimulator","clipboard.roundTrip","geolocation.disabled.rejects"]}',
153 ...REQUIRED_CASES.ios.map(name => `CRAFT-E2E {"event":"case","name":"${name}","status":"pass","detail":""}`),
154 'CRAFT-E2E {"event":"done","passed":99,"failed":0}',
155 ].join('\n')
156
157 const verdict = evaluateRun('ios', text)
158 expect(verdict.ok).toBe(false)
159 expect(verdict.failures).toContain('done says 99 passed / 0 failed, transcript shows 4 / 0')
160 })
161
162 it('fails when the app is not the platform the leg was built for', () => {
163 const text = iosTranscript().replace('"platform":"ios"', '"platform":"android"')
164 const verdict = evaluateRun('ios', text)
165 expect(verdict.ok).toBe(false)
166 expect(verdict.failures).toContain('the app reports platform android, expected ios')
167 })
168
169 // Android reads its transcript out of a ring buffer that survives an
170 // uninstall, so a leg whose app printed nothing could otherwise be evaluated
171 // against the previous run's output.
172 it('refuses a transcript left over from an earlier run', () => {
173 const verdict = evaluateRun('ios', iosTranscript({ run: 'craft-e2e-ios-shim-OLD' }), RUN)
174 expect(verdict.ok).toBe(false)
175 expect(verdict.failures).toContain(
176 `the transcript is from run "craft-e2e-ios-shim-OLD", not "${RUN}"; this is stale output, not this run's`,
177 )
178 })
179
180 it('accepts the transcript when the run matches', () => {
181 expect(evaluateRun('ios', iosTranscript(), RUN).ok).toBe(true)
182 })
183
184 it('does not check the run when none was given', () => {
185 expect(evaluateRun('ios', iosTranscript({ run: 'anything' })).ok).toBe(true)
186 })
187
188 // "Planned, and did report" is not the same as "passed". A third status
189 // satisfies both of the other checks while leaving the case unasserted.
190 it('refuses a status that is neither pass nor fail', () => {
191 const text = [
192 `CRAFT-E2E ${JSON.stringify({ event: 'plan', platform: 'ios', run: RUN, cases: REQUIRED_CASES.ios })}`,
193 ...REQUIRED_CASES.ios.map((name, index) => `CRAFT-E2E ${JSON.stringify({
194 event: 'case',
195 name,
196 status: index === 0 ? 'skipped' : 'pass',
197 detail: '',
198 })}`),
199 `CRAFT-E2E ${JSON.stringify({ event: 'done', passed: 3, failed: 0 })}`,
200 ].join('\n')
201
202 const verdict = evaluateRun('ios', text, RUN)
203 expect(verdict.ok).toBe(false)
204 expect(verdict.failures).toContain('case bridge.ready reported status "skipped", which is neither pass nor fail')
205 })
206
207 it('reports the driver fatal when the bridge never appeared', () => {
208 const verdict = evaluateRun('ios', 'CRAFT-E2E {"event":"fatal","reason":"window.craft never appeared"}\n')
209 expect(verdict.ok).toBe(false)
210 expect(verdict.failures).toContain('driver reported a fatal condition: window.craft never appeared')
211 })
212
213 it('evaluates an android transcript against the android list', () => {
214 const lines = [
215 `CRAFT-E2E ${JSON.stringify({ event: 'plan', platform: 'android', run: RUN, cases: REQUIRED_CASES.android })}`,
216 ...REQUIRED_CASES.android.map(name => `CRAFT-E2E ${JSON.stringify({ event: 'case', name, status: 'pass', detail: '' })}`),
217 `CRAFT-E2E ${JSON.stringify({ event: 'done', passed: REQUIRED_CASES.android.length, failed: 0 })}`,
218 ].join('\n')
219
220 expect(evaluateRun('android', lines).ok).toBe(true)
221 // The same transcript must not satisfy the iOS leg.
222 expect(evaluateRun('ios', lines).ok).toBe(false)
223 })
224})
225
226describe('hasTerminated', () => {
227 it('waits for the whole event, not the first bytes of it', () => {
228 // The harness kills the app the moment this is true, so a substring match
229 // would cut the line it was waiting for and report it as malformed.
230 expect(hasTerminated('[Craft Web] CRAFT-E2E {"event":"done","pa')).toBe(false)
231 expect(hasTerminated('[Craft Web] CRAFT-E2E {"event":"done","passed":4,"failed":0}')).toBe(true)
232 })
233
234 it('treats a fatal as terminal too', () => {
235 expect(hasTerminated('CRAFT-E2E {"event":"fatal","reason":"window.craft never appeared"}')).toBe(true)
236 })
237
238 it('is false while cases are still arriving', () => {
239 expect(hasTerminated('CRAFT-E2E {"event":"case","name":"a","status":"pass"}')).toBe(false)
240 })
241})
242
243describe('zig attribution', () => {
244 it('names the actions the Zig dispatcher saw', () => {
245 const text = [
246 'info: craft-bridge dispatch t=mobile a=getDeviceInfo i=3',
247 `${ESC}[32minfo: craft-bridge dispatch t=mobile a=clipboardWrite i=4${ESC}[0m`,
248 'info: something else entirely',
249 ].join('\n')
250
251 expect(zigDispatchedActions(text)).toEqual(['clipboardWrite', 'getDeviceInfo'])
252 })
253
254 // The reason this counts actions and not lines: every report the page makes
255 // goes through craft.log, and log is itself dispatched. A bare count is
256 // satisfied by the reporting traffic while Swift serves everything tested.
257 it('is not satisfied by the page reporting through craft.log', () => {
258 const logOnly = Array.from({ length: 12 }, () => 'info: craft-bridge dispatch t=mobile a=log i=null').join('\n')
259 const dispatched = zigDispatchedActions(logOnly)
260
261 expect(dispatched).toEqual(['log'])
262 expect(ZIG_TESTED_ACTIONS.filter(action => !dispatched.includes(action))).toEqual(ZIG_TESTED_ACTIONS)
263 })
264
265 it('is empty for a run the platform shim served alone', () => {
266 expect(zigDispatchedActions(iosTranscript())).toEqual([])
267 expect(zigRefusals(iosTranscript())).toEqual([])
268 })
269
270 it('reads the refusal Zig writes through its own capability gate', () => {
271 const text = 'info: ios: refusing getCurrentPosition; enableGeolocation is not enabled in craft.config.json'
272 expect(zigRefusals(text)).toEqual([ZIG_REFUSED_ACTION])
273 })
274
275 it('expects a refusal for an action the suite actually exercises', () => {
276 expect(ZIG_TESTED_ACTIONS).toContain(ZIG_REFUSED_ACTION)
277 })
278})
scripts/mobile-e2e/protocol.tsadded+359-0
Changes to scripts/mobile-e2e/protocol.ts
@@ -0,0 +1,359 @@
1/**
2 * The wire between the test page and the harness, and the rules for reading it.
3 *
4 * Kept separate from the two runners because this is the part that can be
5 * tested without a simulator: `scripts/mobile-e2e/protocol.test.ts` feeds it
6 * transcripts a device would have produced, including the ones that matter
7 * most - a suite that died halfway, a case that never ran, a run with no
8 * output at all. The runners stay untested; the reading of their output does
9 * not. Same split as scripts/native-lifecycle-plan.ts and its test.
10 */
11
12/** Emitted once, before any case runs, so a truncated run is detectable. */
13export interface PlanEvent {
14 event: 'plan'
15 platform: string
16 cases: string[]
17 /** This run's nonce, so a stale transcript cannot be mistaken for this one. */
18 run?: string
19}
20
21export interface CaseEvent {
22 event: 'case'
23 name: string
24 status: 'pass' | 'fail'
25 /** Empty on pass; on failure, what the page actually saw. */
26 detail?: string
27}
28
29export interface DoneEvent {
30 event: 'done'
31 passed: number
32 failed: number
33}
34
35export interface FatalEvent {
36 event: 'fatal'
37 reason: string
38}
39
40export interface ObservedEvent {
41 event: 'observed'
42 name: string
43 value: unknown
44}
45
46export type DriverEvent = PlanEvent | CaseEvent | DoneEvent | FatalEvent | ObservedEvent
47
48export type MobilePlatform = 'ios' | 'android'
49
50/**
51 * The cases each platform must run, named here and nowhere else.
52 *
53 * The page decides what a case *asserts*; this decides which ones have to
54 * exist. Both halves are needed: without the page there is nothing to assert,
55 * and without this list a suite can be hollowed out to one trivially passing
56 * case and still report green - which is the failure mode the workflow this
57 * replaces actually had, for months, with `if: hashFiles(...)` guards that
58 * skipped every real step.
59 *
60 * Each platform's list must contain at least one success path and one
61 * rejection path; `requiredCaseProblems` enforces that rather than trusting
62 * whoever edits it next.
63 */
64export const REQUIRED_CASES: Record<MobilePlatform, string[]> = {
65 ios: [
66 'bridge.ready',
67 'deviceInfo.isSimulator',
68 'clipboard.roundTrip',
69 'geolocation.disabled.rejects',
70 ],
71 android: [
72 'bridge.ready',
73 'deviceInfo.isEmulator',
74 'clipboard.roundTrip',
75 'push.disabled.rejects',
76 ],
77}
78
79/** Cases whose whole point is that the native side *refuses*. */
80const REJECTION_SUFFIX = '.rejects'
81
82/**
83 * Guard the list above against being watered down.
84 *
85 * A platform that lost its rejection case would still look like a passing
86 * suite - every remaining case is a success path, and success paths are the
87 * easy half. Run from the protocol test.
88 */
89export function requiredCaseProblems(): string[] {
90 const problems: string[] = []
91
92 for (const [platform, names] of Object.entries(REQUIRED_CASES)) {
93 if (!names.some(name => name.endsWith(REJECTION_SUFFIX)))
94 problems.push(`${platform} has no ${REJECTION_SUFFIX} case; a suite of success paths only proves half the bridge`)
95
96 if (!names.some(name => !name.endsWith(REJECTION_SUFFIX) && name !== 'bridge.ready'))
97 problems.push(`${platform} has no success case beyond bridge.ready`)
98
99 const duplicated = names.filter((name, index) => names.indexOf(name) !== index)
100 if (duplicated.length)
101 problems.push(`${platform} lists ${duplicated.join(', ')} more than once`)
102 }
103
104 return problems
105}
106
107const MARKER = 'CRAFT-E2E '
108
109/**
110 * Colour codes the iOS console adds, stripped before matching.
111 *
112 * Built from a char code rather than written as an escape so this file stays
113 * free of literal control characters.
114 */
115const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g')
116
117/**
118 * The first complete JSON object in `text`, or null.
119 *
120 * Taking the rest of the line would be enough on iOS, where the console shows
121 * exactly what the app printed. It is not enough on Android: the page's
122 * `console.log` reaches logcat through Chromium, which wraps it -
123 *
124 * [INFO:CONSOLE(1)] "CRAFT-E2E {...}", source: https://appassets... (1)
125 *
126 * - so the rest of the line carries a trailing quote and a source reference.
127 * Scanning to the matching brace reads the event out of either shape, while
128 * still rejecting a line that was genuinely truncated.
129 */
130function firstJsonObject(text: string): string | null {
131 const start = text.indexOf('{')
132 if (start === -1) return null
133
134 let depth = 0
135 let inString = false
136 let escaped = false
137
138 for (let index = start; index < text.length; index++) {
139 const character = text[index]!
140
141 if (escaped) { escaped = false; continue }
142 if (character === '\\' && inString) { escaped = true; continue }
143 if (character === '"') { inString = !inString; continue }
144 if (inString) continue
145
146 if (character === '{') depth++
147 else if (character === '}') {
148 depth--
149 if (depth === 0) return text.slice(start, index + 1)
150 }
151 }
152
153 return null
154}
155
156/**
157 * Pull the driver's events out of a device log.
158 *
159 * The log is not ours: on iOS it is the app's whole stderr as
160 * `simctl launch --console-pty` saw it, on Android a logcat slice, both
161 * carrying unrelated system chatter and, on iOS, ANSI colouring. So lines are
162 * found by marker rather than by position, colour codes are stripped first,
163 * and a marker line with no readable event is reported rather than skipped -
164 * a garbled event is evidence of a problem, not something to shrug at.
165 *
166 * The page emits each event twice on purpose, through `console.log` and
167 * through the bridge's own `log` action, so that a run whose bridge never
168 * arrived can still say so. Only one of those reaches the log on iOS; both do
169 * on Android. Duplicates are dropped here rather than being left for
170 * `evaluateRun` to trip over.
171 */
172export function parseDriverOutput(text: string): { events: DriverEvent[], malformed: string[] } {
173 const events: DriverEvent[] = []
174 const malformed: string[] = []
175 const seen = new Set<string>()
176
177 for (const rawLine of text.split('\n')) {
178 const line = rawLine.replace(ANSI, '').replace(/\r$/, '')
179 const at = line.indexOf(MARKER)
180 if (at === -1) continue
181
182 const rest = line.slice(at + MARKER.length).trim()
183 const payload = firstJsonObject(rest)
184 if (payload === null) {
185 malformed.push(rest)
186 continue
187 }
188
189 let parsed: DriverEvent
190 try {
191 parsed = JSON.parse(payload) as DriverEvent
192 }
193 catch {
194 malformed.push(payload)
195 continue
196 }
197
198 if (!parsed || typeof parsed !== 'object' || typeof parsed.event !== 'string') {
199 malformed.push(payload)
200 continue
201 }
202
203 if (seen.has(payload)) continue
204 seen.add(payload)
205 events.push(parsed)
206 }
207
208 return { events, malformed }
209}
210
211/**
212 * Has the suite finished, as a complete event rather than a substring?
213 *
214 * The callers kill the app the moment this is true, so matching `"event":"done"`
215 * in a half-written line would truncate the very event being waited for, and
216 * the truncation would then be reported as a malformed event - a passing run
217 * failing on a race. Requiring the object to parse means the line is whole.
218 */
219export function hasTerminated(text: string): boolean {
220 return parseDriverOutput(text).events.some(event => event.event === 'done' || event.event === 'fatal')
221}
222
223export interface RunVerdict {
224 ok: boolean
225 /** One line per reason the run is not a pass. Empty when ok. */
226 failures: string[]
227 planned: string[]
228 passed: string[]
229 failed: string[]
230}
231
232/**
233 * Decide whether a captured run is a pass.
234 *
235 * Silence is the thing being guarded against. A run that produced no output
236 * at all, or announced a plan and then died, or ran every case it planned but
237 * planned only one - each of those reads as "nothing went wrong" to a naive
238 * reader, and each is a failure here.
239 */
240export function evaluateRun(platform: MobilePlatform, text: string, expectedRun?: string): RunVerdict {
241 const { events, malformed } = parseDriverOutput(text)
242 const failures: string[] = []
243
244 for (const line of malformed)
245 failures.push(`driver emitted an unparseable event: ${line}`)
246
247 const fatal = events.find((event): event is FatalEvent => event.event === 'fatal')
248 if (fatal) failures.push(`driver reported a fatal condition: ${fatal.reason}`)
249
250 const plan = events.find((event): event is PlanEvent => event.event === 'plan')
251 const cases = events.filter((event): event is CaseEvent => event.event === 'case')
252 const done = events.find((event): event is DoneEvent => event.event === 'done')
253
254 const planned = plan?.cases ?? []
255 const passed = cases.filter(entry => entry.status === 'pass').map(entry => entry.name)
256 const failed = cases.filter(entry => entry.status === 'fail').map(entry => entry.name)
257
258 if (!plan) {
259 // No plan means the page never reached the bridge - the app crashed, the
260 // page never loaded, or the bridge refused it. Everything below would be
261 // vacuously true, so say this and stop adding noise.
262 failures.push('the test page never announced a plan; it did not reach the native bridge')
263 return { ok: false, failures, planned, passed, failed }
264 }
265
266 if (plan.platform !== platform)
267 failures.push(`the app reports platform ${plan.platform}, expected ${platform}`)
268
269 // Android reads its transcript out of the device's logcat ring buffer, which
270 // survives an uninstall and is cleared only on a best-effort basis. Without
271 // this, a leg whose app printed nothing at all could be evaluated against —
272 // and pass on — the previous run's output still sitting in the buffer.
273 if (expectedRun !== undefined && plan.run !== expectedRun)
274 failures.push(`the transcript is from run ${JSON.stringify(plan.run)}, not ${JSON.stringify(expectedRun)}; this is stale output, not this run's`)
275
276 for (const required of REQUIRED_CASES[platform]) {
277 if (!planned.includes(required))
278 failures.push(`required case ${required} is not in the suite the page ran`)
279 }
280
281 for (const name of planned) {
282 if (!cases.some(entry => entry.name === name))
283 failures.push(`case ${name} was planned but never reported; the suite stopped early`)
284 }
285
286 for (const entry of cases) {
287 if (entry.status === 'fail')
288 failures.push(`case ${entry.name} failed: ${entry.detail || 'no detail'}`)
289 // Neither a pass nor a fail is not a third outcome to be tolerated. A case
290 // reporting anything else satisfies "was planned and did report" while
291 // never appearing in `passed`, which would leave it unasserted inside a
292 // green run - the precise shape of failure this harness exists to remove.
293 else if (entry.status !== 'pass')
294 failures.push(`case ${entry.name} reported status ${JSON.stringify(entry.status)}, which is neither pass nor fail`)
295 }
296
297 if (!done)
298 failures.push('the suite never reported done')
299 else if (done.failed !== failed.length || done.passed !== passed.length)
300 failures.push(`done says ${done.passed} passed / ${done.failed} failed, transcript shows ${passed.length} / ${failed.length}`)
301
302 return { ok: failures.length === 0, failures, planned, passed, failed }
303}
304
305/**
306 * The actions the Zig dispatcher saw, by name.
307 *
308 * Zig and the platform shim answer the same actions with the same shapes -
309 * that is the point of the hand-off design - so the only way to tell them
310 * apart from outside is what Zig's own logger writes
311 * (`packages/zig/src/ios_dispatch.zig`). With no archive linked, `dlsym`
312 * misses and none of this is emitted at all, which is what separates the two
313 * iOS legs.
314 *
315 * By name rather than by count, because the page routes every report through
316 * `craft.log`, and `log` is itself a dispatched action: a bare count is
317 * satisfied by the reporting traffic alone and would pass while Swift served
318 * every capability under test.
319 */
320export function zigDispatchedActions(text: string): string[] {
321 const plain = text.replace(ANSI, '')
322 const seen = new Set<string>()
323 for (const match of plain.matchAll(/craft-bridge dispatch t=\w+ a=(\w+)/g)) seen.add(match[1]!)
324 return [...seen].sort()
325}
326
327/**
328 * The actions Zig refused through its own capability gate.
329 *
330 * A dispatch line proves Zig was *offered* an action - it is logged on entry,
331 * before the decision. This proves Zig *answered* one: `ios_config`'s gate
332 * writes the line, and the rejection the page then sees came off Zig's error
333 * route rather than Swift's.
334 */
335export function zigRefusals(text: string): string[] {
336 const plain = text.replace(ANSI, '')
337 const seen = new Set<string>()
338 for (const match of plain.matchAll(/ios: refusing (\w+);/g)) seen.add(match[1]!)
339 return [...seen].sort()
340}
341
342/**
343 * The capability actions the page exercises, which the runtime leg expects to
344 * see Zig dispatch.
345 *
346 * Kept in step with the cases in driver.html by hand. Derived from the page it
347 * would have to be parsed out of HTML; named here it is at least checkable,
348 * and a case removed from the page fails the leg loudly rather than quietly
349 * reducing what the assertion covers.
350 */
351export const ZIG_TESTED_ACTIONS = [
352 'clipboardRead',
353 'clipboardWrite',
354 'getCurrentPosition',
355 'getDeviceInfo',
356]
357
358/** The one action the suite configures off, so Zig must refuse it by name. */
359export const ZIG_REFUSED_ACTION = 'getCurrentPosition'
scripts/mobile-e2e/support.tsadded+126-0
Changes to scripts/mobile-e2e/support.ts
@@ -0,0 +1,126 @@
1import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
2import { join } from 'node:path'
3
4const HERE = import.meta.dir
5
6export interface CommandOptions {
7 cwd?: string
8 /** Append the command line and its output here, for the evidence upload. */
9 logPath?: string
10 /** Record the failure and carry on, for teardown steps whose failure is not the story. */
11 allowFailure?: boolean
12 stdin?: string
13 /**
14 * Write raw stdout bytes here instead of returning them as text.
15 *
16 * `adb exec-out screencap -p` emits a PNG; decoding that as UTF-8 to hand it
17 * back as a string is how a screenshot becomes an unopenable file.
18 */
19 outPath?: string
20}
21
22export interface CommandResult {
23 exitCode: number
24 stdout: string
25 stderr: string
26}
27
28/**
29 * Run a command, keep its output, and fail loudly by default.
30 *
31 * Everything a leg does to a device goes through here so that the evidence
32 * upload contains the actual command lines. A failing `xcodebuild` whose log
33 * is three screens of Swift diagnostics is useless as an exception message and
34 * invaluable as a file.
35 */
36export async function command(argv: string[], options: CommandOptions = {}): Promise<CommandResult> {
37 const spawned = Bun.spawn(argv, {
38 cwd: options.cwd,
39 stdin: options.stdin === undefined ? 'ignore' : new TextEncoder().encode(options.stdin),
40 stdout: 'pipe',
41 stderr: 'pipe',
42 })
43
44 const [raw, stderr] = await Promise.all([
45 new Response(spawned.stdout).arrayBuffer(),
46 new Response(spawned.stderr).text(),
47 ])
48 const exitCode = await spawned.exited
49
50 if (options.outPath) writeFileSync(options.outPath, new Uint8Array(raw))
51 const stdout = options.outPath ? '' : new TextDecoder().decode(raw)
52
53 if (options.logPath) {
54 appendFileSync(options.logPath, [
55 `$ ${argv.join(' ')}`,
56 options.cwd ? `# cwd ${options.cwd}` : '',
57 stdout,
58 stderr,
59 `# exit ${exitCode}`,
60 '',
61 ].filter(Boolean).join('\n'))
62 }
63
64 if (exitCode !== 0 && !options.allowFailure) {
65 const tail = `${stdout}\n${stderr}`.trim().split('\n').slice(-25).join('\n')
66 throw new Error(`${argv.join(' ')} exited with ${exitCode}\n${tail}`)
67 }
68
69 return { exitCode, stdout, stderr }
70}
71
72/**
73 * Wait for a file to say what we are waiting for.
74 *
75 * `refresh` exists because the two platforms deliver output differently: iOS
76 * streams the app's stderr straight into the file, so there is nothing to do
77 * between polls, while Android has to be asked - `adb logcat -d` dumps and
78 * exits. Streaming logcat instead would mean killing it at exactly the right
79 * moment, and the dump costs nothing.
80 *
81 * Returns false on the deadline rather than throwing, because a run that
82 * produced no terminating event still has a log worth keeping and a screenshot
83 * worth taking - and the caller does both before it decides anything. Throwing
84 * here would skip the evidence for the failure that most needs it.
85 */
86export async function waitForFile(
87 path: string,
88 timeoutMs: number,
89 done: (text: string) => boolean,
90 refresh?: () => Promise<void>,
91): Promise<boolean> {
92 const deadline = Date.now() + timeoutMs
93
94 while (Date.now() < deadline) {
95 if (refresh) await refresh()
96 if (existsSync(path)) {
97 const text = readFileSync(path, 'utf8')
98 if (done(text)) return true
99 }
100 await Bun.sleep(refresh ? 1000 : 250)
101 }
102
103 return false
104}
105
106/**
107 * The test page, with this run's nonce baked in.
108 *
109 * One page serves both platforms: `window.craft` is injected by each runtime
110 * with the same shape, so the suite that runs on a simulator is the same file
111 * that runs on an emulator, and a case can only diverge where it says it does.
112 */
113export function driverPage(nonce: string, platform: string): string {
114 const template = readFileSync(join(HERE, 'driver.html'), 'utf8')
115
116 // Both placeholders checked, because a silently unsubstituted one turns its
117 // case into an assertion about a constant the page chose for itself.
118 for (const placeholder of ['__CRAFT_E2E_NONCE__', '__CRAFT_E2E_PLATFORM__']) {
119 if (!template.includes(placeholder))
120 throw new Error(`driver.html no longer carries the ${placeholder} placeholder`)
121 }
122
123 return template
124 .replaceAll('__CRAFT_E2E_NONCE__', nonce)
125 .replaceAll('__CRAFT_E2E_PLATFORM__', platform)
126}
scripts/mobile-e2e/types.tsadded+28-0
Changes to scripts/mobile-e2e/types.ts
@@ -0,0 +1,28 @@
1export interface RunnerOptions {
2 /** Repository root, used to shorten evidence paths in the report. */
3 root: string
4 /** artifacts/mobile-e2e/<label>, created before any leg runs. */
5 evidenceDir: string
6 /** Scratch space for generated projects; kept under the evidence dir in CI. */
7 workDir: string
8 /** How long one app may run before its leg is called a failure. */
9 timeoutMs: number
10 /** Distinguishes this run's clipboard nonce from a stale one. */
11 runId: string
12 /** Directory holding the Zig iOS simulator archives. */
13 iosRuntimeDir: string | null
14}
15
16export interface LegOutcome {
17 name: string
18 status: 'passed' | 'failed'
19 /** Empty on a pass. Each entry is one reason, in the words of what was seen. */
20 failures: string[]
21 planned: string[]
22 passed: string[]
23 failed: string[]
24 /** Which actions reached the Zig dispatcher, i.e. which side answered. */
25 zigActions: string[]
26 /** Repo-relative path to this leg's evidence. */
27 evidence: string
28}