also looking at this

test(mobile): run the generated apps on a simulator and an emulator

#199

Merged glennmichael123 wants to merge fix/mobile-e2e-real-assertions into main

26 files +3009 -293

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

.github/workflows/README.mdmodified+45-9

Changes to .github/workflows/README.md
		@@ -1,13 +1,49 @@
1	1	# GitHub Actions
2	2
3		This folder contains the following GitHub Actions:
	3	\| Workflow \| What it proves \|
	4	\|---\|---\|
	5	\| [CI](./ci.yml) \| Lint, typecheck, the Zig and TypeScript suites, and that the generated iOS, Watch, Live Activity and Android projects still compile \|
	6	\| [Mobile E2E Testing](./mobile-e2e.yml) \| That a generated app runs: a call from JavaScript reaches native code and comes back, on a real simulator and a real emulator \|
	7	\| [Native Package Lifecycle](./native-lifecycle.yml) \| Install, update, rollback and uninstall of the packaged app on macOS, Linux and Windows \|
	8	\| [Benchmark](./benchmarks.yml) \| Startup and runtime numbers, tracked over time \|
	9	\| [Binary Size](./binary-size.yml) \| The shipped binary has not grown unexpectedly \|
	10	\| [SBOM](./sbom.yml) \| A software bill of materials for each build \|
	11	\| [Release](./release.yml) \| Tagging, changelog, pantry publish and npm publish \|
4	12
5		- [CI][CI] - all CI jobs for the project
6		- lints the code
7		- `typecheck`s the code
8		- runs test suite
9		- runs on `ubuntu-latest`
10		- [Release][Release] - automates the release process & changelog generation
	13	## Running the mobile E2E locally
11	14
12		[CI]: ./workflows/ci.yml
13		[Release]: ./workflows/release.yml
	15	The harness generates an app, drops a self-driving page into its web assets,
	16	builds it, installs it on a device and reads the page's report back off the
	17	console. It asserts a required list of cases per platform — at least one
	18	success path and one rejection path — and nothing in it skips: a missing
	19	toolchain or an app that printed nothing is a failure with a named cause.
	20
	21	iOS needs Xcode, `xcodegen` (`brew install xcodegen`), and the Zig archives,
	22	because the suite runs twice — once with the Zig runtime linked and once
	23	without. `build-ios-all` rather than `build-ios-simulator`: the generated
	24	project links a device slice too, and project generation refuses a runtime
	25	directory that has none.
	26
	27	```bash
	28	cd packages/zig && zig build build-ios-all -Doptimize=ReleaseSafe && cd -
	29	bun run test:mobile-e2e:ios
	30	```
	31
	32	```
	33	ok ios-shim — 4/4 cases, no zig
	34	ok ios-runtime — 4/4 cases, zig served clipboardRead, clipboardWrite, getCurrentPosition, getDeviceInfo, log
	35	```
	36
	37	Android needs JDK 17, Gradle 8.11.1, an Android SDK with `platforms;android-36`
	38	and `build-tools;36.0.0`, and a booted emulator or attached device:
	39
	40	```bash
	41	bun run test:mobile-e2e:android
	42	```
	43
	44	Both write everything they saw — console logs, build logs, a screenshot, the
	45	exact page that ran and a `report.json` — to
	46	`artifacts/mobile-e2e/<label>/`, which is what CI uploads.
	47
	48	The cases themselves are in [`scripts/mobile-e2e/driver.html`](../../scripts/mobile-e2e/driver.html);
	49	which of them are mandatory is in [`scripts/mobile-e2e/protocol.ts`](../../scripts/mobile-e2e/protocol.ts).

.github/workflows/mobile-e2e.ymlmodified+260-183

Changes to .github/workflows/mobile-e2e.yml
		@@ -1,193 +1,208 @@
1	1	name: Mobile E2E Testing
2	2
3		# Manual only. This workflow has been red since May and cannot go green from
4		# a workflow edit: the iOS Zig sources stopped compiling in the 0.17 migration
5		# (1b6f285) and need real source work — see the notes in the PR that made this
6		# manual. It also runs no end-to-end tests today; every step that would has an
7		# `if: hashFiles(...)` guard pointing at a TestApp/project that is not in the
8		# tree, so they all skip. Firing it on push meant a red X on unrelated commits
9		# for a signal nobody could act on, and two of its path filters
10		# (packages/zig/src/ios/, packages/zig/src/android/) match directories that
11		# do not exist at all.
	3	# Runs a generated Craft app on a real iOS simulator and a real Android
	4	# emulator and asserts that a call made from JavaScript reached native code and
	5	# came back.
	6	#
	7	# The version this replaces asserted nothing. Every iOS step that would have
	8	# tested something was guarded by
	9	# `if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''` against a
	10	# project that has never existed in the tree, and the Android job's only test
	11	# invocation was gated on `packages/android/gradlew`, which the generator does
	12	# not write. Both skipped on every run, and the job reported success. Its
	13	# header also blamed the iOS Zig sources for not compiling since the 0.17
	14	# migration (1b6f285); that is no longer true —
	15	# `zig build build-ios-simulator` succeeds at this commit, and the iOS job
	16	# below depends on it.
	17	#
	18	# What the harness asserts, per platform, is in scripts/mobile-e2e/protocol.ts:
	19	# a required list of cases that must all run and all pass, covering at least
	20	# one success path and one rejection path. Nothing here skips. A missing
	21	# toolchain, an absent device, or an app that printed nothing is a failure with
	22	# a named cause.
	23
	24	# GitHub Actions does not resolve YAML anchors, so the two path filters are
	25	# written out rather than shared. Keep them identical.
12	26	on:
	27	push:
	28	branches: [main]
	29	paths:
	30	- packages/ios/**
	31	- packages/android/**
	32	# The mobile Zig sources are flat files under src/, not src/ios/ and
	33	# src/android/ as the previous filters claimed — those two directories
	34	# have never existed, so the filter matched nothing.
	35	- packages/zig/src/ios*.zig
	36	- packages/zig/src/android*.zig
	37	- packages/zig/src/mobile*.zig
	38	- packages/zig/src/bridge_mobile_*.zig
	39	- packages/zig/src/bridge_android_*.zig
	40	- packages/zig/src/jni*.zig
	41	- packages/zig/build.zig
	42	- scripts/mobile-e2e.ts
	43	- scripts/mobile-e2e/**
	44	- .github/workflows/mobile-e2e.yml
	45	pull_request:
	46	paths:
	47	- packages/ios/**
	48	- packages/android/**
	49	- packages/zig/src/ios*.zig
	50	- packages/zig/src/android*.zig
	51	- packages/zig/src/mobile*.zig
	52	- packages/zig/src/bridge_mobile_*.zig
	53	- packages/zig/src/bridge_android_*.zig
	54	- packages/zig/src/jni*.zig
	55	- packages/zig/build.zig
	56	- scripts/mobile-e2e.ts
	57	- scripts/mobile-e2e/**
	58	- .github/workflows/mobile-e2e.yml
13	59	workflow_dispatch:
14	60
15	61	concurrency:
16	62	group: mobile-e2e-${{ github.head_ref \|\| github.ref }}
17	63	cancel-in-progress: true
18	64
	65	permissions:
	66	contents: read
	67
19	68	defaults:
20	69	run:
21	70	shell: bash
22	71
23	72	jobs:
24		ios-simulator:
25		# name: iOS Simulator Tests
26		runs-on: macos-14 # M1 runner for better iOS simulator performance
	73	# The transcript reader and the required-case list are the part of this
	74	# harness that can be tested without a device, and nothing ran those tests:
	75	# every `bun test` in this repository is scoped to a package directory or to
	76	# one explicit file, and the root `test` script is the Zig suite. So the
	77	# ratchet that stops the suite being hollowed out was itself unenforced.
	78	harness:
	79	name: Harness unit tests
	80	runs-on: ubuntu-latest
	81	timeout-minutes: 10
27	82
28	83	steps:
29	84	- uses: actions/checkout@v6
30	85
31		- name: Setup Zig
	86	- name: Setup Pantry (provides Bun, Zig, etc.)
32	87	uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
33		- name: Setup Xcode
34		uses: maxim-lobanov/setup-xcode@v1
35		with:
36		xcode-version: latest-stable
37	88
38		- name: Cache Zig artifacts
39		uses: actions/cache@v5
40		with:
41		path: \|
42		~/.cache/zig
43		packages/zig/.zig-cache
44		key: ios-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }}
	89	- name: Install dependencies
	90	run: bun install --frozen-lockfile
45	91
46		- name: List available simulators
47		run: \|
48		xcrun simctl list devices available
	92	# The explicit path is load-bearing: a bare filter makes bun walk the
	93	# repository and run out of file descriptors under the hoisted linker.
	94	# See the note in native-lifecycle.yml.
	95	- name: Test the report protocol
	96	run: bun test ./scripts/mobile-e2e/protocol.test.ts
49	97
50		- name: Boot iOS Simulator
51		run: \|
52		# Find an iPhone 15 simulator
53		DEVICE_ID=$(xcrun simctl list devices available \| grep "iPhone 15" \| head -1 \| grep -oE '[0-9A-F-]{36}')
	98	ios-simulator:
	99	name: iOS simulator
	100	runs-on: macos-15
	101	# A hung simulator launch would otherwise burn the six-hour default. The
	102	# harness has its own per-app timeout; this is the backstop for everything
	103	# around it.
	104	timeout-minutes: 45
	105	env:
	106	CRAFT_EVIDENCE_LABEL: ios-simulator
54	107
55		if [ -z "$DEVICE_ID" ]; then
56		# Create one if not found
57		DEVICE_ID=$(xcrun simctl create "iPhone 15 Test" "iPhone 15" "iOS17.0")
58		fi
	108	steps:
	109	- uses: actions/checkout@v6
59	110
60		echo "DEVICE_ID=$DEVICE_ID" >> $GITHUB_ENV
	111	- name: Setup Pantry (provides Bun, Zig, etc.)
	112	uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
61	113
62		# Boot the simulator
63		xcrun simctl boot "$DEVICE_ID" \|\| true
	114	- name: Install XcodeGen
	115	run: brew install xcodegen
64	116
65		# Wait for simulator to finish booting. `bootstatus -b` blocks until
66		# the device reaches a Booted state, eliminating the flaky fixed sleep.
67		xcrun simctl bootstatus "$DEVICE_ID" -b
	117	- name: Install dependencies
	118	run: bun install --frozen-lockfile
68	119
69		- name: First-party Zig dependencies
70		uses: ./.github/actions/first-party-zig-deps
71
72		- name: Build iOS library
73		working-directory: packages/zig
74		run: \|
75		# Build iOS simulator static libraries using dedicated build step
76		# (not the default target which builds macOS executables)
77		zig build build-ios-simulator -Doptimize=ReleaseSafe -Dmacos-sdk="$(xcrun --sdk iphonesimulator --show-sdk-path)"
	120	- name: Cache Zig artifacts
	121	uses: actions/cache@v5
	122	with:
	123	path: \|
	124	~/.cache/zig
	125	packages/zig/.zig-cache
	126	key: mobile-e2e-ios-${{ runner.os }}-${{ hashFiles('packages/zig/build.zig') }}
78	127
79	128	- name: First-party Zig dependencies
80	129	uses: ./.github/actions/first-party-zig-deps
81	130
82		- name: Run iOS unit tests
	131	# build-ios-all, not build-ios-simulator: installRuntime() resolves every
	132	# SDK in RUNTIME_ARCHIVES before it writes anything and throws when one
	133	# has no archive at all, so a zig-out holding only the simulator slices
	134	# fails at project generation. It passes locally only because a device
	135	# archive is usually already sitting there from an earlier build.
	136	#
	137	# No -Dmacos-sdk: the iOS steps resolve their SDK through xcrun at
	138	# configure time (build.zig's iosSdkPath), and the option is read only by
	139	# desktop and host-test artifacts, so passing it was a no-op.
	140	- name: Build the Zig iOS runtime
83	141	working-directory: packages/zig
84		run: \|
85		# `zig build test` reports through its exit code. The old form piped
86		# it to a file and grepped for "mobile"/"ios", which fails under
87		# `set -e` whenever the run is clean and the file is therefore empty.
88		zig build test
89
90		- name: Build iOS test app
91		if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''
92		run: \|
93		cd packages/ios/TestApp
94		xcodebuild build-for-testing \
95		-scheme TestApp \
96		-destination "platform=iOS Simulator,id=$DEVICE_ID" \
97		-configuration Debug \
98		CODE_SIGN_IDENTITY="" \
99		CODE_SIGNING_REQUIRED=NO
100
101		- name: Run iOS E2E tests
102		if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''
103		run: \|
104		cd packages/ios/TestApp
105		xcodebuild test-without-building \
106		-scheme TestApp \
107		-destination "platform=iOS Simulator,id=$DEVICE_ID" \
108		-configuration Debug
109
110		- name: Capture iOS screenshots
111		run: \|
112		mkdir -p screenshots/ios
113
114		# Take a screenshot of the simulator
115		xcrun simctl io "$DEVICE_ID" screenshot screenshots/ios/simulator.png \|\| true
116
117		- name: Shutdown simulator
118		if: always()
119		run: \|
120		xcrun simctl shutdown "$DEVICE_ID" \|\| true
121
122		- name: Upload iOS screenshots
	142	run: zig build build-ios-all -Doptimize=ReleaseSafe
	143
	144	- name: Run mobile E2E on the simulator
	145	id: harness
	146	run: bun scripts/mobile-e2e.ts --platform ios --ios-runtime packages/zig/zig-out/lib
	147
	148	- name: Upload iOS evidence
	149	# Not a bare `always()`: when an earlier step fails the harness never
	150	# runs, the evidence directory is never created, and `if-no-files-found:
	151	# error` would then add a "No files were found" error on top of the real
	152	# one. Skipped-means-skipped; ran-and-produced-nothing still fails.
	153	if: always() && steps.harness.conclusion != 'skipped'
123	154	uses: actions/upload-artifact@v4
124	155	with:
125		name: ios-screenshots
126		path: screenshots/ios/
127		retention-days: 7
128
129		- name: Generate iOS test report
130		run: \|
131		echo "## iOS Simulator Tests" >> $GITHUB_STEP_SUMMARY
132		echo "" >> $GITHUB_STEP_SUMMARY
133		echo "- Simulator: iPhone 15" >> $GITHUB_STEP_SUMMARY
134		echo "- iOS Version: 17.0" >> $GITHUB_STEP_SUMMARY
135		echo "- Status: Completed" >> $GITHUB_STEP_SUMMARY
	156	name: mobile-e2e-ios
	157	# Console logs, build logs, screenshots, the generated project.yml and
	158	# the exact page that ran. Excludes work/ — the generated projects and
	159	# their DerivedData are hundreds of megabytes and the interesting
	160	# parts are copied out.
	161	path: \|
	162	artifacts/mobile-e2e/ios-simulator/**
	163	!artifacts/mobile-e2e/ios-simulator/work/**
	164	# A run that produced no evidence is a broken harness, not a pass.
	165	if-no-files-found: error
	166	retention-days: 30
136	167
137	168	android-emulator:
138		# name: Android Emulator Tests
	169	name: Android emulator
139	170	runs-on: ubuntu-latest
	171	timeout-minutes: 60
	172	env:
	173	CRAFT_EVIDENCE_LABEL: android-emulator
140	174
141	175	steps:
142	176	- uses: actions/checkout@v6
143	177
144		- name: Setup Zig
	178	- name: Setup Pantry (provides Bun, Zig, etc.)
145	179	uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
	180
146	181	- name: Setup Java
147		uses: actions/setup-java@v4
	182	uses: actions/setup-java@v6
148	183	with:
149		distribution: 'temurin'
	184	distribution: temurin
150	185	java-version: '17'
151	186
152	187	- name: Setup Android SDK
153		uses: android-actions/setup-android@v3
154
155		- name: Cache Zig artifacts
156		uses: actions/cache@v5
	188	uses: android-actions/setup-android@v4
157	189	with:
158		path: \|
159		~/.cache/zig
160		packages/zig/.zig-cache
161		key: android-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }}
162
163		- name: Install Android NDK
164		run: \|
165		sdkmanager --install "ndk;26.1.10909125"
166		echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/26.1.10909125" >> $GITHUB_ENV
	190	# The action defaults to `tools platform-tools`, but the obsolete
	191	# `tools` package is no longer offered by the Android SDK repository.
	192	packages: platform-tools
167	193
168		- name: First-party Zig dependencies
169		uses: ./.github/actions/first-party-zig-deps
	194	# Same platform and build-tools as ci.yml's android-builder, so the
	195	# emulator job and the compile gate cannot drift onto different SDKs.
	196	- name: Install Android platform
	197	run: sdkmanager "platforms;android-36" "build-tools;36.0.0"
170	198
171		- name: Build Android library
172		working-directory: packages/zig
173		run: \|
174		# Use the dedicated android step, which resolves its own target.
175		# Passing -Dtarget steers the default install step instead, which
176		# builds the desktop binaries — and since an android triple reports
177		# os_tag == .linux, that links GTK3/WebKit2GTK against a sysroot that
178		# does not exist for Android.
179		zig build build-android -Doptimize=ReleaseSafe
	199	- name: Setup Gradle
	200	uses: gradle/actions/setup-gradle@v6
	201	with:
	202	gradle-version: '8.11.1'
180	203
181		- name: First-party Zig dependencies
182		uses: ./.github/actions/first-party-zig-deps
183
184		- name: Run Android unit tests
185		working-directory: packages/zig
186		run: \|
187		# `zig build test` reports through its exit code. The old form piped
188		# it to a file and grepped for "mobile"/"android", which fails under
189		# `set -e` whenever the run is clean and the file is therefore empty.
190		zig build test
	204	- name: Install dependencies
	205	run: bun install --frozen-lockfile
191	206
192	207	- name: Enable KVM
193	208	run: \|
		@@ -202,9 +217,17 @@ jobs:
202	217	path: \|
203	218	~/.android/avd/*
204	219	~/.android/adb*
205		key: avd-api-34-${{ runner.os }}
206
207		- name: Create AVD and run tests
	220	# The device, not just the API level. With force-avd-creation off the
	221	# action reuses whatever the cache restored, so a key that ignores
	222	# arch and profile would keep testing the old device after someone
	223	# changed them.
	224	key: avd-api-34-x86_64-pixel_6-${{ runner.os }}
	225
	226	# `script:` is the only place a command can run with the emulator
	227	# attached, so the harness is invoked from inside it rather than as its
	228	# own step.
	229	- name: Run mobile E2E on the emulator
	230	id: harness
208	231	uses: reactivecircus/android-emulator-runner@v2
209	232	with:
210	233	api-level: 34
		@@ -214,56 +237,110 @@ jobs:
214	237	emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
215	238	disable-animations: true
216	239	script: \|
217		echo "Emulator started"
218	240	adb wait-for-device
219		mkdir -p screenshots/android
220		adb exec-out screencap -p > screenshots/android/emulator.png \|\| true
221		if [ -x "packages/android/gradlew" ]; then
222		cd packages/android
223		./gradlew test
224		else
225		echo "Android Gradle wrapper not present; skipping project-level tests"
226		fi
227
228		- name: Upload Android screenshots
	241	bun scripts/mobile-e2e.ts --platform android
	242
	243	- name: Upload Android evidence
	244	if: always() && steps.harness.conclusion != 'skipped'
229	245	uses: actions/upload-artifact@v4
230	246	with:
231		name: android-screenshots
232		path: screenshots/android/
233		retention-days: 7
	247	name: mobile-e2e-android
	248	path: \|
	249	artifacts/mobile-e2e/android-emulator/**
	250	!artifacts/mobile-e2e/android-emulator/work/**
	251	if-no-files-found: error
	252	retention-days: 30
234	253
235		- name: Generate Android test report
	254	# The Zig runtime's own simulator fixture: a hand-linked four-file app that
	255	# drives libcraft-ios-simulator directly, with assertions the generated-app
	256	# suite cannot make — the host hand-off, the error route's escaping, a
	257	# Keychain round trip, in-process SQLite, and a sensor the simulator does not
	258	# have refusing rather than fabricating a stream. It has been in the tree,
	259	# passing, and wired into no workflow.
	260	ios-zig-slice:
	261	name: iOS Zig runtime slice
	262	runs-on: macos-15
	263	timeout-minutes: 45
	264
	265	steps:
	266	- uses: actions/checkout@v6
	267
	268	- name: Setup Pantry (provides Bun, Zig, etc.)
	269	uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
	270
	271	- name: Cache Zig artifacts
	272	uses: actions/cache@v5
	273	with:
	274	path: \|
	275	~/.cache/zig
	276	packages/zig/.zig-cache
	277	key: mobile-e2e-slice-${{ runner.os }}-${{ hashFiles('packages/zig/build.zig') }}
	278
	279	- name: First-party Zig dependencies
	280	uses: ./.github/actions/first-party-zig-deps
	281
	282	# The script defaults CRAFT_ZIG to a pinned toolchain under
	283	# ~/.cache/craft-ci-local, which exists on the author's machine and not on
	284	# a runner. Point it at the one pantry provides.
	285	- name: Exercise the Zig runtime on a simulator
	286	id: harness
	287	# The fixture polls for its late markers against a wall-clock deadline.
	288	# A runner is several times slower than a laptop here — the first CI run
	289	# of this job reached marker 36 of 42 before the old iteration-count
	290	# budget ran out — so give it room. It exits as soon as the markers
	291	# arrive; the budget only matters when something is genuinely stuck.
	292	env:
	293	CRAFT_SLICE_TIMEOUT: '900'
236	294	run: \|
237		echo "## Android Emulator Tests" >> $GITHUB_STEP_SUMMARY
238		echo "" >> $GITHUB_STEP_SUMMARY
239		echo "- Emulator: Pixel 6" >> $GITHUB_STEP_SUMMARY
240		echo "- API Level: 34" >> $GITHUB_STEP_SUMMARY
241		echo "- Status: Completed" >> $GITHUB_STEP_SUMMARY
	295	CRAFT_ZIG="$(command -v zig)" \
	296	packages/ios/fixtures/zig-slice/build-and-run.sh \
	297	"$PWD/artifacts/mobile-e2e/ios-zig-slice"
	298
	299	- name: Upload slice evidence
	300	if: always() && steps.harness.conclusion != 'skipped'
	301	uses: actions/upload-artifact@v4
	302	with:
	303	name: mobile-e2e-ios-zig-slice
	304	path: \|
	305	artifacts/mobile-e2e/ios-zig-slice/console.log
	306	artifacts/mobile-e2e/ios-zig-slice/console.plain
	307	if-no-files-found: error
	308	retention-days: 30
242	309
243	310	mobile-summary:
244		# name: Mobile Test Summary
245		needs: [ios-simulator, android-emulator]
	311	name: Mobile E2E summary
	312	needs: [harness, ios-simulator, android-emulator, ios-zig-slice]
246	313	runs-on: ubuntu-latest
247	314	if: always()
248	315
249	316	steps:
250		- name: Check test results
	317	- name: Report
251	318	run: \|
252		echo "## Mobile E2E Test Summary" >> $GITHUB_STEP_SUMMARY
253		echo "" >> $GITHUB_STEP_SUMMARY
254		echo "\| Platform \| Status \|" >> $GITHUB_STEP_SUMMARY
255		echo "\|----------\|--------\|" >> $GITHUB_STEP_SUMMARY
256		echo "\| iOS Simulator \| $ \|" >> $GITHUB_STEP_SUMMARY
257		echo "\| Android Emulator \| $ \|" >> $GITHUB_STEP_SUMMARY
258
259		- name: Download all screenshots
260		uses: actions/download-artifact@v4
261		with:
262		path: all-screenshots
263
264		- name: Upload combined screenshots
265		uses: actions/upload-artifact@v4
266		with:
267		name: mobile-screenshots
268		path: all-screenshots/
269		retention-days: 30
	319	{
	320	echo "## Mobile E2E"
	321	echo
	322	echo "\| Leg \| Result \|"
	323	echo "\|-----\|--------\|"
	324	echo "\| Harness unit tests \| $ \|"
	325	echo "\| iOS simulator (generated app) \| $ \|"
	326	echo "\| Android emulator (generated app) \| $ \|"
	327	echo "\| iOS Zig runtime slice \| $ \|"
	328	} >> "$GITHUB_STEP_SUMMARY"
	329
	330	# The summary job used to be the whole story, and it reported whatever it
	331	# was given without ever failing. `needs` does not fail a job on a failed
	332	# dependency when `if: always()` is set, so say it explicitly.
	333	#
	334	# `failure` and `skipped`, not `!= success`: with cancel-in-progress on,
	335	# pushing twice in quick succession cancels the first run, and treating
	336	# `cancelled` as a failure would put a red X on it — a signal about
	337	# nothing, which is how the workflow this replaces earned its reputation.
	338	- name: Fail if any leg failed
	339	if: >-
	340	contains(fromJSON('["failure", "skipped"]'), needs.harness.result)
	341	\|\| contains(fromJSON('["failure", "skipped"]'), needs.ios-simulator.result)
	342	\|\| contains(fromJSON('["failure", "skipped"]'), needs.android-emulator.result)
	343	\|\| contains(fromJSON('["failure", "skipped"]'), needs.ios-zig-slice.result)
	344	run: \|
	345	echo "::error::at least one mobile E2E leg did not succeed"
	346	exit 1

package.jsonmodified+2-0

Changes to package.json
		@@ -35,6 +35,8 @@
35	35	"test": "command -v zig >/dev/null 2>&1 \|\| eval \"$(pantry env \| sed -n '/^export /,$p')\"; cd packages/zig && zig build test",
36	36	"test:sdk": "cd packages/typescript && bun test",
37	37	"test:native-lifecycle": "bun scripts/native-lifecycle.ts",
	38	"test:mobile-e2e:ios": "bun scripts/mobile-e2e.ts --platform ios --ios-runtime packages/zig/zig-out/lib",
	39	"test:mobile-e2e:android": "bun scripts/mobile-e2e.ts --platform android",
38	40	"typecheck": "cd packages/typescript && bun run typecheck && cd ../create-craft && bun run typecheck && cd ../ts-maps && bun run typecheck",
39	41	"run": "command -v zig >/dev/null 2>&1 \|\| eval \"$(pantry env \| sed -n '/^export /,$p')\"; cd packages/zig && zig build run",
40	42	"run:demo": "command -v zig >/dev/null 2>&1 \|\| eval \"$(pantry env \| sed -n '/^export /,$p')\"; cd packages/zig && zig build run-demo",

scripts/mobile-e2e.tsadded+125-0

Changes to scripts/mobile-e2e.ts
		@@ -0,0 +1,125 @@
	1	#!/usr/bin/env bun
	2
	3	/**
	4	* Mobile end-to-end: generate an app, run it on a simulator or emulator, and
	5	* assert that a call made from JavaScript reached native code and came back.
	6	*
	7	* What this replaces ran nothing. Every step in the old mobile-e2e workflow
	8	* that would have tested something was guarded by
	9	* `if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''`, and that
	10	* project has never existed in the tree, so each one skipped and the job went
	11	* green. The Android half looked for `packages/android/gradlew`, which the
	12	* generator does not write either, and echoed "skipping project-level tests".
	13	* Months of green with no coverage behind it.
	14	*
	15	* So the rules here are the opposite ones. Nothing skips: a missing toolchain,
	16	* an absent device, an app that never printed anything are all failures with a
	17	* named cause. The page announces which cases it is going to run before it
	18	* runs them, so a suite that dies halfway cannot be read as a shorter suite
	19	* that passed. And every leg writes its console log, its build log and a
	20	* screenshot under artifacts/mobile-e2e/, uploaded with
	21	* `if-no-files-found: error` so a run that produced no evidence is red rather
	22	* than quietly empty.
	23	*
	24	* bun scripts/mobile-e2e.ts --platform ios --ios-runtime packages/zig/zig-out/lib
	25	* bun scripts/mobile-e2e.ts --platform android
	26	*/
	27
	28	import type { LegOutcome, RunnerOptions } from './mobile-e2e/types'
	29	import { mkdirSync, rmSync, writeFileSync } from 'node:fs'
	30	import { join, resolve } from 'node:path'
	31	import { runAndroid } from './mobile-e2e/android'
	32	import { runIos } from './mobile-e2e/ios'
	33
	34	const root = resolve(import.meta.dir, '..')
	35
	36	function flag(name: string): string \| undefined {
	37	const prefixed = `--${name}`
	38	const index = process.argv.indexOf(prefixed)
	39	if (index !== -1 && index + 1 < process.argv.length) return process.argv[index + 1]
	40
	41	const inline = process.argv.find(argument => argument.startsWith(`${prefixed}=`))
	42	return inline?.slice(prefixed.length + 1)
	43	}
	44
	45	const platform = flag('platform')
	46	if (platform !== 'ios' && platform !== 'android')
	47	throw new Error('pass --platform ios or --platform android')
	48
	49	const evidenceLabel = process.env.CRAFT_EVIDENCE_LABEL \|\| `${platform}-${process.arch}`
	50	const evidenceDir = join(root, 'artifacts', 'mobile-e2e', evidenceLabel)
	51	const workDir = join(evidenceDir, 'work')
	52
	53	// A single run id, reused as the clipboard nonce suffix. Deliberately not a
	54	// timestamp alone: two legs run minutes apart and each needs its own value, so
	55	// the leg name is mixed in downstream.
	56	const runId = process.env.GITHUB_RUN_ID \|\| String(process.pid)
	57
	58	const options: RunnerOptions = {
	59	root,
	60	evidenceDir,
	61	workDir,
	62	timeoutMs: Number(flag('timeout') ?? 180_000),
	63	runId,
	64	iosRuntimeDir: flag('ios-runtime')
	65	? resolve(root, flag('ios-runtime')!)
	66	: (process.env.CRAFT_IOS_RUNTIME ? resolve(root, process.env.CRAFT_IOS_RUNTIME) : null),
	67	}
	68
	69	async function main(): Promise<void> {
	70	rmSync(evidenceDir, { force: true, recursive: true })
	71	mkdirSync(workDir, { recursive: true })
	72
	73	let outcomes: LegOutcome[] = []
	74	let error: string \| undefined
	75
	76	try {
	77	outcomes = platform === 'ios' ? await runIos(options) : await runAndroid(options)
	78	}
	79	catch (caught) {
	80	error = caught instanceof Error ? caught.stack \|\| caught.message : String(caught)
	81	}
	82
	83	// A run with no legs is the shape the old workflow had, and it must not read
	84	// as success. Named here rather than left to the caller, because "zero
	85	// failures" is true of an empty list.
	86	if (!error && outcomes.length === 0)
	87	error = 'no legs ran; the harness produced no result at all'
	88
	89	const failedLegs = outcomes.filter(leg => leg.status === 'failed')
	90	if (!error && failedLegs.length)
	91	error = failedLegs.map(leg => `${leg.name}: ${leg.failures.join('; ')}`).join('\n')
	92
	93	const report = {
	94	schemaVersion: 1,
	95	generatedAt: new Date().toISOString(),
	96	revision: process.env.GITHUB_SHA \|\| 'local',
	97	platform,
	98	label: evidenceLabel,
	99	runner: { os: process.platform, arch: process.arch, bun: Bun.version },
	100	status: error ? 'failed' : 'passed',
	101	legs: outcomes,
	102	error,
	103	}
	104
	105	mkdirSync(evidenceDir, { recursive: true })
	106	writeFileSync(join(evidenceDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`)
	107
	108	for (const leg of outcomes) {
	109	const cases = `${leg.passed.length}/${leg.planned.length} cases`
	110	const zig = leg.zigActions.length ? `zig served ${leg.zigActions.join(', ')}` : 'no zig'
	111	console.log(`${leg.status === 'passed' ? 'ok ' : 'FAIL'} ${leg.name} — ${cases}, ${zig}`)
	112	for (const failure of leg.failures) console.log(` ${failure}`)
	113	}
	114
	115	if (error) {
	116	// The annotation is what makes the reason visible on the run summary
	117	// rather than only inside a step log the reader has to expand.
	118	console.log(`::error::mobile e2e (${evidenceLabel}) failed: ${error.split('\n')[0]}`)
	119	throw new Error(error)
	120	}
	121
	122	console.log(`\nmobile e2e passed on ${platform}; evidence in artifacts/mobile-e2e/${evidenceLabel}`)
	123	}
	124
	125	await main()

scripts/mobile-e2e/android.tsadded+191-0

Changes to scripts/mobile-e2e/android.ts
		@@ -0,0 +1,191 @@
	1	import type { LegOutcome, RunnerOptions } from './types'
	2	import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
	3	import { join } from 'node:path'
	4	import { init } from '../../packages/android/src/index'
	5	import { evaluateRun, hasTerminated } from './protocol'
	6	import { command, driverPage, waitForFile } from './support'
	7
	8	/**
	9	* Push off, which is what makes the rejection case deterministic: with it
	10	* disabled the generator writes a literal `_craftPushReject('Push
	11	* notifications are disabled')` into CraftBridge.kt rather than a Firebase
	12	* block, so the refusal is fixed at generation time and needs no network, no
	13	* Play Services and no account on the emulator.
	14	*
	15	* Clipboard is not a configurable capability on Android - CraftBridge serves
	16	* clipboardRead/clipboardWrite unconditionally - so the success case needs no
	17	* flag, and asking for one would imply a gate that is not there.
	18	*/
	19	const CONFIG = {
	20	enablePushNotifications: false,
	21	}
	22
	23	const PACKAGE = 'dev.craft.e2e.probe'
	24	const APP_NAME = 'CraftE2EProbe'
	25
	26	/** Both tags matter: the bridge's own, and the WebView's for a page that never reached the bridge. */
	27	const LOGCAT_FILTER = ['CraftBridge:D', 'chromium:I', 'AndroidRuntime:E', '*:S']
	28
	29	async function adb(argv: string[], options: { serial: string, logPath?: string, allowFailure?: boolean }) {
	30	return command(['adb', '-s', options.serial, ...argv], { logPath: options.logPath, allowFailure: options.allowFailure })
	31	}
	32
	33	async function resolveSerial(): Promise<string> {
	34	const listed = await command(['adb', 'devices'])
	35	const serials = listed.stdout
	36	.split('\n')
	37	.slice(1)
	38	.map(line => line.trim().split(/\s+/))
	39	.filter(parts => parts.length >= 2 && parts[1] === 'device')
	40	.map(parts => parts[0]!)
	41
	42	if (!serials.length)
	43	throw new Error(`no Android device is attached; \`adb devices\` reported:\n${listed.stdout.trim()}`)
	44
	45	return serials[0]!
	46	}
	47
	48	async function runLeg(options: RunnerOptions): Promise<LegOutcome> {
	49	const label = 'android-shim'
	50	const evidence = join(options.evidenceDir, label)
	51	const project = join(options.workDir, label)
	52	const failures: string[] = []
	53
	54	mkdirSync(evidence, { recursive: true })
	55	rmSync(project, { force: true, recursive: true })
	56
	57	await init({
	58	name: APP_NAME,
	59	packageName: PACKAGE,
	60	output: project,
	61	config: CONFIG,
	62	})
	63
	64	const nonce = `craft-e2e-${label}-${options.runId}`
	65	const assets = join(project, 'app', 'src', 'main', 'assets')
	66	writeFileSync(join(assets, 'index.html'), driverPage(nonce, 'android'))
	67	copyFileSync(join(assets, 'index.html'), join(evidence, 'index.html'))
	68	copyFileSync(join(assets, 'craft.config.json'), join(evidence, 'craft.config.json'))
	69
	70	// The generator writes gradle-wrapper.properties but never gradlew, so there
	71	// is no wrapper to invoke. Call Gradle directly, the way the templates gate
	72	// already does, and honour the same override so both use one toolchain.
	73	const gradle = process.env.GRADLE_EXECUTABLE \|\| 'gradle'
	74	await command([gradle, '--no-daemon', '--stacktrace', ':app:assembleDebug'], {
	75	cwd: project,
	76	logPath: join(evidence, 'gradle.log'),
	77	})
	78
	79	const apk = join(project, 'app', 'build', 'outputs', 'apk', 'debug', 'app-debug.apk')
	80	if (!existsSync(apk))
	81	throw new Error(`gradle reported success but ${apk} is missing`)
	82
	83	// Which side is about to answer, established from the APK rather than
	84	// assumed. Nothing in the generator copies libcraft.so into jniLibs (#200),
	85	// so `System.loadLibrary("craft")` throws, `CraftNative.isAvailable` is
	86	// false, and Kotlin serves every action. The leg reports that as fact, so it
	87	// checks it: when #200 lands this assertion fails, and whoever fixes it has
	88	// to come back and teach this leg to assert the Zig path instead of
	89	// declaring there isn't one.
	90	const listed = await command(['unzip', '-l', apk], { logPath: join(evidence, 'apk.txt') })
	91	const zigLibraries = listed.stdout
	92	.split('\n')
	93	.map(line => line.trim().split(/\s+/).pop() ?? '')
	94	.filter(name => /^lib\/[^/]+\/libcraft\.so$/.test(name))
	95
	96	if (zigLibraries.length) {
	97	failures.push(
	98	`the APK ships ${zigLibraries.join(', ')}, so the Zig runtime may be serving these calls. `
	99	+ 'This leg reports zigActions: [] on the basis that it never loads — teach it to assert which side answered before trusting that again.',
	100	)
	101	}
	102
	103	const serial = await resolveSerial()
	104	writeFileSync(join(evidence, 'device.txt'), `${serial}\n`)
	105
	106	// Ground truth from outside the app, to check getDeviceInfo against.
	107	const sdk = (await adb(['shell', 'getprop', 'ro.build.version.sdk'], { serial })).stdout.trim()
	108
	109	await adb(['uninstall', PACKAGE], { serial, allowFailure: true })
	110	await adb(['install', '-r', apk], { serial, logPath: join(evidence, 'adb.log') })
	111
	112	// Not best-effort: the ring buffer outlives an uninstall, and a transcript
	113	// left in it would otherwise be read as this run's. The plan event carries
	114	// this run's nonce as a second guard.
	115	await adb(['logcat', '-c'], { serial })
	116	await adb(['shell', 'am', 'force-stop', PACKAGE], { serial, allowFailure: true })
	117	await adb(['shell', 'am', 'start', '-n', `${PACKAGE}/.MainActivity`], { serial, logPath: join(evidence, 'adb.log') })
	118
	119	// logcat -d dumps and exits, so poll it into the evidence file rather than
	120	// streaming: a stream would have to be killed at exactly the right moment,
	121	// and the dump is cheap.
	122	const logPath = join(evidence, 'logcat.txt')
	123	const finished = await waitForFile(logPath, options.timeoutMs, hasTerminated, async () => {
	124	const dumped = await command(['adb', '-s', serial, 'logcat', '-d', ...LOGCAT_FILTER], { allowFailure: true })
	125	writeFileSync(logPath, dumped.stdout)
	126	})
	127
	128	await command(['adb', '-s', serial, 'exec-out', 'screencap', '-p'], { allowFailure: true, outPath: join(evidence, 'screen.png') })
	129	const full = await command(['adb', '-s', serial, 'logcat', '-d', '-b', 'all'], { allowFailure: true })
	130	writeFileSync(join(evidence, 'logcat-full.txt'), full.stdout)
	131	await adb(['shell', 'am', 'force-stop', PACKAGE], { serial, allowFailure: true })
	132
	133	const logText = existsSync(logPath) ? readFileSync(logPath, 'utf8') : ''
	134	if (!finished)
	135	failures.push(`the app produced no terminating event within ${options.timeoutMs}ms; see ${label}/logcat.txt`)
	136
	137	const verdict = evaluateRun('android', logText, nonce)
	138	failures.push(...verdict.failures)
	139
	140	// The page reported the SDK level the bridge told it; the host asked the
	141	// device directly. Agreement means the value crossed the bridge rather than
	142	// being invented in JavaScript - this leg's substitute for the pasteboard
	143	// read the iOS leg does, since Android has no equally cheap host-side
	144	// clipboard read.
	145	const observed = logText.match(/"event":"observed","name":"sdkVersion","value":(\d+)/)
	146	if (!sdk) {
	147	// Not skipped when the probe comes back empty. An unverifiable check that
	148	// quietly passes is worse than one that fails: it reads as corroboration
	149	// and is not.
	150	failures.push('`adb shell getprop ro.build.version.sdk` returned nothing, so the page\'s sdkVersion could not be corroborated')
	151	}
	152	else if (!observed) {
	153	failures.push('the page never reported the sdkVersion it got from the bridge')
	154	}
	155	else if (observed[1] !== sdk) {
	156	failures.push(`the bridge reported sdkVersion ${observed[1]} but the device says ${sdk}`)
	157	}
	158
	159	return {
	160	name: label,
	161	status: failures.length ? 'failed' : 'passed',
	162	failures,
	163	planned: verdict.planned,
	164	passed: verdict.passed,
	165	failed: verdict.failed,
	166	// Nothing copies libcraft.so into the generated app's jniLibs (#200), so
	167	// CraftNative.isAvailable is always false and Kotlin serves every action.
	168	// Reported as empty rather than omitted, so the report says which side
	169	// answered instead of leaving it open.
	170	zigActions: [],
	171	evidence: evidence.replace(`${options.root}/`, ''),
	172	}
	173	}
	174
	175	export async function runAndroid(options: RunnerOptions): Promise<LegOutcome[]> {
	176	try {
	177	return [await runLeg(options)]
	178	}
	179	catch (caught) {
	180	return [{
	181	name: 'android-shim',
	182	status: 'failed',
	183	failures: [caught instanceof Error ? caught.message : String(caught)],
	184	planned: [],
	185	passed: [],
	186	failed: [],
	187	zigActions: [],
	188	evidence: join(options.evidenceDir, 'android-shim').replace(`${options.root}/`, ''),
	189	}]
	190	}
	191	}

scripts/mobile-e2e/ios.tsadded+241-0

Changes to scripts/mobile-e2e/ios.ts
		@@ -0,0 +1,241 @@
	1	import type { LegOutcome, RunnerOptions } from './types'
	2	import { closeSync, copyFileSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
	3	import { join } from 'node:path'
	4	import { bootSimulator, init, pickSimulator } from '../../packages/ios/src/index'
	5	import { evaluateRun, hasTerminated, ZIG_REFUSED_ACTION, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigRefusals } from './protocol'
	6	import { command, driverPage, waitForFile } from './support'
	7
	8	/**
	9	* The app the suite runs against.
	10	*
	11	* Clipboard on, geolocation off, and that pairing is the whole point: the
	12	* success case needs a capability that is enabled and reachable without a
	13	* permission prompt, and the rejection case needs one that is switched off so
	14	* the refusal is a property of the configuration rather than of the machine.
	15	* Reading a pasteboard the app itself just wrote raises no iOS paste prompt,
	16	* which is what makes the round trip scriptable.
	17	*/
	18	const CONFIG = {
	19	enableClipboard: true,
	20	enableGeolocation: false,
	21	}
	22
	23	const BUNDLE_ID = 'dev.craft.e2e.probe'
	24	const APP_NAME = 'CraftE2EProbe'
	25
	26	/**
	27	* One leg is one answer to "who served the call".
	28	*
	29	* `shim` is what every generated app is today: no Zig archive, every dlsym
	30	* miss, Swift serving the whole surface. `runtime` links the archive so Zig
	31	* takes the actions it owns. They assert the same cases, because the hand-off
	32	* is supposed to be invisible to the page - and that is exactly why the
	33	* runtime leg also counts Zig's dispatch log lines. Identical behaviour is the
	34	* requirement; identical behaviour with the archive silently absent is the bug
	35	* that requirement would otherwise hide.
	36	*/
	37	interface Leg {
	38	name: 'shim' \| 'runtime'
	39	runtimeDir: string \| null
	40	requireZigDispatch: boolean
	41	}
	42
	43	function legs(runtimeDir: string \| null): Leg[] {
	44	return [
	45	{ name: 'shim', runtimeDir: null, requireZigDispatch: false },
	46	{ name: 'runtime', runtimeDir, requireZigDispatch: true },
	47	]
	48	}
	49
	50	async function runLeg(leg: Leg, options: RunnerOptions): Promise<LegOutcome> {
	51	const label = `ios-${leg.name}`
	52	const evidence = join(options.evidenceDir, label)
	53	const project = join(options.workDir, label)
	54	const failures: string[] = []
	55
	56	mkdirSync(evidence, { recursive: true })
	57	rmSync(project, { force: true, recursive: true })
	58
	59	await init({
	60	name: APP_NAME,
	61	bundleId: BUNDLE_ID,
	62	output: project,
	63	config: CONFIG,
	64	runtimeDir: leg.runtimeDir,
	65	})
	66
	67	// The archive is linked through project.yml's ,
	68	// which init() leaves empty when it installed no runtime. Checking the
	69	// generated file rather than trusting the option means a leg cannot quietly
	70	// become a second copy of the shim leg.
	71	const projectYml = readFileSync(join(project, 'project.yml'), 'utf8')
	72	const linksRuntime = projectYml.includes('-lcraft-ios')
	73	if (leg.requireZigDispatch && !linksRuntime)
	74	failures.push(`the ${leg.name} leg generated a project that does not link the Zig archive; check ${leg.runtimeDir}`)
	75	if (!leg.requireZigDispatch && linksRuntime)
	76	failures.push(`the ${leg.name} leg was supposed to have no runtime but its project links one`)
	77
	78	const nonce = `craft-e2e-${label}-${options.runId}`
	79	writeFileSync(join(project, 'dist', 'index.html'), driverPage(nonce, 'ios'))
	80	copyFileSync(join(project, 'dist', 'index.html'), join(evidence, 'index.html'))
	81	copyFileSync(join(project, 'craft.config.json'), join(evidence, 'craft.config.json'))
	82	copyFileSync(join(project, 'project.yml'), join(evidence, 'project.yml'))
	83
	84	await command(['xcodegen', 'generate'], { cwd: project, logPath: join(evidence, 'xcodegen.log') })
	85
	86	const derived = join(project, 'DerivedData')
	87	await command([
	88	'xcodebuild',
	89	'-project', `${APP_NAME}.xcodeproj`,
	90	'-scheme', APP_NAME,
	91	'-configuration', 'Debug',
	92	// The simulator SDK, which no existing job builds against: the templates
	93	// gate compiles -sdk iphoneos only. A generated app that links for a
	94	// device and not for a simulator would have gone unnoticed.
	95	'-sdk', 'iphonesimulator',
	96	'-derivedDataPath', derived,
	97	'CODE_SIGNING_ALLOWED=NO',
	98	'build',
	99	], { cwd: project, logPath: join(evidence, 'xcodebuild.log') })
	100
	101	const app = join(derived, 'Build', 'Products', 'Debug-iphonesimulator', `${APP_NAME}.app`)
	102	if (!existsSync(app))
	103	throw new Error(`xcodebuild reported success but ${app} is missing`)
	104
	105	const device = await pickSimulator()
	106	if (!device)
	107	throw new Error('no iOS simulator is available; `xcrun simctl list devices available` found none')
	108	await bootSimulator(device)
	109	writeFileSync(join(evidence, 'device.json'), `${JSON.stringify(device, null, 2)}\n`)
	110
	111	await command(['xcrun', 'simctl', 'uninstall', device.udid, BUNDLE_ID], { allowFailure: true })
	112	await command(['xcrun', 'simctl', 'install', device.udid, app])
	113
	114	// Clear the pasteboard first. Without this a nonce left by the previous leg
	115	// would still be there, and the external check below would pass on a value
	116	// this run never wrote.
	117	await command(['xcrun', 'simctl', 'pbcopy', device.udid], { stdin: 'craft-e2e-pasteboard-cleared', allowFailure: true })
	118
	119	// The app's own stdout and stderr, straight to a file. `--console-pty` is
	120	// what makes Swift's `print` - and therefore the page's `craft.log` - leave
	121	// the device at all; without it the launch returns a pid and says nothing.
	122	// A raw descriptor rather than a pipe, so the writing outlives this process
	123	// reading it and the poll below sees the file grow.
	124	const consolePath = join(evidence, 'console.log')
	125	const consoleFd = openSync(consolePath, 'w')
	126	const launched = Bun.spawn(['xcrun', 'simctl', 'launch', '--console-pty', device.udid, BUNDLE_ID], {
	127	stdout: consoleFd,
	128	stderr: consoleFd,
	129	})
	130
	131	// The suite is quick once the app is up, but a cold simulator is not, so
	132	// poll for the terminator rather than sleeping a fixed amount.
	133	//
	134	// `hasTerminated` parses rather than matching a substring, because the very
	135	// next statement kills the writer: a poll landing mid-write of the final
	136	// line would otherwise leave a truncated event in the file and turn a
	137	// passing suite into "driver emitted an unparseable event".
	138	const finished = await waitForFile(consolePath, options.timeoutMs, hasTerminated)
	139
	140	launched.kill()
	141	closeSync(consoleFd)
	142	await command(['xcrun', 'simctl', 'terminate', device.udid, BUNDLE_ID], { allowFailure: true })
	143
	144	// Evidence before assertions, always: a leg that dies here is the one whose
	145	// screenshot is worth the most.
	146	await command(['xcrun', 'simctl', 'io', device.udid, 'screenshot', join(evidence, 'screen.png')], { allowFailure: true })
	147	const pasteboard = await command(['xcrun', 'simctl', 'pbpaste', device.udid], { allowFailure: true })
	148	writeFileSync(join(evidence, 'pasteboard.txt'), pasteboard.stdout)
	149
	150	const consoleText = existsSync(consolePath) ? readFileSync(consolePath, 'utf8') : ''
	151	if (!finished)
	152	failures.push(`the app produced no terminating event within ${options.timeoutMs}ms; see ${label}/console.log`)
	153
	154	const verdict = evaluateRun('ios', consoleText, nonce)
	155	failures.push(...verdict.failures)
	156
	157	// The one assertion the page cannot fake. simctl reads the device's own
	158	// pasteboard from the host, so a nonce found here travelled from JavaScript
	159	// through the bridge into real UIPasteboard state.
	160	if (!pasteboard.stdout.includes(nonce))
	161	failures.push(`the simulator pasteboard does not hold the nonce this run wrote; it holds ${JSON.stringify(pasteboard.stdout.trim().slice(0, 80))}`)
	162
	163	const dispatched = zigDispatchedActions(consoleText)
	164	const refused = zigRefusals(consoleText)
	165
	166	if (leg.requireZigDispatch) {
	167	const missing = ZIG_TESTED_ACTIONS.filter(action => !dispatched.includes(action))
	168	if (missing.length)
	169	failures.push(`the runtime leg links the Zig archive but ${missing.join(', ')} never reached the Zig dispatcher; Swift served ${missing.length === 1 ? 'it' : 'them'} and the archive was not in the loop for the cases under test`)
	170
	171	// A dispatch line is logged on entry, before Zig decides whether it owns
	172	// the action, so it alone does not prove Zig answered. The refusal line is
	173	// written by Zig's own capability gate, which means the rejection the page
	174	// saw came off Zig's error route.
	175	if (!refused.includes(ZIG_REFUSED_ACTION))
	176	failures.push(`Zig never refused ${ZIG_REFUSED_ACTION}; the rejection the page saw came from Swift, not from the Zig gate`)
	177	}
	178	else {
	179	if (dispatched.length)
	180	failures.push(`the shim leg reached the Zig dispatcher for ${dispatched.join(', ')}; it was supposed to link no runtime`)
	181	if (refused.length)
	182	failures.push(`the shim leg saw Zig refuse ${refused.join(', ')}; it was supposed to link no runtime`)
	183	}
	184
	185	return {
	186	name: label,
	187	status: failures.length ? 'failed' : 'passed',
	188	failures,
	189	planned: verdict.planned,
	190	passed: verdict.passed,
	191	failed: verdict.failed,
	192	zigActions: dispatched,
	193	evidence: evidence.replace(`${options.root}/`, ''),
	194	}
	195	}
	196
	197	export async function runIos(options: RunnerOptions): Promise<LegOutcome[]> {
	198	const runtimeDir = options.iosRuntimeDir
	199	if (!runtimeDir)
	200	throw new Error('no iOS Zig runtime directory; run `zig build build-ios-all` in packages/zig and pass --ios-runtime <dir>')
	201
	202	// The device archive as well as the simulator ones, because that is what
	203	// `installRuntime` requires: it resolves every SDK in RUNTIME_ARCHIVES before
	204	// writing anything and throws when one has none. Checking only the simulator
	205	// slices here would accept a zig-out that project generation then rejects,
	206	// several minutes later, with a message about a different build step.
	207	const simulator = ['libcraft-ios-simulator-arm64.a', 'libcraft-ios-simulator-x64.a']
	208	const missing: string[] = []
	209	if (!simulator.some(name => existsSync(join(runtimeDir, name)))) missing.push(...simulator)
	210	if (!existsSync(join(runtimeDir, 'libcraft-ios.a'))) missing.push('libcraft-ios.a')
	211
	212	if (missing.length) {
	213	throw new Error(
	214	`${runtimeDir} is missing ${missing.join(', ')}. `
	215	+ 'Run `zig build build-ios-all -Doptimize=ReleaseSafe` in packages/zig first — '
	216	+ 'build-ios-simulator alone is not enough, because the generated project links a device slice too. '
	217	+ 'Skipping the runtime leg instead would leave the Zig bridge untested while the suite still reported green.',
	218	)
	219	}
	220
	221	const outcomes: LegOutcome[] = []
	222	for (const leg of legs(runtimeDir)) {
	223	try {
	224	outcomes.push(await runLeg(leg, options))
	225	}
	226	catch (caught) {
	227	outcomes.push({
	228	name: `ios-${leg.name}`,
	229	status: 'failed',
	230	failures: [caught instanceof Error ? caught.message : String(caught)],
	231	planned: [],
	232	passed: [],
	233	failed: [],
	234	zigActions: [],
	235	evidence: join(options.evidenceDir, `ios-${leg.name}`).replace(`${options.root}/`, ''),
	236	})
	237	}
	238	}
	239
	240	return outcomes
	241	}