ReviewOS
Reload

also looking at this

craft-native/craft

test(mobile): run the generated apps on a simulator and an emulator

#199
Merged glennmichael123 wants to merge fix/mobile-e2e-real-assertions into main
26 files +3009 -293
.github/workflows/mobile-e2e.ymlmodified+276-173
@@ -1,156 +1,208 @@
11name: Mobile E2E Testing
22
3# Manual only. This workflow has been red since May and cannot go green from
4# a workflow edit: the iOS Zig sources stopped compiling in the 0.17 migration
5# (1b6f285) and need real source work — see the notes in the PR that made this
6# manual. It also runs no end-to-end tests today; every step that would has an
7# `if: hashFiles(...)` guard pointing at a TestApp/project that is not in the
8# tree, so they all skip. Firing it on push meant a red X on unrelated commits
9# for a signal nobody could act on, and two of its path filters
10# (packages/zig/src/ios/**, packages/zig/src/android/**) match directories that
11# do not exist at all.
3# Runs a generated Craft app on a real iOS simulator and a real Android
4# emulator and asserts that a call made from JavaScript reached native code and
5# came back.
6#
7# The version this replaces asserted nothing. Every iOS step that would have
8# tested something was guarded by
9# `if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''` against a
10# project that has never existed in the tree, and the Android job's only test
11# invocation was gated on `packages/android/gradlew`, which the generator does
12# not write. Both skipped on every run, and the job reported success. Its
13# header also blamed the iOS Zig sources for not compiling since the 0.17
14# migration (1b6f285); that is no longer true —
15# `zig build build-ios-simulator` succeeds at this commit, and the iOS job
16# below depends on it.
17#
18# What the harness asserts, per platform, is in scripts/mobile-e2e/protocol.ts:
19# a required list of cases that must all run and all pass, covering at least
20# one success path and one rejection path. Nothing here skips. A missing
21# toolchain, an absent device, or an app that printed nothing is a failure with
22# a named cause.
23
24# GitHub Actions does not resolve YAML anchors, so the two path filters are
25# written out rather than shared. Keep them identical.
1226on:
27 push:
28 branches: [main]
29 paths:
30 - packages/ios/**
31 - packages/android/**
32 # The mobile Zig sources are flat files under src/, not src/ios/ and
33 # src/android/ as the previous filters claimed — those two directories
34 # have never existed, so the filter matched nothing.
35 - packages/zig/src/ios*.zig
36 - packages/zig/src/android*.zig
37 - packages/zig/src/mobile*.zig
38 - packages/zig/src/bridge_mobile_*.zig
39 - packages/zig/src/bridge_android_*.zig
40 - packages/zig/src/jni*.zig
41 - packages/zig/build.zig
42 - scripts/mobile-e2e.ts
43 - scripts/mobile-e2e/**
44 - .github/workflows/mobile-e2e.yml
45 pull_request:
46 paths:
47 - packages/ios/**
48 - packages/android/**
49 - packages/zig/src/ios*.zig
50 - packages/zig/src/android*.zig
51 - packages/zig/src/mobile*.zig
52 - packages/zig/src/bridge_mobile_*.zig
53 - packages/zig/src/bridge_android_*.zig
54 - packages/zig/src/jni*.zig
55 - packages/zig/build.zig
56 - scripts/mobile-e2e.ts
57 - scripts/mobile-e2e/**
58 - .github/workflows/mobile-e2e.yml
1359 workflow_dispatch:
1460
1561concurrency:
1662 group: mobile-e2e-${{ github.head_ref || github.ref }}
1763 cancel-in-progress: true
1864
65permissions:
66 contents: read
67
1968defaults:
2069 run:
2170 shell: bash
2271
2372jobs:
24 ios-simulator:
25 # name: iOS Simulator Tests
26 runs-on: macos-14 # M1 runner for better iOS simulator performance
73 # The transcript reader and the required-case list are the part of this
74 # harness that can be tested without a device, and nothing ran those tests:
75 # every `bun test` in this repository is scoped to a package directory or to
76 # one explicit file, and the root `test` script is the Zig suite. So the
77 # ratchet that stops the suite being hollowed out was itself unenforced.
78 harness:
79 name: Harness unit tests
80 runs-on: ubuntu-latest
81 timeout-minutes: 10
2782
2883 steps:
2984 - uses: actions/checkout@v6
3085
31 - name: Setup Zig
86 - name: Setup Pantry (provides Bun, Zig, etc.)
3287 uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
33 - name: Setup Xcode
34 uses: maxim-lobanov/setup-xcode@v1
35 with:
36 xcode-version: latest-stable
37
38 - name: Cache Zig artifacts
39 uses: actions/cache@v5
40 with:
41 path: |
42 ~/.cache/zig
43 packages/zig/.zig-cache
44 key: ios-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }}
4588
46 - name: List available simulators
47 run: |
48 xcrun simctl list devices available
89 - name: Install dependencies
90 run: bun install --frozen-lockfile
4991
50 - name: Boot iOS Simulator
51 run: |
52 # Find an iPhone 15 simulator
53 DEVICE_ID=$(xcrun simctl list devices available | grep "iPhone 15" | head -1 | grep -oE '[0-9A-F-]{36}')
92 # The explicit path is load-bearing: a bare filter makes bun walk the
93 # repository and run out of file descriptors under the hoisted linker.
94 # See the note in native-lifecycle.yml.
95 - name: Test the report protocol
96 run: bun test ./scripts/mobile-e2e/protocol.test.ts
5497
55 if [ -z "$DEVICE_ID" ]; then
56 # Create one if not found
57 DEVICE_ID=$(xcrun simctl create "iPhone 15 Test" "iPhone 15" "iOS17.0")
58 fi
98 ios-simulator:
99 name: iOS simulator
100 runs-on: macos-15
101 # A hung simulator launch would otherwise burn the six-hour default. The
102 # harness has its own per-app timeout; this is the backstop for everything
103 # around it.
104 timeout-minutes: 45
105 env:
106 CRAFT_EVIDENCE_LABEL: ios-simulator
59107
60 echo "DEVICE_ID=$DEVICE_ID" >> $GITHUB_ENV
108 steps:
109 - uses: actions/checkout@v6
61110
62 # Boot the simulator
63 xcrun simctl boot "$DEVICE_ID" || true
111 - name: Setup Pantry (provides Bun, Zig, etc.)
112 uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
64113
65 # Wait for simulator to finish booting. `bootstatus -b` blocks until
66 # the device reaches a Booted state, eliminating the flaky fixed sleep.
67 xcrun simctl bootstatus "$DEVICE_ID" -b
114 - name: Install XcodeGen
115 run: brew install xcodegen
68116
69 - name: First-party Zig dependencies
70 uses: ./.github/actions/first-party-zig-deps
117 - name: Install dependencies
118 run: bun install --frozen-lockfile
71119
72 - name: Build iOS library
73 working-directory: packages/zig
74 run: |
75 # Build iOS simulator static libraries using dedicated build step
76 # (not the default target which builds macOS executables)
77 zig build build-ios-simulator -Doptimize=ReleaseSafe -Dmacos-sdk="$(xcrun --sdk iphonesimulator --show-sdk-path)"
120 - name: Cache Zig artifacts
121 uses: actions/cache@v5
122 with:
123 path: |
124 ~/.cache/zig
125 packages/zig/.zig-cache
126 key: mobile-e2e-ios-${{ runner.os }}-${{ hashFiles('packages/zig/build.zig') }}
78127
79128 - name: First-party Zig dependencies
80129 uses: ./.github/actions/first-party-zig-deps
81130
82 - name: Run iOS unit tests
131 # build-ios-all, not build-ios-simulator: installRuntime() resolves every
132 # SDK in RUNTIME_ARCHIVES before it writes anything and throws when one
133 # has no archive at all, so a zig-out holding only the simulator slices
134 # fails at project generation. It passes locally only because a device
135 # archive is usually already sitting there from an earlier build.
136 #
137 # No -Dmacos-sdk: the iOS steps resolve their SDK through xcrun at
138 # configure time (build.zig's iosSdkPath), and the option is read only by
139 # desktop and host-test artifacts, so passing it was a no-op.
140 - name: Build the Zig iOS runtime
83141 working-directory: packages/zig
84 run: |
85 # `zig build test` reports through its exit code. The old form piped
86 # it to a file and grepped for "mobile"/"ios", which fails under
87 # `set -e` whenever the run is clean and the file is therefore empty.
88 zig build test
89
90 - name: Build iOS test app
91 if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''
92 run: |
93 cd packages/ios/TestApp
94 xcodebuild build-for-testing \
95 -scheme TestApp \
96 -destination "platform=iOS Simulator,id=$DEVICE_ID" \
97 -configuration Debug \
98 CODE_SIGN_IDENTITY="" \
99 CODE_SIGNING_REQUIRED=NO
100
101 - name: Run iOS E2E tests
102 if: hashFiles('packages/ios/TestApp/TestApp.xcodeproj') != ''
103 run: |
104 cd packages/ios/TestApp
105 xcodebuild test-without-building \
106 -scheme TestApp \
107 -destination "platform=iOS Simulator,id=$DEVICE_ID" \
108 -configuration Debug
109
110 - name: Capture iOS screenshots
111 run: |
112 mkdir -p screenshots/ios
113
114 # Take a screenshot of the simulator
115 xcrun simctl io "$DEVICE_ID" screenshot screenshots/ios/simulator.png || true
116
117 - name: Shutdown simulator
118 if: always()
119 run: |
120 xcrun simctl shutdown "$DEVICE_ID" || true
121
122 - name: Upload iOS screenshots
142 run: zig build build-ios-all -Doptimize=ReleaseSafe
143
144 - name: Run mobile E2E on the simulator
145 id: harness
146 run: bun scripts/mobile-e2e.ts --platform ios --ios-runtime packages/zig/zig-out/lib
147
148 - name: Upload iOS evidence
149 # Not a bare `always()`: when an earlier step fails the harness never
150 # runs, the evidence directory is never created, and `if-no-files-found:
151 # error` would then add a "No files were found" error on top of the real
152 # one. Skipped-means-skipped; ran-and-produced-nothing still fails.
153 if: always() && steps.harness.conclusion != 'skipped'
123154 uses: actions/upload-artifact@v4
124155 with:
125 name: ios-screenshots
126 path: screenshots/ios/
127 retention-days: 7
128
129 - name: Generate iOS test report
130 run: |
131 echo "## iOS Simulator Tests" >> $GITHUB_STEP_SUMMARY
132 echo "" >> $GITHUB_STEP_SUMMARY
133 echo "- Simulator: iPhone 15" >> $GITHUB_STEP_SUMMARY
134 echo "- iOS Version: 17.0" >> $GITHUB_STEP_SUMMARY
135 echo "- Status: Completed" >> $GITHUB_STEP_SUMMARY
156 name: mobile-e2e-ios
157 # Console logs, build logs, screenshots, the generated project.yml and
158 # the exact page that ran. Excludes work/ — the generated projects and
159 # their DerivedData are hundreds of megabytes and the interesting
160 # parts are copied out.
161 path: |
162 artifacts/mobile-e2e/ios-simulator/**
163 !artifacts/mobile-e2e/ios-simulator/work/**
164 # A run that produced no evidence is a broken harness, not a pass.
165 if-no-files-found: error
166 retention-days: 30
136167
137168 android-emulator:
138 # name: Android Emulator Tests
169 name: Android emulator
139170 runs-on: ubuntu-latest
171 timeout-minutes: 60
172 env:
173 CRAFT_EVIDENCE_LABEL: android-emulator
140174
141175 steps:
142176 - uses: actions/checkout@v6
143177
144 - name: Setup Zig
178 - name: Setup Pantry (provides Bun, Zig, etc.)
145179 uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
180
146181 - name: Setup Java
147 uses: actions/setup-java@v4
182 uses: actions/setup-java@v6
148183 with:
149 distribution: 'temurin'
184 distribution: temurin
150185 java-version: '17'
151186
152187 - name: Setup Android SDK
153 uses: android-actions/setup-android@v3
188 uses: android-actions/setup-android@v4
189 with:
190 # The action defaults to `tools platform-tools`, but the obsolete
191 # `tools` package is no longer offered by the Android SDK repository.
192 packages: platform-tools
193
194 # Same platform and build-tools as ci.yml's android-builder, so the
195 # emulator job and the compile gate cannot drift onto different SDKs.
196 - name: Install Android platform
197 run: sdkmanager "platforms;android-36" "build-tools;36.0.0"
198
199 - name: Setup Gradle
200 uses: gradle/actions/setup-gradle@v6
201 with:
202 gradle-version: '8.11.1'
203
204 - name: Install dependencies
205 run: bun install --frozen-lockfile
154206
155207 - name: Cache Zig artifacts
156208 uses: actions/cache@v5
@@ -158,36 +210,25 @@ jobs:
158210 path: |
159211 ~/.cache/zig
160212 packages/zig/.zig-cache
161 key: android-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }}
162
163 - name: Install Android NDK
164 run: |
165 sdkmanager --install "ndk;26.1.10909125"
166 echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/26.1.10909125" >> $GITHUB_ENV
213 key: mobile-e2e-android-${{ runner.os }}-${{ hashFiles('packages/zig/build.zig') }}
167214
168215 - name: First-party Zig dependencies
169216 uses: ./.github/actions/first-party-zig-deps
170217
171 - name: Build Android library
172 working-directory: packages/zig
218 # The NDK is what supplies bionic. Without it the JNI library builds and
219 # then fails at dlopen with `cannot locate symbol "__tls_get_addr"`, which
220 # CraftNative catches and turns into a silent fallback to the Kotlin shim
221 # — the failure this whole leg exists to catch.
222 - name: Install the Android NDK
173223 run: |
174 # Use the dedicated android step, which resolves its own target.
175 # Passing -Dtarget steers the *default* install step instead, which
176 # builds the desktop binaries — and since an android triple reports
177 # os_tag == .linux, that links GTK3/WebKit2GTK against a sysroot that
178 # does not exist for Android.
179 zig build build-android -Doptimize=ReleaseSafe
180
181 - name: First-party Zig dependencies
182 uses: ./.github/actions/first-party-zig-deps
224 sdkmanager --install "ndk;26.1.10909125"
225 echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/26.1.10909125" >> "$GITHUB_ENV"
183226
184 - name: Run Android unit tests
227 # Both ABIs. x86_64 is the one the emulator actually loads; arm64-v8a is
228 # built so the runtime leg installs the same APK a phone would get.
229 - name: Build the Zig Android runtime
185230 working-directory: packages/zig
186 run: |
187 # `zig build test` reports through its exit code. The old form piped
188 # it to a file and grepped for "mobile"/"android", which fails under
189 # `set -e` whenever the run is clean and the file is therefore empty.
190 zig build test
231 run: zig build build-android-all -Doptimize=ReleaseSafe -Dandroid-ndk="$ANDROID_NDK_HOME"
191232
192233 - name: Enable KVM
193234 run: |
@@ -202,9 +243,17 @@ jobs:
202243 path: |
203244 ~/.android/avd/*
204245 ~/.android/adb*
205 key: avd-api-34-${{ runner.os }}
206
207 - name: Create AVD and run tests
246 # The device, not just the API level. With force-avd-creation off the
247 # action reuses whatever the cache restored, so a key that ignores
248 # arch and profile would keep testing the old device after someone
249 # changed them.
250 key: avd-api-34-x86_64-pixel_6-${{ runner.os }}
251
252 # `script:` is the only place a command can run with the emulator
253 # attached, so the harness is invoked from inside it rather than as its
254 # own step.
255 - name: Run mobile E2E on the emulator
256 id: harness
208257 uses: reactivecircus/android-emulator-runner@v2
209258 with:
210259 api-level: 34
@@ -214,56 +263,110 @@ jobs:
214263 emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
215264 disable-animations: true
216265 script: |
217 echo "Emulator started"
218266 adb wait-for-device
219 mkdir -p screenshots/android
220 adb exec-out screencap -p > screenshots/android/emulator.png || true
221 if [ -x "packages/android/gradlew" ]; then
222 cd packages/android
223 ./gradlew test
224 else
225 echo "Android Gradle wrapper not present; skipping project-level tests"
226 fi
227
228 - name: Upload Android screenshots
267 bun scripts/mobile-e2e.ts --platform android --android-runtime packages/zig/zig-out/android
268
269 - name: Upload Android evidence
270 if: always() && steps.harness.conclusion != 'skipped'
229271 uses: actions/upload-artifact@v4
230272 with:
231 name: android-screenshots
232 path: screenshots/android/
233 retention-days: 7
273 name: mobile-e2e-android
274 path: |
275 artifacts/mobile-e2e/android-emulator/**
276 !artifacts/mobile-e2e/android-emulator/work/**
277 if-no-files-found: error
278 retention-days: 30
279
280 # The Zig runtime's own simulator fixture: a hand-linked four-file app that
281 # drives libcraft-ios-simulator directly, with assertions the generated-app
282 # suite cannot make — the host hand-off, the error route's escaping, a
283 # Keychain round trip, in-process SQLite, and a sensor the simulator does not
284 # have refusing rather than fabricating a stream. It has been in the tree,
285 # passing, and wired into no workflow.
286 ios-zig-slice:
287 name: iOS Zig runtime slice
288 runs-on: macos-15
289 timeout-minutes: 45
290
291 steps:
292 - uses: actions/checkout@v6
293
294 - name: Setup Pantry (provides Bun, Zig, etc.)
295 uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
234296
235 - name: Generate Android test report
297 - name: Cache Zig artifacts
298 uses: actions/cache@v5
299 with:
300 path: |
301 ~/.cache/zig
302 packages/zig/.zig-cache
303 key: mobile-e2e-slice-${{ runner.os }}-${{ hashFiles('packages/zig/build.zig') }}
304
305 - name: First-party Zig dependencies
306 uses: ./.github/actions/first-party-zig-deps
307
308 # The script defaults CRAFT_ZIG to a pinned toolchain under
309 # ~/.cache/craft-ci-local, which exists on the author's machine and not on
310 # a runner. Point it at the one pantry provides.
311 - name: Exercise the Zig runtime on a simulator
312 id: harness
313 # The fixture polls for its late markers against a wall-clock deadline.
314 # A runner is several times slower than a laptop here — the first CI run
315 # of this job reached marker 36 of 42 before the old iteration-count
316 # budget ran out — so give it room. It exits as soon as the markers
317 # arrive; the budget only matters when something is genuinely stuck.
318 env:
319 CRAFT_SLICE_TIMEOUT: '900'
236320 run: |
237 echo "## Android Emulator Tests" >> $GITHUB_STEP_SUMMARY
238 echo "" >> $GITHUB_STEP_SUMMARY
239 echo "- Emulator: Pixel 6" >> $GITHUB_STEP_SUMMARY
240 echo "- API Level: 34" >> $GITHUB_STEP_SUMMARY
241 echo "- Status: Completed" >> $GITHUB_STEP_SUMMARY
321 CRAFT_ZIG="$(command -v zig)" \
322 packages/ios/fixtures/zig-slice/build-and-run.sh \
323 "$PWD/artifacts/mobile-e2e/ios-zig-slice"
324
325 - name: Upload slice evidence
326 if: always() && steps.harness.conclusion != 'skipped'
327 uses: actions/upload-artifact@v4
328 with:
329 name: mobile-e2e-ios-zig-slice
330 path: |
331 artifacts/mobile-e2e/ios-zig-slice/console.log
332 artifacts/mobile-e2e/ios-zig-slice/console.plain
333 if-no-files-found: error
334 retention-days: 30
242335
243336 mobile-summary:
244 # name: Mobile Test Summary
245 needs: [ios-simulator, android-emulator]
337 name: Mobile E2E summary
338 needs: [harness, ios-simulator, android-emulator, ios-zig-slice]
246339 runs-on: ubuntu-latest
247340 if: always()
248341
249342 steps:
250 - name: Check test results
343 - name: Report
251344 run: |
252 echo "## Mobile E2E Test Summary" >> $GITHUB_STEP_SUMMARY
253 echo "" >> $GITHUB_STEP_SUMMARY
254 echo "| Platform | Status |" >> $GITHUB_STEP_SUMMARY
255 echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
256 echo "| iOS Simulator | $ |" >> $GITHUB_STEP_SUMMARY
257 echo "| Android Emulator | $ |" >> $GITHUB_STEP_SUMMARY
258
259 - name: Download all screenshots
260 uses: actions/download-artifact@v4
261 with:
262 path: all-screenshots
263
264 - name: Upload combined screenshots
265 uses: actions/upload-artifact@v4
266 with:
267 name: mobile-screenshots
268 path: all-screenshots/
269 retention-days: 30
345 {
346 echo "## Mobile E2E"
347 echo
348 echo "| Leg | Result |"
349 echo "|-----|--------|"
350 echo "| Harness unit tests | $ |"
351 echo "| iOS simulator (generated app) | $ |"
352 echo "| Android emulator (generated app) | $ |"
353 echo "| iOS Zig runtime slice | $ |"
354 } >> "$GITHUB_STEP_SUMMARY"
355
356 # The summary job used to be the whole story, and it reported whatever it
357 # was given without ever failing. `needs` does not fail a job on a failed
358 # dependency when `if: always()` is set, so say it explicitly.
359 #
360 # `failure` and `skipped`, not `!= success`: with cancel-in-progress on,
361 # pushing twice in quick succession cancels the first run, and treating
362 # `cancelled` as a failure would put a red X on it — a signal about
363 # nothing, which is how the workflow this replaces earned its reputation.
364 - name: Fail if any leg failed
365 if: >-
366 contains(fromJSON('["failure", "skipped"]'), needs.harness.result)
367 || contains(fromJSON('["failure", "skipped"]'), needs.ios-simulator.result)
368 || contains(fromJSON('["failure", "skipped"]'), needs.android-emulator.result)
369 || contains(fromJSON('["failure", "skipped"]'), needs.ios-zig-slice.result)
370 run: |
371 echo "::error::at least one mobile E2E leg did not succeed"
372 exit 1