also looking at this
test(mobile): run the generated apps on a simulator and an emulator
#199
26 files
+3009
-293
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -35,12 +35,20 @@ ok ios-runtime — 4/4 cases, zig served clipboardRead, clipboardWrite, getCur | ||
| 35 | 35 | ``` |
| 36 | 36 | |
| 37 | 37 | Android needs JDK 17, Gradle 8.11.1, an Android SDK with `platforms;android-36` |
| 38 | and `build-tools;36.0.0`, and a booted emulator or attached device: | |
| 38 | and `build-tools;36.0.0`, a booted emulator or attached device, and the Zig | |
| 39 | Android libraries — it runs twice as well, once with `libcraft.so` packaged | |
| 40 | into the APK and once without: | |
| 39 | 41 | |
| 40 | 42 | ```bash |
| 43 | cd packages/zig && zig build build-android-all -Doptimize=ReleaseSafe && cd - | |
| 41 | 44 | bun run test:mobile-e2e:android |
| 42 | 45 | ``` |
| 43 | 46 | |
| 47 | ``` | |
| 48 | ok android-shim — 4/4 cases, no zig | |
| 49 | ok android-runtime — 4/4 cases, zig served registered:59 | |
| 50 | ``` | |
| 51 | ||
| 44 | 52 | Both write everything they saw — console logs, build logs, a screenshot, the |
| 45 | 53 | exact page that ran and a `report.json` — to |
| 46 | 54 | `artifacts/mobile-e2e/<label>/`, which is what CI uploads. |
| @@ -204,6 +204,25 @@ jobs: | ||
| 204 | 204 | - name: Install dependencies |
| 205 | 205 | run: bun install --frozen-lockfile |
| 206 | 206 | |
| 207 | - name: Cache Zig artifacts | |
| 208 | uses: actions/cache@v5 | |
| 209 | with: | |
| 210 | path: | | |
| 211 | ~/.cache/zig | |
| 212 | packages/zig/.zig-cache | |
| 213 | key: mobile-e2e-android-${{ runner.os }}-${{ hashFiles('packages/zig/build.zig') }} | |
| 214 | ||
| 215 | - name: First-party Zig dependencies | |
| 216 | uses: ./.github/actions/first-party-zig-deps | |
| 217 | ||
| 218 | # Both ABIs. x86_64 is the one the emulator actually loads; arm64-v8a is | |
| 219 | # built so the runtime leg installs the same APK a phone would get. | |
| 220 | # No NDK: the JNI library links no libc on purpose, which is what lets | |
| 221 | # Zig build it alone (android_dispatch.zig records why). | |
| 222 | - name: Build the Zig Android runtime | |
| 223 | working-directory: packages/zig | |
| 224 | run: zig build build-android-all -Doptimize=ReleaseSafe | |
| 225 | ||
| 207 | 226 | - name: Enable KVM |
| 208 | 227 | run: | |
| 209 | 228 | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules |
| @@ -238,7 +257,7 @@ jobs: | ||
| 238 | 257 | disable-animations: true |
| 239 | 258 | script: | |
| 240 | 259 | adb wait-for-device |
| 241 | bun scripts/mobile-e2e.ts --platform android | |
| 260 | bun scripts/mobile-e2e.ts --platform android --android-runtime packages/zig/zig-out/android | |
| 242 | 261 | |
| 243 | 262 | - name: Upload Android evidence |
| 244 | 263 | if: always() && steps.harness.conclusion != 'skipped' |
| @@ -36,7 +36,7 @@ | ||
| 36 | 36 | "test:sdk": "cd packages/typescript && bun test", |
| 37 | 37 | "test:native-lifecycle": "bun scripts/native-lifecycle.ts", |
| 38 | 38 | "test:mobile-e2e:ios": "bun scripts/mobile-e2e.ts --platform ios --ios-runtime packages/zig/zig-out/lib", |
| 39 | "test:mobile-e2e:android": "bun scripts/mobile-e2e.ts --platform android", | |
| 39 | "test:mobile-e2e:android": "bun scripts/mobile-e2e.ts --platform android --android-runtime packages/zig/zig-out/android", | |
| 40 | 40 | "typecheck": "cd packages/typescript && bun run typecheck && cd ../create-craft && bun run typecheck && cd ../ts-maps && bun run typecheck", |
| 41 | 41 | "run": "command -v zig >/dev/null 2>&1 || eval \"$(pantry env | sed -n '/^export /,$p')\"; cd packages/zig && zig build run", |
| 42 | 42 | "run:demo": "command -v zig >/dev/null 2>&1 || eval \"$(pantry env | sed -n '/^export /,$p')\"; cd packages/zig && zig build run-demo", |
| @@ -64,6 +64,9 @@ const options: RunnerOptions = { | ||
| 64 | 64 | iosRuntimeDir: flag('ios-runtime') |
| 65 | 65 | ? resolve(root, flag('ios-runtime')!) |
| 66 | 66 | : (process.env.CRAFT_IOS_RUNTIME ? resolve(root, process.env.CRAFT_IOS_RUNTIME) : null), |
| 67 | androidRuntimeDir: flag('android-runtime') | |
| 68 | ? resolve(root, flag('android-runtime')!) | |
| 69 | : (process.env.CRAFT_ANDROID_RUNTIME ? resolve(root, process.env.CRAFT_ANDROID_RUNTIME) : null), | |
| 67 | 70 | } |
| 68 | 71 | |
| 69 | 72 | async function main(): Promise<void> { |
| @@ -23,8 +23,38 @@ const CONFIG = { | ||
| 23 | 23 | const PACKAGE = 'dev.craft.e2e.probe' |
| 24 | 24 | const APP_NAME = 'CraftE2EProbe' |
| 25 | 25 | |
| 26 | /** Both tags matter: the bridge's own, and the WebView's for a page that never reached the bridge. */ | |
| 27 | const LOGCAT_FILTER = ['CraftBridge:D', 'chromium:I', 'AndroidRuntime:E', '*:S'] | |
| 26 | /** | |
| 27 | * Tags worth reading. CraftBridge is the Kotlin shim's, CraftNative is Zig's | |
| 28 | * own — that pair is how this leg tells which side answered — chromium carries | |
| 29 | * the page's console for a run whose bridge never arrived, and AndroidRuntime | |
| 30 | * carries the crash if it did not get that far. | |
| 31 | */ | |
| 32 | const LOGCAT_FILTER = ['CraftBridge:D', 'CraftNative:D', 'chromium:I', 'AndroidRuntime:E', '*:S'] | |
| 33 | ||
| 34 | /** | |
| 35 | * One leg is one answer to "who served the call". | |
| 36 | * | |
| 37 | * The same split the iOS suite runs, and it exists here for a sharper reason: | |
| 38 | * until this change nothing copied libcraft.so into a generated app at all, so | |
| 39 | * every Android app anyone has ever generated was the shim leg and the whole | |
| 40 | * Zig bridge had never run. The runtime leg is what stops that being true | |
| 41 | * again without anybody noticing. | |
| 42 | */ | |
| 43 | interface Leg { | |
| 44 | name: 'shim' | 'runtime' | |
| 45 | runtimeDir: string | null | |
| 46 | requireZig: boolean | |
| 47 | } | |
| 48 | ||
| 49 | function legs(runtimeDir: string | null): Leg[] { | |
| 50 | return [ | |
| 51 | { name: 'shim', runtimeDir: null, requireZig: false }, | |
| 52 | { name: 'runtime', runtimeDir, requireZig: true }, | |
| 53 | ] | |
| 54 | } | |
| 55 | ||
| 56 | /** What JNI_OnLoad logs once it has bound the natives. */ | |
| 57 | const REGISTERED = /craft: registered (\d+) natives on com\/craft\/runtime\/CraftNative/ | |
| 28 | 58 | |
| 29 | 59 | async function adb(argv: string[], options: { serial: string, logPath?: string, allowFailure?: boolean }) { |
| 30 | 60 | return command(['adb', '-s', options.serial, ...argv], { logPath: options.logPath, allowFailure: options.allowFailure }) |
| @@ -45,8 +75,8 @@ async function resolveSerial(): Promise<string> { | ||
| 45 | 75 | return serials[0]! |
| 46 | 76 | } |
| 47 | 77 | |
| 48 | async function runLeg(options: RunnerOptions): Promise<LegOutcome> { | |
| 49 | const label = 'android-shim' | |
| 78 | async function runLeg(leg: Leg, options: RunnerOptions): Promise<LegOutcome> { | |
| 79 | const label = `android-${leg.name}` | |
| 50 | 80 | const evidence = join(options.evidenceDir, label) |
| 51 | 81 | const project = join(options.workDir, label) |
| 52 | 82 | const failures: string[] = [] |
| @@ -59,6 +89,7 @@ async function runLeg(options: RunnerOptions): Promise<LegOutcome> { | ||
| 59 | 89 | packageName: PACKAGE, |
| 60 | 90 | output: project, |
| 61 | 91 | config: CONFIG, |
| 92 | runtimeDir: leg.runtimeDir, | |
| 62 | 93 | }) |
| 63 | 94 | |
| 64 | 95 | const nonce = `craft-e2e-${label}-${options.runId}` |
| @@ -81,24 +112,26 @@ async function runLeg(options: RunnerOptions): Promise<LegOutcome> { | ||
| 81 | 112 | throw new Error(`gradle reported success but ${apk} is missing`) |
| 82 | 113 | |
| 83 | 114 | // Which side is about to answer, established from the APK rather than |
| 84 | // assumed. Nothing in the generator copies libcraft.so into jniLibs (#200), | |
| 85 | // so `System.loadLibrary("craft")` throws, `CraftNative.isAvailable` is | |
| 86 | // false, and Kotlin serves every action. The leg reports that as fact, so it | |
| 87 | // checks it: when #200 lands this assertion fails, and whoever fixes it has | |
| 88 | // to come back and teach this leg to assert the Zig path instead of | |
| 89 | // declaring there isn't one. | |
| 115 | // assumed. `System.loadLibrary("craft")` finding nothing is a designed | |
| 116 | // no-op — CraftNative catches it and every action falls through to Kotlin — | |
| 117 | // so a runtime leg whose library never shipped would look exactly like the | |
| 118 | // shim leg and pass. The APK is the only place to see the difference before | |
| 119 | // the app runs. | |
| 90 | 120 | const listed = await command(['unzip', '-l', apk], { logPath: join(evidence, 'apk.txt') }) |
| 91 | 121 | const zigLibraries = listed.stdout |
| 92 | 122 | .split('\n') |
| 93 | 123 | .map(line => line.trim().split(/\s+/).pop() ?? '') |
| 94 | 124 | .filter(name => /^lib\/[^/]+\/libcraft\.so$/.test(name)) |
| 95 | 125 | |
| 96 | if (zigLibraries.length) { | |
| 126 | if (leg.requireZig && !zigLibraries.some(name => name.includes('/x86_64/'))) { | |
| 97 | 127 | failures.push( |
| 98 | `the APK ships ${zigLibraries.join(', ')}, so the Zig runtime may be serving these calls. ` | |
| 99 | + 'This leg reports zigActions: [] on the basis that it never loads — teach it to assert which side answered before trusting that again.', | |
| 128 | `the runtime leg's APK carries ${zigLibraries.length ? zigLibraries.join(', ') : 'no libcraft.so at all'}, ` | |
| 129 | + 'and nothing for x86_64 — which is the architecture every emulator runs, so the runtime could not load here whatever else shipped', | |
| 100 | 130 | ) |
| 101 | 131 | } |
| 132 | if (!leg.requireZig && zigLibraries.length) { | |
| 133 | failures.push(`the shim leg's APK ships ${zigLibraries.join(', ')}; it was generated with no runtime`) | |
| 134 | } | |
| 102 | 135 | |
| 103 | 136 | const serial = await resolveSerial() |
| 104 | 137 | writeFileSync(join(evidence, 'device.txt'), `${serial}\n`) |
| @@ -156,6 +189,34 @@ async function runLeg(options: RunnerOptions): Promise<LegOutcome> { | ||
| 156 | 189 | failures.push(`the bridge reported sdkVersion ${observed[1]} but the device says ${sdk}`) |
| 157 | 190 | } |
| 158 | 191 | |
| 192 | // Did the Zig runtime actually load and bind? JNI_OnLoad says so itself, on | |
| 193 | // the happy path as well as the failure ones — which is deliberate, because | |
| 194 | // "loaded and bound" and "never shipped" are otherwise both silent, and this | |
| 195 | // is also the only line that proves the log channel is working rather than | |
| 196 | // merely quiet. | |
| 197 | // | |
| 198 | // There is no per-action dispatch line to count the way the iOS legs do. | |
| 199 | // Binding is the gate instead: once the natives are registered, every | |
| 200 | // CraftNative wrapper returns a value and the Kotlin falls through to Zig | |
| 201 | // rather than the other way round. | |
| 202 | const registered = logText.match(REGISTERED) | |
| 203 | const zigActions: string[] = [] | |
| 204 | ||
| 205 | if (leg.requireZig) { | |
| 206 | if (!registered) { | |
| 207 | failures.push( | |
| 208 | 'the Zig runtime never reported registering its natives. Either libcraft.so did not load, ' | |
| 209 | + `or JNI_OnLoad refused a descriptor — see ${label}/logcat.txt under the CraftNative tag, which now carries the reason`, | |
| 210 | ) | |
| 211 | } | |
| 212 | else { | |
| 213 | zigActions.push(`registered:${registered[1]}`) | |
| 214 | } | |
| 215 | } | |
| 216 | else if (registered) { | |
| 217 | failures.push(`the shim leg registered ${registered[1]} natives; it was generated with no runtime`) | |
| 218 | } | |
| 219 | ||
| 159 | 220 | return { |
| 160 | 221 | name: label, |
| 161 | 222 | status: failures.length ? 'failed' : 'passed', |
| @@ -163,29 +224,47 @@ async function runLeg(options: RunnerOptions): Promise<LegOutcome> { | ||
| 163 | 224 | planned: verdict.planned, |
| 164 | 225 | passed: verdict.passed, |
| 165 | 226 | failed: verdict.failed, |
| 166 | // Nothing copies libcraft.so into the generated app's jniLibs (#200), so | |
| 167 | // CraftNative.isAvailable is always false and Kotlin serves every action. | |
| 168 | // Reported as empty rather than omitted, so the report says which side | |
| 169 | // answered instead of leaving it open. | |
| 170 | zigActions: [], | |
| 227 | zigActions, | |
| 171 | 228 | evidence: evidence.replace(`${options.root}/`, ''), |
| 172 | 229 | } |
| 173 | 230 | } |
| 174 | 231 | |
| 175 | 232 | export async function runAndroid(options: RunnerOptions): Promise<LegOutcome[]> { |
| 176 | try { | |
| 177 | return [await runLeg(options)] | |
| 233 | const runtimeDir = options.androidRuntimeDir | |
| 234 | if (!runtimeDir) { | |
| 235 | throw new Error( | |
| 236 | 'no Android Zig runtime directory; run `zig build build-android-all -Doptimize=ReleaseSafe` ' | |
| 237 | + 'in packages/zig and pass --android-runtime packages/zig/zig-out/android', | |
| 238 | ) | |
| 178 | 239 | } |
| 179 | catch (caught) { | |
| 180 | return [{ | |
| 181 | name: 'android-shim', | |
| 182 | status: 'failed', | |
| 183 | failures: [caught instanceof Error ? caught.message : String(caught)], | |
| 184 | planned: [], | |
| 185 | passed: [], | |
| 186 | failed: [], | |
| 187 | zigActions: [], | |
| 188 | evidence: join(options.evidenceDir, 'android-shim').replace(`${options.root}/`, ''), | |
| 189 | }] | |
| 240 | ||
| 241 | const abi = join(runtimeDir, 'x86_64', 'libcraft.so') | |
| 242 | if (!existsSync(abi)) { | |
| 243 | throw new Error( | |
| 244 | `${abi} is missing. Every Android emulator is x86_64, so without it the runtime leg would ` | |
| 245 | + 'install an APK that cannot load the library and pass as though the shim were the only option. ' | |
| 246 | + 'Run `zig build build-android-all -Doptimize=ReleaseSafe` in packages/zig.', | |
| 247 | ) | |
| 190 | 248 | } |
| 249 | ||
| 250 | const outcomes: LegOutcome[] = [] | |
| 251 | for (const leg of legs(runtimeDir)) { | |
| 252 | try { | |
| 253 | outcomes.push(await runLeg(leg, options)) | |
| 254 | } | |
| 255 | catch (caught) { | |
| 256 | outcomes.push({ | |
| 257 | name: `android-${leg.name}`, | |
| 258 | status: 'failed', | |
| 259 | failures: [caught instanceof Error ? caught.message : String(caught)], | |
| 260 | planned: [], | |
| 261 | passed: [], | |
| 262 | failed: [], | |
| 263 | zigActions: [], | |
| 264 | evidence: join(options.evidenceDir, `android-${leg.name}`).replace(`${options.root}/`, ''), | |
| 265 | }) | |
| 266 | } | |
| 267 | } | |
| 268 | ||
| 269 | return outcomes | |
| 191 | 270 | } |
| @@ -11,6 +11,8 @@ export interface RunnerOptions { | ||
| 11 | 11 | runId: string |
| 12 | 12 | /** Directory holding the Zig iOS simulator archives. */ |
| 13 | 13 | iosRuntimeDir: string | null |
| 14 | /** Directory holding the Zig Android `<abi>/libcraft.so` tree. */ | |
| 15 | androidRuntimeDir: string | null | |
| 14 | 16 | } |
| 15 | 17 | |
| 16 | 18 | export interface LegOutcome { |