ReviewOS

also looking at this

craft-native/craft

test(mobile): run the generated apps on a simulator and an emulator

#199
Merged glennmichael123 wants to merge fix/mobile-e2e-real-assertions into main
26 files +3009 -293

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

.github/workflows/README.mdmodified+5-1
Changes to .github/workflows/README.md
@@ -39,8 +39,12 @@ and `build-tools;36.0.0`, a booted emulator or attached device, and the Zig
3939Android libraries — it runs twice as well, once with `libcraft.so` packaged
4040into the APK and once without:
4141
42The Zig Android library links bionic, so building it needs the NDK —
43`sdkmanager --install "ndk;26.1.10909125"`. Without it `build-android` refuses
44rather than producing a library that installs and then fails to load.
45
4246```bash
43cd packages/zig && zig build build-android-all -Doptimize=ReleaseSafe && cd -
47cd packages/zig && zig build build-android-all -Doptimize=ReleaseSafe -Dandroid-ndk="$ANDROID_NDK_HOME" && cd -
4448bun run test:mobile-e2e:android
4549```
4650
.github/workflows/mobile-e2e.ymlmodified+10-3
Changes to .github/workflows/mobile-e2e.yml
@@ -215,13 +215,20 @@ jobs:
215215 - name: First-party Zig dependencies
216216 uses: ./.github/actions/first-party-zig-deps
217217
218 # The NDK is what supplies bionic. Without it the JNI library builds and
219 # then fails at dlopen with `cannot locate symbol "__tls_get_addr"`, which
220 # CraftNative catches and turns into a silent fallback to the Kotlin shim
221 # — the failure this whole leg exists to catch.
222 - name: Install the Android NDK
223 run: |
224 sdkmanager --install "ndk;26.1.10909125"
225 echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/26.1.10909125" >> "$GITHUB_ENV"
226
218227 # Both ABIs. x86_64 is the one the emulator actually loads; arm64-v8a is
219228 # built so the runtime leg installs the same APK a phone would get.
220 # No NDK: the JNI library links no libc on purpose, which is what lets
221 # Zig build it alone (android_dispatch.zig records why).
222229 - name: Build the Zig Android runtime
223230 working-directory: packages/zig
224 run: zig build build-android-all -Doptimize=ReleaseSafe
231 run: zig build build-android-all -Doptimize=ReleaseSafe -Dandroid-ndk="$ANDROID_NDK_HOME"
225232
226233 - name: Enable KVM
227234 run: |
packages/android/src/index.test.tsmodified+26-0
Changes to packages/android/src/index.test.ts
@@ -1299,3 +1299,29 @@ describe('Zig runtime installation', () => {
12991299 }
13001300 })
13011301})
1302
1303describe('the Zig library and the generated app agree on how old a device may be', () => {
1304 it('builds the JNI library for no newer an API than minSdk', () => {
1305 // One prebuilt libcraft.so serves every generated app, so it has to be
1306 // linked against the *oldest* API craft's default configuration claims to
1307 // support. Build it against a newer one and the linker happily binds
1308 // symbols that are simply absent on an older phone — and the failure is
1309 // dlopen refusing the library at startup, on exactly the devices the app
1310 // said it ran on.
1311 //
1312 // Nothing else connects a Zig constant to a TypeScript one, so this reads
1313 // build.zig and compares.
1314 const buildZig = readFileSync(join(import.meta.dir, '../../zig/build.zig'), 'utf8')
1315 const declared = buildZig.match(/const android_api_level: u32 = (\d+);/)
1316
1317 expect(declared).not.toBeNull()
1318
1319 // DEFAULT_CONFIG is not exported, so read it the same way: from the source
1320 // that defines it, which is the thing that would have to change.
1321 const generator = readFileSync(join(import.meta.dir, 'index.ts'), 'utf8')
1322 const minSdk = generator.match(/minSdk:\s*(\d+)/)
1323
1324 expect(minSdk).not.toBeNull()
1325 expect(Number(declared![1])).toBeLessThanOrEqual(Number(minSdk![1]))
1326 })
1327})
packages/android/templates/CraftNative.kt.templatemodified+14-0
Changes to packages/android/templates/CraftNative.kt.template
@@ -110,10 +110,24 @@ object CraftNative {
110110 */
111111 val isAvailable: Boolean = try {
112112 System.loadLibrary("craft")
113 // Said out loud, because the two outcomes below are otherwise
114 // indistinguishable from outside: a library that never shipped and a
115 // library that loaded and then failed to bind both end with the shim
116 // answering every call. Zig logs its own registration under the same
117 // tag, so the pair reads as a sequence.
118 android.util.Log.i("CraftNative", "craft: libcraft.so loaded")
113119 true
114120 } catch (e: UnsatisfiedLinkError) {
121 // The reason, not just the fact. This used to be a bare `false`, and
122 // the first time the runtime was genuinely shipped and genuinely
123 // failed to load, the only evidence anywhere was that nothing
124 // happened — no linker line, no exception, nothing in logcat at all.
125 // The message carries the dynamic linker's own complaint, which is the
126 // one thing that says *why*.
127 android.util.Log.w("CraftNative", "craft: libcraft.so did not load: ${e.message}")
115128 false
116129 } catch (e: SecurityException) {
130 android.util.Log.w("CraftNative", "craft: libcraft.so was refused: ${e.message}")
117131 false
118132 }
119133
packages/zig/build.zigmodified+182-30
Changes to packages/zig/build.zig
@@ -15,6 +15,19 @@ pub fn build(b: *std.Build) void {
1515 // to avoid Zig bug where --sysroot breaks @cImport (ziglang/zig#22704, #25010)
1616 const macos_sdk = b.option([]const u8, "macos-sdk", "macOS SDK path for cross-compilation");
1717
18 // The Android NDK, needed by the JNI library and nothing else. An explicit
19 // path rather than a read of ANDROID_NDK_HOME, matching -Dmacos-sdk: the
20 // caller passes "$ANDROID_NDK_HOME" when that is what they mean, and a
21 // build that depends on an environment variable is a build that behaves
22 // differently in two shells for reasons neither of them shows.
23 //
24 // It stopped being optional because without bionic the library builds,
25 // ships, and then fails to load — std references getauxval and
26 // __tls_get_addr, both of which live there, and Zig answers "unable to
27 // provide libc" for an android target. The failure was a silent fallback
28 // to the Kotlin shim, which is the quietest of the three outcomes.
29 const android_ndk = b.option([]const u8, "android-ndk", "Android NDK root, required by build-android");
30
1831 // Set when the opt-in JavaScript tests are wired up, so `test` can include
1932 // them without the declaration being scoped inside the `if`.
2033 var js_test_run: ?*std.Build.Step.Run = null;
@@ -2391,6 +2404,11 @@ pub fn build(b: *std.Build) void {
23912404 .cpu_arch = .aarch64,
23922405 .os_tag = .linux,
23932406 .abi = .android,
2407 // Named explicitly, because Zig's default is 29 and the stubs this
2408 // links against come from the API-26 directory. A target and a sysroot
2409 // that disagree is how a library binds a symbol the device does not
2410 // have.
2411 .android_api_level = android_api_level,
23942412 });
23952413
23962414 const android_arm64_lib = b.addLibrary(.{
@@ -2425,46 +2443,74 @@ pub fn build(b: *std.Build) void {
24252443 // the arch-suffixed name beside it: the loader picks the ABI directory,
24262444 // and a per-arch name would make the Kotlin need to know its own
24272445 // architecture.
2428 const android_arm64_jni = b.addLibrary(.{
2429 .linkage = .dynamic,
2430 .name = "craft",
2431 .root_module = b.createModule(.{
2432 .root_source_file = b.path("src/android_dispatch.zig"),
2433 .target = android_arm64_target,
2434 .optimize = optimize,
2435 }),
2436 });
2446 if (android_ndk) |ndk| {
2447 const android_arm64_jni = b.addLibrary(.{
2448 .linkage = .dynamic,
2449 .name = "craft",
2450 .root_module = b.createModule(.{
2451 .root_source_file = b.path("src/android_dispatch.zig"),
2452 .target = android_arm64_target,
2453 .optimize = optimize,
2454 }),
2455 });
2456 linkAndroidLibc(b, android_arm64_jni, ndk, "aarch64-linux-android");
24372457
2438 const android_arm64_jni_install = b.addInstallArtifact(android_arm64_jni, .{
2439 .dest_dir = .{ .override = .{ .custom = "android/arm64-v8a" } },
2440 });
2441 build_android.dependOn(&android_arm64_jni_install.step);
2442 build_android_all.dependOn(&android_arm64_jni_install.step);
2458 const android_arm64_jni_install = b.addInstallArtifact(android_arm64_jni, .{
2459 .dest_dir = .{ .override = .{ .custom = "android/arm64-v8a" } },
2460 });
2461 build_android.dependOn(&android_arm64_jni_install.step);
2462 build_android_all.dependOn(&android_arm64_jni_install.step);
2463 } else {
2464 // Not built at all rather than built without bionic. The library would
2465 // compile, install, ship in the APK and then fail at `dlopen` with two
2466 // unresolved symbols, at which point `CraftNative` catches the
2467 // `UnsatisfiedLinkError` and every action quietly uses the Kotlin shim.
2468 // A step that refuses, naming the flag, is the same information
2469 // delivered where someone can act on it.
2470 //
2471 // A `Fail` step rather than a panic, so this is only reached by
2472 // *asking* for an Android build: `zig build test` on a machine with no
2473 // NDK must still work, and it did not while this was a configure-time
2474 // panic.
2475 const missing = b.addFail(
2476 "build-android needs the NDK: pass -Dandroid-ndk=\"$ANDROID_NDK_HOME\" " ++
2477 "(sdkmanager --install \"ndk;26.1.10909125\"). Without bionic the JNI library " ++
2478 "builds and then loads nowhere — std references getauxval and __tls_get_addr, " ++
2479 "both of which live there, and Zig cannot supply libc for an android target.",
2480 );
2481 build_android.dependOn(&missing.step);
2482 build_android_x86.dependOn(&missing.step);
2483 build_android_all.dependOn(&missing.step);
2484 }
24432485
24442486 // Android Emulator (x86_64)
24452487 const android_x86_target = b.resolveTargetQuery(.{
24462488 .cpu_arch = .x86_64,
24472489 .os_tag = .linux,
24482490 .abi = .android,
2491 .android_api_level = android_api_level,
24492492 });
24502493
2451 const android_x86_jni = b.addLibrary(.{
2452 .linkage = .dynamic,
2453 .name = "craft",
2454 .root_module = b.createModule(.{
2455 .root_source_file = b.path("src/android_dispatch.zig"),
2456 .target = android_x86_target,
2457 .optimize = optimize,
2458 }),
2459 });
2494 if (android_ndk) |ndk| {
2495 const android_x86_jni = b.addLibrary(.{
2496 .linkage = .dynamic,
2497 .name = "craft",
2498 .root_module = b.createModule(.{
2499 .root_source_file = b.path("src/android_dispatch.zig"),
2500 .target = android_x86_target,
2501 .optimize = optimize,
2502 }),
2503 });
2504 linkAndroidLibc(b, android_x86_jni, ndk, "x86_64-linux-android");
24602505
2461 // `x86_64`, matching the jniLibs directory the loader looks in — not
2462 // `x86`, which is the 32-bit ABI and would be silently ignored.
2463 const android_x86_jni_install = b.addInstallArtifact(android_x86_jni, .{
2464 .dest_dir = .{ .override = .{ .custom = "android/x86_64" } },
2465 });
2466 build_android_x86.dependOn(&android_x86_jni_install.step);
2467 build_android_all.dependOn(&android_x86_jni_install.step);
2506 // `x86_64`, matching the jniLibs directory the loader looks in — not
2507 // `x86`, which is the 32-bit ABI and would be silently ignored.
2508 const android_x86_jni_install = b.addInstallArtifact(android_x86_jni, .{
2509 .dest_dir = .{ .override = .{ .custom = "android/x86_64" } },
2510 });
2511 build_android_x86.dependOn(&android_x86_jni_install.step);
2512 build_android_all.dependOn(&android_x86_jni_install.step);
2513 }
24682514
24692515 const android_x86_lib = b.addLibrary(.{
24702516 .linkage = .static,
@@ -2524,6 +2570,112 @@ pub fn build(b: *std.Build) void {
25242570/// Link platform-specific system libraries for a build module.
25252571/// Centralizes the per-OS library linking that is shared across exe, craft_exe,
25262572/// lib_unit_tests, and cross-compilation targets.
2573/// The Android API level the JNI library targets.
2574///
2575/// Must not exceed the generator's default `minSdk`, because one prebuilt
2576/// library serves every generated app: building against a newer API would let
2577/// the linker bind symbols that are simply absent on an older device, and the
2578/// failure is `dlopen` refusing the library at startup on exactly the phones
2579/// the app claims to support. `packages/android/src/index.test.ts` asserts the
2580/// two numbers agree, since nothing else connects a Zig constant to a
2581/// TypeScript one.
2582const android_api_level: u32 = 26;
2583
2584/// The NDK's sysroot, or null when `ndk` does not contain one.
2585///
2586/// The prebuilt directory is named for the *host* and the NDK ships exactly
2587/// one, so the candidates are tried in turn rather than guessed from
2588/// `builtin.os`. An Apple-silicon Mac still gets `darwin-x86_64` from most NDK
2589/// releases, which is the guess that would have been wrong.
2590///
2591/// Null rather than a panic: this runs at configure time, so a panic here
2592/// would break `zig build test` for anyone who passed a wrong -Dandroid-ndk,
2593/// not just the Android steps.
2594fn androidSysroot(b: *std.Build, ndk: []const u8) ?[]const u8 {
2595 const io = b.graph.io;
2596 for ([_][]const u8{ "darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-x86_64" }) |host| {
2597 const candidate = b.pathJoin(&.{ ndk, "toolchains/llvm/prebuilt", host, "sysroot" });
2598 std.Io.Dir.cwd().access(io, candidate, .{}) catch continue;
2599 return candidate;
2600 }
2601 return null;
2602}
2603
2604/// Link one Android artifact against the NDK's bionic.
2605///
2606/// A `--libc` file, not include and library paths. Paths alone are what
2607/// `applySdkPaths` does for Apple SDKs and they are not enough here: with
2608/// `link_libc` set and no libc file, Zig looks the target up in its own table
2609/// of libcs it can build, does not find android, and stops with
2610///
2611/// error: unable to provide libc for target "x86_64-linux...-android.29"
2612///
2613/// before it ever consults a search path. The libc file is how Zig is told the
2614/// libc already exists.
2615///
2616/// `gcc_dir` is not optional, and its absence is reported under the wrong
2617/// name. For any `.linux` target — Android is one — Zig takes crtbegin and
2618/// crtend from `gcc_dir`, not `crt_dir`, and returns
2619/// `LibCInstallationMissingCrtDir` when `gcc_dir` is empty
2620/// (std/zig/LibCInstallation.zig, `resolveCrtPaths`). So a file with a correct
2621/// `crt_dir` and a blank `gcc_dir` fails with "libc installation is missing crt
2622/// directory", which sends you checking a directory that is fine. Bionic keeps
2623/// crtbegin_so.o and crtend_so.o beside its stubs, so both fields point at the
2624/// same API-level directory. Zig 0.17.0-dev.1441 read them from `crt_dir`,
2625/// which is why this passed locally and failed on CI's 1963.
2626///
2627/// Still not `--sysroot`, which breaks `@cImport` (ziglang/zig#22704, #25010)
2628/// and is why the Apple side takes the path route too.
2629fn linkAndroidLibc(b: *std.Build, compile: *std.Build.Step.Compile, ndk: []const u8, triple: []const u8) void {
2630 const io = b.graph.io;
2631
2632 const sysroot = androidSysroot(b, ndk) orelse {
2633 compile.step.dependOn(&b.addFail(b.fmt(
2634 "-Dandroid-ndk={s} has no toolchains/llvm/prebuilt/<host>/sysroot. " ++
2635 "Point it at an NDK root, not at the SDK.",
2636 .{ndk},
2637 )).step);
2638 return;
2639 };
2640
2641 const api = b.fmt("{d}", .{android_api_level});
2642 const lib_dir = b.pathJoin(&.{ sysroot, "usr/lib", triple, api });
2643
2644 // Checked here so a missing API level fails with the path and the fix,
2645 // rather than as a linker error about crtbegin_so.o. Lazy, like the NDK
2646 // check: only a build that asks for this artifact is refused.
2647 std.Io.Dir.cwd().access(io, b.pathJoin(&.{ lib_dir, "crtbegin_so.o" }), .{}) catch {
2648 compile.step.dependOn(&b.addFail(b.fmt(
2649 "no crtbegin_so.o in {s}: this NDK does not carry API {s} for {s}. " ++
2650 "Install one that does, or lower android_api_level.",
2651 .{ lib_dir, api, triple },
2652 )).step);
2653 return;
2654 };
2655
2656 const libc_file = b.addWriteFile("android-libc.conf", b.fmt(
2657 \\include_dir={s}
2658 \\sys_include_dir={s}
2659 \\crt_dir={s}
2660 \\msvc_lib_dir=
2661 \\kernel32_lib_dir=
2662 \\gcc_dir={s}
2663 \\
2664 , .{
2665 b.pathJoin(&.{ sysroot, "usr/include" }),
2666 // The triple-specific headers: bionic keeps its architecture-dependent
2667 // definitions there and the generic ones include them by relative path.
2668 b.pathJoin(&.{ sysroot, "usr/include", triple }),
2669 lib_dir,
2670 lib_dir,
2671 }));
2672
2673 compile.setLibCFile(libc_file.getDirectory().path(b, "android-libc.conf"));
2674 compile.step.dependOn(&libc_file.step);
2675 compile.root_module.link_libc = true;
2676 compile.root_module.addLibraryPath(.{ .cwd_relative = lib_dir });
2677}
2678
25272679/// The iOS SDK sysroot, resolved through xcrun at configure time.
25282680///
25292681/// Needed because the iOS archives compile the vendored sqlite3.c, and C
packages/zig/src/android_dispatch.zigmodified+18-6
Changes to packages/zig/src/android_dispatch.zig
@@ -77,12 +77,24 @@ const Jni = jni.Jni;
7777
7878/// The allocator every reply is built with.
7979///
80/// `page_allocator`, and the choice is a build constraint rather than a
81/// preference. `c_allocator` would pull in bionic, and Zig cannot provide
82/// bionic — a static Android library links nothing and builds anyway, but this
83/// one is shared and has to resolve its symbols, so libc here means the whole
84/// library needs the NDK to build at all. Standing free of libc keeps it
85/// buildable with nothing but Zig.
80/// This used to carry a note explaining that `page_allocator` was chosen over
81/// `c_allocator` to keep the library buildable with nothing but Zig, since
82/// bionic would mean needing the NDK. That reasoning was sound and the
83/// conclusion was wrong: standing free of libc does not make the references go
84/// away, it only defers them. `page_allocator` needs `getauxval` for the page
85/// size, any `threadlocal` needs `__tls_get_addr`, and so does `std.fmt` — so
86/// the library linked clean, shipped, and then failed at `dlopen` with
87///
88/// cannot locate symbol "__tls_get_addr" referenced by ".../libcraft.so"
89///
90/// which `CraftNative` caught as an `UnsatisfiedLinkError` and turned into a
91/// silent fallback to the Kotlin shim. Every generated Android app was in that
92/// state; the mobile E2E suite is what finally said so out loud.
93///
94/// So the library links bionic now and `build-android` requires the NDK
95/// (`build.zig`'s -Dandroid-ndk). The allocator is free to be whichever one
96/// suits; it stays `page_allocator` because the arena below wants pages
97/// anyway.
8698///
8799/// Every native call wraps this in an arena, so the page granularity costs one
88100/// page per call rather than one per allocation, and nothing has to be freed