| @@ -0,0 +1,767 @@ |
| 1 | +//! The `mobile` namespace's three Vision actions: `classifyImage`, |
| 2 | +//! `detectObjects` and `recognizeText`. |
| 3 | +//! |
| 4 | +//! All three were unservable until `ios_config.zig` landed — their Swift arms |
| 5 | +//! are conditional on `config.enableMLKit`, and the dispatcher's guard has no |
| 6 | +//! `else`, so a page calling one in an app that left the flag off gets a |
| 7 | +//! promise that never settles. `ios_dispatch.route` now refuses on the flag |
| 8 | +//! before this module runs, and what is left is Swift's own behaviour. |
| 9 | +//! |
| 10 | +//! ## Why these three are the cheapest remaining tier |
| 11 | +//! |
| 12 | +//! No permission, no presented UI, no delegate, no long-lived subscription. |
| 13 | +//! A page hands over a base64 image and gets an array back. The whole |
| 14 | +//! interaction is a pure function of its input, which also makes it the one |
| 15 | +//! tier a fixture can prove end to end without a human tapping anything: the |
| 16 | +//! test page draws a word on a `<canvas>`, sends `toDataURL`'s payload, and |
| 17 | +//! asserts the word comes back. |
| 18 | +//! |
| 19 | +//! ## `detectObjects` detects animals |
| 20 | +//! |
| 21 | +//! `CraftApp.swift:4656` builds a `VNRecognizeAnimalsRequest`, whose entire |
| 22 | +//! vocabulary is cat and dog. The action is named `detectObjects` and its TS |
| 23 | +//! signature promises objects, but what it does is find pets. That is carried |
| 24 | +//! across unchanged and written down here rather than quietly widened to |
| 25 | +//! `VNDetectRectanglesRequest` or a Core ML model: the reply shape, the label |
| 26 | +//! strings and the empty result for a photo of a chair are all observable |
| 27 | +//! behaviour that a page may already depend on. Renaming the action or |
| 28 | +//! changing the request is a decision for whoever owns the API, not a |
| 29 | +//! translation detail. |
| 30 | +//! |
| 31 | +//! ## Threading, and the one place a shortcut was refused |
| 32 | +//! |
| 33 | +//! Swift performs every request on `DispatchQueue.global(qos: .userInitiated)`, |
| 34 | +//! and so does this. Running Vision inline on the main thread would have been |
| 35 | +//! materially less code — no context to hand across, no retain/release pair — |
| 36 | +//! and would freeze the UI for as long as recognition takes, which for |
| 37 | +//! `.accurate` text on a full-page image is comfortably visible. The author of |
| 38 | +//! the Swift moved it off the main thread deliberately; a port that quietly |
| 39 | +//! moved it back would be trading the user's responsiveness for the porter's |
| 40 | +//! convenience. |
| 41 | +//! |
| 42 | +//! The hand-across is a heap-allocated `Work`, passed straight to |
| 43 | +//! `dispatch_async_f` as its context pointer, rather than the index-into-a-pool |
| 44 | +//! shape `ios_events` uses. The pool exists there because a location stream can |
| 45 | +//! burst faster than the main queue drains; a Vision request is user-initiated |
| 46 | +//! and arrives one at a time, so a bounded pool would only add an exhaustion |
| 47 | +//! path nothing reaches. `ios_async`'s ticket pool still bounds the number of |
| 48 | +//! outstanding *replies*, which is the resource that actually needs a ceiling. |
| 49 | +//! |
| 50 | +//! ## Two of the three do not work on a simulator |
| 51 | +//! |
| 52 | +//! `VNClassifyImageRequest` and `VNRecognizeAnimalsRequest` are Core ML |
| 53 | +//! backed, and the simulator cannot build an inference context for either — |
| 54 | +//! they fail with "Failed to create espresso context" and "Could not create |
| 55 | +//! inference context" respectively. `VNRecognizeTextRequest` uses a different |
| 56 | +//! path and works there. |
| 57 | +//! |
| 58 | +//! That is a property of the simulator, not of this module, and the right |
| 59 | +//! response is the one below: report the failure with the framework's own |
| 60 | +//! description in the log and settle the page's promise with |
| 61 | +//! `NATIVE_CALL_FAILED`. An empty array would have been the tempting answer |
| 62 | +//! and the wrong one — "I found no cats" and "I could not look" are different |
| 63 | +//! facts, and only one of them is true. The fixture asserts that both actions |
| 64 | +//! *answer*, by either route, because answering at all is what this phase |
| 65 | +//! changes: their Swift arms are gated with no `else`. |
| 66 | +//! |
| 67 | +//! The request handler is built on the calling thread and retained across the |
| 68 | +//! hop. `[UIImage imageWithData:]` returns an autoreleased object, and the main |
| 69 | +//! runloop drains its pool at the end of the turn — so a `CGImage` merely |
| 70 | +//! borrowed from it would be freed while the background queue was still reading |
| 71 | +//! it. `VNImageRequestHandler` retains the image it is initialised with, so |
| 72 | +//! retaining the handler is enough to keep the pixels alive, and it is released |
| 73 | +//! on every path out of the background function. |
| 74 | + |
| 75 | +const std = ; |
| 76 | +const builtin = ; |
| 77 | +const capabilities = ; |
| 78 | +const bridge_error = ; |
| 79 | +const objc_runtime = ; |
| 80 | +const ios_async = ; |
| 81 | + |
| 82 | +const objc = objc_runtime.objc; |
| 83 | +const BridgeError = bridge_error.BridgeError; |
| 84 | +const Id = ?*anyopaque; |
| 85 | +const is_darwin = builtin.target.os.tag.isDarwin(); |
| 86 | + |
| 87 | +pub const A = struct { |
| 88 | + pub const classify_image = "classifyImage"; |
| 89 | + pub const detect_objects = "detectObjects"; |
| 90 | + pub const recognize_text = "recognizeText"; |
| 91 | +}; |
| 92 | + |
| 93 | +/// `.result`: each action resolves with an array or rejects, and the injected |
| 94 | +/// JS returns a promise the page awaits. |
| 95 | +pub const capability_actions = [_]capabilities.ActionDecl{ |
| 96 | + .{ .name = A.classify_image, .reply = .result }, |
| 97 | + .{ .name = A.detect_objects, .reply = .result }, |
| 98 | + .{ .name = A.recognize_text, .reply = .result }, |
| 99 | +}; |
| 100 | + |
| 101 | +/// Which request to build, and how to shape what it returns. |
| 102 | +const Kind = enum { |
| 103 | + classify, |
| 104 | + detect, |
| 105 | + recognize, |
| 106 | + |
| 107 | + fn action(self: Kind) []const u8 { |
| 108 | + return switch (self) { |
| 109 | + .classify => A.classify_image, |
| 110 | + .detect => A.detect_objects, |
| 111 | + .recognize => A.recognize_text, |
| 112 | + }; |
| 113 | + } |
| 114 | + |
| 115 | + /// The Vision request class. `detect` is `VNRecognizeAnimalsRequest` on |
| 116 | + /// purpose — see the module comment. |
| 117 | + fn className(self: Kind) [*:0]const u8 { |
| 118 | + return switch (self) { |
| 119 | + .classify => "VNClassifyImageRequest", |
| 120 | + .detect => "VNRecognizeAnimalsRequest", |
| 121 | + .recognize => "VNRecognizeTextRequest", |
| 122 | + }; |
| 123 | + } |
| 124 | +}; |
| 125 | + |
| 126 | +/// Swift keeps the first ten classifications and all of everything else. |
| 127 | +/// |
| 128 | +/// `observations.prefix(10)` at `CraftApp.swift:4632`. Vision returns well over |
| 129 | +/// a thousand classifications for a single image, ordered by confidence, so the |
| 130 | +/// cap is what keeps the reply from being a megabyte of labels with confidence |
| 131 | +/// 0.0001. It applies to `classifyImage` alone. |
| 132 | +const max_classifications = 10; |
| 133 | + |
| 134 | +pub const VisionBridge = struct { |
| 135 | + allocator: std.mem.Allocator, |
| 136 | + |
| 137 | + const Self = @This(); |
| 138 | + |
| 139 | + pub fn init(allocator: std.mem.Allocator) Self { |
| 140 | + return .{ .allocator = allocator }; |
| 141 | + } |
| 142 | + |
| 143 | + pub fn deinit(_: *Self) void {} |
| 144 | + |
| 145 | + pub fn handleMessage(self: *Self, action: []const u8, data: []const u8) !void { |
| 146 | + const kind: Kind = if (std.mem.eql(u8, action, A.classify_image)) |
| 147 | + .classify |
| 148 | + else if (std.mem.eql(u8, action, A.detect_objects)) |
| 149 | + .detect |
| 150 | + else if (std.mem.eql(u8, action, A.recognize_text)) |
| 151 | + .recognize |
| 152 | + else |
| 153 | + return BridgeError.UnknownAction; |
| 154 | + |
| 155 | + try self.perform(kind, data); |
| 156 | + } |
| 157 | + |
| 158 | + /// Decode the image, build the request, and hand both to a background |
| 159 | + /// queue. |
| 160 | + /// |
| 161 | + /// Everything that can fail synchronously does so here, before a ticket is |
| 162 | + /// taken: a bad payload, an image Vision cannot read, a process with no |
| 163 | + /// Vision framework. That ordering matters — a ticket acquired and then |
| 164 | + /// dropped on an early return is a slot that stays in use until the pool |
| 165 | + /// wraps, and the pool is what bounds concurrent replies. |
| 166 | + fn perform(self: *Self, kind: Kind, data: []const u8) !void { |
| 167 | + if (!is_darwin) return BridgeError.PlatformNotSupported; |
| 168 | + |
| 169 | + const base64 = try readImageField(self.allocator, data); |
| 170 | + defer self.allocator.free(base64); |
| 171 | + |
| 172 | + const handler = try makeRequestHandler(self.allocator, base64); |
| 173 | + errdefer release(handler); |
| 174 | + |
| 175 | + const request = try makeRequest(kind); |
| 176 | + errdefer release(request); |
| 177 | + |
| 178 | + // The process allocator, explicitly, and not `self.allocator`. This |
| 179 | + // outlives the dispatch frame by definition — the background function |
| 180 | + // is what frees it — so it must not come from anything scoped to the |
| 181 | + // call. The two happen to be the same allocator today |
| 182 | + // (`ios_dispatch` dispatches with `c_allocator`), which is exactly why |
| 183 | + // the coupling is worth spelling out rather than inheriting. |
| 184 | + const work = std.heap.c_allocator.create(Work) catch return BridgeError.AllocationFailed; |
| 185 | + errdefer std.heap.c_allocator.destroy(work); |
| 186 | + |
| 187 | + // Last, so every failure above answers synchronously with its own |
| 188 | + // error rather than through a ticket nobody would deliver. |
| 189 | + const ticket = ios_async.acquire(kind.action()) orelse { |
| 190 | + std.log.warn( |
| 191 | + "{s}: no free reply slot; {d} native calls are already awaiting one", |
| 192 | + .{ kind.action(), ios_async.max_in_flight }, |
| 193 | + ); |
| 194 | + return BridgeError.NativeCallFailed; |
| 195 | + }; |
| 196 | + |
| 197 | + work.* = .{ .handler = handler, .request = request, .kind = kind, .ticket = ticket }; |
| 198 | + dispatch_async_f(userInitiatedQueue(), work, performOnBackground); |
| 199 | + } |
| 200 | +}; |
| 201 | + |
| 202 | +/// What crosses the queue boundary. |
| 203 | +/// |
| 204 | +/// Heap-allocated per call and owned by `performOnBackground`, which frees it |
| 205 | +/// and releases both Objective-C objects on every path out. |
| 206 | +const Work = struct { |
| 207 | + handler: Id, |
| 208 | + request: Id, |
| 209 | + kind: Kind, |
| 210 | + ticket: ios_async.Ticket, |
| 211 | +}; |
| 212 | + |
| 213 | +/// Run the request and answer, off the main thread. |
| 214 | +/// |
| 215 | +/// `dispatch_async_f` guarantees exactly one call, so this owns `work` |
| 216 | +/// outright. Nothing here touches the main thread directly: |
| 217 | +/// `ios_async.deliverJson` does its own hop. |
| 218 | +fn performOnBackground(context: ?*anyopaque) callconv(.c) void { |
| 219 | + const allocator = std.heap.c_allocator; |
| 220 | + const work: *Work = @ptrCast(@alignCast(context orelse return)); |
| 221 | + defer { |
| 222 | + release(work.handler); |
| 223 | + release(work.request); |
| 224 | + allocator.destroy(work); |
| 225 | + } |
| 226 | + |
| 227 | + const sel_perform = objc.sel_registerName("performRequests:error:") orelse { |
| 228 | + ios_async.deliverError(work.ticket); |
| 229 | + return; |
| 230 | + }; |
| 231 | + const NSArray = objc.objc_getClass("NSArray") orelse { |
| 232 | + ios_async.deliverError(work.ticket); |
| 233 | + return; |
| 234 | + }; |
| 235 | + const sel_with_object = objc.sel_registerName("arrayWithObject:") orelse { |
| 236 | + ios_async.deliverError(work.ticket); |
| 237 | + return; |
| 238 | + }; |
| 239 | + const requests = objc.msgSendId1(NSArray, sel_with_object, work.request) orelse { |
| 240 | + ios_async.deliverError(work.ticket); |
| 241 | + return; |
| 242 | + }; |
| 243 | + |
| 244 | + var err: Id = null; |
| 245 | + const PerformFn = *const fn (Id, objc.SEL, Id, *Id) callconv(.c) bool; |
| 246 | + const performFn: PerformFn = @ptrCast(&objc.objc_msgSend); |
| 247 | + if (!performFn(work.handler, sel_perform, requests, &err)) { |
| 248 | + // Swift rejects with `error.localizedDescription`. The protocol has no |
| 249 | + // slot for a native message, so the code carries what it can and the |
| 250 | + // description goes to the log, where it is at least readable — rather |
| 251 | + // than being dropped on the floor as "something went wrong". |
| 252 | + logNSError(work.kind.action(), err); |
| 253 | + ios_async.deliverErrorCode(work.ticket, BridgeError.NativeCallFailed); |
| 254 | + return; |
| 255 | + } |
| 256 | + |
| 257 | + const json = shapeResults(allocator, work.kind, work.request) catch |shape_err| { |
| 258 | + std.log.warn("{s}: could not shape the results: {}", .{ work.kind.action(), shape_err }); |
| 259 | + ios_async.deliverError(work.ticket); |
| 260 | + return; |
| 261 | + }; |
| 262 | + defer allocator.free(json); |
| 263 | + |
| 264 | + ios_async.deliverJson(work.ticket, json); |
| 265 | +} |
| 266 | + |
| 267 | +/// The `image` field, as Swift reads it. |
| 268 | +/// |
| 269 | +/// `body["image"] as? String` with no `else` in the dispatcher: a missing or |
| 270 | +/// mistyped field replies nothing at all and the page waits out its timeout. |
| 271 | +/// That is not carried across — every path here ends in a value or an error. |
| 272 | +fn readImageField(allocator: std.mem.Allocator, data: []const u8) ![]u8 { |
| 273 | + var parsed = std.json.parseFromSlice(std.json.Value, allocator, data, .{}) catch |err| switch (err) { |
| 274 | + error.OutOfMemory => return err, |
| 275 | + else => return BridgeError.InvalidJSON, |
| 276 | + }; |
| 277 | + defer parsed.deinit(); |
| 278 | + |
| 279 | + const object = switch (parsed.value) { |
| 280 | + .object => |o| o, |
| 281 | + else => return BridgeError.InvalidJSON, |
| 282 | + }; |
| 283 | + const value = object.get("image") orelse return BridgeError.MissingData; |
| 284 | + const text = switch (value) { |
| 285 | + .string => |s| s, |
| 286 | + else => return BridgeError.InvalidParameter, |
| 287 | + }; |
| 288 | + if (text.len == 0) return BridgeError.InvalidParameter; |
| 289 | + |
| 290 | + return allocator.dupe(u8, text) catch BridgeError.AllocationFailed; |
| 291 | +} |
| 292 | + |
| 293 | +/// `VNImageRequestHandler(cgImage:options:)`, +1, or the reason there is none. |
| 294 | +/// |
| 295 | +/// Follows Swift's three-step guard exactly — base64 to `NSData`, `NSData` to |
| 296 | +/// `UIImage`, `UIImage` to `CGImage` — and collapses all three into |
| 297 | +/// `InvalidParameter`, which is what "Invalid image data" means in a protocol |
| 298 | +/// that has no message field. |
| 299 | +/// |
| 300 | +/// The base64 is handed to `NSData` rather than decoded in Zig so the accepted |
| 301 | +/// input is exactly Swift's. `Data(base64Encoded:)` uses default options, |
| 302 | +/// which reject any character outside the alphabet — including the newlines |
| 303 | +/// some encoders wrap at column 64, and including the `data:image/png;base64,` |
| 304 | +/// prefix a page might forward from a canvas. A Zig decoder written to be |
| 305 | +/// "helpful" about either would accept images the Swift arm refuses. |
| 306 | +fn makeRequestHandler(allocator: std.mem.Allocator, base64: []const u8) !Id { |
| 307 | + const ns_base64 = objc.createNSString(base64, allocator) catch |
| 308 | + return BridgeError.AllocationFailed; |
| 309 | + |
| 310 | + const NSData = objc.objc_getClass("NSData") orelse return BridgeError.NativeCallFailed; |
| 311 | + const sel_alloc = objc.sel_registerName("alloc") orelse return BridgeError.NativeCallFailed; |
| 312 | + const sel_init_b64 = objc.sel_registerName("initWithBase64EncodedString:options:") orelse |
| 313 | + return BridgeError.NativeCallFailed; |
| 314 | + |
| 315 | + const data_alloc = objc.msgSendId(NSData, sel_alloc) orelse return BridgeError.NativeCallFailed; |
| 316 | + const InitB64Fn = *const fn (Id, objc.SEL, Id, c_ulong) callconv(.c) Id; |
| 317 | + const initB64: InitB64Fn = @ptrCast(&objc.objc_msgSend); |
| 318 | + const ns_data = initB64(data_alloc, sel_init_b64, ns_base64, 0) orelse { |
| 319 | + std.log.warn("vision: the image field is not valid base64", .{}); |
| 320 | + return BridgeError.InvalidParameter; |
| 321 | + }; |
| 322 | + defer release(ns_data); |
| 323 | + |
| 324 | + // UIKit, so this is null on a host that links Cocoa — the honest refusal |
| 325 | + // rather than a link error, the same route `takeScreenshot` takes. |
| 326 | + const UIImage = objc.objc_getClass("UIImage") orelse { |
| 327 | + std.log.warn("vision: UIImage is not in this process", .{}); |
| 328 | + return BridgeError.PlatformNotSupported; |
| 329 | + }; |
| 330 | + const sel_with_data = objc.sel_registerName("imageWithData:") orelse |
| 331 | + return BridgeError.NativeCallFailed; |
| 332 | + const image = objc.msgSendId1(UIImage, sel_with_data, ns_data) orelse { |
| 333 | + std.log.warn("vision: the decoded bytes are not an image UIKit can read", .{}); |
| 334 | + return BridgeError.InvalidParameter; |
| 335 | + }; |
| 336 | + |
| 337 | + const sel_cg = objc.sel_registerName("CGImage") orelse return BridgeError.NativeCallFailed; |
| 338 | + const cg_image = objc.msgSendId(image, sel_cg) orelse { |
| 339 | + // A UIImage backed by a CIImage has no CGImage. Swift's third guard. |
| 340 | + std.log.warn("vision: the image has no CGImage representation", .{}); |
| 341 | + return BridgeError.InvalidParameter; |
| 342 | + }; |
| 343 | + |
| 344 | + const VNImageRequestHandler = objc.objc_getClass("VNImageRequestHandler") orelse { |
| 345 | + std.log.warn( |
| 346 | + "vision: VNImageRequestHandler is not in this process; the app does not link Vision", |
| 347 | + .{}, |
| 348 | + ); |
| 349 | + return BridgeError.PlatformNotSupported; |
| 350 | + }; |
| 351 | + const NSDictionary = objc.objc_getClass("NSDictionary") orelse |
| 352 | + return BridgeError.NativeCallFailed; |
| 353 | + const sel_dictionary = objc.sel_registerName("dictionary") orelse |
| 354 | + return BridgeError.NativeCallFailed; |
| 355 | + const options = objc.msgSendId(NSDictionary, sel_dictionary) orelse |
| 356 | + return BridgeError.NativeCallFailed; |
| 357 | + |
| 358 | + const sel_init_handler = objc.sel_registerName("initWithCGImage:options:") orelse |
| 359 | + return BridgeError.NativeCallFailed; |
| 360 | + const handler_alloc = objc.msgSendId(VNImageRequestHandler, sel_alloc) orelse |
| 361 | + return BridgeError.NativeCallFailed; |
| 362 | + |
| 363 | + // +1 and kept: this is what holds the pixels alive across the queue hop. |
| 364 | + return objc.msgSendId2(handler_alloc, sel_init_handler, cg_image, options) orelse |
| 365 | + BridgeError.NativeCallFailed; |
| 366 | +} |
| 367 | + |
| 368 | +/// `VNRequestTextRecognitionLevelAccurate`. Swift sets it explicitly at |
| 369 | +/// `CraftApp.swift:4731`, and the default is the same value — but the default |
| 370 | +/// is Vision's to change and the spec's is not. |
| 371 | +const text_recognition_level_accurate: c_long = 0; |
| 372 | + |
| 373 | +/// The request object, +1, with no completion handler. |
| 374 | +/// |
| 375 | +/// Swift constructs its requests with a completion block and then performs |
| 376 | +/// them; the block is optional, and `performRequests:error:` is synchronous, |
| 377 | +/// so reading `results` after it returns gets the same observations without a |
| 378 | +/// block. That matters beyond tidiness: a completion block would have to be |
| 379 | +/// global to be legal here, and a global block cannot carry the ticket that |
| 380 | +/// says which page call it is answering. |
| 381 | +fn makeRequest(kind: Kind) !Id { |
| 382 | + const class = objc.objc_getClass(kind.className()) orelse { |
| 383 | + std.log.warn( |
| 384 | + "{s}: {s} is not in this process; the app does not link Vision", |
| 385 | + .{ kind.action(), kind.className() }, |
| 386 | + ); |
| 387 | + return BridgeError.PlatformNotSupported; |
| 388 | + }; |
| 389 | + const request = objc.allocInit(class) catch return BridgeError.NativeCallFailed; |
| 390 | + |
| 391 | + if (kind == .recognize) { |
| 392 | + const sel_level = objc.sel_registerName("setRecognitionLevel:") orelse |
| 393 | + return BridgeError.NativeCallFailed; |
| 394 | + const SetLevelFn = *const fn (Id, objc.SEL, c_long) callconv(.c) void; |
| 395 | + const setLevel: SetLevelFn = @ptrCast(&objc.objc_msgSend); |
| 396 | + setLevel(request, sel_level, text_recognition_level_accurate); |
| 397 | + } |
| 398 | + |
| 399 | + return request; |
| 400 | +} |
| 401 | + |
| 402 | +/// The reply body: a bare JSON array, matching `resolveCallback`'s |
| 403 | +/// `.fragmentsAllowed` serialisation of a Swift array. |
| 404 | +fn shapeResults(allocator: std.mem.Allocator, kind: Kind, request: Id) ![]u8 { |
| 405 | + const sel_results = objc.sel_registerName("results") orelse return error.SelectorNotFound; |
| 406 | + const results = objc.msgSendId(request, sel_results) orelse { |
| 407 | + // Swift's `as? [VN…Observation]` failing is a rejection there. A nil |
| 408 | + // `results` is not the same as an empty one: no request ran. |
| 409 | + return error.NoResults; |
| 410 | + }; |
| 411 | + |
| 412 | + var out: std.ArrayListUnmanaged(u8) = .empty; |
| 413 | + errdefer out.deinit(allocator); |
| 414 | + try out.append(allocator, '['); |
| 415 | + |
| 416 | + const count = try arrayCount(results); |
| 417 | + const limit = if (kind == .classify) @min(count, max_classifications) else count; |
| 418 | + |
| 419 | + var i: usize = 0; |
| 420 | + while (i < limit) : (i += 1) { |
| 421 | + const observation = try arrayObjectAt(results, i); |
| 422 | + if (i != 0) try out.append(allocator, ','); |
| 423 | + switch (kind) { |
| 424 | + .classify => try appendClassification(allocator, &out, observation), |
| 425 | + .detect => try appendDetection(allocator, &out, observation), |
| 426 | + .recognize => try appendRecognizedText(allocator, &out, observation), |
| 427 | + } |
| 428 | + } |
| 429 | + |
| 430 | + try out.append(allocator, ']'); |
| 431 | + return out.toOwnedSlice(allocator); |
| 432 | +} |
| 433 | + |
| 434 | +/// `{"label": identifier, "confidence": confidence}`. |
| 435 | +fn appendClassification( |
| 436 | + allocator: std.mem.Allocator, |
| 437 | + out: *std.ArrayListUnmanaged(u8), |
| 438 | + observation: Id, |
| 439 | +) !void { |
| 440 | + const identifier = try readIdentifier(observation); |
| 441 | + try out.appendSlice(allocator, "{\"label\":"); |
| 442 | + try appendJsonString(allocator, out, identifier); |
| 443 | + try out.appendSlice(allocator, ",\"confidence\":"); |
| 444 | + try appendConfidence(allocator, out, try readConfidence(observation)); |
| 445 | + try out.append(allocator, '}'); |
| 446 | +} |
| 447 | + |
| 448 | +/// `{"labels": [...], "boundingBox": {...}, "confidence": c}`. |
| 449 | +fn appendDetection( |
| 450 | + allocator: std.mem.Allocator, |
| 451 | + out: *std.ArrayListUnmanaged(u8), |
| 452 | + observation: Id, |
| 453 | +) !void { |
| 454 | + const sel_labels = objc.sel_registerName("labels") orelse return error.SelectorNotFound; |
| 455 | + const labels = objc.msgSendId(observation, sel_labels); |
| 456 | + |
| 457 | + try out.appendSlice(allocator, "{\"labels\":["); |
| 458 | + if (labels) |list| { |
| 459 | + const count = try arrayCount(list); |
| 460 | + var i: usize = 0; |
| 461 | + while (i < count) : (i += 1) { |
| 462 | + if (i != 0) try out.append(allocator, ','); |
| 463 | + try appendClassification(allocator, out, try arrayObjectAt(list, i)); |
| 464 | + } |
| 465 | + } |
| 466 | + try out.appendSlice(allocator, "],\"boundingBox\":"); |
| 467 | + try appendBoundingBox(allocator, out, try readBoundingBox(observation)); |
| 468 | + try out.appendSlice(allocator, ",\"confidence\":"); |
| 469 | + try appendConfidence(allocator, out, try readConfidence(observation)); |
| 470 | + try out.append(allocator, '}'); |
| 471 | +} |
| 472 | + |
| 473 | +/// `{"text": s, "confidence": c, "boundingBox": {...}}`. |
| 474 | +/// |
| 475 | +/// Swift takes `topCandidates(1).first` and **skips the observation entirely** |
| 476 | +/// when there is none, so the reply can be shorter than the observation list. |
| 477 | +/// Emitting a null-text entry instead would be a different array. |
| 478 | +fn appendRecognizedText( |
| 479 | + allocator: std.mem.Allocator, |
| 480 | + out: *std.ArrayListUnmanaged(u8), |
| 481 | + observation: Id, |
| 482 | +) !void { |
| 483 | + const sel_top = objc.sel_registerName("topCandidates:") orelse return error.SelectorNotFound; |
| 484 | + const TopFn = *const fn (Id, objc.SEL, c_ulong) callconv(.c) Id; |
| 485 | + const topFn: TopFn = @ptrCast(&objc.objc_msgSend); |
| 486 | + const candidates = topFn(observation, sel_top, 1) orelse return error.NoCandidates; |
| 487 | + |
| 488 | + const sel_first = objc.sel_registerName("firstObject") orelse return error.SelectorNotFound; |
| 489 | + const candidate = objc.msgSendId(candidates, sel_first) orelse return error.NoCandidates; |
| 490 | + |
| 491 | + const sel_string = objc.sel_registerName("string") orelse return error.SelectorNotFound; |
| 492 | + const ns_text = objc.msgSendId(candidate, sel_string) orelse return error.NoCandidates; |
| 493 | + const text = objc.getNSStringUTF8(ns_text) orelse return error.NoCandidates; |
| 494 | + |
| 495 | + try out.appendSlice(allocator, "{\"text\":"); |
| 496 | + try appendJsonString(allocator, out, std.mem.span(text)); |
| 497 | + try out.appendSlice(allocator, ",\"confidence\":"); |
| 498 | + // The candidate's confidence, not the observation's — Swift reads |
| 499 | + // `candidate.confidence`, and the two differ: one scores the transcription, |
| 500 | + // the other scores having found text at all. |
| 501 | + try appendConfidence(allocator, out, try readConfidence(candidate)); |
| 502 | + try out.appendSlice(allocator, ",\"boundingBox\":"); |
| 503 | + try appendBoundingBox(allocator, out, try readBoundingBox(observation)); |
| 504 | + try out.append(allocator, '}'); |
| 505 | +} |
| 506 | + |
| 507 | +/// `{"x":…,"y":…,"width":…,"height":…}`, Swift's four keys in Swift's order. |
| 508 | +fn appendBoundingBox( |
| 509 | + allocator: std.mem.Allocator, |
| 510 | + out: *std.ArrayListUnmanaged(u8), |
| 511 | + box: objc.CGRect, |
| 512 | +) !void { |
| 513 | + try out.appendSlice(allocator, "{\"x\":"); |
| 514 | + try appendDouble(allocator, out, box.origin.x); |
| 515 | + try out.appendSlice(allocator, ",\"y\":"); |
| 516 | + try appendDouble(allocator, out, box.origin.y); |
| 517 | + try out.appendSlice(allocator, ",\"width\":"); |
| 518 | + try appendDouble(allocator, out, box.size.width); |
| 519 | + try out.appendSlice(allocator, ",\"height\":"); |
| 520 | + try appendDouble(allocator, out, box.size.height); |
| 521 | + try out.append(allocator, '}'); |
| 522 | +} |
| 523 | + |
| 524 | +/// A `VNConfidence`, which is a `float`. |
| 525 | +/// |
| 526 | +/// Widened to `f64` before formatting rather than printed as an `f32`: the |
| 527 | +/// value crosses into JavaScript, whose numbers are doubles, and the shortest |
| 528 | +/// representation that round-trips an `f32` is not the shortest that |
| 529 | +/// round-trips the double a page will compare against. |
| 530 | +fn appendConfidence( |
| 531 | + allocator: std.mem.Allocator, |
| 532 | + out: *std.ArrayListUnmanaged(u8), |
| 533 | + confidence: f32, |
| 534 | +) !void { |
| 535 | + try appendDouble(allocator, out, @floatCast(confidence)); |
| 536 | +} |
| 537 | + |
| 538 | +fn appendDouble( |
| 539 | + allocator: std.mem.Allocator, |
| 540 | + out: *std.ArrayListUnmanaged(u8), |
| 541 | + value: f64, |
| 542 | +) !void { |
| 543 | + // NaN and the infinities are not JSON. Vision does not produce them for a |
| 544 | + // confidence or a normalised box, but a value that reached the page as a |
| 545 | + // bare `nan` would be a syntax error in the source `evaluateJavaScript:` |
| 546 | + // parses — the whole reply lost, not one field. |
| 547 | + if (std.math.isNan(value) or std.math.isInf(value)) { |
| 548 | + try out.append(allocator, '0'); |
| 549 | + return; |
| 550 | + } |
| 551 | + var buf: [64]u8 = undefined; |
| 552 | + const text = try std.fmt.bufPrint(&buf, "{d}", .{value}); |
| 553 | + try out.appendSlice(allocator, text); |
| 554 | +} |
| 555 | + |
| 556 | +fn appendJsonString( |
| 557 | + allocator: std.mem.Allocator, |
| 558 | + out: *std.ArrayListUnmanaged(u8), |
| 559 | + s: []const u8, |
| 560 | +) !void { |
| 561 | + try out.append(allocator, '"'); |
| 562 | + try bridge_error.appendJsonEscaped(allocator, out, s); |
| 563 | + try out.append(allocator, '"'); |
| 564 | +} |
| 565 | + |
| 566 | +fn readIdentifier(observation: Id) ![]const u8 { |
| 567 | + const sel = objc.sel_registerName("identifier") orelse return error.SelectorNotFound; |
| 568 | + const ns = objc.msgSendId(observation, sel) orelse return error.NoIdentifier; |
| 569 | + const utf8 = objc.getNSStringUTF8(ns) orelse return error.NoIdentifier; |
| 570 | + return std.mem.span(utf8); |
| 571 | +} |
| 572 | + |
| 573 | +fn readConfidence(observation: Id) !f32 { |
| 574 | + const sel = objc.sel_registerName("confidence") orelse return error.SelectorNotFound; |
| 575 | + const ConfidenceFn = *const fn (Id, objc.SEL) callconv(.c) f32; |
| 576 | + const confidenceFn: ConfidenceFn = @ptrCast(&objc.objc_msgSend); |
| 577 | + return confidenceFn(observation, sel); |
| 578 | +} |
| 579 | + |
| 580 | +fn readBoundingBox(observation: Id) !objc.CGRect { |
| 581 | + const sel = objc.sel_registerName("boundingBox") orelse return error.SelectorNotFound; |
| 582 | + const BoxFn = *const fn (Id, objc.SEL) callconv(.c) objc.CGRect; |
| 583 | + const boxFn: BoxFn = @ptrCast(&objc.objc_msgSend); |
| 584 | + return boxFn(observation, sel); |
| 585 | +} |
| 586 | + |
| 587 | +fn arrayCount(array: Id) !usize { |
| 588 | + const sel = objc.sel_registerName("count") orelse return error.SelectorNotFound; |
| 589 | + const CountFn = *const fn (Id, objc.SEL) callconv(.c) c_ulong; |
| 590 | + const countFn: CountFn = @ptrCast(&objc.objc_msgSend); |
| 591 | + return @intCast(countFn(array, sel)); |
| 592 | +} |
| 593 | + |
| 594 | +fn arrayObjectAt(array: Id, index: usize) !Id { |
| 595 | + const sel = objc.sel_registerName("objectAtIndex:") orelse return error.SelectorNotFound; |
| 596 | + const AtFn = *const fn (Id, objc.SEL, c_ulong) callconv(.c) Id; |
| 597 | + const atFn: AtFn = @ptrCast(&objc.objc_msgSend); |
| 598 | + return atFn(array, sel, @intCast(index)) orelse error.NilObservation; |
| 599 | +} |
| 600 | + |
| 601 | +fn logNSError(action: []const u8, err: Id) void { |
| 602 | + const ns_error = err orelse { |
| 603 | + std.log.warn("{s}: the request failed and reported no error", .{action}); |
| 604 | + return; |
| 605 | + }; |
| 606 | + const sel = objc.sel_registerName("localizedDescription") orelse return; |
| 607 | + const ns_description = objc.msgSendId(ns_error, sel) orelse return; |
| 608 | + const utf8 = objc.getNSStringUTF8(ns_description) orelse return; |
| 609 | + std.log.warn("{s}: {s}", .{ action, std.mem.span(utf8) }); |
| 610 | +} |
| 611 | + |
| 612 | +fn release(object: Id) void { |
| 613 | + const target = object orelse return; |
| 614 | + const sel = objc.sel_registerName("release") orelse return; |
| 615 | + objc.msgSend(target, sel); |
| 616 | +} |
| 617 | + |
| 618 | +// --------------------------------------------------------------------------- |
| 619 | +// The background queue |
| 620 | +// --------------------------------------------------------------------------- |
| 621 | + |
| 622 | +const dispatch_function_t = *const fn (?*anyopaque) callconv(.c) void; |
| 623 | +extern "c" fn dispatch_async_f(queue: ?*anyopaque, context: ?*anyopaque, work: dispatch_function_t) void; |
| 624 | +extern "c" fn dispatch_get_global_queue(identifier: c_long, flags: c_ulong) ?*anyopaque; |
| 625 | + |
| 626 | +/// `QOS_CLASS_USER_INITIATED`, the class Swift names. |
| 627 | +/// |
| 628 | +/// The page is waiting on the answer, which is what this class means; the |
| 629 | +/// default would deprioritise a request a user is watching a spinner for. |
| 630 | +const qos_class_user_initiated: c_long = 0x19; |
| 631 | + |
| 632 | +fn userInitiatedQueue() ?*anyopaque { |
| 633 | + return dispatch_get_global_queue(qos_class_user_initiated, 0); |
| 634 | +} |
| 635 | + |
| 636 | +const testing = std.testing; |
| 637 | + |
| 638 | +test "the action names match the Swift case labels exactly" { |
| 639 | + // The conformance ratchet compares these against `case "…":` in |
| 640 | + // CraftApp.swift in both directions; a prettier spelling would register as |
| 641 | + // Zig serving an action the spec does not have. |
| 642 | + try testing.expectEqualStrings("classifyImage", A.classify_image); |
| 643 | + try testing.expectEqualStrings("detectObjects", A.detect_objects); |
| 644 | + try testing.expectEqualStrings("recognizeText", A.recognize_text); |
| 645 | +} |
| 646 | + |
| 647 | +test "detectObjects builds an animal request, which is what the spec does" { |
| 648 | + // Pinned because it looks like a mistake. `CraftApp.swift:4656` uses |
| 649 | + // VNRecognizeAnimalsRequest, whose whole vocabulary is cat and dog, for an |
| 650 | + // action named detectObjects. Substituting a general detector here would |
| 651 | + // change the labels, the confidences and the empty-vs-non-empty answer for |
| 652 | + // every image a page already sends. |
| 653 | + try testing.expectEqualStrings("VNRecognizeAnimalsRequest", std.mem.span(Kind.detect.className())); |
| 654 | + try testing.expectEqualStrings("VNClassifyImageRequest", std.mem.span(Kind.classify.className())); |
| 655 | + try testing.expectEqualStrings("VNRecognizeTextRequest", std.mem.span(Kind.recognize.className())); |
| 656 | +} |
| 657 | + |
| 658 | +test "every declared action maps to a kind" { |
| 659 | + // A name in the table `handleMessage` never compares against would reach |
| 660 | + // the shim as UnknownAction while the manifest claimed Zig served it. |
| 661 | + var bridge = VisionBridge.init(testing.allocator); |
| 662 | + defer bridge.deinit(); |
| 663 | + |
| 664 | + for (capability_actions) |decl| { |
| 665 | + bridge.handleMessage(decl.name, "{}") catch |err| { |
| 666 | + try testing.expect(err != BridgeError.UnknownAction); |
| 667 | + continue; |
| 668 | + }; |
| 669 | + } |
| 670 | +} |
| 671 | + |
| 672 | +test "an action this module does not serve is passed on" { |
| 673 | + var bridge = VisionBridge.init(testing.allocator); |
| 674 | + defer bridge.deinit(); |
| 675 | + try testing.expectError(BridgeError.UnknownAction, bridge.handleMessage("getDeviceInfo", "{}")); |
| 676 | +} |
| 677 | + |
| 678 | +test "the image field is required, and its absence is not a hang" { |
| 679 | + // Swift's arm is `if config.enableMLKit, let imageBase64 = body["image"] |
| 680 | + // as? String` with no else: a missing field settles nothing and the page |
| 681 | + // waits out its timeout. Every path here ends in a value or an error. |
| 682 | + try testing.expectError(BridgeError.MissingData, readImageField(testing.allocator, "{}")); |
| 683 | + try testing.expectError( |
| 684 | + BridgeError.InvalidParameter, |
| 685 | + readImageField(testing.allocator, "{\"image\":42}"), |
| 686 | + ); |
| 687 | + try testing.expectError( |
| 688 | + BridgeError.InvalidParameter, |
| 689 | + readImageField(testing.allocator, "{\"image\":\"\"}"), |
| 690 | + ); |
| 691 | + try testing.expectError(BridgeError.InvalidJSON, readImageField(testing.allocator, "not json")); |
| 692 | + try testing.expectError(BridgeError.InvalidJSON, readImageField(testing.allocator, "[]")); |
| 693 | + |
| 694 | + const ok = try readImageField(testing.allocator, "{\"image\":\"aGk=\"}"); |
| 695 | + defer testing.allocator.free(ok); |
| 696 | + try testing.expectEqualStrings("aGk=", ok); |
| 697 | +} |
| 698 | + |
| 699 | +test "a bounding box carries Swift's four keys in Swift's order" { |
| 700 | + var out: std.ArrayListUnmanaged(u8) = .empty; |
| 701 | + defer out.deinit(testing.allocator); |
| 702 | + |
| 703 | + try appendBoundingBox(testing.allocator, &out, .{ |
| 704 | + .origin = .{ .x = 0.25, .y = 0.5 }, |
| 705 | + .size = .{ .width = 0.125, .height = 1 }, |
| 706 | + }); |
| 707 | + |
| 708 | + try testing.expectEqualStrings( |
| 709 | + "{\"x\":0.25,\"y\":0.5,\"width\":0.125,\"height\":1}", |
| 710 | + out.items, |
| 711 | + ); |
| 712 | +} |
| 713 | + |
| 714 | +test "a confidence is widened to double before it is printed" { |
| 715 | + // The value crosses into JavaScript, whose numbers are doubles. Printing |
| 716 | + // the shortest form that round-trips an f32 gives a different decimal from |
| 717 | + // the one that round-trips the double a page compares against. |
| 718 | + var out: std.ArrayListUnmanaged(u8) = .empty; |
| 719 | + defer out.deinit(testing.allocator); |
| 720 | + |
| 721 | + try appendConfidence(testing.allocator, &out, 0.5); |
| 722 | + try testing.expectEqualStrings("0.5", out.items); |
| 723 | + |
| 724 | + out.clearRetainingCapacity(); |
| 725 | + try appendConfidence(testing.allocator, &out, 1); |
| 726 | + try testing.expectEqualStrings("1", out.items); |
| 727 | +} |
| 728 | + |
| 729 | +test "a non-finite number is written as 0 rather than breaking the whole reply" { |
| 730 | + // `nan` and `inf` are not JSON. The reply is replayed into the source |
| 731 | + // `evaluateJavaScript:` parses, so one of them would be a syntax error |
| 732 | + // that loses every field, not just its own. |
| 733 | + var out: std.ArrayListUnmanaged(u8) = .empty; |
| 734 | + defer out.deinit(testing.allocator); |
| 735 | + |
| 736 | + try appendDouble(testing.allocator, &out, std.math.nan(f64)); |
| 737 | + try appendDouble(testing.allocator, &out, std.math.inf(f64)); |
| 738 | + try appendDouble(testing.allocator, &out, -std.math.inf(f64)); |
| 739 | + try testing.expectEqualStrings("000", out.items); |
| 740 | +} |
| 741 | + |
| 742 | +test "a label with a quote survives into the reply escaped" { |
| 743 | + var out: std.ArrayListUnmanaged(u8) = .empty; |
| 744 | + defer out.deinit(testing.allocator); |
| 745 | + |
| 746 | + try appendJsonString(testing.allocator, &out, "a \"b\" \\ c"); |
| 747 | + try testing.expectEqualStrings("\"a \\\"b\\\" \\\\ c\"", out.items); |
| 748 | +} |
| 749 | + |
| 750 | +test "the classification cap is Swift's prefix(10)" { |
| 751 | + // Vision returns well over a thousand classifications ordered by |
| 752 | + // confidence. Without the cap the reply is a megabyte of labels scoring |
| 753 | + // 0.0001, and it applies to classifyImage alone. |
| 754 | + try testing.expectEqual(@as(usize, 10), max_classifications); |
| 755 | +} |
| 756 | + |
| 757 | +test "an image cannot be built on a host that has no UIKit" { |
| 758 | + // The host test binaries link Cocoa, so UIImage is absent. Refusing is the |
| 759 | + // honest answer; a module that fabricated an empty result array here would |
| 760 | + // report "no text found" for an image it never looked at. |
| 761 | + if (!is_darwin) return error.SkipZigTest; |
| 762 | + |
| 763 | + try testing.expectError( |
| 764 | + BridgeError.PlatformNotSupported, |
| 765 | + makeRequestHandler(testing.allocator, "aGVsbG8="), |
| 766 | + ); |
| 767 | +} |