test(ios): every field the spec's replies carry, not just getDeviceInfo's

packages/zig/test/ios_conformance_test.zigmodified+374-1

Changes to packages/zig/test/ios_conformance_test.zig

		⋯802 linesShow the 802 lines above this hunk@@ -803,7 +803,9 @@ test "getDeviceInfo answers every field the spec answers" {
803	803	const end = std.mem.indexOf(u8, after, "\n ]") orelse return error.SpecShapeChanged;
804	804	const block = after[0..end];
805	805
806		- const zig_src = @embedFile("src/bridge_mobile.zig");
	806	+ // Narrowed to the implementation for the reason `implementationOf`
	807	+ // documents: `bridge_mobile.zig`'s own tests quote these keys back.
	808	+ const zig_src = implementationOf(@embedFile("src/bridge_mobile.zig"));
807	809
808	810	var it = std.mem.splitScalar(u8, block, '\n');
809	811	var checked: usize = 0;
		⋯279 linesShow the 279 lines above this hunk@@ -1089,3 +1091,374 @@ test "every recorded dead subscription is real, and still dead" {
1089	1091	// Non-vacuity: the loop above is satisfied by an empty table.
1090	1092	try testing.expect(dead_subscriptions.len >= 4);
1091	1093	}
	1094	+
	1095	+// ---------------------------------------------------------------------------
	1096	+// Reply shape
	1097	+//
	1098	+// `getDeviceInfo answers every field the spec answers` exists because Zig's
	1099	+// `getDeviceInfo` once answered four of the spec's fourteen fields. The action
	1100	+// was `.live`, so the seam preferred it; it reported success; a page reading
	1101	+// `screenWidth` or `locale` got `undefined`. Nothing else in the repo could
	1102	+// catch that, because from every other angle the action works — it just works
	1103	+// less.
	1104	+//
	1105	+// That test was written for one action, by hand, and the same hazard applies
	1106	+// to every action whose reply carries more than one field. This generalises
	1107	+// it: the spec's reply dictionaries are read out of `CraftApp.swift`, matched
	1108	+// to the action whose case arm produces them, and every key is required of the
	1109	+// Zig module that declares that action.
	1110	+//
	1111	+// Attribution is the whole difficulty, and it is worth doing properly rather
	1112	+// than checking keys against the union of all Zig sources. Half these keys are
	1113	+// words like `value`, `key`, `id` and `success` that appear in a dozen
	1114	+// modules; a union check would pass for every one of them no matter which
	1115	+// module dropped which field, which is the shape of a test that is green
	1116	+// because it is vacuous.
	1117	+// ---------------------------------------------------------------------------
	1118	+
	1119	+/// The Swift function enclosing `offset`, by name.
	1120	+///
	1121	+/// Anchored on ` func <name>(` so a `private func` matches at the `func` and a
	1122	+/// word merely ending in "func" does not. The last one before the offset is
	1123	+/// the enclosing one — Swift's functions here are flat, not nested.
	1124	+fn enclosingFunc(offset: usize) ?[]const u8 {
	1125	+ var best: ?[]const u8 = null;
	1126	+ var search: usize = 0;
	1127	+ while (std.mem.indexOfPos(u8, swift_spec, search, "func ")) |at| {
	1128	+ if (at >= offset) break;
	1129	+ search = at + "func ".len;
	1130	+ if (at == 0 or (swift_spec[at - 1] != ' ' and swift_spec[at - 1] != '\n')) continue;
	1131	+
	1132	+ const paren = std.mem.indexOfScalarPos(u8, swift_spec, search, '(') orelse continue;
	1133	+ const name = swift_spec[search..paren];
	1134	+ if (name.len == 0) continue;
	1135	+ var ok = true;
	1136	+ for (name) |c| {
	1137	+ if (!std.ascii.isAlphanumeric(c) and c != '_') ok = false;
	1138	+ }
	1139	+ if (ok) best = name;
	1140	+ }
	1141	+ return best;
	1142	+}
	1143	+
	1144	+/// The bracket-balanced literal beginning at `start`, which must be a `[`.
	1145	+///
	1146	+/// A string value containing a bracket would run this long. That fails in the
	1147	+/// safe direction — a block that swallows the next statement contributes keys
	1148	+/// Zig was never asked for, and the test says so loudly — rather than quietly
	1149	+/// checking nothing.
	1150	+fn balancedBracket(start: usize) ?[]const u8 {
	1151	+ var depth: i32 = 0;
	1152	+ var i = start;
	1153	+ while (i < swift_spec.len) : (i += 1) {
	1154	+ switch (swift_spec[i]) {
	1155	+ '[' => depth += 1,
	1156	+ ']' => {
	1157	+ depth -= 1;
	1158	+ if (depth == 0) return swift_spec[start .. i + 1];
	1159	+ },
	1160	+ else => {},
	1161	+ }
	1162	+ }
	1163	+ return null;
	1164	+}
	1165	+
	1166	+/// The next `"key":` in `block` at or after `from`, and where to resume.
	1167	+fn nextDictKey(block: []const u8, from: usize) ?struct { key: []const u8, next: usize } {
	1168	+ var search = from;
	1169	+ while (std.mem.indexOfScalarPos(u8, block, search, '"')) |open| {
	1170	+ const close = std.mem.indexOfScalarPos(u8, block, open + 1, '"') orelse return null;
	1171	+ search = close + 1;
	1172	+
	1173	+ var after = close + 1;
	1174	+ while (after < block.len and (block[after] == ' ' or block[after] == '\n')) : (after += 1) {}
	1175	+ if (after < block.len and block[after] == ':') {
	1176	+ return .{ .key = block[open + 1 .. close], .next = search };
	1177	+ }
	1178	+ }
	1179	+ return null;
	1180	+}
	1181	+
	1182	+/// The part of a module that is its implementation, not its tests.
	1183	+///
	1184	+/// The needle these checks use is the emitted JSON key — `\\"residentSize\\":` —
	1185	+/// and a module's own test literals are full of exactly that. Searching the
	1186	+/// whole file lets a module keep passing on the strength of a test asserting a
	1187	+/// field the implementation has stopped emitting, which is the one arrangement
	1188	+/// worse than no check: the assertion and the bug live in the same file and
	1189	+/// agree with each other.
	1190	+///
	1191	+/// Verified against the mutation it exists for — deleting `residentSize` from
	1192	+/// `getMemoryUsage`'s format string passes a whole-file search and fails this
	1193	+/// one.
	1194	+fn implementationOf(source: []const u8) []const u8 {
	1195	+ const at = std.mem.indexOf(u8, source, "\ntest \"") orelse return source;
	1196	+ return source[0..at];
	1197	+}
	1198	+
	1199	+/// Does `source` declare `action` in its `A` block?
	1200	+fn declaresAction(source: []const u8, action: []const u8) bool {
	1201	+ const block_start = std.mem.indexOf(u8, source, "pub const A = struct {") orelse return false;
	1202	+ var it = std.mem.splitScalar(u8, source[block_start..], '\n');
	1203	+ _ = it.next();
	1204	+ while (it.next()) |line| {
	1205	+ const trimmed = std.mem.trim(u8, line, " \t\r");
	1206	+ if (std.mem.eql(u8, trimmed, "};")) break;
	1207	+ const open = std.mem.indexOfScalar(u8, trimmed, '"') orelse continue;
	1208	+ const close = std.mem.indexOfScalarPos(u8, trimmed, open + 1, '"') orelse continue;
	1209	+ if (std.mem.eql(u8, trimmed[open + 1 .. close], action)) return true;
	1210	+ }
	1211	+ return false;
	1212	+}
	1213	+
	1214	+/// The module that serves `action`, or null for one no module declares.
	1215	+fn zigSourceFor(action: []const u8) ?[]const u8 {
	1216	+ for (zig_sources) |source| {
	1217	+ if (declaresAction(source, action)) return source;
	1218	+ }
	1219	+ return null;
	1220	+}
	1221	+
	1222	+/// The action whose dispatcher case arm calls `func_name`, or reaches `offset`.
	1223	+///
	1224	+/// Two shapes to cover: most replies are built inside a private helper the
	1225	+/// case arm calls, and a few are built inline in the arm itself. The inline
	1226	+/// ones have `userContentController` as their enclosing function, which no
	1227	+/// case calls, so they are attributed by position instead.
	1228	+fn actionProducing(func_name: []const u8, offset: usize) ?[]const u8 {
	1229	+ const region = dispatcherRegion();
	1230	+ const region_start = @intFromPtr(region.ptr) - @intFromPtr(swift_spec.ptr);
	1231	+
	1232	+ var found: ?[]const u8 = null;
	1233	+ var search: usize = 0;
	1234	+ while (std.mem.indexOfPos(u8, region, search, "case \"")) |at| {
	1235	+ const name_start = at + "case \"".len;
	1236	+ const name_end = std.mem.indexOfScalarPos(u8, region, name_start, '"') orelse break;
	1237	+ const action = region[name_start..name_end];
	1238	+ search = name_end;
	1239	+
	1240	+ const block_end = std.mem.indexOfPos(u8, region, name_end, "case \"") orelse region.len;
	1241	+
	1242	+ // Inline: the dictionary itself sits in this arm.
	1243	+ if (offset >= region_start + name_end and offset < region_start + block_end) return action;
	1244	+
	1245	+ // Delegated: the arm calls the helper that builds the dictionary.
	1246	+ var call_buf: [96]u8 = undefined;
	1247	+ if (func_name.len + 1 > call_buf.len) continue;
	1248	+ const call = std.fmt.bufPrint(&call_buf, "{s}(", .{func_name}) catch continue;
	1249	+ if (std.mem.indexOf(u8, region[name_end..block_end], call) != null) {
	1250	+ if (found == null) found = action;
	1251	+ }
	1252	+ }
	1253	+ return found;
	1254	+}
	1255	+
	1256	+const ReplyDivergence = struct {
	1257	+ action: []const u8,
	1258	+ key: []const u8,
	1259	+ reason: []const u8,
	1260	+};
	1261	+
	1262	+/// Keys the spec's reply carries that Zig deliberately does not answer.
	1263	+///
	1264	+/// A reply that drops a field is a bug by default — that is what the check
	1265	+/// below is for — so a row here has to argue that Zig is *right* and the spec
	1266	+/// is wrong, not merely that they differ.
	1267	+const deliberate_reply_divergences = [_]ReplyDivergence{
	1268	+ // Swift's failure arm resolves with `["usedMB": 0, "error": ...]`, which
	1269	+ // settles the page's promise carrying a fabricated zero: a caller cannot
	1270	+ // tell "this app uses no memory" from "the reading could not be taken".
	1271	+ // Zig returns `NativeCallFailed`, so there is no success reply for an
	1272	+ // `error` key to live in. Documented at the `task_info` call site.
	1273	+ .{ .action = "getMemoryUsage", .key = "error", .reason = "Swift resolves the failure with a fabricated usedMB: 0; Zig rejects, so there is no reply to carry it" },
	1274	+
	1275	+ // Declared `.status = .unavailable`: Zig names the action so the seam can
	1276	+ // refuse it explicitly, and the Swift shim answers. There is no Zig reply
	1277	+ // to compare, and the shim's own `isConnected` is hardcoded `true` — which
	1278	+ // is why Zig refuses rather than reproducing it.
	1279	+ .{ .action = "getNetworkStatus", .key = "isConnected", .reason = "Zig declares it .status = .unavailable; the shim answers, and answers it wrong" },
	1280	+ .{ .action = "getNetworkStatus", .key = "type", .reason = "Zig declares it .status = .unavailable; the shim answers, and answers it wrong" },
	1281	+};
	1282	+
	1283	+fn divergenceRecorded(action: []const u8, key: []const u8) bool {
	1284	+ for (deliberate_reply_divergences) |d| {
	1285	+ if (std.mem.eql(u8, d.action, action) and std.mem.eql(u8, d.key, key)) return true;
	1286	+ }
	1287	+ return false;
	1288	+}
	1289	+
	1290	+/// Walk every multi-key reply dictionary in the spec, calling `visit` with the
	1291	+/// action it belongs to and each of its keys.
	1292	+///
	1293	+/// Shared by the check and its own anti-rot test so the two cannot disagree
	1294	+/// about what the scan found.
	1295	+fn forEachReplyKey(
	1296	+ context: anytype,
	1297	+ comptime visit: fn (@TypeOf(context), action: []const u8, key: []const u8) anyerror!void,
	1298	+) !void {
	1299	+ const anchors = [_][]const u8{
	1300	+ "resolveCallback(callbackId, result: [",
	1301	+ "resolveCallbackJSON(callbackId, json: [",
	1302	+ ": [String: Any] = [",
	1303	+ };
	1304	+
	1305	+ for (anchors) |anchor| {
	1306	+ var search: usize = 0;
	1307	+ while (std.mem.indexOfPos(u8, swift_spec, search, anchor)) |at| {
	1308	+ const bracket = at + anchor.len - 1;
	1309	+ search = at + anchor.len;
	1310	+
	1311	+ const block = balancedBracket(bracket) orelse continue;
	1312	+
	1313	+ // One key is a single-field reply — `["reachable": x]` — which the
	1314	+ // action's own module test already pins and which cannot lose a
	1315	+ // field without losing the whole reply.
	1316	+ var count: usize = 0;
	1317	+ var at_key: usize = 0;
	1318	+ while (nextDictKey(block, at_key)) |k| : (at_key = k.next) count += 1;
	1319	+ if (count < 2) continue;
	1320	+
	1321	+ const func = enclosingFunc(at) orelse continue;
	1322	+ const action = actionProducing(func, at) orelse continue;
	1323	+
	1324	+ at_key = 0;
	1325	+ while (nextDictKey(block, at_key)) |k| : (at_key = k.next) {
	1326	+ try visit(context, action, k.key);
	1327	+ }
	1328	+ }
	1329	+ }
	1330	+}
	1331	+
	1332	+const ReplyCounter = struct {
	1333	+ checked: usize = 0,
	1334	+ actions_seen: usize = 0,
	1335	+};
	1336	+
	1337	+/// Does `source` emit `key` as a JSON key?
	1338	+///
	1339	+/// Two spellings, because modules legitimately use both. Most build the reply
	1340	+/// as a format string and the key appears as the escaped `\\"key\\":` it will
	1341	+/// emit. `bridge_mobile_bgtasks.zig` instead names it — `const scheduled_key =
	1342	+/// "scheduled";` — and passes it to a shared `taskReply` helper, so the
	1343	+/// literal never appears beside a colon. Demanding the first spelling would
	1344	+/// fail a module that is correct, which is the failure that gets a check
	1345	+/// deleted rather than fixed.
	1346	+fn emitsJsonKey(source: []const u8, key: []const u8) !bool {
	1347	+ var buf: [96]u8 = undefined;
	1348	+ if (key.len + 8 > buf.len) return true;
	1349	+
	1350	+ const escaped = try std.fmt.bufPrint(&buf, "\\\"{s}\\\":", .{key});
	1351	+ if (std.mem.indexOf(u8, source, escaped) != null) return true;
	1352	+
	1353	+ var buf2: [96]u8 = undefined;
	1354	+ const named = try std.fmt.bufPrint(&buf2, "= \"{s}\";", .{key});
	1355	+ return std.mem.indexOf(u8, source, named) != null;
	1356	+}
	1357	+
	1358	+fn checkReplyKey(counter: *ReplyCounter, action: []const u8, key: []const u8) !void {
	1359	+ if (divergenceRecorded(action, key)) return;
	1360	+
	1361	+ // An action no Zig module declares is still owed, not broken — the ratchet
	1362	+ // above is what tracks those.
	1363	+ const source = implementationOf(zigSourceFor(action) orelse return);
	1364	+
	1365	+ if (!try emitsJsonKey(source, key)) {
	1366	+ std.debug.print(
	1367	+ "{s}: the spec's reply carries `{s}` and the Zig module serving it never emits that key.\n" ++
	1368	+ " The action reports success and the page reads undefined.\n",
	1369	+ .{ action, key },
	1370	+ );
	1371	+ return error.ReplyDropsAFieldTheSpecAnswers;
	1372	+ }
	1373	+ counter.checked += 1;
	1374	+}
	1375	+
	1376	+test "every field the spec's replies carry is one Zig's reply carries" {
	1377	+ // The generalisation of the `getDeviceInfo` check, which found exactly this
	1378	+ // and could only ever find it for one action.
	1379	+ var counter = ReplyCounter{};
	1380	+ try forEachReplyKey(&counter, checkReplyKey);
	1381	+
	1382	+ // Non-vacuity, and a real floor rather than a token one: the scan finds
	1383	+ // twenty-odd multi-key replies today across device info, the keychain,
	1384	+ // biometric persistence, background tasks, PDF, SQLite, health, Siri and
	1385	+ // AR. A parsing change that dropped it to a handful would otherwise leave
	1386	+ // this test green and checking almost nothing.
	1387	+ //
	1388	+ // 52 keys reach this check as of this commit, out of the 64 the scan
	1389	+ // attributes — the gap is actions Zig does not serve yet, whose absence is
	1390	+ // the ratchet's business rather than this test's. The floor is set below
	1391	+ // that with room for the spec to lose a field honestly, and well above
	1392	+ // what any broken sub-parser would leave behind.
	1393	+ try testing.expect(counter.checked >= 40);
	1394	+}
	1395	+
	1396	+fn countReplyKey(counter: *ReplyCounter, action: []const u8, key: []const u8) !void {
	1397	+ _ = key;
	1398	+ _ = action;
	1399	+ counter.checked += 1;
	1400	+}
	1401	+
	1402	+test "the reply scan attributes dictionaries to actions, not to nothing" {
	1403	+ // The scan has three failure modes that all look like success: an anchor
	1404	+ // that stops matching, a bracket walk that returns null, and an
	1405	+ // attribution that never resolves. Each would leave the check above
	1406	+ // iterating an empty set.
	1407	+ var counter = ReplyCounter{};
	1408	+ try forEachReplyKey(&counter, countReplyKey);
	1409	+ try testing.expect(counter.checked >= 50); // 64 today
	1410	+
	1411	+ // And the attribution has to reach the two shapes it was written for: a
	1412	+ // reply built in a private helper, and one built inline in the case arm.
	1413	+ const helper = actionProducing("getDeviceInfo", 0) orelse return error.AttributionLostTheHelperShape;
	1414	+ try testing.expectEqualStrings("getDeviceInfo", helper);
	1415	+
	1416	+ const inline_at = std.mem.indexOf(u8, swift_spec, "\"isConnected\": isConnected") orelse
	1417	+ return error.SpecShapeChanged;
	1418	+ const inline_action = actionProducing("userContentController", inline_at) orelse
	1419	+ return error.AttributionLostTheInlineShape;
	1420	+ try testing.expectEqualStrings("getNetworkStatus", inline_action);
	1421	+}
	1422	+
	1423	+const PairProbe = struct {
	1424	+ action: []const u8,
	1425	+ key: []const u8,
	1426	+ found: bool = false,
	1427	+};
	1428	+
	1429	+fn probePair(probe: *PairProbe, action: []const u8, key: []const u8) !void {
	1430	+ if (std.mem.eql(u8, probe.action, action) and std.mem.eql(u8, probe.key, key)) probe.found = true;
	1431	+}
	1432	+
	1433	+test "every recorded reply divergence is real, and still a divergence" {
	1434	+ // The anti-rot half, in both directions the deferral table checks. A row
	1435	+ // whose key Zig has started answering argues against code that no longer
	1436	+ // exists; a row naming a key the spec stopped answering argues against
	1437	+ // nothing at all, and the next reader has no way to tell which.
	1438	+ for (deliberate_reply_divergences) |d| {
	1439	+ try testing.expect(d.reason.len > 0);
	1440	+
	1441	+ var probe = PairProbe{ .action = d.action, .key = d.key };
	1442	+ try forEachReplyKey(&probe, probePair);
	1443	+ if (!probe.found) {
	1444	+ std.debug.print(
	1445	+ "the divergence table says {s} deliberately drops `{s}`, and the spec's reply " ++
	1446	+ "does not carry that key — delete the row.\n",
	1447	+ .{ d.action, d.key },
	1448	+ );
	1449	+ return error.ReplyDivergenceNamesNoSuchKey;
	1450	+ }
	1451	+
	1452	+ const source = implementationOf(zigSourceFor(d.action) orelse continue);
	1453	+ if (try emitsJsonKey(source, d.key)) {
	1454	+ std.debug.print(
	1455	+ "{s} is recorded as deliberately not answering `{s}`, and it answers it now — " ++
	1456	+ "delete the row.\n",
	1457	+ .{ d.action, d.key },
	1458	+ );
	1459	+ return error.ReplyDivergenceIsSpent;
	1460	+ }
	1461	+ }
	1462	+
	1463	+ try testing.expect(deliberate_reply_divergences.len >= 3);
	1464	+}