fix(android): text going to the JVM is modified UTF-8, and was not being

packages/zig/src/jni_runtime.zigmodified+190-7

Changes to packages/zig/src/jni_runtime.zig

		⋯476 linesShow the 476 lines above this hunk@@ -477,6 +477,9 @@ pub const JniError = error{
477	477	StringUnavailable,
478	478	/// The JVM handed back bytes that are not well-formed modified UTF-8.
479	479	MalformedString,
	480	+ /// Text on its way *to* the JVM that is not well-formed UTF-8. A bug in
	481	+ /// whatever built it, rather than something a device did.
	482	+ InvalidUtf8,
480	483	OutOfMemory,
481	484	};
482	485
		⋯296 linesShow the 296 lines above this hunk@@ -779,14 +782,14 @@ pub const Jni = struct {
779	782	try self.check();
780	783	}
781	784
782		- /// Make a Java `String` from UTF-8.
	785	+ /// Make a Java `String` from text already in *modified* UTF-8.
783	786	///
784		- /// The JVM wants *modified* UTF-8 here, the same encoding `stringToUtf8`
785		- /// decodes on the way back — so a string carrying a NUL or an astral
786		- /// character has to be re-encoded rather than handed over. Callers pass
787		- /// NUL-terminated literals and ASCII package names today, which are
788		- /// identical in both encodings; `encodeModifiedUtf8` is the function to
789		- /// add when that stops being true, and this is where it goes.
	787	+ /// Use this only for ASCII the source file holds as a literal — a class
	788	+ /// name, a column name, a preference key. Anything that came from the page
	789	+ /// or from the device goes through `newStringUtf8`, which re-encodes:
	790	+ /// standard UTF-8 and modified UTF-8 agree on ASCII and disagree on a NUL
	791	+ /// and on every character outside the BMP, and `NewStringUTF` given a
	792	+ /// four-byte sequence does not produce the string that was meant.
790	793	pub fn newStringUtf(self: Self, text: [*:0]const u8) JniError!jstring {
791	794	const new: *const fn (JNIEnv, [*:0]const u8) callconv(.c) jstring =
792	795	@ptrCast(self.table().NewStringUTF orelse return JniError.NotFound);
		⋯2 linesShow the 2 lines above this hunk@@ -795,6 +798,22 @@ pub const Jni = struct {
795	798	return str orelse JniError.OutOfMemory;
796	799	}
797	800
	801	+ /// Make a Java `String` from arbitrary UTF-8.
	802	+ ///
	803	+ /// This is the one to reach for whenever the text is not a literal in this
	804	+ /// repository: a contact's name, a widget's title, a SQL statement, a
	805	+ /// preference value. `NewStringUTF` reads *modified* UTF-8, so an emoji —
	806	+ /// four bytes in the standard encoding, two three-byte halves in Java's —
	807	+ /// has to be re-encoded rather than handed over, and a NUL has to become
	808	+ /// `C0 80` or the string ends early.
	809	+ ///
	810	+ /// The allocation is freed before returning; the `jstring` is the JVM's.
	811	+ pub fn newStringUtf8(self: Self, allocator: std.mem.Allocator, text: []const u8) JniError!jstring {
	812	+ const encoded = try encodeModifiedUtf8(allocator, text);
	813	+ defer allocator.free(encoded);
	814	+ return self.newStringUtf(encoded.ptr);
	815	+ }
	816	+
798	817	pub fn callIntMethod(self: Self, obj: jobject, id: jmethodID) JniError!jint {
799	818	const call: *const fn (JNIEnv, jobject, jmethodID) callconv(.c) jint =
800	819	@ptrCast(self.table().CallIntMethod orelse return JniError.NotFound);
		⋯165 linesShow the 165 lines above this hunk@@ -966,6 +985,81 @@ fn decodeThreeByte(bytes: *const [3]u8) u32 {
966	985	@as(u32, bytes[2] & 0x3F);
967	986	}
968	987
	988	+/// Convert standard UTF-8 to Java's modified UTF-8, NUL-terminated.
	989	+///
	990	+/// The inverse of `decodeModifiedUtf8`, and needed for the same two reasons in
	991	+/// the other direction:
	992	+///
	993	+/// - **U+0000 becomes `C0 80`**, so the result can be NUL-terminated without
	994	+/// the terminator being ambiguous. `NewStringUTF` takes a C string, so a
	995	+/// text carrying a NUL would otherwise be truncated at it silently.
	996	+/// - **A character outside the BMP becomes two three-byte sequences**, one
	997	+/// per UTF-16 surrogate. Handing `NewStringUTF` the standard four-byte form
	998	+/// does not produce the character that was meant — an emoji in a contact's
	999	+/// name is the everyday case.
	1000	+///
	1001	+/// Invalid UTF-8 is an error rather than something to pass through. Every
	1002	+/// caller's text either came from `stringToUtf8`, which produces valid UTF-8,
	1003	+/// or was built here — so invalid input means a bug upstream, and quietly
	1004	+/// writing malformed bytes into the JVM would hide it.
	1005	+pub fn encodeModifiedUtf8(allocator: std.mem.Allocator, input: []const u8) JniError![:0]u8 {
	1006	+ // Validated once rather than per sequence, so the loop below can copy the
	1007	+ // two- and three-byte forms without re-deciding whether they are legal.
	1008	+ // This is also what refuses a lone surrogate: valid CESU-8, and not UTF-8,
	1009	+ // so it can only have come from something already confused about which
	1010	+ // encoding it holds.
	1011	+ if (!std.unicode.utf8ValidateSlice(input)) return JniError.InvalidUtf8;
	1012	+
	1013	+ var out: std.ArrayListUnmanaged(u8) = .empty;
	1014	+ errdefer out.deinit(allocator);
	1015	+
	1016	+ var i: usize = 0;
	1017	+ while (i < input.len) {
	1018	+ const len = std.unicode.utf8ByteSequenceLength(input[i]) catch return JniError.InvalidUtf8;
	1019	+ const bytes = input[i .. i + len];
	1020	+ i += len;
	1021	+
	1022	+ switch (len) {
	1023	+ 1 => {
	1024	+ if (bytes[0] == 0) {
	1025	+ // The overlong form Java uses, and standard UTF-8 forbids.
	1026	+ out.appendSlice(allocator, &.{ 0xC0, 0x80 }) catch return JniError.OutOfMemory;
	1027	+ } else {
	1028	+ out.append(allocator, bytes[0]) catch return JniError.OutOfMemory;
	1029	+ }
	1030	+ },
	1031	+ 2, 3 => out.appendSlice(allocator, bytes) catch return JniError.OutOfMemory,
	1032	+ 4 => {
	1033	+ const codepoint = std.unicode.utf8Decode(bytes) catch return JniError.InvalidUtf8;
	1034	+ const offset = codepoint - 0x10000;
	1035	+ const high: u32 = 0xD800 + (offset >> 10);
	1036	+ const low: u32 = 0xDC00 + (offset & 0x3FF);
	1037	+ appendThreeByte(allocator, &out, high) catch return JniError.OutOfMemory;
	1038	+ appendThreeByte(allocator, &out, low) catch return JniError.OutOfMemory;
	1039	+ },
	1040	+ else => return JniError.InvalidUtf8,
	1041	+ }
	1042	+ }
	1043	+
	1044	+ return out.toOwnedSliceSentinel(allocator, 0) catch JniError.OutOfMemory;
	1045	+}
	1046	+
	1047	+/// One UTF-16 code unit as the three-byte form, surrogates included.
	1048	+///
	1049	+/// `std.unicode.utf8Encode` refuses a lone surrogate, which is precisely what
	1050	+/// each half of a CESU-8 pair is — so the bytes are written directly.
	1051	+fn appendThreeByte(
	1052	+ allocator: std.mem.Allocator,
	1053	+ out: *std.ArrayListUnmanaged(u8),
	1054	+ unit: u32,
	1055	+) !void {
	1056	+ try out.appendSlice(allocator, &.{
	1057	+ @intCast(0xE0 | (unit >> 12)),
	1058	+ @intCast(0x80 | ((unit >> 6) & 0x3F)),
	1059	+ @intCast(0x80 | (unit & 0x3F)),
	1060	+ });
	1061	+}
	1062	+
969	1063	pub fn decodeModifiedUtf8(allocator: std.mem.Allocator, input: []const u8) JniError![]u8 {
970	1064	var out: std.ArrayListUnmanaged(u8) = .empty;
971	1065	errdefer out.deinit(allocator);
		⋯422 linesShow the 422 lines above this hunk@@ -1394,3 +1488,92 @@ test "a field read that throws reports the exception, not the value" {
1394	1488	var obj: u8 = 0;
1395	1489	try testing.expectError(JniError.JavaException, jni.intField(@ptrCast(&obj), null));
1396	1490	}
	1491	+
	1492	+test "encoding is the exact inverse of decoding" {
	1493	+ // The property that matters: whatever goes to the JVM comes back the same.
	1494	+ // Written as a round trip rather than as byte assertions, because the two
	1495	+ // functions are the pair that has to agree — and a shared misunderstanding
	1496	+ // of the format would survive either one being tested alone.
	1497	+ for ([_][]const u8{
	1498	+ "",
	1499	+ "plain ascii",
	1500	+ "café",
	1501	+ "日本語",
	1502	+ "🙂",
	1503	+ "a🙂b",
	1504	+ "🙂🙂",
	1505	+ // The astral range's ends.
	1506	+ "\u{10000}",
	1507	+ "\u{10FFFF}",
	1508	+ // BMP characters either side of the surrogate block, which must pass
	1509	+ // through as three bytes rather than being re-paired.
	1510	+ "\u{D7FF}\u{E000}",
	1511	+ "mixed café 🙂 日本語 x",
	1512	+ }) |original| {
	1513	+ const encoded = try encodeModifiedUtf8(testing.allocator, original);
	1514	+ defer testing.allocator.free(encoded);
	1515	+
	1516	+ const decoded = try decodeModifiedUtf8(testing.allocator, encoded);
	1517	+ defer testing.allocator.free(decoded);
	1518	+
	1519	+ try testing.expectEqualStrings(original, decoded);
	1520	+ }
	1521	+}
	1522	+
	1523	+test "a NUL becomes C0 80, so the terminator stays unambiguous" {
	1524	+ // `NewStringUTF` takes a C string. A text carrying a NUL encoded as one
	1525	+ // byte would be truncated there and nothing would say so.
	1526	+ const encoded = try encodeModifiedUtf8(testing.allocator, "a\x00b");
	1527	+ defer testing.allocator.free(encoded);
	1528	+
	1529	+ try testing.expectEqualSlices(u8, &.{ 'a', 0xC0, 0x80, 'b' }, encoded);
	1530	+ try testing.expectEqual(@as(usize, 4), encoded.len);
	1531	+ try testing.expectEqual(@as(u8, 0), encoded.ptr[encoded.len]);
	1532	+
	1533	+ // And the NUL survives the round trip rather than ending the string.
	1534	+ const decoded = try decodeModifiedUtf8(testing.allocator, encoded);
	1535	+ defer testing.allocator.free(decoded);
	1536	+ try testing.expectEqualSlices(u8, &.{ 'a', 0, 'b' }, decoded);
	1537	+}
	1538	+
	1539	+test "an astral character becomes two three-byte halves" {
	1540	+ // U+1F642, which JSON.stringify hands over as four bytes and Java holds as
	1541	+ // a surrogate pair. Asserted as bytes here because this is the case where
	1542	+ // handing NewStringUTF the standard form produces the wrong string, and a
	1543	+ // round trip through both of our own functions could not tell.
	1544	+ const encoded = try encodeModifiedUtf8(testing.allocator, "🙂");
	1545	+ defer testing.allocator.free(encoded);
	1546	+
	1547	+ // U+1F642 - 0x10000 = 0xF642; high = D800 + (F642 >> 10) = D83D,
	1548	+ // low = DC00 + (F642 & 3FF) = DE42.
	1549	+ try testing.expectEqualSlices(u8, &.{
	1550	+ 0xED, 0xA0, 0xBD, // D83D
	1551	+ 0xED, 0xB9, 0x82, // DE42
	1552	+ }, encoded);
	1553	+ try testing.expectEqual(@as(usize, 6), encoded.len);
	1554	+}
	1555	+
	1556	+test "ASCII and BMP text pass through unchanged" {
	1557	+ // The reason `newStringUtf` on a literal is still correct: the two
	1558	+ // encodings agree everywhere except a NUL and the astral planes.
	1559	+ for ([_][]const u8{ "SELECT 1", "craft_widget_prefs", "café", "日本語" }) |text| {
	1560	+ const encoded = try encodeModifiedUtf8(testing.allocator, text);
	1561	+ defer testing.allocator.free(encoded);
	1562	+ try testing.expectEqualStrings(text, encoded);
	1563	+ }
	1564	+}
	1565	+
	1566	+test "invalid UTF-8 is refused rather than written into the JVM" {
	1567	+ // Every caller's text either came from `stringToUtf8` or was built here,
	1568	+ // so this means a bug upstream — and malformed bytes reaching the JVM
	1569	+ // would hide it somewhere much harder to read.
	1570	+ for ([_][]const u8{
	1571	+ &.{0x80}, // a continuation byte with nothing to continue
	1572	+ &.{0xC3}, // a two-byte lead with no second byte
	1573	+ &.{ 0xE2, 0x82 }, // a three-byte sequence cut short
	1574	+ &.{0xF8}, // a five-byte lead, which UTF-8 does not have
	1575	+ &.{ 0xED, 0xA0, 0xBD }, // a lone surrogate, valid CESU-8 and not UTF-8
	1576	+ }) |bad| {
	1577	+ try testing.expectError(JniError.InvalidUtf8, encodeModifiedUtf8(testing.allocator, bad));
	1578	+ }
	1579	+}