Add reproducible randomized test order

src/test_history.zigmodified+17-2

Changes to src/test_history.zig

		⋯12 linesShow the 12 lines above this hunk@@ -13,6 +13,7 @@ pub const HistoryEntry = struct {
13	13	failed: usize,
14	14	skipped: usize,
15	15	duration_ns: u64,
	16	+ random_seed: ?u64 = null,
16	17	tests: std.ArrayList(TestRecord),
17	18
18	19	pub fn deinit(self: *HistoryEntry, allocator: std.mem.Allocator) void {
		⋯46 linesShow the 46 lines above this hunk@@ -65,6 +66,11 @@ pub const TestHistory = struct {
65	66
66	67	/// Start recording a new test run
67	68	pub fn startRun(self: *Self, total: usize) !void {
	69	+ try self.startRunWithSeed(total, null);
	70	+ }
	71	+
	72	+ /// Start recording a run with optional randomized-order metadata.
	73	+ pub fn startRunWithSeed(self: *Self, total: usize, random_seed: ?u64) !void {
68	74	self.start_time = compat.milliTimestamp();
69	75
70	76	self.current_entry = HistoryEntry{
		⋯4 linesShow the 4 lines above this hunk@@ -75,6 +81,7 @@ pub const TestHistory = struct {
75	81	.failed = 0,
76	82	.skipped = 0,
77	83	.duration_ns = 0,
	84	+ .random_seed = random_seed,
78	85	.tests = std.ArrayList(TestRecord).empty,
79	86	};
80	87	}
		⋯36 linesShow the 36 lines above this hunk@@ -117,6 +124,7 @@ pub const TestHistory = struct {
117	124	self.current_entry.?.flaky = results.flaky;
118	125	self.current_entry.?.failed = results.failed;
119	126	self.current_entry.?.skipped = results.skipped;
	127	+ self.current_entry.?.random_seed = results.random_seed orelse self.current_entry.?.random_seed;
120	128
121	129	try self.saveToFile();
122	130	}
		⋯16 linesShow the 16 lines above this hunk@@ -139,7 +147,10 @@ pub const TestHistory = struct {
139	147	errdefer buffer = allocating.toArrayList();
140	148	const writer = &allocating.writer;
141	149	try writer.writeAll("[\n");
142		- try std.json.Stringify.value(protocol.Event{ .run_start = .{ .total = self.current_entry.?.total } }, .{}, writer);
	150	+ try std.json.Stringify.value(protocol.Event{ .run_start = .{
	151	+ .total = self.current_entry.?.total,
	152	+ .random_seed = self.current_entry.?.random_seed,
	153	+ } }, .{}, writer);
143	154	for (self.current_entry.?.tests.items) |test_record| {
144	155	try writer.writeAll(",\n");
145	156	try std.json.Stringify.value(protocol.Event{ .test_end = .{
		⋯12 linesShow the 12 lines above this hunk@@ -158,6 +169,7 @@ pub const TestHistory = struct {
158	169	.failed = self.current_entry.?.failed,
159	170	.skipped = self.current_entry.?.skipped,
160	171	.duration_ns = self.current_entry.?.duration_ns,
	172	+ .random_seed = self.current_entry.?.random_seed,
161	173	} }, .{}, writer);
162	174	try writer.writeAll("\n]\n");
163	175	try writer.flush();
		⋯28 linesShow the 28 lines above this hunk@@ -192,6 +204,7 @@ pub const TestHistory = struct {
192	204	.failed = 0,
193	205	.skipped = 0,
194	206	.duration_ns = 0,
	207	+ .random_seed = null,
195	208	.tests = std.ArrayList(TestRecord).empty,
196	209	};
197	210	}
		⋯60 linesShow the 60 lines above this hunk@@ -258,7 +271,7 @@ test "TestHistory saves versioned protocol events" {
258	271
259	272	var history = TestHistory.init(allocator, history_dir);
260	273	defer history.deinit();
261		- try history.startRun(1);
	274	+ try history.startRunWithSeed(1, 9876);
262	275	try history.recordTest("quoted \"test\"", "suite\nname", .passed, 42, null);
263	276	var results = reporter_mod.TestResults.init(allocator);
264	277	defer results.deinit();
		⋯12 linesShow the 12 lines above this hunk@@ -277,9 +290,11 @@ test "TestHistory saves versioned protocol events" {
277	290	const events = parsed.value.array.items;
278	291	try std.testing.expectEqual(@as(usize, 3), events.len);
279	292	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	293	+ try std.testing.expectEqual(@as(i64, 9876), events[0].object.get("data").?.object.get("random_seed").?.integer);
280	294	try std.testing.expectEqualStrings("test_end", events[1].object.get("type").?.string);
281	295	try std.testing.expectEqualStrings("quoted \"test\"", events[1].object.get("data").?.object.get("name").?.string);
282	296	try std.testing.expectEqualStrings("run_end", events[2].object.get("type").?.string);
	297	+ try std.testing.expectEqual(@as(i64, 9876), events[2].object.get("data").?.object.get("random_seed").?.integer);
283	298	for (events) |event| {
284	299	try std.testing.expectEqual(@as(i64, protocol.version), event.object.get("protocol_version").?.integer);
285	300	}