Add reproducible randomized test order

src/reporter.zigmodified+25-6

Changes to src/reporter.zig

		⋯30 linesShow the 30 lines above this hunk@@ -31,6 +31,7 @@ pub const Reporter = struct {
31	31	vtable: *const VTable,
32	32	allocator: std.mem.Allocator,
33	33	use_colors: bool = true,
	34	+ random_seed: ?u64 = null,
34	35
35	36	const Self = @This();
36	37
		⋯39 linesShow the 39 lines above this hunk@@ -76,6 +77,7 @@ pub const TestResults = struct {
76	77	failed: usize = 0,
77	78	skipped: usize = 0,
78	79	total_time_ns: u64 = 0,
	80	+ random_seed: ?u64 = null,
79	81	failed_tests: std.ArrayList(suite.TestCase),
80	82	flaky_tests: std.ArrayList(suite.TestCase),
81	83	allocator: std.mem.Allocator,
		⋯81 linesShow the 81 lines above this hunk@@ -163,6 +165,9 @@ pub const SpecReporter = struct {
163	165	} else {
164	166	try s.output().print("Running {d} test(s)...\n\n", .{total_tests});
165	167	}
	168	+ if (reporter.random_seed) |seed| {
	169	+ try s.output().print("Random seed: {d}\n\n", .{seed});
	170	+ }
166	171	}
167	172
168	173	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯19 linesShow the 19 lines above this hunk@@ -188,6 +193,9 @@ pub const SpecReporter = struct {
188	193	}
189	194	}
190	195	try s.output().print("\n", .{});
	196	+ if (results.random_seed) |seed| {
	197	+ try s.output().print("Reproduce with: --seed {d}\n\n", .{seed});
	198	+ }
191	199	}
192	200
193	201	// Print summary
		⋯183 linesShow the 183 lines above this hunk@@ -377,6 +385,7 @@ pub const DotReporter = struct {
377	385	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
378	386	const s = self(reporter);
379	387	try s.output().print("\nRunning {d} tests:\n", .{total_tests});
	388	+ if (reporter.random_seed) |seed| try s.output().print("Random seed: {d}\n", .{seed});
380	389	}
381	390
382	391	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯16 linesShow the 16 lines above this hunk@@ -399,6 +408,9 @@ pub const DotReporter = struct {
399	408	total_time_ms,
400	409	});
401	410	}
	411	+ if (results.failed > 0) {
	412	+ if (results.random_seed) |seed| try s.output().print("Reproduce with: --seed {d}\n", .{seed});
	413	+ }
402	414	}
403	415
404	416	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
		⋯107 linesShow the 107 lines above this hunk@@ -512,7 +524,7 @@ pub const JsonReporter = struct {
512	524	s.emitted_events = false;
513	525	s.suites.clearRetainingCapacity();
514	526	try s.output().writeAll("[\n");
515		- try s.emit(.{ .run_start = .{ .total = total_tests } });
	527	+ try s.emit(.{ .run_start = .{ .total = total_tests, .random_seed = reporter.random_seed } });
516	528	}
517	529
518	530	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯5 linesShow the 5 lines above this hunk@@ -524,6 +536,7 @@ pub const JsonReporter = struct {
524	536	.failed = results.failed,
525	537	.skipped = results.skipped,
526	538	.duration_ns = results.total_time_ns,
	539	+ .random_seed = results.random_seed,
527	540	} });
528	541	try s.output().writeAll("\n]\n");
529	542	}
		⋯88 linesShow the 88 lines above this hunk@@ -618,11 +631,14 @@ pub const TAPReporter = struct {
618	631	fn onRunStart(reporter: *Reporter, total: usize) !void {
619	632	const self: *Self = @fieldParentPtr("reporter", reporter);
620	633	try self.output().print("TAP version 14\n1..{d}\n", .{total});
	634	+ if (reporter.random_seed) |seed| try self.output().print("# random-seed {d}\n", .{seed});
621	635	}
622	636
623	637	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
624		- _ = reporter;
625		- _ = results;
	638	+ const self: *Self = @fieldParentPtr("reporter", reporter);
	639	+ if (results.failed > 0) {
	640	+ if (results.random_seed) |seed| try self.output().print("# reproduce --seed {d}\n", .{seed});
	641	+ }
626	642	}
627	643
628	644	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
		⋯124 linesShow the 124 lines above this hunk@@ -753,14 +769,13 @@ pub const JUnitReporter = struct {
753	769	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
754	770	const self: *Self = @fieldParentPtr("reporter", reporter);
755	771
756		- const suite_result = try self.allocator.create(TestSuiteResult);
757		- suite_result.* = .{
	772	+ const suite_result = TestSuiteResult{
758	773	.name = try self.allocator.dupe(u8, suite_name),
759	774	.tests = .empty,
760	775	.timestamp = compat.milliTimestamp(),
761	776	};
762	777
763		- try self.suites.append(self.allocator, suite_result.*);
	778	+ try self.suites.append(self.allocator, suite_result);
764	779	self.current_suite = &self.suites.items[self.suites.items.len - 1];
765	780	}
766	781
		⋯76 linesShow the 76 lines above this hunk@@ -843,6 +858,10 @@ pub const JUnitReporter = struct {
843	858	suite_time,
844	859	});
845	860
	861	+ if (results.random_seed) |seed| {
	862	+ try buffer.print(self.allocator, " <properties><property name=\"random_seed\" value=\"{d}\"/></properties>\n", .{seed});
	863	+ }
	864	+
846	865	for (suite_result.tests.items) |test_result| {
847	866	try buffer.print(self.allocator, " <testcase name=\"{s}\" time=\"{d:.6}\"", .{
848	867	test_result.name,