Add reproducible randomized test order

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

src/cli.zigmodified+34-0

Changes to src/cli.zig

		⋯48 linesShow the 48 lines above this hunk@@ -49,6 +49,8 @@ pub const CLIOptions = struct {
49	49	retries: usize = 0,
50	50	repeat: usize = 1,
51	51	fail_on_flaky: bool = false,
	52	+ shuffle: bool = false,
	53	+ seed: ?u64 = null,
52	54	};
53	55
54	56	pub const version = build_options.version;
		⋯27 linesShow the 27 lines above this hunk@@ -82,6 +84,8 @@ pub const CLI = struct {
82	84	self.options.retries = config.test_options.retries;
83	85	self.options.repeat = config.test_options.repeat;
84	86	self.options.fail_on_flaky = config.test_options.fail_on_flaky;
	87	+ self.options.shuffle = config.test_options.shuffle or config.test_options.seed != null;
	88	+ self.options.seed = config.test_options.seed;
85	89	self.options.shard_index = config.sharding.index;
86	90	self.options.shard_count = config.sharding.count;
87	91	self.options.parallel = config.parallel.enabled;
		⋯52 linesShow the 52 lines above this hunk@@ -140,6 +144,19 @@ pub const CLI = struct {
140	144	}
141	145	} else if (std.mem.eql(u8, arg, "--fail-on-flaky")) {
142	146	self.options.fail_on_flaky = true;
	147	+ } else if (std.mem.eql(u8, arg, "--shuffle")) {
	148	+ self.options.shuffle = true;
	149	+ } else if (std.mem.eql(u8, arg, "--seed")) {
	150	+ if (i + 1 >= args.len) {
	151	+ std.debug.print("Error: --seed requires a value\n", .{});
	152	+ return CLIError.MissingValue;
	153	+ }
	154	+ i += 1;
	155	+ self.options.seed = std.fmt.parseInt(u64, args[i], 10) catch {
	156	+ std.debug.print("Error: --seed must be a valid unsigned integer\n", .{});
	157	+ return CLIError.InvalidArgument;
	158	+ };
	159	+ self.options.shuffle = true;
143	160	} else if (std.mem.eql(u8, arg, "--verbose")) {
144	161	self.options.verbose = true;
145	162	} else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) {
		⋯240 linesShow the 240 lines above this hunk@@ -386,6 +403,8 @@ pub const CLI = struct {
386	403	\\ --retry <N> Retry each failed test up to N times
387	404	\\ --repeat <N> Require N successful runs of each selected test
388	405	\\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure
	406	+ \\ --shuffle Randomize suite and test order
	407	+ \\ --seed <N> Reproduce a randomized order (implies --shuffle)
389	408	\\ --filter <pattern> Run only tests matching pattern
390	409	\\ --grep <pattern> Same as --filter (alias)
391	410	\\ --reporter <name> Set reporter type (spec, dot, json, tap, junit)
		⋯60 linesShow the 60 lines above this hunk@@ -452,6 +471,8 @@ pub const CLI = struct {
452	471	\\ zig-test --timeout 10000 Set 10s timeout for all tests
453	472	\\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests
454	473	\\ zig-test --repeat 100 --filter parser Stress-run selected tests
	474	+ \\ zig-test --shuffle Run in a generated random order
	475	+ \\ zig-test --seed 12345 Replay a randomized order
455	476	\\ zig-test --profile-memory --fail-on-leak
456	477	\\ zig-test --parallel --jobs 4 Run with 4 parallel workers
457	478	\\ zig-test --config zig-test.json Load config from file
		⋯24 linesShow the 24 lines above this hunk@@ -482,6 +503,8 @@ pub const CLI = struct {
482	503	.retries = self.options.retries,
483	504	.repeat = self.options.repeat,
484	505	.fail_on_flaky = self.options.fail_on_flaky,
	506	+ .shuffle = self.options.shuffle,
	507	+ .seed = self.options.seed,
485	508	};
486	509	}
487	510	};
		⋯236 linesShow the 236 lines above this hunk@@ -724,6 +747,17 @@ test "CLI parses retry repeat and flaky exit policy" {
724	747	try std.testing.expect(runner_options.fail_on_flaky);
725	748	}
726	749
	750	+test "CLI parses shuffle and a seed implies it" {
	751	+ var cli = CLI.init(std.testing.allocator);
	752	+ try cli.parse(&.{ "zig-test", "--seed", "12345" });
	753	+ try std.testing.expect(cli.options.shuffle);
	754	+ try std.testing.expectEqual(@as(?u64, 12345), cli.options.seed);
	755	+
	756	+ const runner_options = cli.toRunnerOptions();
	757	+ try std.testing.expect(runner_options.shuffle);
	758	+ try std.testing.expectEqual(@as(?u64, 12345), runner_options.seed);
	759	+}
	760	+
727	761	test "CLI rejects a zero repeat count" {
728	762	var cli = CLI.init(std.testing.allocator);
729	763	try std.testing.expectError(

src/config.zigmodified+6-0

Changes to src/config.zig

		⋯31 linesShow the 31 lines above this hunk@@ -32,6 +32,8 @@ pub const TestOptions = struct {
32	32	retries: usize = 0,
33	33	repeat: usize = 1,
34	34	fail_on_flaky: bool = false,
	35	+ shuffle: bool = false,
	36	+ seed: ?u64 = null,
35	37	};
36	38
37	39	pub const ParallelOptions = struct {
		⋯169 linesShow the 169 lines above this hunk@@ -207,6 +209,8 @@ test "TestConfig defaults match CLI discovery defaults" {
207	209	try std.testing.expectEqual(@as(usize, 0), config.test_options.retries);
208	210	try std.testing.expectEqual(@as(usize, 1), config.test_options.repeat);
209	211	try std.testing.expect(!config.test_options.fail_on_flaky);
	212	+ try std.testing.expect(!config.test_options.shuffle);
	213	+ try std.testing.expect(config.test_options.seed == null);
210	214	try std.testing.expect(config.sharding.index == null);
211	215	try std.testing.expect(config.sharding.count == null);
212	216	try std.testing.expect(!config.parallel.enabled);
		⋯34 linesShow the 34 lines above this hunk@@ -247,6 +251,8 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" {
247	251	try std.testing.expectEqual(@as(usize, 1), config.test_options.retries);
248	252	try std.testing.expectEqual(@as(usize, 2), config.test_options.repeat);
249	253	try std.testing.expect(!config.test_options.fail_on_flaky);
	254	+ try std.testing.expect(config.test_options.shuffle);
	255	+ try std.testing.expectEqual(@as(?u64, 42), config.test_options.seed);
250	256	try std.testing.expectEqual(@as(?usize, 1), config.sharding.index);
251	257	try std.testing.expectEqual(@as(?usize, 1), config.sharding.count);
252	258

src/event_protocol.zigmodified+5-1

Changes to src/event_protocol.zig

		⋯6 linesShow the 6 lines above this hunk@@ -7,7 +7,10 @@ pub const Status = enum { pending, running, passed, flaky, failed, skipped };
7	7	pub const HookKind = enum { before_all, before_each, after_each, after_all };
8	8	pub const OutputStream = enum { stdout, stderr };
9	9
10		-pub const RunStart = struct { total: usize };
	10	+pub const RunStart = struct {
	11	+ total: usize,
	12	+ random_seed: ?u64 = null,
	13	+};
11	14	pub const RunEnd = struct {
12	15	total: usize,
13	16	passed: usize,
		⋯1 lineShow the 1 line above this hunk@@ -15,6 +18,7 @@ pub const RunEnd = struct {
15	18	failed: usize,
16	19	skipped: usize,
17	20	duration_ns: u64 = 0,
	21	+ random_seed: ?u64 = null,
18	22	};
19	23	pub const Suite = struct { name: []const u8 };
20	24	pub const TestStart = struct { name: []const u8, suite: ?[]const u8 = null };

src/lib.zigmodified+1-0

Changes to src/lib.zig

		⋯26 linesShow the 26 lines above this hunk@@ -27,6 +27,7 @@ pub const timeout = ;
27	27	pub const progress = ;
28	28	pub const time = ;
29	29	pub const compat = ;
	30	+pub const random_order = ;
30	31
31	32	// Re-export commonly used types and functions
32	33	pub const expect = assertions.expect;

src/main.zigmodified+4-0

Changes to src/main.zig

		⋯188 linesShow the 188 lines above this hunk@@ -189,6 +189,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
189	189	.retries = cli_parser.options.retries,
190	190	.repeat = cli_parser.options.repeat,
191	191	.fail_on_flaky = cli_parser.options.fail_on_flaky,
	192	+ .shuffle = cli_parser.options.shuffle,
	193	+ .seed = cli_parser.options.seed,
192	194	.reporter_writer = &stdout_writer.interface,
193	195	.coverage_options = cov_opts,
194	196	.ui_server = if (ui_server) |*server| server else null,
		⋯40 linesShow the 40 lines above this hunk@@ -235,6 +237,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
235	237	.retries = cli_parser.options.retries,
236	238	.repeat = cli_parser.options.repeat,
237	239	.fail_on_flaky = cli_parser.options.fail_on_flaky,
	240	+ .shuffle = cli_parser.options.shuffle,
	241	+ .seed = cli_parser.options.seed,
238	242	.reporter_writer = &stdout_writer.interface,
239	243	.coverage_options = cov_opts,
240	244	.ui_server = if (ui_server) |*server| server else null,

src/parallel.zigmodified+74-8

Changes to src/parallel.zig

		⋯2 linesShow the 2 lines above this hunk@@ -3,6 +3,7 @@ const suite = ;
3	3	const reporter = ;
4	4	const pipeline = ;
5	5	const compat = ;
	6	+const random_order = ;
6	7
7	8	/// Options for bounded parallel test execution.
8	9	pub const ParallelOptions = struct {
		⋯3 linesShow the 3 lines above this hunk@@ -12,6 +13,7 @@ pub const ParallelOptions = struct {
12	13	enabled: bool = false,
13	14	filter: ?[]const u8 = null,
14	15	timeout_ms: ?u64 = null,
	16	+ random_seed: ?u64 = null,
15	17	};
16	18
17	19	pub const ParallelError = error{
		⋯93 linesShow the 93 lines above this hunk@@ -111,6 +113,7 @@ const RunContext = struct {
111	113	rep: *reporter.Reporter,
112	114	options: ParallelOptions,
113	115	worker_limit: usize,
	116	+ shuffler: ?*random_order.Shuffler,
114	117
115	118	fn runSuite(self: *RunContext, test_suite: *suite.TestSuite) !bool {
116	119	try self.rep.onSuiteStart(test_suite.name);
		⋯14 linesShow the 14 lines above this hunk@@ -131,7 +134,12 @@ const RunContext = struct {
131	134	var scheduled: std.ArrayList(*suite.TestCase) = .empty;
132	135	defer scheduled.deinit(self.allocator);
133	136
134		- for (test_suite.tests.items) |*test_case| {
	137	+ var ordered_tests: std.ArrayList(*suite.TestCase) = .empty;
	138	+ defer ordered_tests.deinit(self.allocator);
	139	+ for (test_suite.tests.items) |*test_case| try ordered_tests.append(self.allocator, test_case);
	140	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, ordered_tests.items);
	141	+
	142	+ for (ordered_tests.items) |test_case| {
135	143	if (!self.shouldRun(test_suite, test_case)) {
136	144	test_case.status = .skipped;
137	145	continue;
		⋯5 linesShow the 5 lines above this hunk@@ -143,12 +151,16 @@ const RunContext = struct {
143	151	try self.runBatch(test_suite, scheduled.items);
144	152
145	153	var all_passed = true;
146		- for (test_suite.tests.items) |*test_case| {
	154	+ for (ordered_tests.items) |test_case| {
147	155	try self.rep.onTestEnd(test_case);
148	156	if (test_case.status == .failed) all_passed = false;
149	157	}
150	158
151		- for (test_suite.suites.items) |nested_suite| {
	159	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	160	+ defer nested_suites.deinit(self.allocator);
	161	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	162	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	163	+ for (nested_suites.items) |nested_suite| {
152	164	if (!try self.runSuite(nested_suite)) all_passed = false;
153	165	}
154	166
		⋯46 linesShow the 46 lines above this hunk@@ -201,11 +213,19 @@ const RunContext = struct {
201	213	}
202	214
203	215	fn reportSkippedSuite(self: *RunContext, test_suite: *suite.TestSuite) !void {
204		- for (test_suite.tests.items) |*test_case| {
	216	+ var tests: std.ArrayList(*suite.TestCase) = .empty;
	217	+ defer tests.deinit(self.allocator);
	218	+ for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case);
	219	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items);
	220	+ for (tests.items) |test_case| {
205	221	test_case.status = .skipped;
206	222	try self.rep.onTestEnd(test_case);
207	223	}
208		- for (test_suite.suites.items) |nested| {
	224	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	225	+ defer nested_suites.deinit(self.allocator);
	226	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	227	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	228	+ for (nested_suites.items) |nested| {
209	229	try self.rep.onSuiteStart(nested.name);
210	230	try self.reportSkippedSuite(nested);
211	231	try self.rep.onSuiteEnd(nested.name);
		⋯1 lineShow the 1 line above this hunk@@ -213,9 +233,9 @@ const RunContext = struct {
213	233	}
214	234	};
215	235
216		-/// Run suites in declaration order while executing each suite's direct tests
	236	+/// Run suites in the selected deterministic order while executing each suite's direct tests
217	237	/// with a bounded worker set. Reporter callbacks are serialized and emitted in
218		-/// declaration order after each batch completes.
	238	+/// the selected order after each batch completes.
219	239	pub fn runTestsParallel(
220	240	allocator: std.mem.Allocator,
221	241	test_registry: *suite.TestRegistry,
		⋯4 linesShow the 4 lines above this hunk@@ -226,23 +246,36 @@ pub fn runTestsParallel(
226	246	const worker_limit = try resolveWorkerCount(options.n_jobs);
227	247
228	248	const total_tests = test_registry.countAllTests();
	249	+ rep.random_seed = options.random_seed;
229	250	try rep.onRunStart(total_tests);
230	251
	252	+ var shuffler: random_order.Shuffler = undefined;
	253	+ const shuffler_ptr: ?*random_order.Shuffler = if (options.random_seed) |seed| blk: {
	254	+ shuffler = .init(seed);
	255	+ break :blk &shuffler;
	256	+ } else null;
	257	+
231	258	var context = RunContext{
232	259	.allocator = allocator,
233	260	.registry = test_registry,
234	261	.rep = rep,
235	262	.options = options,
236	263	.worker_limit = worker_limit,
	264	+ .shuffler = shuffler_ptr,
237	265	};
238	266
	267	+ var root_suites: std.ArrayList(*suite.TestSuite) = .empty;
	268	+ defer root_suites.deinit(allocator);
	269	+ try root_suites.appendSlice(allocator, test_registry.root_suites.items);
	270	+ if (shuffler_ptr) |randomizer| randomizer.shuffle(*suite.TestSuite, root_suites.items);
239	271	var all_passed = true;
240		- for (test_registry.root_suites.items) |test_suite| {
	272	+ for (root_suites.items) |test_suite| {
241	273	if (!try context.runSuite(test_suite)) all_passed = false;
242	274	}
243	275
244	276	var results = reporter.TestResults.init(allocator);
245	277	defer results.deinit();
	278	+ results.random_seed = options.random_seed;
246	279	for (test_registry.root_suites.items) |test_suite| {
247	280	try addSuiteResults(&results, test_suite);
248	281	}
		⋯98 linesShow the 98 lines above this hunk@@ -347,6 +380,7 @@ const RecordingReporter = struct {
347	380	passed: usize = 0,
348	381	failed: usize = 0,
349	382	skipped: usize = 0,
	383	+ random_seed: ?u64 = null,
350	384
351	385	fn init(allocator: std.mem.Allocator) RecordingReporter {
352	386	return .{
		⋯26 linesShow the 26 lines above this hunk@@ -379,6 +413,7 @@ const RecordingReporter = struct {
379	413	recording.passed = results.passed;
380	414	recording.failed = results.failed;
381	415	recording.skipped = results.skipped;
	416	+ recording.random_seed = results.random_seed;
382	417	}
383	418
384	419	fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void {
		⋯83 linesShow the 83 lines above this hunk@@ -468,6 +503,37 @@ test "parallel reporting is serialized, ordered, and complete" {
468	503	try std.testing.expectEqual(@as(usize, 1), recording.skipped);
469	504	}
470	505
	506	+test "parallel order is reproducible with a fixed seed" {
	507	+ const allocator = std.heap.smp_allocator;
	508	+ var registry = suite.TestRegistry.init(allocator);
	509	+ defer registry.deinit();
	510	+ const test_suite = try suite.TestSuite.init(allocator, "randomized");
	511	+ inline for (.{ "one", "two", "three", "four", "five", "six" }) |name| {
	512	+ try test_suite.addTest(suite.TestCase.init(name, passingTest));
	513	+ }
	514	+ try registry.registerSuite(test_suite);
	515	+
	516	+ var first = RecordingReporter.init(allocator);
	517	+ try std.testing.expect(try runTestsParallel(allocator, &registry, &first.reporter, .{
	518	+ .enabled = true,
	519	+ .n_jobs = 2,
	520	+ .random_seed = 8675309,
	521	+ }));
	522	+ var second = RecordingReporter.init(allocator);
	523	+ try std.testing.expect(try runTestsParallel(allocator, &registry, &second.reporter, .{
	524	+ .enabled = true,
	525	+ .n_jobs = 2,
	526	+ .random_seed = 8675309,
	527	+ }));
	528	+
	529	+ try std.testing.expectEqual(first.ended_count, second.ended_count);
	530	+ for (first.ended_names[0..first.ended_count], second.ended_names[0..second.ended_count]) |a, b| {
	531	+ try std.testing.expectEqualStrings(a, b);
	532	+ }
	533	+ try std.testing.expectEqual(@as(?u64, 8675309), first.random_seed);
	534	+ try std.testing.expectEqual(@as(?u64, 8675309), second.random_seed);
	535	+}
	536	+
471	537	test "parallel executor preserves nested hook lifecycles" {
472	538	const counters = [_]*std.atomic.Value(usize){
473	539	&parent_before_each,

src/random_order.zigadded+42-0

Changes to src/random_order.zig

		@@ -0,0 +1,42 @@
	1	+const std = ;
	2	+const compat = ;
	3	+
	4	+/// Return the seed that should control this run. Supplying a seed enables
	5	+/// shuffling even when `shuffle` is false; otherwise random order is opt-in.
	6	+pub fn resolveSeed(shuffle: bool, configured: ?u64) ?u64 {
	7	+ if (configured) |seed| return seed;
	8	+ if (!shuffle) return null;
	9	+ return @truncate(@as(u128, @bitCast(compat.nanoTimestamp())));
	10	+}
	11	+
	12	+pub const Shuffler = struct {
	13	+ prng: std.Random.DefaultPrng,
	14	+
	15	+ pub fn init(seed: u64) Shuffler {
	16	+ return .{ .prng = std.Random.DefaultPrng.init(seed) };
	17	+ }
	18	+
	19	+ pub fn shuffle(self: *Shuffler, comptime T: type, items: []T) void {
	20	+ self.prng.random().shuffleWithIndex(T, items, u64);
	21	+ }
	22	+};
	23	+
	24	+test "random order is opt-in and an explicit seed enables it" {
	25	+ try std.testing.expect(resolveSeed(false, null) == null);
	26	+ try std.testing.expect(resolveSeed(true, null) != null);
	27	+ try std.testing.expectEqual(@as(?u64, 0), resolveSeed(false, 0));
	28	+ try std.testing.expectEqual(@as(?u64, 42), resolveSeed(true, 42));
	29	+}
	30	+
	31	+test "the same seed produces the same order" {
	32	+ var first = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
	33	+ var second = first;
	34	+ var first_shuffler = Shuffler.init(123456789);
	35	+ var second_shuffler = Shuffler.init(123456789);
	36	+
	37	+ first_shuffler.shuffle(u8, &first);
	38	+ second_shuffler.shuffle(u8, &second);
	39	+
	40	+ try std.testing.expectEqualSlices(u8, &first, &second);
	41	+ try std.testing.expect(!std.mem.eql(u8, &first, &[_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }));
	42	+}

src/reporter.zigmodified+25-6

Changes to src/reporter.zig

		⋯30 linesShow the 30 lines above this hunk@@ -31,6 +31,7 @@ pub const Reporter = struct {
31	31	vtable: *const VTable,
32	32	allocator: std.mem.Allocator,
33	33	use_colors: bool = true,
	34	+ random_seed: ?u64 = null,
34	35
35	36	const Self = @This();
36	37
		⋯39 linesShow the 39 lines above this hunk@@ -76,6 +77,7 @@ pub const TestResults = struct {
76	77	failed: usize = 0,
77	78	skipped: usize = 0,
78	79	total_time_ns: u64 = 0,
	80	+ random_seed: ?u64 = null,
79	81	failed_tests: std.ArrayList(suite.TestCase),
80	82	flaky_tests: std.ArrayList(suite.TestCase),
81	83	allocator: std.mem.Allocator,
		⋯81 linesShow the 81 lines above this hunk@@ -163,6 +165,9 @@ pub const SpecReporter = struct {
163	165	} else {
164	166	try s.output().print("Running {d} test(s)...\n\n", .{total_tests});
165	167	}
	168	+ if (reporter.random_seed) |seed| {
	169	+ try s.output().print("Random seed: {d}\n\n", .{seed});
	170	+ }
166	171	}
167	172
168	173	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯19 linesShow the 19 lines above this hunk@@ -188,6 +193,9 @@ pub const SpecReporter = struct {
188	193	}
189	194	}
190	195	try s.output().print("\n", .{});
	196	+ if (results.random_seed) |seed| {
	197	+ try s.output().print("Reproduce with: --seed {d}\n\n", .{seed});
	198	+ }
191	199	}
192	200
193	201	// Print summary
		⋯183 linesShow the 183 lines above this hunk@@ -377,6 +385,7 @@ pub const DotReporter = struct {
377	385	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
378	386	const s = self(reporter);
379	387	try s.output().print("\nRunning {d} tests:\n", .{total_tests});
	388	+ if (reporter.random_seed) |seed| try s.output().print("Random seed: {d}\n", .{seed});
380	389	}
381	390
382	391	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯16 linesShow the 16 lines above this hunk@@ -399,6 +408,9 @@ pub const DotReporter = struct {
399	408	total_time_ms,
400	409	});
401	410	}
	411	+ if (results.failed > 0) {
	412	+ if (results.random_seed) |seed| try s.output().print("Reproduce with: --seed {d}\n", .{seed});
	413	+ }
402	414	}
403	415
404	416	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
		⋯107 linesShow the 107 lines above this hunk@@ -512,7 +524,7 @@ pub const JsonReporter = struct {
512	524	s.emitted_events = false;
513	525	s.suites.clearRetainingCapacity();
514	526	try s.output().writeAll("[\n");
515		- try s.emit(.{ .run_start = .{ .total = total_tests } });
	527	+ try s.emit(.{ .run_start = .{ .total = total_tests, .random_seed = reporter.random_seed } });
516	528	}
517	529
518	530	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯5 linesShow the 5 lines above this hunk@@ -524,6 +536,7 @@ pub const JsonReporter = struct {
524	536	.failed = results.failed,
525	537	.skipped = results.skipped,
526	538	.duration_ns = results.total_time_ns,
	539	+ .random_seed = results.random_seed,
527	540	} });
528	541	try s.output().writeAll("\n]\n");
529	542	}
		⋯88 linesShow the 88 lines above this hunk@@ -618,11 +631,14 @@ pub const TAPReporter = struct {
618	631	fn onRunStart(reporter: *Reporter, total: usize) !void {
619	632	const self: *Self = @fieldParentPtr("reporter", reporter);
620	633	try self.output().print("TAP version 14\n1..{d}\n", .{total});
	634	+ if (reporter.random_seed) |seed| try self.output().print("# random-seed {d}\n", .{seed});
621	635	}
622	636
623	637	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
624		- _ = reporter;
625		- _ = results;
	638	+ const self: *Self = @fieldParentPtr("reporter", reporter);
	639	+ if (results.failed > 0) {
	640	+ if (results.random_seed) |seed| try self.output().print("# reproduce --seed {d}\n", .{seed});
	641	+ }
626	642	}
627	643
628	644	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
		⋯124 linesShow the 124 lines above this hunk@@ -753,14 +769,13 @@ pub const JUnitReporter = struct {
753	769	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
754	770	const self: *Self = @fieldParentPtr("reporter", reporter);
755	771
756		- const suite_result = try self.allocator.create(TestSuiteResult);
757		- suite_result.* = .{
	772	+ const suite_result = TestSuiteResult{
758	773	.name = try self.allocator.dupe(u8, suite_name),
759	774	.tests = .empty,
760	775	.timestamp = compat.milliTimestamp(),
761	776	};
762	777
763		- try self.suites.append(self.allocator, suite_result.*);
	778	+ try self.suites.append(self.allocator, suite_result);
764	779	self.current_suite = &self.suites.items[self.suites.items.len - 1];
765	780	}
766	781
		⋯76 linesShow the 76 lines above this hunk@@ -843,6 +858,10 @@ pub const JUnitReporter = struct {
843	858	suite_time,
844	859	});
845	860
	861	+ if (results.random_seed) |seed| {
	862	+ try buffer.print(self.allocator, " <properties><property name=\"random_seed\" value=\"{d}\"/></properties>\n", .{seed});
	863	+ }
	864	+
846	865	for (suite_result.tests.items) |test_result| {
847	866	try buffer.print(self.allocator, " <testcase name=\"{s}\" time=\"{d:.6}\"", .{
848	867	test_result.name,

src/test_history.zigmodified+17-2

Changes to src/test_history.zig

		⋯12 linesShow the 12 lines above this hunk@@ -13,6 +13,7 @@ pub const HistoryEntry = struct {
13	13	failed: usize,
14	14	skipped: usize,
15	15	duration_ns: u64,
	16	+ random_seed: ?u64 = null,
16	17	tests: std.ArrayList(TestRecord),
17	18
18	19	pub fn deinit(self: *HistoryEntry, allocator: std.mem.Allocator) void {
		⋯46 linesShow the 46 lines above this hunk@@ -65,6 +66,11 @@ pub const TestHistory = struct {
65	66
66	67	/// Start recording a new test run
67	68	pub fn startRun(self: *Self, total: usize) !void {
	69	+ try self.startRunWithSeed(total, null);
	70	+ }
	71	+
	72	+ /// Start recording a run with optional randomized-order metadata.
	73	+ pub fn startRunWithSeed(self: *Self, total: usize, random_seed: ?u64) !void {
68	74	self.start_time = compat.milliTimestamp();
69	75
70	76	self.current_entry = HistoryEntry{
		⋯4 linesShow the 4 lines above this hunk@@ -75,6 +81,7 @@ pub const TestHistory = struct {
75	81	.failed = 0,
76	82	.skipped = 0,
77	83	.duration_ns = 0,
	84	+ .random_seed = random_seed,
78	85	.tests = std.ArrayList(TestRecord).empty,
79	86	};
80	87	}
		⋯36 linesShow the 36 lines above this hunk@@ -117,6 +124,7 @@ pub const TestHistory = struct {
117	124	self.current_entry.?.flaky = results.flaky;
118	125	self.current_entry.?.failed = results.failed;
119	126	self.current_entry.?.skipped = results.skipped;
	127	+ self.current_entry.?.random_seed = results.random_seed orelse self.current_entry.?.random_seed;
120	128
121	129	try self.saveToFile();
122	130	}
		⋯16 linesShow the 16 lines above this hunk@@ -139,7 +147,10 @@ pub const TestHistory = struct {
139	147	errdefer buffer = allocating.toArrayList();
140	148	const writer = &allocating.writer;
141	149	try writer.writeAll("[\n");
142		- try std.json.Stringify.value(protocol.Event{ .run_start = .{ .total = self.current_entry.?.total } }, .{}, writer);
	150	+ try std.json.Stringify.value(protocol.Event{ .run_start = .{
	151	+ .total = self.current_entry.?.total,
	152	+ .random_seed = self.current_entry.?.random_seed,
	153	+ } }, .{}, writer);
143	154	for (self.current_entry.?.tests.items) |test_record| {
144	155	try writer.writeAll(",\n");
145	156	try std.json.Stringify.value(protocol.Event{ .test_end = .{
		⋯12 linesShow the 12 lines above this hunk@@ -158,6 +169,7 @@ pub const TestHistory = struct {
158	169	.failed = self.current_entry.?.failed,
159	170	.skipped = self.current_entry.?.skipped,
160	171	.duration_ns = self.current_entry.?.duration_ns,
	172	+ .random_seed = self.current_entry.?.random_seed,
161	173	} }, .{}, writer);
162	174	try writer.writeAll("\n]\n");
163	175	try writer.flush();
		⋯28 linesShow the 28 lines above this hunk@@ -192,6 +204,7 @@ pub const TestHistory = struct {
192	204	.failed = 0,
193	205	.skipped = 0,
194	206	.duration_ns = 0,
	207	+ .random_seed = null,
195	208	.tests = std.ArrayList(TestRecord).empty,
196	209	};
197	210	}
		⋯60 linesShow the 60 lines above this hunk@@ -258,7 +271,7 @@ test "TestHistory saves versioned protocol events" {
258	271
259	272	var history = TestHistory.init(allocator, history_dir);
260	273	defer history.deinit();
261		- try history.startRun(1);
	274	+ try history.startRunWithSeed(1, 9876);
262	275	try history.recordTest("quoted \"test\"", "suite\nname", .passed, 42, null);
263	276	var results = reporter_mod.TestResults.init(allocator);
264	277	defer results.deinit();
		⋯12 linesShow the 12 lines above this hunk@@ -277,9 +290,11 @@ test "TestHistory saves versioned protocol events" {
277	290	const events = parsed.value.array.items;
278	291	try std.testing.expectEqual(@as(usize, 3), events.len);
279	292	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	293	+ try std.testing.expectEqual(@as(i64, 9876), events[0].object.get("data").?.object.get("random_seed").?.integer);
280	294	try std.testing.expectEqualStrings("test_end", events[1].object.get("type").?.string);
281	295	try std.testing.expectEqualStrings("quoted \"test\"", events[1].object.get("data").?.object.get("name").?.string);
282	296	try std.testing.expectEqualStrings("run_end", events[2].object.get("type").?.string);
	297	+ try std.testing.expectEqual(@as(i64, 9876), events[2].object.get("data").?.object.get("random_seed").?.integer);
283	298	for (events) |event| {
284	299	try std.testing.expectEqual(@as(i64, protocol.version), event.object.get("protocol_version").?.integer);
285	300	}

src/test_loader.zigmodified+18-2

Changes to src/test_loader.zig

		⋯7 linesShow the 7 lines above this hunk@@ -8,6 +8,7 @@ const coverage = ;
8	8	const ui_server = ;
9	9	const compat = ;
10	10	const protocol = ;
	11	+const random_order = ;
11	12
12	13	/// Options for running discovered tests
13	14	pub const LoaderOptions = struct {
		⋯19 linesShow the 19 lines above this hunk@@ -33,6 +34,10 @@ pub const LoaderOptions = struct {
33	34	repeat: usize = 1,
34	35	/// Treat recovered failures as an unsuccessful run.
35	36	fail_on_flaky: bool = false,
	37	+ /// Shuffle selected files. A generated seed is printed for replay.
	38	+ shuffle: bool = false,
	39	+ /// Reproduce a shuffled file order. Supplying a seed enables shuffle.
	40	+ seed: ?u64 = null,
36	41	/// Optional output supplied by a CLI host. Embedded/test callers default
37	42	/// to stderr so they do not interfere with Zig's stdout test protocol.
38	43	reporter_writer: ?*std.Io.Writer = null,
		⋯16 linesShow the 16 lines above this hunk@@ -55,6 +60,11 @@ pub fn runDiscoveredTests(
55	60
56	61	var plan = try discoveryPlan(allocator, discovered, options.shard);
57	62	defer plan.deinit();
	63	+ const random_seed = random_order.resolveSeed(options.shuffle, options.seed);
	64	+ if (random_seed) |seed| {
	65	+ var shuffler = random_order.Shuffler.init(seed);
	66	+ shuffler.shuffle(pipeline.PlanItem, plan.items.items);
	67	+ }
58	68	const selected_files = plan.items.items.len;
59	69	if (options.shard) |shard| {
60	70	std.debug.print(
		⋯23 linesShow the 23 lines above this hunk@@ -84,7 +94,9 @@ pub fn runDiscoveredTests(
84	94	options.use_colors,
85	95	);
86	96	defer reporters.deinit();
87		- const events = pipeline.EventStream{ .rep = reporters.selected() };
	97	+ const selected_reporter = reporters.selected();
	98	+ selected_reporter.random_seed = random_seed;
	99	+ const events = pipeline.EventStream{ .rep = selected_reporter };
88	100	const policy = pipeline.ExecutionPolicy{
89	101	.bail = options.bail,
90	102	.filter = options.filter,
		⋯1 lineShow the 1 line above this hunk@@ -92,6 +104,7 @@ pub fn runDiscoveredTests(
92	104	};
93	105	var results = reporter.TestResults.init(allocator);
94	106	defer results.deinit();
	107	+ results.random_seed = random_seed;
95	108	var executed_cases: std.ArrayList(suite.TestCase) = .empty;
96	109	defer {
97	110	for (executed_cases.items) |*test_case| test_case.deinit(allocator);
		⋯4 linesShow the 4 lines above this hunk@@ -102,7 +115,7 @@ pub fn runDiscoveredTests(
102	115
103	116	// Notify UI of run start
104	117	if (options.ui_server) |server| {
105		- try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files } });
	118	+ try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files, .random_seed = random_seed } });
106	119	}
107	120
108	121	var files_run: usize = 0;
		⋯132 linesShow the 132 lines above this hunk@@ -241,6 +254,7 @@ pub fn runDiscoveredTests(
241	254	.failed = results.failed,
242	255	.skipped = results.skipped,
243	256	.duration_ns = results.total_time_ns,
	257	+ .random_seed = results.random_seed,
244	258	} });
245	259	}
246	260
		⋯108 linesShow the 108 lines above this hunk@@ -355,6 +369,8 @@ test "LoaderOptions default values" {
355	369	try std.testing.expectEqual(@as(?sharding.ShardOptions, null), options.shard);
356	370	try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type);
357	371	try std.testing.expectEqualStrings("test-results.xml", options.junit_output);
	372	+ try std.testing.expect(!options.shuffle);
	373	+ try std.testing.expect(options.seed == null);
358	374	try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
359	375	try std.testing.expectEqual(@as(usize, 0), options.retries);
360	376	try std.testing.expectEqual(@as(usize, 1), options.repeat);

src/test_runner.zigmodified+133-13

Changes to src/test_runner.zig

		⋯3 linesShow the 3 lines above this hunk@@ -4,6 +4,7 @@ const reporter_mod = ;
4	4	const pipeline = ;
5	5	const parallel = ;
6	6	const compat = ;
	7	+const random_order = ;
7	8
8	9	pub const RunnerError = error{
9	10	NoTestsFound,
		⋯15 linesShow the 15 lines above this hunk@@ -25,6 +26,10 @@ pub const RunnerOptions = struct {
25	26	repeat: usize = 1,
26	27	/// Treat tests that pass after a retry as an unsuccessful run.
27	28	fail_on_flaky: bool = false,
	29	+ /// Shuffle suites and tests. A generated seed is printed for replay.
	30	+ shuffle: bool = false,
	31	+ /// Reproduce a shuffled run. Supplying a seed implicitly enables shuffle.
	32	+ seed: ?u64 = null,
28	33	/// Optional output supplied by a CLI host. Embedded/test callers default
29	34	/// to stderr so they do not interfere with Zig's stdout test protocol.
30	35	reporter_writer: ?*std.Io.Writer = null,
		⋯40 linesShow the 40 lines above this hunk@@ -71,6 +76,9 @@ pub const TestRunner = struct {
71	76	);
72	77	defer reporters.deinit();
73	78	const current_reporter = reporters.selected();
	79	+ const random_seed = random_order.resolveSeed(self.options.shuffle, self.options.seed);
	80	+ current_reporter.random_seed = random_seed;
	81	+ self.results.random_seed = random_seed;
74	82	const events = pipeline.EventStream{ .rep = current_reporter };
75	83
76	84	var plan = try pipeline.TestPlan.fromRegistry(self.allocator, self.registry);
		⋯10 linesShow the 10 lines above this hunk@@ -87,6 +95,7 @@ pub const TestRunner = struct {
87	95	.n_jobs = self.options.n_jobs,
88	96	.filter = self.options.filter,
89	97	.timeout_ms = self.options.timeout_ms,
	98	+ .random_seed = random_seed,
90	99	};
91	100
92	101	const all_passed = try parallel.runTestsParallel(
		⋯13 linesShow the 13 lines above this hunk@@ -106,9 +115,19 @@ pub const TestRunner = struct {
106	115	}
107	116
108	117	// Sequential execution (original behavior)
	118	+ var shuffler: random_order.Shuffler = undefined;
	119	+ const shuffler_ptr: ?*random_order.Shuffler = if (random_seed) |seed| blk: {
	120	+ shuffler = .init(seed);
	121	+ break :blk &shuffler;
	122	+ } else null;
	123	+ var root_suites: std.ArrayList(*suite.TestSuite) = .empty;
	124	+ defer root_suites.deinit(self.allocator);
	125	+ try root_suites.appendSlice(self.allocator, self.registry.root_suites.items);
	126	+ if (shuffler_ptr) |randomizer| randomizer.shuffle(*suite.TestSuite, root_suites.items);
	127	+
109	128	try events.emit(.{ .run_started = total_tests });
110		- for (self.registry.root_suites.items) |test_suite| {
111		- try self.runSuite(test_suite, events);
	129	+ for (root_suites.items) |test_suite| {
	130	+ try self.runSuite(test_suite, events, shuffler_ptr);
112	131	if (self.policy().shouldStop(&self.results)) {
113	132	break;
114	133	}
		⋯15 linesShow the 15 lines above this hunk@@ -130,15 +149,20 @@ pub const TestRunner = struct {
130	149	}
131	150
132	151	/// Run a single test suite
133		- fn runSuite(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
	152	+ fn runSuite(
	153	+ self: *Self,
	154	+ test_suite: *suite.TestSuite,
	155	+ events: pipeline.EventStream,
	156	+ shuffler: ?*random_order.Shuffler,
	157	+ ) !void {
134	158	// Skip if marked as skip or if has_only and this isn't marked as only
135	159	if (test_suite.shouldSkip()) {
136		- try self.skipAllTests(test_suite, events);
	160	+ try self.skipAllTests(test_suite, events, shuffler);
137	161	return;
138	162	}
139	163
140	164	if (self.registry.has_only and !test_suite.hasOnly()) {
141		- try self.skipAllTests(test_suite, events);
	165	+ try self.skipAllTests(test_suite, events, shuffler);
142	166	return;
143	167	}
144	168
		⋯3 linesShow the 3 lines above this hunk@@ -148,13 +172,17 @@ pub const TestRunner = struct {
148	172	// Run beforeAll hooks
149	173	test_suite.runBeforeAllHooks(self.allocator) catch |err| {
150	174	std.debug.print("beforeAll hook failed: {any}\n", .{err});
151		- try self.skipAllTests(test_suite, events);
	175	+ try self.skipAllTests(test_suite, events, shuffler);
152	176	try events.emit(.{ .suite_finished = test_suite.name });
153	177	return;
154	178	};
155	179
156	180	// Run tests in this suite
157		- for (test_suite.tests.items) |*test_case| {
	181	+ var tests: std.ArrayList(*suite.TestCase) = .empty;
	182	+ defer tests.deinit(self.allocator);
	183	+ for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case);
	184	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items);
	185	+ for (tests.items) |test_case| {
158	186	if (test_case.skip or (self.registry.has_only and !test_case.only)) {
159	187	test_case.status = .skipped;
160	188	try events.emit(.{ .test_finished = test_case });
		⋯17 linesShow the 17 lines above this hunk@@ -178,8 +206,12 @@ pub const TestRunner = struct {
178	206	}
179	207
180	208	// Run nested suites
181		- for (test_suite.suites.items) |nested_suite| {
182		- try self.runSuite(nested_suite, events);
	209	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	210	+ defer nested_suites.deinit(self.allocator);
	211	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	212	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	213	+ for (nested_suites.items) |nested_suite| {
	214	+ try self.runSuite(nested_suite, events, shuffler);
183	215	if (self.policy().shouldStop(&self.results)) {
184	216	break;
185	217	}
		⋯112 linesShow the 112 lines above this hunk@@ -298,15 +330,28 @@ pub const TestRunner = struct {
298	330	}
299	331
300	332	/// Skip all tests in a suite
301		- fn skipAllTests(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
302		- for (test_suite.tests.items) |*test_case| {
	333	+ fn skipAllTests(
	334	+ self: *Self,
	335	+ test_suite: *suite.TestSuite,
	336	+ events: pipeline.EventStream,
	337	+ shuffler: ?*random_order.Shuffler,
	338	+ ) !void {
	339	+ var tests: std.ArrayList(*suite.TestCase) = .empty;
	340	+ defer tests.deinit(self.allocator);
	341	+ for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case);
	342	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items);
	343	+ for (tests.items) |test_case| {
303	344	test_case.status = .skipped;
304	345	try self.results.addTest(test_case);
305	346	try events.emit(.{ .test_finished = test_case });
306	347	}
307	348
308		- for (test_suite.suites.items) |nested_suite| {
309		- try self.skipAllTests(nested_suite, events);
	349	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	350	+ defer nested_suites.deinit(self.allocator);
	351	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	352	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	353	+ for (nested_suites.items) |nested_suite| {
	354	+ try self.skipAllTests(nested_suite, events, shuffler);
310	355	}
311	356	}
312	357
		⋯42 linesShow the 42 lines above this hunk@@ -355,6 +400,8 @@ test "RunnerOptions default values" {
355	400	try std.testing.expectEqual(@as(usize, 0), options.retries);
356	401	try std.testing.expectEqual(@as(usize, 1), options.repeat);
357	402	try std.testing.expect(!options.fail_on_flaky);
	403	+ try std.testing.expect(!options.shuffle);
	404	+ try std.testing.expect(options.seed == null);
358	405	}
359	406
360	407	test "RunnerOptions custom values" {
		⋯15 linesShow the 15 lines above this hunk@@ -376,6 +423,37 @@ var failing_fixture_runs: usize = 0;
376	423	var skipped_by_bail_runs: usize = 0;
377	424	var repeated_fixture_runs: usize = 0;
378	425	var timeout_fixture_runs: usize = 0;
	426	+var randomized_log: [6]u8 = undefined;
	427	+var randomized_log_len: usize = 0;
	428	+
	429	+fn recordRandomized(id: u8) void {
	430	+ randomized_log[randomized_log_len] = id;
	431	+ randomized_log_len += 1;
	432	+}
	433	+
	434	+fn randomizedOne(_: std.mem.Allocator) !void {
	435	+ recordRandomized(1);
	436	+}
	437	+
	438	+fn randomizedTwo(_: std.mem.Allocator) !void {
	439	+ recordRandomized(2);
	440	+}
	441	+
	442	+fn randomizedThree(_: std.mem.Allocator) !void {
	443	+ recordRandomized(3);
	444	+}
	445	+
	446	+fn randomizedFour(_: std.mem.Allocator) !void {
	447	+ recordRandomized(4);
	448	+}
	449	+
	450	+fn randomizedFive(_: std.mem.Allocator) !void {
	451	+ recordRandomized(5);
	452	+}
	453	+
	454	+fn randomizedSix(_: std.mem.Allocator) !void {
	455	+ recordRandomized(6);
	456	+}
379	457
380	458	fn flakyFixture(_: std.mem.Allocator) !void {
381	459	flaky_fixture_runs += 1;
		⋯9 linesShow the 9 lines above this hunk@@ -391,6 +469,48 @@ fn skippedByBailFixture(_: std.mem.Allocator) !void {
391	469	skipped_by_bail_runs += 1;
392	470	}
393	471
	472	+test "sequential order is reproducible with a fixed seed" {
	473	+ const allocator = std.testing.allocator;
	474	+ var registry = suite.TestRegistry.init(allocator);
	475	+ defer registry.deinit();
	476	+ const test_suite = try suite.TestSuite.init(allocator, "randomized");
	477	+ try test_suite.addTest(suite.TestCase.init("one", randomizedOne));
	478	+ try test_suite.addTest(suite.TestCase.init("two", randomizedTwo));
	479	+ try test_suite.addTest(suite.TestCase.init("three", randomizedThree));
	480	+ try test_suite.addTest(suite.TestCase.init("four", randomizedFour));
	481	+ try test_suite.addTest(suite.TestCase.init("five", randomizedFive));
	482	+ try test_suite.addTest(suite.TestCase.init("six", randomizedSix));
	483	+ try registry.registerSuite(test_suite);
	484	+
	485	+ var first_output: [4096]u8 = undefined;
	486	+ var first_writer: std.Io.Writer = .fixed(&first_output);
	487	+ randomized_log_len = 0;
	488	+ var first_runner = TestRunner.init(allocator, &registry, .{
	489	+ .seed = 112358,
	490	+ .use_colors = false,
	491	+ .reporter_writer = &first_writer,
	492	+ });
	493	+ defer first_runner.deinit();
	494	+ try std.testing.expect(try first_runner.run());
	495	+ const first_order = randomized_log;
	496	+
	497	+ var second_output: [4096]u8 = undefined;
	498	+ var second_writer: std.Io.Writer = .fixed(&second_output);
	499	+ randomized_log_len = 0;
	500	+ var second_runner = TestRunner.init(allocator, &registry, .{
	501	+ .seed = 112358,
	502	+ .use_colors = false,
	503	+ .reporter_writer = &second_writer,
	504	+ });
	505	+ defer second_runner.deinit();
	506	+ try std.testing.expect(try second_runner.run());
	507	+
	508	+ try std.testing.expectEqual(@as(usize, randomized_log.len), randomized_log_len);
	509	+ try std.testing.expectEqualSlices(u8, &first_order, &randomized_log);
	510	+ try std.testing.expectEqual(@as(?u64, 112358), first_runner.results.random_seed);
	511	+ try std.testing.expectEqual(@as(?u64, 112358), second_runner.results.random_seed);
	512	+}
	513	+
394	514	fn repeatedFixture(_: std.mem.Allocator) !void {
395	515	repeated_fixture_runs += 1;
396	516	}

src/ui_server.zigmodified+7-2

Changes to src/ui_server.zig

		⋯320 linesShow the 320 lines above this hunk@@ -321,7 +321,10 @@ pub const UIReporter = struct {
321	321
322	322	fn onRunStart(reporter: *reporter_mod.Reporter, total: usize) !void {
323	323	const self: *Self = @fieldParentPtr("reporter", reporter);
324		- try self.server.broadcastEvent(reporter.allocator, .{ .run_start = .{ .total = total } });
	324	+ try self.server.broadcastEvent(reporter.allocator, .{ .run_start = .{
	325	+ .total = total,
	326	+ .random_seed = reporter.random_seed,
	327	+ } });
325	328	}
326	329
327	330	fn onRunEnd(reporter: *reporter_mod.Reporter, results: *reporter_mod.TestResults) !void {
		⋯6 linesShow the 6 lines above this hunk@@ -334,6 +337,7 @@ pub const UIReporter = struct {
334	337	.failed = results.failed,
335	338	.skipped = results.skipped,
336	339	.duration_ns = results.total_time_ns,
	340	+ .random_seed = results.random_seed,
337	341	} });
338	342	}
339	343
		⋯67 linesShow the 67 lines above this hunk@@ -407,6 +411,7 @@ pub const MultiReporter = struct {
407	411	fn onRunStart(reporter: *reporter_mod.Reporter, total: usize) !void {
408	412	const self: *Self = @fieldParentPtr("reporter", reporter);
409	413	for (self.reporters) |rep| {
	414	+ rep.random_seed = reporter.random_seed;
410	415	try rep.onRunStart(total);
411	416	}
412	417	}
		⋯123 linesShow the 123 lines above this hunk@@ -536,7 +541,7 @@ test "UI server streams broadcast events over SSE" {
536	541
537	542	try server.broadcastEvent(std.testing.allocator, .{ .run_start = .{ .total = 2 } });
538	543	try expectLine(&stream_reader.interface, "event: run_start");
539		- try expectLine(&stream_reader.interface, "data: {\"protocol_version\":1,\"type\":\"run_start\",\"data\":{\"total\":2}}");
	544	+ try expectLine(&stream_reader.interface, "data: {\"protocol_version\":1,\"type\":\"run_start\",\"data\":{\"total\":2,\"random_seed\":null}}");
540	545	try expectLine(&stream_reader.interface, "");
541	546	}
542	547

tests/fixtures/zig-test.jsonmodified+3-1

Changes to tests/fixtures/zig-test.json

		⋯5 linesShow the 5 lines above this hunk@@ -6,7 +6,9 @@
6	6	"filter": "selected test passes",
7	7	"retries": 1,
8	8	"repeat": 2,
9		- "fail_on_flaky": false
	9	+ "fail_on_flaky": false,
	10	+ "shuffle": true,
	11	+ "seed": 42
10	12	},
11	13	"sharding": {
12	14	"index": 1,

tests/reporter_test.zigmodified+33-0

Changes to tests/reporter_test.zig

		⋯55 linesShow the 55 lines above this hunk@@ -56,6 +56,8 @@ test "reporter json output uses versioned event envelopes" {
56	56	defer results.deinit();
57	57	results.total = 1;
58	58	results.passed = 1;
	59	+ results.random_seed = 4242;
	60	+ reporter.reporter.random_seed = 4242;
59	61	var test_case = ztf.TestCase.init("passes \"quoted\"\nname", passingTest);
60	62	test_case.status = .passed;
61	63
		⋯11 linesShow the 11 lines above this hunk@@ -73,12 +75,14 @@ test "reporter json output uses versioned event envelopes" {
73	75	try std.testing.expectEqual(@as(usize, 6), events.len);
74	76	try std.testing.expectEqual(@as(i64, ztf.protocol_version), events[0].object.get("protocol_version").?.integer);
75	77	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	78	+ try std.testing.expectEqual(@as(i64, 4242), events[0].object.get("data").?.object.get("random_seed").?.integer);
76	79	try std.testing.expectEqualStrings("test_start", events[2].object.get("type").?.string);
77	80	const test_data = events[3].object.get("data").?.object;
78	81	try std.testing.expectEqualStrings(test_case.name, test_data.get("name").?.string);
79	82	try std.testing.expectEqualStrings("suite\tone", test_data.get("suite").?.string);
80	83	try std.testing.expectEqualStrings("run_end", events[5].object.get("type").?.string);
81	84	try std.testing.expectEqual(@as(i64, 1), events[5].object.get("data").?.object.get("passed").?.integer);
	85	+ try std.testing.expectEqual(@as(i64, 4242), events[5].object.get("data").?.object.get("random_seed").?.integer);
82	86	}
83	87
84	88	test "reporter json output includes flaky attempt history" {
		⋯55 linesShow the 55 lines above this hunk@@ -140,3 +144,32 @@ test "reporter set selects the same built-ins for every executor" {
140	144	try std.testing.expect(reporters.selected() == &reporters.tap.reporter);
141	145	try std.testing.expect(!reporters.selected().use_colors);
142	146	}
	147	+
	148	+test "JUnit reporter records the random seed as a suite property" {
	149	+ const allocator = std.testing.allocator;
	150	+ var tmp = std.testing.tmpDir(.{});
	151	+ defer tmp.cleanup();
	152	+ const output_path = try std.fmt.allocPrint(allocator, ".zig-cache/tmp/{s}/results.xml", .{tmp.sub_path});
	153	+ defer allocator.free(output_path);
	154	+
	155	+ var reporter = ztf.JUnitReporter.init(allocator, output_path);
	156	+ defer reporter.deinit();
	157	+ reporter.reporter.random_seed = 73;
	158	+ var results = ztf.TestResults.init(allocator);
	159	+ defer results.deinit();
	160	+ results.total = 1;
	161	+ results.passed = 1;
	162	+ results.random_seed = 73;
	163	+ var test_case = ztf.TestCase.init("passes", passingTest);
	164	+ test_case.status = .passed;
	165	+
	166	+ try reporter.reporter.onRunStart(1);
	167	+ try reporter.reporter.onSuiteStart("suite");
	168	+ try reporter.reporter.onTestEnd(&test_case);
	169	+ try reporter.reporter.onSuiteEnd("suite");
	170	+ try reporter.reporter.onRunEnd(&results);
	171	+
	172	+ const output = try ztf.compat.readFileAlloc(allocator, output_path);
	173	+ defer allocator.free(output);
	174	+ try std.testing.expect(std.mem.indexOf(u8, output, "<property name=\"random_seed\" value=\"73\"/>") != null);
	175	+}