Add reproducible randomized test order

src/test_runner.zigmodified+133-13

Changes to src/test_runner.zig

		⋯3 linesShow the 3 lines above this hunk@@ -4,6 +4,7 @@ const reporter_mod = ;
4	4	const pipeline = ;
5	5	const parallel = ;
6	6	const compat = ;
	7	+const random_order = ;
7	8
8	9	pub const RunnerError = error{
9	10	NoTestsFound,
		⋯15 linesShow the 15 lines above this hunk@@ -25,6 +26,10 @@ pub const RunnerOptions = struct {
25	26	repeat: usize = 1,
26	27	/// Treat tests that pass after a retry as an unsuccessful run.
27	28	fail_on_flaky: bool = false,
	29	+ /// Shuffle suites and tests. A generated seed is printed for replay.
	30	+ shuffle: bool = false,
	31	+ /// Reproduce a shuffled run. Supplying a seed implicitly enables shuffle.
	32	+ seed: ?u64 = null,
28	33	/// Optional output supplied by a CLI host. Embedded/test callers default
29	34	/// to stderr so they do not interfere with Zig's stdout test protocol.
30	35	reporter_writer: ?*std.Io.Writer = null,
		⋯40 linesShow the 40 lines above this hunk@@ -71,6 +76,9 @@ pub const TestRunner = struct {
71	76	);
72	77	defer reporters.deinit();
73	78	const current_reporter = reporters.selected();
	79	+ const random_seed = random_order.resolveSeed(self.options.shuffle, self.options.seed);
	80	+ current_reporter.random_seed = random_seed;
	81	+ self.results.random_seed = random_seed;
74	82	const events = pipeline.EventStream{ .rep = current_reporter };
75	83
76	84	var plan = try pipeline.TestPlan.fromRegistry(self.allocator, self.registry);
		⋯10 linesShow the 10 lines above this hunk@@ -87,6 +95,7 @@ pub const TestRunner = struct {
87	95	.n_jobs = self.options.n_jobs,
88	96	.filter = self.options.filter,
89	97	.timeout_ms = self.options.timeout_ms,
	98	+ .random_seed = random_seed,
90	99	};
91	100
92	101	const all_passed = try parallel.runTestsParallel(
		⋯13 linesShow the 13 lines above this hunk@@ -106,9 +115,19 @@ pub const TestRunner = struct {
106	115	}
107	116
108	117	// Sequential execution (original behavior)
	118	+ var shuffler: random_order.Shuffler = undefined;
	119	+ const shuffler_ptr: ?*random_order.Shuffler = if (random_seed) |seed| blk: {
	120	+ shuffler = .init(seed);
	121	+ break :blk &shuffler;
	122	+ } else null;
	123	+ var root_suites: std.ArrayList(*suite.TestSuite) = .empty;
	124	+ defer root_suites.deinit(self.allocator);
	125	+ try root_suites.appendSlice(self.allocator, self.registry.root_suites.items);
	126	+ if (shuffler_ptr) |randomizer| randomizer.shuffle(*suite.TestSuite, root_suites.items);
	127	+
109	128	try events.emit(.{ .run_started = total_tests });
110		- for (self.registry.root_suites.items) |test_suite| {
111		- try self.runSuite(test_suite, events);
	129	+ for (root_suites.items) |test_suite| {
	130	+ try self.runSuite(test_suite, events, shuffler_ptr);
112	131	if (self.policy().shouldStop(&self.results)) {
113	132	break;
114	133	}
		⋯15 linesShow the 15 lines above this hunk@@ -130,15 +149,20 @@ pub const TestRunner = struct {
130	149	}
131	150
132	151	/// Run a single test suite
133		- fn runSuite(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
	152	+ fn runSuite(
	153	+ self: *Self,
	154	+ test_suite: *suite.TestSuite,
	155	+ events: pipeline.EventStream,
	156	+ shuffler: ?*random_order.Shuffler,
	157	+ ) !void {
134	158	// Skip if marked as skip or if has_only and this isn't marked as only
135	159	if (test_suite.shouldSkip()) {
136		- try self.skipAllTests(test_suite, events);
	160	+ try self.skipAllTests(test_suite, events, shuffler);
137	161	return;
138	162	}
139	163
140	164	if (self.registry.has_only and !test_suite.hasOnly()) {
141		- try self.skipAllTests(test_suite, events);
	165	+ try self.skipAllTests(test_suite, events, shuffler);
142	166	return;
143	167	}
144	168
		⋯3 linesShow the 3 lines above this hunk@@ -148,13 +172,17 @@ pub const TestRunner = struct {
148	172	// Run beforeAll hooks
149	173	test_suite.runBeforeAllHooks(self.allocator) catch |err| {
150	174	std.debug.print("beforeAll hook failed: {any}\n", .{err});
151		- try self.skipAllTests(test_suite, events);
	175	+ try self.skipAllTests(test_suite, events, shuffler);
152	176	try events.emit(.{ .suite_finished = test_suite.name });
153	177	return;
154	178	};
155	179
156	180	// Run tests in this suite
157		- for (test_suite.tests.items) |*test_case| {
	181	+ var tests: std.ArrayList(*suite.TestCase) = .empty;
	182	+ defer tests.deinit(self.allocator);
	183	+ for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case);
	184	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items);
	185	+ for (tests.items) |test_case| {
158	186	if (test_case.skip or (self.registry.has_only and !test_case.only)) {
159	187	test_case.status = .skipped;
160	188	try events.emit(.{ .test_finished = test_case });
		⋯17 linesShow the 17 lines above this hunk@@ -178,8 +206,12 @@ pub const TestRunner = struct {
178	206	}
179	207
180	208	// Run nested suites
181		- for (test_suite.suites.items) |nested_suite| {
182		- try self.runSuite(nested_suite, events);
	209	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	210	+ defer nested_suites.deinit(self.allocator);
	211	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	212	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	213	+ for (nested_suites.items) |nested_suite| {
	214	+ try self.runSuite(nested_suite, events, shuffler);
183	215	if (self.policy().shouldStop(&self.results)) {
184	216	break;
185	217	}
		⋯112 linesShow the 112 lines above this hunk@@ -298,15 +330,28 @@ pub const TestRunner = struct {
298	330	}
299	331
300	332	/// Skip all tests in a suite
301		- fn skipAllTests(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
302		- for (test_suite.tests.items) |*test_case| {
	333	+ fn skipAllTests(
	334	+ self: *Self,
	335	+ test_suite: *suite.TestSuite,
	336	+ events: pipeline.EventStream,
	337	+ shuffler: ?*random_order.Shuffler,
	338	+ ) !void {
	339	+ var tests: std.ArrayList(*suite.TestCase) = .empty;
	340	+ defer tests.deinit(self.allocator);
	341	+ for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case);
	342	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items);
	343	+ for (tests.items) |test_case| {
303	344	test_case.status = .skipped;
304	345	try self.results.addTest(test_case);
305	346	try events.emit(.{ .test_finished = test_case });
306	347	}
307	348
308		- for (test_suite.suites.items) |nested_suite| {
309		- try self.skipAllTests(nested_suite, events);
	349	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	350	+ defer nested_suites.deinit(self.allocator);
	351	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	352	+ if (shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	353	+ for (nested_suites.items) |nested_suite| {
	354	+ try self.skipAllTests(nested_suite, events, shuffler);
310	355	}
311	356	}
312	357
		⋯42 linesShow the 42 lines above this hunk@@ -355,6 +400,8 @@ test "RunnerOptions default values" {
355	400	try std.testing.expectEqual(@as(usize, 0), options.retries);
356	401	try std.testing.expectEqual(@as(usize, 1), options.repeat);
357	402	try std.testing.expect(!options.fail_on_flaky);
	403	+ try std.testing.expect(!options.shuffle);
	404	+ try std.testing.expect(options.seed == null);
358	405	}
359	406
360	407	test "RunnerOptions custom values" {
		⋯15 linesShow the 15 lines above this hunk@@ -376,6 +423,37 @@ var failing_fixture_runs: usize = 0;
376	423	var skipped_by_bail_runs: usize = 0;
377	424	var repeated_fixture_runs: usize = 0;
378	425	var timeout_fixture_runs: usize = 0;
	426	+var randomized_log: [6]u8 = undefined;
	427	+var randomized_log_len: usize = 0;
	428	+
	429	+fn recordRandomized(id: u8) void {
	430	+ randomized_log[randomized_log_len] = id;
	431	+ randomized_log_len += 1;
	432	+}
	433	+
	434	+fn randomizedOne(_: std.mem.Allocator) !void {
	435	+ recordRandomized(1);
	436	+}
	437	+
	438	+fn randomizedTwo(_: std.mem.Allocator) !void {
	439	+ recordRandomized(2);
	440	+}
	441	+
	442	+fn randomizedThree(_: std.mem.Allocator) !void {
	443	+ recordRandomized(3);
	444	+}
	445	+
	446	+fn randomizedFour(_: std.mem.Allocator) !void {
	447	+ recordRandomized(4);
	448	+}
	449	+
	450	+fn randomizedFive(_: std.mem.Allocator) !void {
	451	+ recordRandomized(5);
	452	+}
	453	+
	454	+fn randomizedSix(_: std.mem.Allocator) !void {
	455	+ recordRandomized(6);
	456	+}
379	457
380	458	fn flakyFixture(_: std.mem.Allocator) !void {
381	459	flaky_fixture_runs += 1;
		⋯9 linesShow the 9 lines above this hunk@@ -391,6 +469,48 @@ fn skippedByBailFixture(_: std.mem.Allocator) !void {
391	469	skipped_by_bail_runs += 1;
392	470	}
393	471
	472	+test "sequential order is reproducible with a fixed seed" {
	473	+ const allocator = std.testing.allocator;
	474	+ var registry = suite.TestRegistry.init(allocator);
	475	+ defer registry.deinit();
	476	+ const test_suite = try suite.TestSuite.init(allocator, "randomized");
	477	+ try test_suite.addTest(suite.TestCase.init("one", randomizedOne));
	478	+ try test_suite.addTest(suite.TestCase.init("two", randomizedTwo));
	479	+ try test_suite.addTest(suite.TestCase.init("three", randomizedThree));
	480	+ try test_suite.addTest(suite.TestCase.init("four", randomizedFour));
	481	+ try test_suite.addTest(suite.TestCase.init("five", randomizedFive));
	482	+ try test_suite.addTest(suite.TestCase.init("six", randomizedSix));
	483	+ try registry.registerSuite(test_suite);
	484	+
	485	+ var first_output: [4096]u8 = undefined;
	486	+ var first_writer: std.Io.Writer = .fixed(&first_output);
	487	+ randomized_log_len = 0;
	488	+ var first_runner = TestRunner.init(allocator, &registry, .{
	489	+ .seed = 112358,
	490	+ .use_colors = false,
	491	+ .reporter_writer = &first_writer,
	492	+ });
	493	+ defer first_runner.deinit();
	494	+ try std.testing.expect(try first_runner.run());
	495	+ const first_order = randomized_log;
	496	+
	497	+ var second_output: [4096]u8 = undefined;
	498	+ var second_writer: std.Io.Writer = .fixed(&second_output);
	499	+ randomized_log_len = 0;
	500	+ var second_runner = TestRunner.init(allocator, &registry, .{
	501	+ .seed = 112358,
	502	+ .use_colors = false,
	503	+ .reporter_writer = &second_writer,
	504	+ });
	505	+ defer second_runner.deinit();
	506	+ try std.testing.expect(try second_runner.run());
	507	+
	508	+ try std.testing.expectEqual(@as(usize, randomized_log.len), randomized_log_len);
	509	+ try std.testing.expectEqualSlices(u8, &first_order, &randomized_log);
	510	+ try std.testing.expectEqual(@as(?u64, 112358), first_runner.results.random_seed);
	511	+ try std.testing.expectEqual(@as(?u64, 112358), second_runner.results.random_seed);
	512	+}
	513	+
394	514	fn repeatedFixture(_: std.mem.Allocator) !void {
395	515	repeated_fixture_runs += 1;
396	516	}