Add reproducible randomized test order

src/parallel.zigmodified+74-8

Changes to src/parallel.zig

		⋯2 linesShow the 2 lines above this hunk@@ -3,6 +3,7 @@ const suite = ;
3	3	const reporter = ;
4	4	const pipeline = ;
5	5	const compat = ;
	6	+const random_order = ;
6	7
7	8	/// Options for bounded parallel test execution.
8	9	pub const ParallelOptions = struct {
		⋯3 linesShow the 3 lines above this hunk@@ -12,6 +13,7 @@ pub const ParallelOptions = struct {
12	13	enabled: bool = false,
13	14	filter: ?[]const u8 = null,
14	15	timeout_ms: ?u64 = null,
	16	+ random_seed: ?u64 = null,
15	17	};
16	18
17	19	pub const ParallelError = error{
		⋯93 linesShow the 93 lines above this hunk@@ -111,6 +113,7 @@ const RunContext = struct {
111	113	rep: *reporter.Reporter,
112	114	options: ParallelOptions,
113	115	worker_limit: usize,
	116	+ shuffler: ?*random_order.Shuffler,
114	117
115	118	fn runSuite(self: *RunContext, test_suite: *suite.TestSuite) !bool {
116	119	try self.rep.onSuiteStart(test_suite.name);
		⋯14 linesShow the 14 lines above this hunk@@ -131,7 +134,12 @@ const RunContext = struct {
131	134	var scheduled: std.ArrayList(*suite.TestCase) = .empty;
132	135	defer scheduled.deinit(self.allocator);
133	136
134		- for (test_suite.tests.items) |*test_case| {
	137	+ var ordered_tests: std.ArrayList(*suite.TestCase) = .empty;
	138	+ defer ordered_tests.deinit(self.allocator);
	139	+ for (test_suite.tests.items) |*test_case| try ordered_tests.append(self.allocator, test_case);
	140	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, ordered_tests.items);
	141	+
	142	+ for (ordered_tests.items) |test_case| {
135	143	if (!self.shouldRun(test_suite, test_case)) {
136	144	test_case.status = .skipped;
137	145	continue;
		⋯5 linesShow the 5 lines above this hunk@@ -143,12 +151,16 @@ const RunContext = struct {
143	151	try self.runBatch(test_suite, scheduled.items);
144	152
145	153	var all_passed = true;
146		- for (test_suite.tests.items) |*test_case| {
	154	+ for (ordered_tests.items) |test_case| {
147	155	try self.rep.onTestEnd(test_case);
148	156	if (test_case.status == .failed) all_passed = false;
149	157	}
150	158
151		- for (test_suite.suites.items) |nested_suite| {
	159	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	160	+ defer nested_suites.deinit(self.allocator);
	161	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	162	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	163	+ for (nested_suites.items) |nested_suite| {
152	164	if (!try self.runSuite(nested_suite)) all_passed = false;
153	165	}
154	166
		⋯46 linesShow the 46 lines above this hunk@@ -201,11 +213,19 @@ const RunContext = struct {
201	213	}
202	214
203	215	fn reportSkippedSuite(self: *RunContext, test_suite: *suite.TestSuite) !void {
204		- for (test_suite.tests.items) |*test_case| {
	216	+ var tests: std.ArrayList(*suite.TestCase) = .empty;
	217	+ defer tests.deinit(self.allocator);
	218	+ for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case);
	219	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items);
	220	+ for (tests.items) |test_case| {
205	221	test_case.status = .skipped;
206	222	try self.rep.onTestEnd(test_case);
207	223	}
208		- for (test_suite.suites.items) |nested| {
	224	+ var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	225	+ defer nested_suites.deinit(self.allocator);
	226	+ try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	227	+ if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	228	+ for (nested_suites.items) |nested| {
209	229	try self.rep.onSuiteStart(nested.name);
210	230	try self.reportSkippedSuite(nested);
211	231	try self.rep.onSuiteEnd(nested.name);
		⋯1 lineShow the 1 line above this hunk@@ -213,9 +233,9 @@ const RunContext = struct {
213	233	}
214	234	};
215	235
216		-/// Run suites in declaration order while executing each suite's direct tests
	236	+/// Run suites in the selected deterministic order while executing each suite's direct tests
217	237	/// with a bounded worker set. Reporter callbacks are serialized and emitted in
218		-/// declaration order after each batch completes.
	238	+/// the selected order after each batch completes.
219	239	pub fn runTestsParallel(
220	240	allocator: std.mem.Allocator,
221	241	test_registry: *suite.TestRegistry,
		⋯4 linesShow the 4 lines above this hunk@@ -226,23 +246,36 @@ pub fn runTestsParallel(
226	246	const worker_limit = try resolveWorkerCount(options.n_jobs);
227	247
228	248	const total_tests = test_registry.countAllTests();
	249	+ rep.random_seed = options.random_seed;
229	250	try rep.onRunStart(total_tests);
230	251
	252	+ var shuffler: random_order.Shuffler = undefined;
	253	+ const shuffler_ptr: ?*random_order.Shuffler = if (options.random_seed) |seed| blk: {
	254	+ shuffler = .init(seed);
	255	+ break :blk &shuffler;
	256	+ } else null;
	257	+
231	258	var context = RunContext{
232	259	.allocator = allocator,
233	260	.registry = test_registry,
234	261	.rep = rep,
235	262	.options = options,
236	263	.worker_limit = worker_limit,
	264	+ .shuffler = shuffler_ptr,
237	265	};
238	266
	267	+ var root_suites: std.ArrayList(*suite.TestSuite) = .empty;
	268	+ defer root_suites.deinit(allocator);
	269	+ try root_suites.appendSlice(allocator, test_registry.root_suites.items);
	270	+ if (shuffler_ptr) |randomizer| randomizer.shuffle(*suite.TestSuite, root_suites.items);
239	271	var all_passed = true;
240		- for (test_registry.root_suites.items) |test_suite| {
	272	+ for (root_suites.items) |test_suite| {
241	273	if (!try context.runSuite(test_suite)) all_passed = false;
242	274	}
243	275
244	276	var results = reporter.TestResults.init(allocator);
245	277	defer results.deinit();
	278	+ results.random_seed = options.random_seed;
246	279	for (test_registry.root_suites.items) |test_suite| {
247	280	try addSuiteResults(&results, test_suite);
248	281	}
		⋯98 linesShow the 98 lines above this hunk@@ -347,6 +380,7 @@ const RecordingReporter = struct {
347	380	passed: usize = 0,
348	381	failed: usize = 0,
349	382	skipped: usize = 0,
	383	+ random_seed: ?u64 = null,
350	384
351	385	fn init(allocator: std.mem.Allocator) RecordingReporter {
352	386	return .{
		⋯26 linesShow the 26 lines above this hunk@@ -379,6 +413,7 @@ const RecordingReporter = struct {
379	413	recording.passed = results.passed;
380	414	recording.failed = results.failed;
381	415	recording.skipped = results.skipped;
	416	+ recording.random_seed = results.random_seed;
382	417	}
383	418
384	419	fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void {
		⋯83 linesShow the 83 lines above this hunk@@ -468,6 +503,37 @@ test "parallel reporting is serialized, ordered, and complete" {
468	503	try std.testing.expectEqual(@as(usize, 1), recording.skipped);
469	504	}
470	505
	506	+test "parallel order is reproducible with a fixed seed" {
	507	+ const allocator = std.heap.smp_allocator;
	508	+ var registry = suite.TestRegistry.init(allocator);
	509	+ defer registry.deinit();
	510	+ const test_suite = try suite.TestSuite.init(allocator, "randomized");
	511	+ inline for (.{ "one", "two", "three", "four", "five", "six" }) |name| {
	512	+ try test_suite.addTest(suite.TestCase.init(name, passingTest));
	513	+ }
	514	+ try registry.registerSuite(test_suite);
	515	+
	516	+ var first = RecordingReporter.init(allocator);
	517	+ try std.testing.expect(try runTestsParallel(allocator, &registry, &first.reporter, .{
	518	+ .enabled = true,
	519	+ .n_jobs = 2,
	520	+ .random_seed = 8675309,
	521	+ }));
	522	+ var second = RecordingReporter.init(allocator);
	523	+ try std.testing.expect(try runTestsParallel(allocator, &registry, &second.reporter, .{
	524	+ .enabled = true,
	525	+ .n_jobs = 2,
	526	+ .random_seed = 8675309,
	527	+ }));
	528	+
	529	+ try std.testing.expectEqual(first.ended_count, second.ended_count);
	530	+ for (first.ended_names[0..first.ended_count], second.ended_names[0..second.ended_count]) |a, b| {
	531	+ try std.testing.expectEqualStrings(a, b);
	532	+ }
	533	+ try std.testing.expectEqual(@as(?u64, 8675309), first.random_seed);
	534	+ try std.testing.expectEqual(@as(?u64, 8675309), second.random_seed);
	535	+}
	536	+
471	537	test "parallel executor preserves nested hook lifecycles" {
472	538	const counters = [_]*std.atomic.Value(usize){
473	539	&parent_before_each,