Add reproducible randomized test order

src/test_loader.zigmodified+18-2

Changes to src/test_loader.zig

		⋯7 linesShow the 7 lines above this hunk@@ -8,6 +8,7 @@ const coverage = ;
8	8	const ui_server = ;
9	9	const compat = ;
10	10	const protocol = ;
	11	+const random_order = ;
11	12
12	13	/// Options for running discovered tests
13	14	pub const LoaderOptions = struct {
		⋯19 linesShow the 19 lines above this hunk@@ -33,6 +34,10 @@ pub const LoaderOptions = struct {
33	34	repeat: usize = 1,
34	35	/// Treat recovered failures as an unsuccessful run.
35	36	fail_on_flaky: bool = false,
	37	+ /// Shuffle selected files. A generated seed is printed for replay.
	38	+ shuffle: bool = false,
	39	+ /// Reproduce a shuffled file order. Supplying a seed enables shuffle.
	40	+ seed: ?u64 = null,
36	41	/// Optional output supplied by a CLI host. Embedded/test callers default
37	42	/// to stderr so they do not interfere with Zig's stdout test protocol.
38	43	reporter_writer: ?*std.Io.Writer = null,
		⋯16 linesShow the 16 lines above this hunk@@ -55,6 +60,11 @@ pub fn runDiscoveredTests(
55	60
56	61	var plan = try discoveryPlan(allocator, discovered, options.shard);
57	62	defer plan.deinit();
	63	+ const random_seed = random_order.resolveSeed(options.shuffle, options.seed);
	64	+ if (random_seed) |seed| {
	65	+ var shuffler = random_order.Shuffler.init(seed);
	66	+ shuffler.shuffle(pipeline.PlanItem, plan.items.items);
	67	+ }
58	68	const selected_files = plan.items.items.len;
59	69	if (options.shard) |shard| {
60	70	std.debug.print(
		⋯23 linesShow the 23 lines above this hunk@@ -84,7 +94,9 @@ pub fn runDiscoveredTests(
84	94	options.use_colors,
85	95	);
86	96	defer reporters.deinit();
87		- const events = pipeline.EventStream{ .rep = reporters.selected() };
	97	+ const selected_reporter = reporters.selected();
	98	+ selected_reporter.random_seed = random_seed;
	99	+ const events = pipeline.EventStream{ .rep = selected_reporter };
88	100	const policy = pipeline.ExecutionPolicy{
89	101	.bail = options.bail,
90	102	.filter = options.filter,
		⋯1 lineShow the 1 line above this hunk@@ -92,6 +104,7 @@ pub fn runDiscoveredTests(
92	104	};
93	105	var results = reporter.TestResults.init(allocator);
94	106	defer results.deinit();
	107	+ results.random_seed = random_seed;
95	108	var executed_cases: std.ArrayList(suite.TestCase) = .empty;
96	109	defer {
97	110	for (executed_cases.items) |*test_case| test_case.deinit(allocator);
		⋯4 linesShow the 4 lines above this hunk@@ -102,7 +115,7 @@ pub fn runDiscoveredTests(
102	115
103	116	// Notify UI of run start
104	117	if (options.ui_server) |server| {
105		- try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files } });
	118	+ try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files, .random_seed = random_seed } });
106	119	}
107	120
108	121	var files_run: usize = 0;
		⋯132 linesShow the 132 lines above this hunk@@ -241,6 +254,7 @@ pub fn runDiscoveredTests(
241	254	.failed = results.failed,
242	255	.skipped = results.skipped,
243	256	.duration_ns = results.total_time_ns,
	257	+ .random_seed = results.random_seed,
244	258	} });
245	259	}
246	260
		⋯108 linesShow the 108 lines above this hunk@@ -355,6 +369,8 @@ test "LoaderOptions default values" {
355	369	try std.testing.expectEqual(@as(?sharding.ShardOptions, null), options.shard);
356	370	try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type);
357	371	try std.testing.expectEqualStrings("test-results.xml", options.junit_output);
	372	+ try std.testing.expect(!options.shuffle);
	373	+ try std.testing.expect(options.seed == null);
358	374	try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
359	375	try std.testing.expectEqual(@as(usize, 0), options.retries);
360	376	try std.testing.expectEqual(@as(usize, 1), options.repeat);