Pull request #29 · zig-utils/zig-test-framework

README.mdmodified+17-3

Changes to README.md
		@@ -9,6 +9,7 @@ A modern, feature-rich testing framework for Zig inspired by Jest, Vitest, and B
9	9	- Async Test Support - Full async test execution with concurrent and sequential modes
10	10	- Timeout Handling - Configurable timeouts at test, suite, and global levels with extension support
11	11	- Retries & Repeats - Re-run failures, stress-run selected tests, and report flaky outcomes
	12	- Reproducible Randomization - Shuffle suites and tests with replayable seeds
12	13	- Familiar API - Describe/it syntax similar to Jest and Vitest
13	14	- Rich Assertions - Comprehensive assertion library with `.expect()` and matchers
14	15	- Error Assertions - `toThrow()` and `toThrowError()` for testing error handling
		@@ -102,6 +103,8 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d
102	103	- `--retry <N>` - Retry a failed test or discovered test file up to N times
103	104	- `--repeat <N>` - Require N successful repetitions of every selected test
104	105	- `--fail-on-flaky` - Return a failing exit status when a retry recovers a failure
	106	- `--shuffle` - Randomize suites, tests, or discovered files with a generated seed
	107	- `--seed <N>` - Replay a randomized order; supplying a seed implies `--shuffle`
105	108	- `--shard-index <N>` - Run one one-based file shard
106	109	- `--shard-count <N>` - Set the total number of file shards
107	110	- `--no-color` - Disable color in child Zig test processes
		@@ -139,6 +142,10 @@ zig-test --test-dir tests --retry 2 --fail-on-flaky
139	142	# Stress-run matching tests 100 times
140	143	zig-test --test-dir tests --filter parser --repeat 100
141	144
	145	# Expose order dependencies, then replay the exact order with the printed seed
	146	zig-test --test-dir tests --shuffle
	147	zig-test --test-dir tests --seed 8675309
	148
142	149	# Run the second of four deterministic file shards
143	150	zig-test --test-dir tests --shard-index 2 --shard-count 4
144	151	```
		@@ -185,7 +192,9 @@ precedence over configured values.
185	192	"filter": "database",
186	193	"retries": 2,
187	194	"repeat": 10,
188		"fail_on_flaky": true
	195	"fail_on_flaky": true,
	196	"shuffle": true,
	197	"seed": 8675309
189	198	},
190	199	"sharding": {
191	200	"index": 1,
		@@ -204,6 +213,8 @@ limitations as their CLI equivalents; unsupported combinations fail explicitly.
204	213
205	214	See [Retries, repeats, and flaky tests](docs/retries-and-repeats.md) for attempt
206	215	semantics, exit codes, reporter output, and programmatic configuration.
	216	See [Randomized test order](docs/randomized-order.md) for replay and ordering
	217	guarantees.
207	218
208	219	### Parallel execution semantics
209	220
		@@ -212,7 +223,8 @@ and set `n_jobs`. A configured value is the exact maximum number of active test
212	223	workers; omitting it uses the host's logical CPU count, with a one-worker
213	224	fallback when detection is unavailable.
214	225
215		Suites and nested suites are visited in declaration order. `beforeAll` and
	226	Suites and nested suites are visited in the selected order: declaration order
	227	by default, or deterministic seed order when randomization is enabled. `beforeAll` and
216	228	`afterAll` run once on the coordinating thread around that suite and its nested
217	229	suites. Direct tests within a suite may overlap. Their inherited `beforeEach`
218	230	and `afterEach` hooks run on the same worker as the test, so per-test hook order
		@@ -220,7 +232,7 @@ is preserved but hooks from different tests may overlap. Hook state and the
220	232	allocator passed to tests must therefore be thread-safe.
221	233
222	234	Reporter callbacks never run concurrently: test results are emitted in
223		declaration order after each parallel batch, making output and final totals
	235	the selected order after each parallel batch, making output and final totals
224	236	deterministic. A parallel batch is already scheduled as a unit, so `bail` cannot
225	237	cancel tests that have started.
226	238
		@@ -665,6 +677,8 @@ zig-test --no-color
665	677	\| `--retry <N>` \| `--retries <N>` \| Retry each failed test up to N times \|
666	678	\| `--repeat <N>` \| \| Require N successful runs of each selected test \|
667	679	\| `--fail-on-flaky` \| \| Fail the run when a retry recovers a failure \|
	680	\| `--shuffle` \| \| Randomize suite, test, and discovered-file order \|
	681	\| `--seed <N>` \| \| Replay a randomized order; implies `--shuffle` \|
668	682	\| `--verbose` \| \| Enable verbose output \|
669	683	\| `--quiet` \| `-q` \| Minimal output \|
670	684	\| `--no-color` \| \| Disable colored output \|

docs/execution-pipeline.mdmodified+6-2

Changes to docs/execution-pipeline.md
		@@ -34,6 +34,9 @@ reporters, including the configured JUnit output path.
34	34	child process.
35	35	- Every logical result owns ordered attempt history. A failure followed by a
36	36	successful retry is classified as `flaky`, distinct from an ordinary pass.
	37	- Optional randomized execution uses one run seed across the selected plan.
	38	The seed is carried by reporters and `TestResults`; see
	39	[Randomized test order](randomized-order.md) for mode-specific guarantees.
37	40	- Summaries always come from `TestResults`, rather than executor-local counters.
38	41
39	42	## Extension points
		@@ -53,10 +56,11 @@ model without coupling the execution API to JSON.
53	56	Existing programmatic APIs remain source compatible:
54	57
55	58	- `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged.
56		- `RunnerOptions` gains optional reporter, timeout, retry, repeat, and flaky-exit
	59	- `RunnerOptions` gains optional reporter, timeout, retry, repeat, flaky-exit,
	60	shuffle, and seed
57	61	fields with backward-compatible defaults.
58	62	- `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter,
59		JUnit path, timeout, retry, repeat, flaky-exit, and writer fields default to
	63	JUnit path, timeout, retry, repeat, flaky-exit, shuffle, seed, and writer fields default to
60	64	the previous spec behavior.
61	65
62	66	CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers

docs/json-event-protocol.mdmodified+4-2

Changes to docs/json-event-protocol.md
		@@ -35,8 +35,8 @@ events are emitted in attempt order immediately before their `test_end`.
35	35
36	36	\| Type \| Data fields \|
37	37	\| --- \| --- \|
38		\| `run_start` \| `total` \|
39		\| `run_end` \| `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns` \|
	38	\| `run_start` \| `total`, optional `random_seed` \|
	39	\| `run_end` \| `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns`, optional `random_seed` \|
40	40	\| `suite_start`, `suite_end` \| `name` \|
41	41	\| `test_start` \| `name`, optional `suite` \|
42	42	\| `test_end` \| `name`, optional `suite`, `status`, `duration_ns`, optional `error_message` \|
		@@ -51,6 +51,8 @@ Output streams are `stdout` or `stderr`.
51	51
52	52	Optional fields are serialized as `null` when no value is available. Durations
53	53	are unsigned integer nanoseconds. Percentages are numbers from 0 through 100.
	54	When randomized ordering is enabled, both run events carry the reproducible
	55	unsigned integer seed.
54	56
55	57	## Compatibility policy
56	58

docs/randomized-order.mdadded+58-0

Changes to docs/randomized-order.md
		@@ -0,0 +1,58 @@
	1	# Randomized test order
	2
	3	Randomized order is opt-in. Use `--shuffle` to generate a seed for the run:
	4
	5	```bash
	6	zig-test --shuffle
	7	```
	8
	9	The selected reporter prints the generated seed at run start. If a test fails,
	10	the human-readable reporters print a replay command as well. Pass that seed to
	11	reproduce the order; `--seed` implies `--shuffle`:
	12
	13	```bash
	14	zig-test --seed 8675309
	15	```
	16
	17	Keep every other order-affecting option the same when replaying a run, including
	18	the test set, filter, shard, execution mode, and suite structure.
	19
	20	## Ordering guarantees
	21
	22	Without `--shuffle` or `--seed`, execution retains declaration or discovery
	23	order.
	24
	25	In sequential programmatic execution, a seed deterministically orders root
	26	suites, each suite's direct tests, and its nested suites. Lifecycle hooks keep
	27	their normal scope: `beforeAll` and `afterAll` still wrap a suite, while
	28	`beforeEach` and `afterEach` still wrap their selected test.
	29
	30	In parallel programmatic execution, the seed determines suite traversal, each
	31	worker queue, and serialized reporter output. Workers claim tests from that
	32	queue in order, but operating-system scheduling means wall-clock start and
	33	completion timing can differ between runs. Reporter order remains reproducible.
	34
	35	In discovery mode, sharding selects files first and randomization orders the
	36	files inside the selected shard. The seed does not change shard membership.
	37	Retries and repetitions stay attached to their logical test or file and do not
	38	trigger another shuffle.
	39
	40	## Metadata
	41
	42	The seed is stored as `random_seed` in JSON `run_start` and `run_end` events,
	43	live UI events, and test-history files. JUnit output records it as a
	44	`random_seed` property on each test suite. A non-randomized run reports a null
	45	seed in JSON-compatible formats and omits the JUnit property.
	46
	47	Programmatic callers can use `RunnerOptions.shuffle` and `RunnerOptions.seed`,
	48	or the matching `LoaderOptions` fields. JSON configuration accepts the same
	49	values under `test`:
	50
	51	```json
	52	{
	53	"test": {
	54	"shuffle": true,
	55	"seed": 8675309
	56	}
	57	}
	58	```

src/cli.zigmodified+34-0

Changes to src/cli.zig
		@@ -49,6 +49,8 @@ pub const CLIOptions = struct {
49	49	retries: usize = 0,
50	50	repeat: usize = 1,
51	51	fail_on_flaky: bool = false,
	52	shuffle: bool = false,
	53	seed: ?u64 = null,
52	54	};
53	55
54	56	pub const version = build_options.version;
		@@ -82,6 +84,8 @@ pub const CLI = struct {
82	84	self.options.retries = config.test_options.retries;
83	85	self.options.repeat = config.test_options.repeat;
84	86	self.options.fail_on_flaky = config.test_options.fail_on_flaky;
	87	self.options.shuffle = config.test_options.shuffle or config.test_options.seed != null;
	88	self.options.seed = config.test_options.seed;
85	89	self.options.shard_index = config.sharding.index;
86	90	self.options.shard_count = config.sharding.count;
87	91	self.options.parallel = config.parallel.enabled;
		@@ -140,6 +144,19 @@ pub const CLI = struct {
140	144	}
141	145	} else if (std.mem.eql(u8, arg, "--fail-on-flaky")) {
142	146	self.options.fail_on_flaky = true;
	147	} else if (std.mem.eql(u8, arg, "--shuffle")) {
	148	self.options.shuffle = true;
	149	} else if (std.mem.eql(u8, arg, "--seed")) {
	150	if (i + 1 >= args.len) {
	151	std.debug.print("Error: --seed requires a value\n", .{});
	152	return CLIError.MissingValue;
	153	}
	154	i += 1;
	155	self.options.seed = std.fmt.parseInt(u64, args[i], 10) catch {
	156	std.debug.print("Error: --seed must be a valid unsigned integer\n", .{});
	157	return CLIError.InvalidArgument;
	158	};
	159	self.options.shuffle = true;
143	160	} else if (std.mem.eql(u8, arg, "--verbose")) {
144	161	self.options.verbose = true;
145	162	} else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) {
		@@ -386,6 +403,8 @@ pub const CLI = struct {
386	403	\\ --retry <N> Retry each failed test up to N times
387	404	\\ --repeat <N> Require N successful runs of each selected test
388	405	\\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure
	406	\\ --shuffle Randomize suite and test order
	407	\\ --seed <N> Reproduce a randomized order (implies --shuffle)
389	408	\\ --filter <pattern> Run only tests matching pattern
390	409	\\ --grep <pattern> Same as --filter (alias)
391	410	\\ --reporter <name> Set reporter type (spec, dot, json, tap, junit)
		@@ -452,6 +471,8 @@ pub const CLI = struct {
452	471	\\ zig-test --timeout 10000 Set 10s timeout for all tests
453	472	\\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests
454	473	\\ zig-test --repeat 100 --filter parser Stress-run selected tests
	474	\\ zig-test --shuffle Run in a generated random order
	475	\\ zig-test --seed 12345 Replay a randomized order
455	476	\\ zig-test --profile-memory --fail-on-leak
456	477	\\ zig-test --parallel --jobs 4 Run with 4 parallel workers
457	478	\\ zig-test --config zig-test.json Load config from file
		@@ -482,6 +503,8 @@ pub const CLI = struct {
482	503	.retries = self.options.retries,
483	504	.repeat = self.options.repeat,
484	505	.fail_on_flaky = self.options.fail_on_flaky,
	506	.shuffle = self.options.shuffle,
	507	.seed = self.options.seed,
485	508	};
486	509	}
487	510	};
		@@ -724,6 +747,17 @@ test "CLI parses retry repeat and flaky exit policy" {
724	747	try std.testing.expect(runner_options.fail_on_flaky);
725	748	}
726	749
	750	test "CLI parses shuffle and a seed implies it" {
	751	var cli = CLI.init(std.testing.allocator);
	752	try cli.parse(&.{ "zig-test", "--seed", "12345" });
	753	try std.testing.expect(cli.options.shuffle);
	754	try std.testing.expectEqual(@as(?u64, 12345), cli.options.seed);
	755
	756	const runner_options = cli.toRunnerOptions();
	757	try std.testing.expect(runner_options.shuffle);
	758	try std.testing.expectEqual(@as(?u64, 12345), runner_options.seed);
	759	}
	760
727	761	test "CLI rejects a zero repeat count" {
728	762	var cli = CLI.init(std.testing.allocator);
729	763	try std.testing.expectError(

src/config.zigmodified+6-0

Changes to src/config.zig
		@@ -32,6 +32,8 @@ pub const TestOptions = struct {
32	32	retries: usize = 0,
33	33	repeat: usize = 1,
34	34	fail_on_flaky: bool = false,
	35	shuffle: bool = false,
	36	seed: ?u64 = null,
35	37	};
36	38
37	39	pub const ParallelOptions = struct {
		@@ -207,6 +209,8 @@ test "TestConfig defaults match CLI discovery defaults" {
207	209	try std.testing.expectEqual(@as(usize, 0), config.test_options.retries);
208	210	try std.testing.expectEqual(@as(usize, 1), config.test_options.repeat);
209	211	try std.testing.expect(!config.test_options.fail_on_flaky);
	212	try std.testing.expect(!config.test_options.shuffle);
	213	try std.testing.expect(config.test_options.seed == null);
210	214	try std.testing.expect(config.sharding.index == null);
211	215	try std.testing.expect(config.sharding.count == null);
212	216	try std.testing.expect(!config.parallel.enabled);
		@@ -247,6 +251,8 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" {
247	251	try std.testing.expectEqual(@as(usize, 1), config.test_options.retries);
248	252	try std.testing.expectEqual(@as(usize, 2), config.test_options.repeat);
249	253	try std.testing.expect(!config.test_options.fail_on_flaky);
	254	try std.testing.expect(config.test_options.shuffle);
	255	try std.testing.expectEqual(@as(?u64, 42), config.test_options.seed);
250	256	try std.testing.expectEqual(@as(?usize, 1), config.sharding.index);
251	257	try std.testing.expectEqual(@as(?usize, 1), config.sharding.count);
252	258

src/event_protocol.zigmodified+5-1

Changes to src/event_protocol.zig
		@@ -7,7 +7,10 @@ pub const Status = enum { pending, running, passed, flaky, failed, skipped };
7	7	pub const HookKind = enum { before_all, before_each, after_each, after_all };
8	8	pub const OutputStream = enum { stdout, stderr };
9	9
10		pub const RunStart = struct { total: usize };
	10	pub const RunStart = struct {
	11	total: usize,
	12	random_seed: ?u64 = null,
	13	};
11	14	pub const RunEnd = struct {
12	15	total: usize,
13	16	passed: usize,
		@@ -15,6 +18,7 @@ pub const RunEnd = struct {
15	18	failed: usize,
16	19	skipped: usize,
17	20	duration_ns: u64 = 0,
	21	random_seed: ?u64 = null,
18	22	};
19	23	pub const Suite = struct { name: []const u8 };
20	24	pub const TestStart = struct { name: []const u8, suite: ?[]const u8 = null };

src/lib.zigmodified+1-0

Changes to src/lib.zig
		@@ -27,6 +27,7 @@ pub const timeout = ;
27	27	pub const progress = ;
28	28	pub const time = ;
29	29	pub const compat = ;
	30	pub const random_order = ;
30	31
31	32	// Re-export commonly used types and functions
32	33	pub const expect = assertions.expect;

src/main.zigmodified+4-0

Changes to src/main.zig
		@@ -189,6 +189,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
189	189	.retries = cli_parser.options.retries,
190	190	.repeat = cli_parser.options.repeat,
191	191	.fail_on_flaky = cli_parser.options.fail_on_flaky,
	192	.shuffle = cli_parser.options.shuffle,
	193	.seed = cli_parser.options.seed,
192	194	.reporter_writer = &stdout_writer.interface,
193	195	.coverage_options = cov_opts,
194	196	.ui_server = if (ui_server) \|*server\| server else null,
		@@ -235,6 +237,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
235	237	.retries = cli_parser.options.retries,
236	238	.repeat = cli_parser.options.repeat,
237	239	.fail_on_flaky = cli_parser.options.fail_on_flaky,
	240	.shuffle = cli_parser.options.shuffle,
	241	.seed = cli_parser.options.seed,
238	242	.reporter_writer = &stdout_writer.interface,
239	243	.coverage_options = cov_opts,
240	244	.ui_server = if (ui_server) \|*server\| server else null,

src/parallel.zigmodified+74-8

Changes to src/parallel.zig
		@@ -3,6 +3,7 @@ const suite = ;
3	3	const reporter = ;
4	4	const pipeline = ;
5	5	const compat = ;
	6	const random_order = ;
6	7
7	8	/// Options for bounded parallel test execution.
8	9	pub const ParallelOptions = struct {
		@@ -12,6 +13,7 @@ pub const ParallelOptions = struct {
12	13	enabled: bool = false,
13	14	filter: ?[]const u8 = null,
14	15	timeout_ms: ?u64 = null,
	16	random_seed: ?u64 = null,
15	17	};
16	18
17	19	pub const ParallelError = error{
		@@ -111,6 +113,7 @@ const RunContext = struct {
111	113	rep: *reporter.Reporter,
112	114	options: ParallelOptions,
113	115	worker_limit: usize,
	116	shuffler: ?*random_order.Shuffler,
114	117
115	118	fn runSuite(self: RunContext, test_suite: suite.TestSuite) !bool {
116	119	try self.rep.onSuiteStart(test_suite.name);
		@@ -131,7 +134,12 @@ const RunContext = struct {
131	134	var scheduled: std.ArrayList(*suite.TestCase) = .empty;
132	135	defer scheduled.deinit(self.allocator);
133	136
134		for (test_suite.tests.items) \|*test_case\| {
	137	var ordered_tests: std.ArrayList(*suite.TestCase) = .empty;
	138	defer ordered_tests.deinit(self.allocator);
	139	for (test_suite.tests.items) \|*test_case\| try ordered_tests.append(self.allocator, test_case);
	140	if (self.shuffler) \|randomizer\| randomizer.shuffle(*suite.TestCase, ordered_tests.items);
	141
	142	for (ordered_tests.items) \|test_case\| {
135	143	if (!self.shouldRun(test_suite, test_case)) {
136	144	test_case.status = .skipped;
137	145	continue;
		@@ -143,12 +151,16 @@ const RunContext = struct {
143	151	try self.runBatch(test_suite, scheduled.items);
144	152
145	153	var all_passed = true;
146		for (test_suite.tests.items) \|*test_case\| {
	154	for (ordered_tests.items) \|test_case\| {
147	155	try self.rep.onTestEnd(test_case);
148	156	if (test_case.status == .failed) all_passed = false;
149	157	}
150	158
151		for (test_suite.suites.items) \|nested_suite\| {
	159	var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	160	defer nested_suites.deinit(self.allocator);
	161	try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	162	if (self.shuffler) \|randomizer\| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	163	for (nested_suites.items) \|nested_suite\| {
152	164	if (!try self.runSuite(nested_suite)) all_passed = false;
153	165	}
154	166
		@@ -201,11 +213,19 @@ const RunContext = struct {
201	213	}
202	214
203	215	fn reportSkippedSuite(self: RunContext, test_suite: suite.TestSuite) !void {
204		for (test_suite.tests.items) \|*test_case\| {
	216	var tests: std.ArrayList(*suite.TestCase) = .empty;
	217	defer tests.deinit(self.allocator);
	218	for (test_suite.tests.items) \|*test_case\| try tests.append(self.allocator, test_case);
	219	if (self.shuffler) \|randomizer\| randomizer.shuffle(*suite.TestCase, tests.items);
	220	for (tests.items) \|test_case\| {
205	221	test_case.status = .skipped;
206	222	try self.rep.onTestEnd(test_case);
207	223	}
208		for (test_suite.suites.items) \|nested\| {
	224	var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	225	defer nested_suites.deinit(self.allocator);
	226	try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	227	if (self.shuffler) \|randomizer\| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	228	for (nested_suites.items) \|nested\| {
209	229	try self.rep.onSuiteStart(nested.name);
210	230	try self.reportSkippedSuite(nested);
211	231	try self.rep.onSuiteEnd(nested.name);
		@@ -213,9 +233,9 @@ const RunContext = struct {
213	233	}
214	234	};
215	235
216		/// Run suites in declaration order while executing each suite's direct tests
	236	/// Run suites in the selected deterministic order while executing each suite's direct tests
217	237	/// with a bounded worker set. Reporter callbacks are serialized and emitted in
218		/// declaration order after each batch completes.
	238	/// the selected order after each batch completes.
219	239	pub fn runTestsParallel(
220	240	allocator: std.mem.Allocator,
221	241	test_registry: *suite.TestRegistry,
		@@ -226,23 +246,36 @@ pub fn runTestsParallel(
226	246	const worker_limit = try resolveWorkerCount(options.n_jobs);
227	247
228	248	const total_tests = test_registry.countAllTests();
	249	rep.random_seed = options.random_seed;
229	250	try rep.onRunStart(total_tests);
230	251
	252	var shuffler: random_order.Shuffler = undefined;
	253	const shuffler_ptr: ?*random_order.Shuffler = if (options.random_seed) \|seed\| blk: {
	254	shuffler = .init(seed);
	255	break :blk &shuffler;
	256	} else null;
	257
231	258	var context = RunContext{
232	259	.allocator = allocator,
233	260	.registry = test_registry,
234	261	.rep = rep,
235	262	.options = options,
236	263	.worker_limit = worker_limit,
	264	.shuffler = shuffler_ptr,
237	265	};
238	266
	267	var root_suites: std.ArrayList(*suite.TestSuite) = .empty;
	268	defer root_suites.deinit(allocator);
	269	try root_suites.appendSlice(allocator, test_registry.root_suites.items);
	270	if (shuffler_ptr) \|randomizer\| randomizer.shuffle(*suite.TestSuite, root_suites.items);
239	271	var all_passed = true;
240		for (test_registry.root_suites.items) \|test_suite\| {
	272	for (root_suites.items) \|test_suite\| {
241	273	if (!try context.runSuite(test_suite)) all_passed = false;
242	274	}
243	275
244	276	var results = reporter.TestResults.init(allocator);
245	277	defer results.deinit();
	278	results.random_seed = options.random_seed;
246	279	for (test_registry.root_suites.items) \|test_suite\| {
247	280	try addSuiteResults(&results, test_suite);
248	281	}
		@@ -347,6 +380,7 @@ const RecordingReporter = struct {
347	380	passed: usize = 0,
348	381	failed: usize = 0,
349	382	skipped: usize = 0,
	383	random_seed: ?u64 = null,
350	384
351	385	fn init(allocator: std.mem.Allocator) RecordingReporter {
352	386	return .{
		@@ -379,6 +413,7 @@ const RecordingReporter = struct {
379	413	recording.passed = results.passed;
380	414	recording.failed = results.failed;
381	415	recording.skipped = results.skipped;
	416	recording.random_seed = results.random_seed;
382	417	}
383	418
384	419	fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void {
		@@ -468,6 +503,37 @@ test "parallel reporting is serialized, ordered, and complete" {
468	503	try std.testing.expectEqual(@as(usize, 1), recording.skipped);
469	504	}
470	505
	506	test "parallel order is reproducible with a fixed seed" {
	507	const allocator = std.heap.smp_allocator;
	508	var registry = suite.TestRegistry.init(allocator);
	509	defer registry.deinit();
	510	const test_suite = try suite.TestSuite.init(allocator, "randomized");
	511	inline for (.{ "one", "two", "three", "four", "five", "six" }) \|name\| {
	512	try test_suite.addTest(suite.TestCase.init(name, passingTest));
	513	}
	514	try registry.registerSuite(test_suite);
	515
	516	var first = RecordingReporter.init(allocator);
	517	try std.testing.expect(try runTestsParallel(allocator, &registry, &first.reporter, .{
	518	.enabled = true,
	519	.n_jobs = 2,
	520	.random_seed = 8675309,
	521	}));
	522	var second = RecordingReporter.init(allocator);
	523	try std.testing.expect(try runTestsParallel(allocator, &registry, &second.reporter, .{
	524	.enabled = true,
	525	.n_jobs = 2,
	526	.random_seed = 8675309,
	527	}));
	528
	529	try std.testing.expectEqual(first.ended_count, second.ended_count);
	530	for (first.ended_names[0..first.ended_count], second.ended_names[0..second.ended_count]) \|a, b\| {
	531	try std.testing.expectEqualStrings(a, b);
	532	}
	533	try std.testing.expectEqual(@as(?u64, 8675309), first.random_seed);
	534	try std.testing.expectEqual(@as(?u64, 8675309), second.random_seed);
	535	}
	536
471	537	test "parallel executor preserves nested hook lifecycles" {
472	538	const counters = [_]*std.atomic.Value(usize){
473	539	&parent_before_each,

src/random_order.zigadded+42-0

Changes to src/random_order.zig
		@@ -0,0 +1,42 @@
	1	const std = ;
	2	const compat = ;
	3
	4	/// Return the seed that should control this run. Supplying a seed enables
	5	/// shuffling even when `shuffle` is false; otherwise random order is opt-in.
	6	pub fn resolveSeed(shuffle: bool, configured: ?u64) ?u64 {
	7	if (configured) \|seed\| return seed;
	8	if (!shuffle) return null;
	9	return @truncate(@as(u128, @bitCast(compat.nanoTimestamp())));
	10	}
	11
	12	pub const Shuffler = struct {
	13	prng: std.Random.DefaultPrng,
	14
	15	pub fn init(seed: u64) Shuffler {
	16	return .{ .prng = std.Random.DefaultPrng.init(seed) };
	17	}
	18
	19	pub fn shuffle(self: *Shuffler, comptime T: type, items: []T) void {
	20	self.prng.random().shuffleWithIndex(T, items, u64);
	21	}
	22	};
	23
	24	test "random order is opt-in and an explicit seed enables it" {
	25	try std.testing.expect(resolveSeed(false, null) == null);
	26	try std.testing.expect(resolveSeed(true, null) != null);
	27	try std.testing.expectEqual(@as(?u64, 0), resolveSeed(false, 0));
	28	try std.testing.expectEqual(@as(?u64, 42), resolveSeed(true, 42));
	29	}
	30
	31	test "the same seed produces the same order" {
	32	var first = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
	33	var second = first;
	34	var first_shuffler = Shuffler.init(123456789);
	35	var second_shuffler = Shuffler.init(123456789);
	36
	37	first_shuffler.shuffle(u8, &first);
	38	second_shuffler.shuffle(u8, &second);
	39
	40	try std.testing.expectEqualSlices(u8, &first, &second);
	41	try std.testing.expect(!std.mem.eql(u8, &first, &[_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }));
	42	}

src/reporter.zigmodified+25-6

Changes to src/reporter.zig
		@@ -31,6 +31,7 @@ pub const Reporter = struct {
31	31	vtable: *const VTable,
32	32	allocator: std.mem.Allocator,
33	33	use_colors: bool = true,
	34	random_seed: ?u64 = null,
34	35
35	36	const Self = @This();
36	37
		@@ -76,6 +77,7 @@ pub const TestResults = struct {
76	77	failed: usize = 0,
77	78	skipped: usize = 0,
78	79	total_time_ns: u64 = 0,
	80	random_seed: ?u64 = null,
79	81	failed_tests: std.ArrayList(suite.TestCase),
80	82	flaky_tests: std.ArrayList(suite.TestCase),
81	83	allocator: std.mem.Allocator,
		@@ -163,6 +165,9 @@ pub const SpecReporter = struct {
163	165	} else {
164	166	try s.output().print("Running {d} test(s)...\n\n", .{total_tests});
165	167	}
	168	if (reporter.random_seed) \|seed\| {
	169	try s.output().print("Random seed: {d}\n\n", .{seed});
	170	}
166	171	}
167	172
168	173	fn onRunEnd(reporter: Reporter, results: TestResults) !void {
		@@ -188,6 +193,9 @@ pub const SpecReporter = struct {
188	193	}
189	194	}
190	195	try s.output().print("\n", .{});
	196	if (results.random_seed) \|seed\| {
	197	try s.output().print("Reproduce with: --seed {d}\n\n", .{seed});
	198	}
191	199	}
192	200
193	201	// Print summary
		@@ -377,6 +385,7 @@ pub const DotReporter = struct {
377	385	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
378	386	const s = self(reporter);
379	387	try s.output().print("\nRunning {d} tests:\n", .{total_tests});
	388	if (reporter.random_seed) \|seed\| try s.output().print("Random seed: {d}\n", .{seed});
380	389	}
381	390
382	391	fn onRunEnd(reporter: Reporter, results: TestResults) !void {
		@@ -399,6 +408,9 @@ pub const DotReporter = struct {
399	408	total_time_ms,
400	409	});
401	410	}
	411	if (results.failed > 0) {
	412	if (results.random_seed) \|seed\| try s.output().print("Reproduce with: --seed {d}\n", .{seed});
	413	}
402	414	}
403	415
404	416	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
		@@ -512,7 +524,7 @@ pub const JsonReporter = struct {
512	524	s.emitted_events = false;
513	525	s.suites.clearRetainingCapacity();
514	526	try s.output().writeAll("[\n");
515		try s.emit(.{ .run_start = .{ .total = total_tests } });
	527	try s.emit(.{ .run_start = .{ .total = total_tests, .random_seed = reporter.random_seed } });
516	528	}
517	529
518	530	fn onRunEnd(reporter: Reporter, results: TestResults) !void {
		@@ -524,6 +536,7 @@ pub const JsonReporter = struct {
524	536	.failed = results.failed,
525	537	.skipped = results.skipped,
526	538	.duration_ns = results.total_time_ns,
	539	.random_seed = results.random_seed,
527	540	} });
528	541	try s.output().writeAll("\n]\n");
529	542	}
		@@ -618,11 +631,14 @@ pub const TAPReporter = struct {
618	631	fn onRunStart(reporter: *Reporter, total: usize) !void {
619	632	const self: *Self = @fieldParentPtr("reporter", reporter);
620	633	try self.output().print("TAP version 14\n1..{d}\n", .{total});
	634	if (reporter.random_seed) \|seed\| try self.output().print("# random-seed {d}\n", .{seed});
621	635	}
622	636
623	637	fn onRunEnd(reporter: Reporter, results: TestResults) !void {
624		_ = reporter;
625		_ = results;
	638	const self: *Self = @fieldParentPtr("reporter", reporter);
	639	if (results.failed > 0) {
	640	if (results.random_seed) \|seed\| try self.output().print("# reproduce --seed {d}\n", .{seed});
	641	}
626	642	}
627	643
628	644	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
		@@ -753,14 +769,13 @@ pub const JUnitReporter = struct {
753	769	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
754	770	const self: *Self = @fieldParentPtr("reporter", reporter);
755	771
756		const suite_result = try self.allocator.create(TestSuiteResult);
757		suite_result.* = .{
	772	const suite_result = TestSuiteResult{
758	773	.name = try self.allocator.dupe(u8, suite_name),
759	774	.tests = .empty,
760	775	.timestamp = compat.milliTimestamp(),
761	776	};
762	777
763		try self.suites.append(self.allocator, suite_result.*);
	778	try self.suites.append(self.allocator, suite_result);
764	779	self.current_suite = &self.suites.items[self.suites.items.len - 1];
765	780	}
766	781
		@@ -843,6 +858,10 @@ pub const JUnitReporter = struct {
843	858	suite_time,
844	859	});
845	860
	861	if (results.random_seed) \|seed\| {
	862	try buffer.print(self.allocator, " <properties><property name=\"random_seed\" value=\"{d}\"/></properties>\n", .{seed});
	863	}
	864
846	865	for (suite_result.tests.items) \|test_result\| {
847	866	try buffer.print(self.allocator, " <testcase name=\"{s}\" time=\"{d:.6}\"", .{
848	867	test_result.name,

src/test_history.zigmodified+17-2

Changes to src/test_history.zig
		@@ -13,6 +13,7 @@ pub const HistoryEntry = struct {
13	13	failed: usize,
14	14	skipped: usize,
15	15	duration_ns: u64,
	16	random_seed: ?u64 = null,
16	17	tests: std.ArrayList(TestRecord),
17	18
18	19	pub fn deinit(self: *HistoryEntry, allocator: std.mem.Allocator) void {
		@@ -65,6 +66,11 @@ pub const TestHistory = struct {
65	66
66	67	/// Start recording a new test run
67	68	pub fn startRun(self: *Self, total: usize) !void {
	69	try self.startRunWithSeed(total, null);
	70	}
	71
	72	/// Start recording a run with optional randomized-order metadata.
	73	pub fn startRunWithSeed(self: *Self, total: usize, random_seed: ?u64) !void {
68	74	self.start_time = compat.milliTimestamp();
69	75
70	76	self.current_entry = HistoryEntry{
		@@ -75,6 +81,7 @@ pub const TestHistory = struct {
75	81	.failed = 0,
76	82	.skipped = 0,
77	83	.duration_ns = 0,
	84	.random_seed = random_seed,
78	85	.tests = std.ArrayList(TestRecord).empty,
79	86	};
80	87	}
		@@ -117,6 +124,7 @@ pub const TestHistory = struct {
117	124	self.current_entry.?.flaky = results.flaky;
118	125	self.current_entry.?.failed = results.failed;
119	126	self.current_entry.?.skipped = results.skipped;
	127	self.current_entry.?.random_seed = results.random_seed orelse self.current_entry.?.random_seed;
120	128
121	129	try self.saveToFile();
122	130	}
		@@ -139,7 +147,10 @@ pub const TestHistory = struct {
139	147	errdefer buffer = allocating.toArrayList();
140	148	const writer = &allocating.writer;
141	149	try writer.writeAll("[\n");
142		try std.json.Stringify.value(protocol.Event{ .run_start = .{ .total = self.current_entry.?.total } }, .{}, writer);
	150	try std.json.Stringify.value(protocol.Event{ .run_start = .{
	151	.total = self.current_entry.?.total,
	152	.random_seed = self.current_entry.?.random_seed,
	153	} }, .{}, writer);
143	154	for (self.current_entry.?.tests.items) \|test_record\| {
144	155	try writer.writeAll(",\n");
145	156	try std.json.Stringify.value(protocol.Event{ .test_end = .{
		@@ -158,6 +169,7 @@ pub const TestHistory = struct {
158	169	.failed = self.current_entry.?.failed,
159	170	.skipped = self.current_entry.?.skipped,
160	171	.duration_ns = self.current_entry.?.duration_ns,
	172	.random_seed = self.current_entry.?.random_seed,
161	173	} }, .{}, writer);
162	174	try writer.writeAll("\n]\n");
163	175	try writer.flush();
		@@ -192,6 +204,7 @@ pub const TestHistory = struct {
192	204	.failed = 0,
193	205	.skipped = 0,
194	206	.duration_ns = 0,
	207	.random_seed = null,
195	208	.tests = std.ArrayList(TestRecord).empty,
196	209	};
197	210	}
		@@ -258,7 +271,7 @@ test "TestHistory saves versioned protocol events" {
258	271
259	272	var history = TestHistory.init(allocator, history_dir);
260	273	defer history.deinit();
261		try history.startRun(1);
	274	try history.startRunWithSeed(1, 9876);
262	275	try history.recordTest("quoted \"test\"", "suite\nname", .passed, 42, null);
263	276	var results = reporter_mod.TestResults.init(allocator);
264	277	defer results.deinit();
		@@ -277,9 +290,11 @@ test "TestHistory saves versioned protocol events" {
277	290	const events = parsed.value.array.items;
278	291	try std.testing.expectEqual(@as(usize, 3), events.len);
279	292	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	293	try std.testing.expectEqual(@as(i64, 9876), events[0].object.get("data").?.object.get("random_seed").?.integer);
280	294	try std.testing.expectEqualStrings("test_end", events[1].object.get("type").?.string);
281	295	try std.testing.expectEqualStrings("quoted \"test\"", events[1].object.get("data").?.object.get("name").?.string);
282	296	try std.testing.expectEqualStrings("run_end", events[2].object.get("type").?.string);
	297	try std.testing.expectEqual(@as(i64, 9876), events[2].object.get("data").?.object.get("random_seed").?.integer);
283	298	for (events) \|event\| {
284	299	try std.testing.expectEqual(@as(i64, protocol.version), event.object.get("protocol_version").?.integer);
285	300	}

src/test_loader.zigmodified+18-2

Changes to src/test_loader.zig
		@@ -8,6 +8,7 @@ const coverage = ;
8	8	const ui_server = ;
9	9	const compat = ;
10	10	const protocol = ;
	11	const random_order = ;
11	12
12	13	/// Options for running discovered tests
13	14	pub const LoaderOptions = struct {
		@@ -33,6 +34,10 @@ pub const LoaderOptions = struct {
33	34	repeat: usize = 1,
34	35	/// Treat recovered failures as an unsuccessful run.
35	36	fail_on_flaky: bool = false,
	37	/// Shuffle selected files. A generated seed is printed for replay.
	38	shuffle: bool = false,
	39	/// Reproduce a shuffled file order. Supplying a seed enables shuffle.
	40	seed: ?u64 = null,
36	41	/// Optional output supplied by a CLI host. Embedded/test callers default
37	42	/// to stderr so they do not interfere with Zig's stdout test protocol.
38	43	reporter_writer: ?*std.Io.Writer = null,
		@@ -55,6 +60,11 @@ pub fn runDiscoveredTests(
55	60
56	61	var plan = try discoveryPlan(allocator, discovered, options.shard);
57	62	defer plan.deinit();
	63	const random_seed = random_order.resolveSeed(options.shuffle, options.seed);
	64	if (random_seed) \|seed\| {
	65	var shuffler = random_order.Shuffler.init(seed);
	66	shuffler.shuffle(pipeline.PlanItem, plan.items.items);
	67	}
58	68	const selected_files = plan.items.items.len;
59	69	if (options.shard) \|shard\| {
60	70	std.debug.print(
		@@ -84,7 +94,9 @@ pub fn runDiscoveredTests(
84	94	options.use_colors,
85	95	);
86	96	defer reporters.deinit();
87		const events = pipeline.EventStream{ .rep = reporters.selected() };
	97	const selected_reporter = reporters.selected();
	98	selected_reporter.random_seed = random_seed;
	99	const events = pipeline.EventStream{ .rep = selected_reporter };
88	100	const policy = pipeline.ExecutionPolicy{
89	101	.bail = options.bail,
90	102	.filter = options.filter,
		@@ -92,6 +104,7 @@ pub fn runDiscoveredTests(
92	104	};
93	105	var results = reporter.TestResults.init(allocator);
94	106	defer results.deinit();
	107	results.random_seed = random_seed;
95	108	var executed_cases: std.ArrayList(suite.TestCase) = .empty;
96	109	defer {
97	110	for (executed_cases.items) \|*test_case\| test_case.deinit(allocator);
		@@ -102,7 +115,7 @@ pub fn runDiscoveredTests(
102	115
103	116	// Notify UI of run start
104	117	if (options.ui_server) \|server\| {
105		try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files } });
	118	try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files, .random_seed = random_seed } });
106	119	}
107	120
108	121	var files_run: usize = 0;
		@@ -241,6 +254,7 @@ pub fn runDiscoveredTests(
241	254	.failed = results.failed,
242	255	.skipped = results.skipped,
243	256	.duration_ns = results.total_time_ns,
	257	.random_seed = results.random_seed,
244	258	} });
245	259	}
246	260
		@@ -355,6 +369,8 @@ test "LoaderOptions default values" {
355	369	try std.testing.expectEqual(@as(?sharding.ShardOptions, null), options.shard);
356	370	try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type);
357	371	try std.testing.expectEqualStrings("test-results.xml", options.junit_output);
	372	try std.testing.expect(!options.shuffle);
	373	try std.testing.expect(options.seed == null);
358	374	try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
359	375	try std.testing.expectEqual(@as(usize, 0), options.retries);
360	376	try std.testing.expectEqual(@as(usize, 1), options.repeat);

src/test_runner.zigmodified+133-13

Changes to src/test_runner.zig
		@@ -4,6 +4,7 @@ const reporter_mod = ;
4	4	const pipeline = ;
5	5	const parallel = ;
6	6	const compat = ;
	7	const random_order = ;
7	8
8	9	pub const RunnerError = error{
9	10	NoTestsFound,
		@@ -25,6 +26,10 @@ pub const RunnerOptions = struct {
25	26	repeat: usize = 1,
26	27	/// Treat tests that pass after a retry as an unsuccessful run.
27	28	fail_on_flaky: bool = false,
	29	/// Shuffle suites and tests. A generated seed is printed for replay.
	30	shuffle: bool = false,
	31	/// Reproduce a shuffled run. Supplying a seed implicitly enables shuffle.
	32	seed: ?u64 = null,
28	33	/// Optional output supplied by a CLI host. Embedded/test callers default
29	34	/// to stderr so they do not interfere with Zig's stdout test protocol.
30	35	reporter_writer: ?*std.Io.Writer = null,
		@@ -71,6 +76,9 @@ pub const TestRunner = struct {
71	76	);
72	77	defer reporters.deinit();
73	78	const current_reporter = reporters.selected();
	79	const random_seed = random_order.resolveSeed(self.options.shuffle, self.options.seed);
	80	current_reporter.random_seed = random_seed;
	81	self.results.random_seed = random_seed;
74	82	const events = pipeline.EventStream{ .rep = current_reporter };
75	83
76	84	var plan = try pipeline.TestPlan.fromRegistry(self.allocator, self.registry);
		@@ -87,6 +95,7 @@ pub const TestRunner = struct {
87	95	.n_jobs = self.options.n_jobs,
88	96	.filter = self.options.filter,
89	97	.timeout_ms = self.options.timeout_ms,
	98	.random_seed = random_seed,
90	99	};
91	100
92	101	const all_passed = try parallel.runTestsParallel(
		@@ -106,9 +115,19 @@ pub const TestRunner = struct {
106	115	}
107	116
108	117	// Sequential execution (original behavior)
	118	var shuffler: random_order.Shuffler = undefined;
	119	const shuffler_ptr: ?*random_order.Shuffler = if (random_seed) \|seed\| blk: {
	120	shuffler = .init(seed);
	121	break :blk &shuffler;
	122	} else null;
	123	var root_suites: std.ArrayList(*suite.TestSuite) = .empty;
	124	defer root_suites.deinit(self.allocator);
	125	try root_suites.appendSlice(self.allocator, self.registry.root_suites.items);
	126	if (shuffler_ptr) \|randomizer\| randomizer.shuffle(*suite.TestSuite, root_suites.items);
	127
109	128	try events.emit(.{ .run_started = total_tests });
110		for (self.registry.root_suites.items) \|test_suite\| {
111		try self.runSuite(test_suite, events);
	129	for (root_suites.items) \|test_suite\| {
	130	try self.runSuite(test_suite, events, shuffler_ptr);
112	131	if (self.policy().shouldStop(&self.results)) {
113	132	break;
114	133	}
		@@ -130,15 +149,20 @@ pub const TestRunner = struct {
130	149	}
131	150
132	151	/// Run a single test suite
133		fn runSuite(self: Self, test_suite: suite.TestSuite, events: pipeline.EventStream) !void {
	152	fn runSuite(
	153	self: *Self,
	154	test_suite: *suite.TestSuite,
	155	events: pipeline.EventStream,
	156	shuffler: ?*random_order.Shuffler,
	157	) !void {
134	158	// Skip if marked as skip or if has_only and this isn't marked as only
135	159	if (test_suite.shouldSkip()) {
136		try self.skipAllTests(test_suite, events);
	160	try self.skipAllTests(test_suite, events, shuffler);
137	161	return;
138	162	}
139	163
140	164	if (self.registry.has_only and !test_suite.hasOnly()) {
141		try self.skipAllTests(test_suite, events);
	165	try self.skipAllTests(test_suite, events, shuffler);
142	166	return;
143	167	}
144	168
		@@ -148,13 +172,17 @@ pub const TestRunner = struct {
148	172	// Run beforeAll hooks
149	173	test_suite.runBeforeAllHooks(self.allocator) catch \|err\| {
150	174	std.debug.print("beforeAll hook failed: {any}\n", .{err});
151		try self.skipAllTests(test_suite, events);
	175	try self.skipAllTests(test_suite, events, shuffler);
152	176	try events.emit(.{ .suite_finished = test_suite.name });
153	177	return;
154	178	};
155	179
156	180	// Run tests in this suite
157		for (test_suite.tests.items) \|*test_case\| {
	181	var tests: std.ArrayList(*suite.TestCase) = .empty;
	182	defer tests.deinit(self.allocator);
	183	for (test_suite.tests.items) \|*test_case\| try tests.append(self.allocator, test_case);
	184	if (shuffler) \|randomizer\| randomizer.shuffle(*suite.TestCase, tests.items);
	185	for (tests.items) \|test_case\| {
158	186	if (test_case.skip or (self.registry.has_only and !test_case.only)) {
159	187	test_case.status = .skipped;
160	188	try events.emit(.{ .test_finished = test_case });
		@@ -178,8 +206,12 @@ pub const TestRunner = struct {
178	206	}
179	207
180	208	// Run nested suites
181		for (test_suite.suites.items) \|nested_suite\| {
182		try self.runSuite(nested_suite, events);
	209	var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	210	defer nested_suites.deinit(self.allocator);
	211	try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	212	if (shuffler) \|randomizer\| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	213	for (nested_suites.items) \|nested_suite\| {
	214	try self.runSuite(nested_suite, events, shuffler);
183	215	if (self.policy().shouldStop(&self.results)) {
184	216	break;
185	217	}
		@@ -298,15 +330,28 @@ pub const TestRunner = struct {
298	330	}
299	331
300	332	/// Skip all tests in a suite
301		fn skipAllTests(self: Self, test_suite: suite.TestSuite, events: pipeline.EventStream) !void {
302		for (test_suite.tests.items) \|*test_case\| {
	333	fn skipAllTests(
	334	self: *Self,
	335	test_suite: *suite.TestSuite,
	336	events: pipeline.EventStream,
	337	shuffler: ?*random_order.Shuffler,
	338	) !void {
	339	var tests: std.ArrayList(*suite.TestCase) = .empty;
	340	defer tests.deinit(self.allocator);
	341	for (test_suite.tests.items) \|*test_case\| try tests.append(self.allocator, test_case);
	342	if (shuffler) \|randomizer\| randomizer.shuffle(*suite.TestCase, tests.items);
	343	for (tests.items) \|test_case\| {
303	344	test_case.status = .skipped;
304	345	try self.results.addTest(test_case);
305	346	try events.emit(.{ .test_finished = test_case });
306	347	}
307	348
308		for (test_suite.suites.items) \|nested_suite\| {
309		try self.skipAllTests(nested_suite, events);
	349	var nested_suites: std.ArrayList(*suite.TestSuite) = .empty;
	350	defer nested_suites.deinit(self.allocator);
	351	try nested_suites.appendSlice(self.allocator, test_suite.suites.items);
	352	if (shuffler) \|randomizer\| randomizer.shuffle(*suite.TestSuite, nested_suites.items);
	353	for (nested_suites.items) \|nested_suite\| {
	354	try self.skipAllTests(nested_suite, events, shuffler);
310	355	}
311	356	}
312	357
		@@ -355,6 +400,8 @@ test "RunnerOptions default values" {
355	400	try std.testing.expectEqual(@as(usize, 0), options.retries);
356	401	try std.testing.expectEqual(@as(usize, 1), options.repeat);
357	402	try std.testing.expect(!options.fail_on_flaky);
	403	try std.testing.expect(!options.shuffle);
	404	try std.testing.expect(options.seed == null);
358	405	}
359	406
360	407	test "RunnerOptions custom values" {
		@@ -376,6 +423,37 @@ var failing_fixture_runs: usize = 0;
376	423	var skipped_by_bail_runs: usize = 0;
377	424	var repeated_fixture_runs: usize = 0;
378	425	var timeout_fixture_runs: usize = 0;
	426	var randomized_log: [6]u8 = undefined;
	427	var randomized_log_len: usize = 0;
	428
	429	fn recordRandomized(id: u8) void {
	430	randomized_log[randomized_log_len] = id;
	431	randomized_log_len += 1;
	432	}
	433
	434	fn randomizedOne(_: std.mem.Allocator) !void {
	435	recordRandomized(1);
	436	}
	437
	438	fn randomizedTwo(_: std.mem.Allocator) !void {
	439	recordRandomized(2);
	440	}
	441
	442	fn randomizedThree(_: std.mem.Allocator) !void {
	443	recordRandomized(3);
	444	}
	445
	446	fn randomizedFour(_: std.mem.Allocator) !void {
	447	recordRandomized(4);
	448	}
	449
	450	fn randomizedFive(_: std.mem.Allocator) !void {
	451	recordRandomized(5);
	452	}
	453
	454	fn randomizedSix(_: std.mem.Allocator) !void {
	455	recordRandomized(6);
	456	}
379	457
380	458	fn flakyFixture(_: std.mem.Allocator) !void {
381	459	flaky_fixture_runs += 1;
		@@ -391,6 +469,48 @@ fn skippedByBailFixture(_: std.mem.Allocator) !void {
391	469	skipped_by_bail_runs += 1;
392	470	}
393	471
	472	test "sequential order is reproducible with a fixed seed" {
	473	const allocator = std.testing.allocator;
	474	var registry = suite.TestRegistry.init(allocator);
	475	defer registry.deinit();
	476	const test_suite = try suite.TestSuite.init(allocator, "randomized");
	477	try test_suite.addTest(suite.TestCase.init("one", randomizedOne));
	478	try test_suite.addTest(suite.TestCase.init("two", randomizedTwo));
	479	try test_suite.addTest(suite.TestCase.init("three", randomizedThree));
	480	try test_suite.addTest(suite.TestCase.init("four", randomizedFour));
	481	try test_suite.addTest(suite.TestCase.init("five", randomizedFive));
	482	try test_suite.addTest(suite.TestCase.init("six", randomizedSix));
	483	try registry.registerSuite(test_suite);
	484
	485	var first_output: [4096]u8 = undefined;
	486	var first_writer: std.Io.Writer = .fixed(&first_output);
	487	randomized_log_len = 0;
	488	var first_runner = TestRunner.init(allocator, &registry, .{
	489	.seed = 112358,
	490	.use_colors = false,
	491	.reporter_writer = &first_writer,
	492	});
	493	defer first_runner.deinit();
	494	try std.testing.expect(try first_runner.run());
	495	const first_order = randomized_log;
	496
	497	var second_output: [4096]u8 = undefined;
	498	var second_writer: std.Io.Writer = .fixed(&second_output);
	499	randomized_log_len = 0;
	500	var second_runner = TestRunner.init(allocator, &registry, .{
	501	.seed = 112358,
	502	.use_colors = false,
	503	.reporter_writer = &second_writer,
	504	});
	505	defer second_runner.deinit();
	506	try std.testing.expect(try second_runner.run());
	507
	508	try std.testing.expectEqual(@as(usize, randomized_log.len), randomized_log_len);
	509	try std.testing.expectEqualSlices(u8, &first_order, &randomized_log);
	510	try std.testing.expectEqual(@as(?u64, 112358), first_runner.results.random_seed);
	511	try std.testing.expectEqual(@as(?u64, 112358), second_runner.results.random_seed);
	512	}
	513
394	514	fn repeatedFixture(_: std.mem.Allocator) !void {
395	515	repeated_fixture_runs += 1;
396	516	}

src/ui_server.zigmodified+7-2

Changes to src/ui_server.zig
		@@ -321,7 +321,10 @@ pub const UIReporter = struct {
321	321
322	322	fn onRunStart(reporter: *reporter_mod.Reporter, total: usize) !void {
323	323	const self: *Self = @fieldParentPtr("reporter", reporter);
324		try self.server.broadcastEvent(reporter.allocator, .{ .run_start = .{ .total = total } });
	324	try self.server.broadcastEvent(reporter.allocator, .{ .run_start = .{
	325	.total = total,
	326	.random_seed = reporter.random_seed,
	327	} });
325	328	}
326	329
327	330	fn onRunEnd(reporter: reporter_mod.Reporter, results: reporter_mod.TestResults) !void {
		@@ -334,6 +337,7 @@ pub const UIReporter = struct {
334	337	.failed = results.failed,
335	338	.skipped = results.skipped,
336	339	.duration_ns = results.total_time_ns,
	340	.random_seed = results.random_seed,
337	341	} });
338	342	}
339	343
		@@ -407,6 +411,7 @@ pub const MultiReporter = struct {
407	411	fn onRunStart(reporter: *reporter_mod.Reporter, total: usize) !void {
408	412	const self: *Self = @fieldParentPtr("reporter", reporter);
409	413	for (self.reporters) \|rep\| {
	414	rep.random_seed = reporter.random_seed;
410	415	try rep.onRunStart(total);
411	416	}
412	417	}
		@@ -536,7 +541,7 @@ test "UI server streams broadcast events over SSE" {
536	541
537	542	try server.broadcastEvent(std.testing.allocator, .{ .run_start = .{ .total = 2 } });
538	543	try expectLine(&stream_reader.interface, "event: run_start");
539		try expectLine(&stream_reader.interface, "data: {\"protocol_version\":1,\"type\":\"run_start\",\"data\":{\"total\":2}}");
	544	try expectLine(&stream_reader.interface, "data: {\"protocol_version\":1,\"type\":\"run_start\",\"data\":{\"total\":2,\"random_seed\":null}}");
540	545	try expectLine(&stream_reader.interface, "");
541	546	}
542	547

tests/fixtures/zig-test.jsonmodified+3-1

Changes to tests/fixtures/zig-test.json
		@@ -6,7 +6,9 @@
6	6	"filter": "selected test passes",
7	7	"retries": 1,
8	8	"repeat": 2,
9		"fail_on_flaky": false
	9	"fail_on_flaky": false,
	10	"shuffle": true,
	11	"seed": 42
10	12	},
11	13	"sharding": {
12	14	"index": 1,

tests/reporter_test.zigmodified+33-0

Changes to tests/reporter_test.zig
		@@ -56,6 +56,8 @@ test "reporter json output uses versioned event envelopes" {
56	56	defer results.deinit();
57	57	results.total = 1;
58	58	results.passed = 1;
	59	results.random_seed = 4242;
	60	reporter.reporter.random_seed = 4242;
59	61	var test_case = ztf.TestCase.init("passes \"quoted\"\nname", passingTest);
60	62	test_case.status = .passed;
61	63
		@@ -73,12 +75,14 @@ test "reporter json output uses versioned event envelopes" {
73	75	try std.testing.expectEqual(@as(usize, 6), events.len);
74	76	try std.testing.expectEqual(@as(i64, ztf.protocol_version), events[0].object.get("protocol_version").?.integer);
75	77	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	78	try std.testing.expectEqual(@as(i64, 4242), events[0].object.get("data").?.object.get("random_seed").?.integer);
76	79	try std.testing.expectEqualStrings("test_start", events[2].object.get("type").?.string);
77	80	const test_data = events[3].object.get("data").?.object;
78	81	try std.testing.expectEqualStrings(test_case.name, test_data.get("name").?.string);
79	82	try std.testing.expectEqualStrings("suite\tone", test_data.get("suite").?.string);
80	83	try std.testing.expectEqualStrings("run_end", events[5].object.get("type").?.string);
81	84	try std.testing.expectEqual(@as(i64, 1), events[5].object.get("data").?.object.get("passed").?.integer);
	85	try std.testing.expectEqual(@as(i64, 4242), events[5].object.get("data").?.object.get("random_seed").?.integer);
82	86	}
83	87
84	88	test "reporter json output includes flaky attempt history" {
		@@ -140,3 +144,32 @@ test "reporter set selects the same built-ins for every executor" {
140	144	try std.testing.expect(reporters.selected() == &reporters.tap.reporter);
141	145	try std.testing.expect(!reporters.selected().use_colors);
142	146	}
	147
	148	test "JUnit reporter records the random seed as a suite property" {
	149	const allocator = std.testing.allocator;
	150	var tmp = std.testing.tmpDir(.{});
	151	defer tmp.cleanup();
	152	const output_path = try std.fmt.allocPrint(allocator, ".zig-cache/tmp/{s}/results.xml", .{tmp.sub_path});
	153	defer allocator.free(output_path);
	154
	155	var reporter = ztf.JUnitReporter.init(allocator, output_path);
	156	defer reporter.deinit();
	157	reporter.reporter.random_seed = 73;
	158	var results = ztf.TestResults.init(allocator);
	159	defer results.deinit();
	160	results.total = 1;
	161	results.passed = 1;
	162	results.random_seed = 73;
	163	var test_case = ztf.TestCase.init("passes", passingTest);
	164	test_case.status = .passed;
	165
	166	try reporter.reporter.onRunStart(1);
	167	try reporter.reporter.onSuiteStart("suite");
	168	try reporter.reporter.onTestEnd(&test_case);
	169	try reporter.reporter.onSuiteEnd("suite");
	170	try reporter.reporter.onRunEnd(&results);
	171
	172	const output = try ztf.compat.readFileAlloc(allocator, output_path);
	173	defer allocator.free(output);
	174	try std.testing.expect(std.mem.indexOf(u8, output, "<property name=\"random_seed\" value=\"73\"/>") != null);
	175	}