also looking at this

Add retries, repeated runs, and flaky-test reporting

#25

Merged glennmichael123 wants to merge codex/issue-7-retries-repeat-flaky into main

15 files +719 -53

CHANGELOG.mdmodified+6-0

Changes to CHANGELOG.md
		@@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
16	16
17	17	## [Unreleased]
18	18
	19	### Added
	20
	21	- Retry failed tests and discovered test files with complete attempt history.
	22	- Repeat mode for stress-running selected tests.
	23	- Flaky outcomes, optional fail-on-flaky exit behavior, and JSON/JUnit attempt data.
	24
19	25	### Planned Features
20	26
21	27	- Snapshot testing

README.mdmodified+21-1

Changes to README.md
		@@ -8,6 +8,7 @@ A modern, feature-rich testing framework for Zig inspired by Jest, Vitest, and B
8	8	- Code Coverage - Line, branch, and function coverage with HTML reports (via kcov/grindcov)
9	9	- Async Test Support - Full async test execution with concurrent and sequential modes
10	10	- Timeout Handling - Configurable timeouts at test, suite, and global levels with extension support
	11	- Retries & Repeats - Re-run failures, stress-run selected tests, and report flaky outcomes
11	12	- Familiar API - Describe/it syntax similar to Jest and Vitest
12	13	- Rich Assertions - Comprehensive assertion library with `.expect()` and matchers
13	14	- Error Assertions - `toThrow()` and `toThrowError()` for testing error handling
		@@ -98,6 +99,9 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d
98	99	- `--reporter <name>` - Use spec, dot, JSON, TAP, or JUnit output
99	100	- `--junit-output <file>` - Set the JUnit XML path
100	101	- `--timeout <ms>` - Mark files that exceed the global time budget as failed
	102	- `--retry <N>` - Retry a failed test or discovered test file up to N times
	103	- `--repeat <N>` - Require N successful repetitions of every selected test
	104	- `--fail-on-flaky` - Return a failing exit status when a retry recovers a failure
101	105	- `--shard-index <N>` - Run one one-based file shard
102	106	- `--shard-count <N>` - Set the total number of file shards
103	107	- `--no-color` - Disable color in child Zig test processes
		@@ -125,6 +129,12 @@ zig-test --test-dir tests --no-recursive
125	129	# Stop on first failure
126	130	zig-test --test-dir tests --bail
127	131
	132	# Retry failures twice and fail CI if any test is flaky
	133	zig-test --test-dir tests --retry 2 --fail-on-flaky
	134
	135	# Stress-run matching tests 100 times
	136	zig-test --test-dir tests --filter parser --repeat 100
	137
128	138	# Run the second of four deterministic file shards
129	139	zig-test --test-dir tests --shard-index 2 --shard-count 4
130	140	```
		@@ -168,7 +178,10 @@ precedence over configured values.
168	178	"test_dir": "tests",
169	179	"pattern": "*.test.zig",
170	180	"recursive": true,
171		"filter": "database"
	181	"filter": "database",
	182	"retries": 2,
	183	"repeat": 10,
	184	"fail_on_flaky": true
172	185	},
173	186	"sharding": {
174	187	"index": 1,
		@@ -185,6 +198,9 @@ The accepted top-level sections are `test`, `sharding`, `parallel`, `reporter`,
185	198	`snapshot`, `watch`, `memory`, `ui`, and `coverage`. Options retain the same execution-mode
186	199	limitations as their CLI equivalents; unsupported combinations fail explicitly.
187	200
	201	See [Retries, repeats, and flaky tests](docs/retries-and-repeats.md) for attempt
	202	semantics, exit codes, reporter output, and programmatic configuration.
	203
188	204	### Parallel execution semantics
189	205
190	206	Programmatic runners can enable bounded execution with `RunnerOptions.parallel`
		@@ -642,6 +658,9 @@ zig-test --no-color
642	658	\| `--reporter <name>` \| \| Set reporter (spec, dot, json, tap, junit) \|
643	659	\| `--junit-output <file>` \| \| Set the JUnit XML output path \|
644	660	\| `--timeout <ms>` \| \| Set the global test timeout \|
	661	\| `--retry <N>` \| `--retries <N>` \| Retry each failed test up to N times \|
	662	\| `--repeat <N>` \| \| Require N successful runs of each selected test \|
	663	\| `--fail-on-flaky` \| \| Fail the run when a retry recovers a failure \|
645	664	\| `--verbose` \| \| Enable verbose output \|
646	665	\| `--quiet` \| `-q` \| Minimal output \|
647	666	\| `--no-color` \| \| Disable colored output \|
		@@ -921,6 +940,7 @@ MIT License - see [LICENSE](LICENSE) for details.
921	940	- [x] Memory profiling
922	941	- [x] Strict JSON configuration files
923	942	- [x] Parallel test execution
	943	- [x] Retries, repeated runs, and flaky-test reporting
924	944	- [ ] Parameterized tests (it.each)
925	945	- [ ] Property-based testing
926	946	- [ ] IDE integration

build.zigmodified+15-0

Changes to build.zig
		@@ -293,6 +293,21 @@ pub fn build(b: *std.Build) void {
293	293	);
294	294	test_step.dependOn(&run_discovery_tap.step);
295	295
	296	const run_discovery_repeat = b.addRunArtifact(exe);
	297	run_discovery_repeat.addArgs(&.{
	298	"--test-dir",
	299	"tests",
	300	"--pattern",
	301	"sample.test.zig",
	302	"--retry",
	303	"1",
	304	"--repeat",
	305	"2",
	306	"--fail-on-flaky",
	307	"--no-color",
	308	});
	309	test_step.dependOn(&run_discovery_repeat.step);
	310
296	311	const run_version = b.addRunArtifact(exe);
297	312	run_version.addArg("--version");
298	313	run_version.expectStdErrEqual(b.fmt("Zig Test Framework v{s}\n", .{package_version}));

docs/execution-pipeline.mdmodified+11-6

Changes to docs/execution-pipeline.md
		@@ -26,10 +26,14 @@ reporters, including the configured JUnit output path.
26	26	- Registered tests match filters directly by test name. Discovery forwards the
27	27	same filter to `zig test`, where Zig applies it to names inside each file.
28	28	- Bail stops scheduling after the shared result model records a failure. Work
29		already started by a parallel batch cannot be cancelled.
	29	already started by a parallel batch cannot be cancelled. With retries, bail
	30	is evaluated only after the retry budget is exhausted.
30	31	- A global timeout compares elapsed execution time with the same policy in both
31		executors. It marks an over-budget result failed after control returns; it
32		does not yet terminate an in-process function or external child process.
	32	executors on every attempt. It marks an over-budget attempt failed after
	33	control returns; it does not yet terminate an in-process function or external
	34	child process.
	35	- Every logical result owns ordered attempt history. A failure followed by a
	36	successful retry is classified as `flaky`, distinct from an ordinary pass.
33	37	- Summaries always come from `TestResults`, rather than executor-local counters.
34	38
35	39	## Extension points
		@@ -48,10 +52,11 @@ depend on one serialization format.
48	52	Existing programmatic APIs remain source compatible:
49	53
50	54	- `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged.
51		- `RunnerOptions` only gains optional `junit_output`, `timeout_ms`, and
52		`reporter_writer` fields.
	55	- `RunnerOptions` gains optional reporter, timeout, retry, repeat, and flaky-exit
	56	fields with backward-compatible defaults.
53	57	- `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter,
54		JUnit path, timeout, and writer fields default to the previous spec behavior.
	58	JUnit path, timeout, retry, repeat, flaky-exit, and writer fields default to
	59	the previous spec behavior.
55	60
56	61	CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers
57	62	may leave it unset; reporter output then uses stderr so it cannot corrupt Zig's

docs/retries-and-repeats.mdadded+92-0

Changes to docs/retries-and-repeats.md
		@@ -0,0 +1,92 @@
	1	# Retries, repeats, and flaky tests
	2
	3	The runner records each execution of a logical test as an attempt. An attempt
	4	contains its one-based number, repetition number, status, duration, and error
	5	message. Registered tests and discovered test files use the same model.
	6
	7	## Command-line use
	8
	9	```bash
	10	# One initial attempt plus as many as two retries after failure
	11	zig-test --test-dir tests --retry 2
	12
	13	# Require every selected test to pass 100 times
	14	zig-test --test-dir tests --filter parser --repeat 100
	15
	16	# Make a recovered failure fail the overall run in CI
	17	zig-test --test-dir tests --retry 2 --fail-on-flaky
	18	```
	19
	20	`--retries` is an alias for `--retry`. A retry count is the number of
	21	additional attempts after a failure. `--repeat N` requires N successful
	22	repetitions; each repetition receives its own retry budget. Attempt history is
	23	ordered across all repetitions.
	24
	25	Discovery operates at test-file granularity because each file is an external
	26	`zig test` process. Programmatic mode operates at individual-test granularity.
	27
	28	## Outcomes and exit status
	29
	30	- `passed`: every attempt passed without a retry.
	31	- `flaky`: at least one attempt failed, but a retry recovered and all required
	32	repetitions eventually passed.
	33	- `failed`: one repetition exhausted its retry budget.
	34	- `skipped`: the test was not selected for execution.
	35
	36	Failed tests always make the process exit unsuccessfully. Flaky tests succeed
	37	by default so retries can recover an intermittent failure. `--fail-on-flaky`
	38	changes that policy and returns an unsuccessful exit status when any test is
	39	flaky.
	40
	41	`--bail` is evaluated after retries, so one failed attempt does not stop the
	42	run. The runner stops scheduling only after a logical test exhausts its retry
	43	budget. The timeout is evaluated independently for every attempt; a timed-out
	44	attempt may be retried.
	45
	46	When retries or repeats are configured for registered tests, execution is
	47	sequential even if parallel mode was requested. This preserves deterministic
	48	attempt ordering and hook behavior.
	49
	50	## Programmatic configuration
	51
	52	Set defaults for a registered-test run with `RunnerOptions`:
	53
	54	```zig
	55	const options = ztf.RunnerOptions{
	56	.retries = 2,
	57	.repeat = 10,
	58	.fail_on_flaky = true,
	59	};
	60	const passed = try ztf.runTestsWithOptions(allocator, registry, options);
	61	```
	62
	63	An individual registered test can override the run-level retry count:
	64
	65	```zig
	66	try suite.addTest(
	67	ztf.TestCase.init("eventually consistent", testEventuallyConsistent)
	68	.withRetries(5),
	69	);
	70	```
	71
	72	`LoaderOptions` exposes the same `retries`, `repeat`, and `fail_on_flaky`
	73	fields for callers that run discovered files directly. Strict JSON
	74	configuration uses the corresponding fields in the `test` object:
	75
	76	```json
	77	{
	78	"test": {
	79	"retries": 2,
	80	"repeat": 10,
	81	"fail_on_flaky": true
	82	}
	83	}
	84	```
	85
	86	## Reporter data
	87
	88	The spec and dot reporters show flaky results separately from passes. JSON test
	89	objects include an `attempts` array with `number`, `repetition`, `status`, and
	90	`durationNs` fields plus an optional `error`. The JSON summary includes a
	91	`flaky` count. JUnit output includes the flaky count on `<testsuites>` and emits
	92	ordered attempt data in each retried test case's `<system-out>` element.

src/cli.zigmodified+70-0

Changes to src/cli.zig
		@@ -45,6 +45,10 @@ pub const CLIOptions = struct {
45	45	junit_output: ?[]const u8 = null,
46	46	// Timeout options
47	47	timeout: ?u64 = null, // Global timeout in milliseconds
	48	// Retry and repetition options
	49	retries: usize = 0,
	50	repeat: usize = 1,
	51	fail_on_flaky: bool = false,
48	52	};
49	53
50	54	pub const version = build_options.version;
		@@ -75,6 +79,9 @@ pub const CLI = struct {
75	79	self.options.no_recursive = !config.test_options.recursive;
76	80	self.options.filter = config.test_options.filter;
77	81	self.options.timeout = config.test_options.timeout;
	82	self.options.retries = config.test_options.retries;
	83	self.options.repeat = config.test_options.repeat;
	84	self.options.fail_on_flaky = config.test_options.fail_on_flaky;
78	85	self.options.shard_index = config.sharding.index;
79	86	self.options.shard_count = config.sharding.count;
80	87	self.options.parallel = config.parallel.enabled;
		@@ -107,6 +114,32 @@ pub const CLI = struct {
107	114	self.options.version = true;
108	115	} else if (std.mem.eql(u8, arg, "--bail") or std.mem.eql(u8, arg, "-b")) {
109	116	self.options.bail = true;
	117	} else if (std.mem.eql(u8, arg, "--retry") or std.mem.eql(u8, arg, "--retries")) {
	118	if (i + 1 >= args.len) {
	119	std.debug.print("Error: {s} requires a value\n", .{arg});
	120	return CLIError.MissingValue;
	121	}
	122	i += 1;
	123	self.options.retries = std.fmt.parseInt(usize, args[i], 10) catch {
	124	std.debug.print("Error: {s} must be a valid number\n", .{arg});
	125	return CLIError.InvalidArgument;
	126	};
	127	} else if (std.mem.eql(u8, arg, "--repeat")) {
	128	if (i + 1 >= args.len) {
	129	std.debug.print("Error: --repeat requires a value\n", .{});
	130	return CLIError.MissingValue;
	131	}
	132	i += 1;
	133	self.options.repeat = std.fmt.parseInt(usize, args[i], 10) catch {
	134	std.debug.print("Error: --repeat must be a valid positive number\n", .{});
	135	return CLIError.InvalidArgument;
	136	};
	137	if (self.options.repeat == 0) {
	138	std.debug.print("Error: --repeat must be greater than zero\n", .{});
	139	return CLIError.InvalidArgument;
	140	}
	141	} else if (std.mem.eql(u8, arg, "--fail-on-flaky")) {
	142	self.options.fail_on_flaky = true;
110	143	} else if (std.mem.eql(u8, arg, "--verbose")) {
111	144	self.options.verbose = true;
112	145	} else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) {
		@@ -351,6 +384,9 @@ pub const CLI = struct {
351	384	\\ -h, --help Show this help message
352	385	\\ -v, --version Show version information
353	386	\\ -b, --bail Stop test execution on first failure
	387	\\ --retry <N> Retry each failed test up to N times
	388	\\ --repeat <N> Require N successful runs of each selected test
	389	\\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure
354	390	\\ --filter <pattern> Run only tests matching pattern
355	391	\\ --grep <pattern> Same as --filter (alias)
356	392	\\ --reporter <name> Set reporter type (spec, dot, json, tap, junit)
		@@ -414,6 +450,8 @@ pub const CLI = struct {
414	450	\\ zig-test --update-snapshots Update all snapshots
415	451	\\ zig-test --watch Run in watch mode
416	452	\\ zig-test --timeout 10000 Set 10s timeout for all tests
	453	\\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests
	454	\\ zig-test --repeat 100 --filter parser Stress-run selected tests
417	455	\\ zig-test --profile-memory --fail-on-leak
418	456	\\ zig-test --parallel --jobs 4 Run with 4 parallel workers
419	457	\\ zig-test --config zig-test.json Load config from file
		@@ -441,6 +479,9 @@ pub const CLI = struct {
441	479	.n_jobs = self.options.jobs,
442	480	.junit_output = self.options.junit_output orelse "test-results.xml",
443	481	.timeout_ms = self.options.timeout,
	482	.retries = self.options.retries,
	483	.repeat = self.options.repeat,
	484	.fail_on_flaky = self.options.fail_on_flaky,
444	485	};
445	486	}
446	487	};
		@@ -662,6 +703,35 @@ test "CLI parse timeout" {
662	703	try std.testing.expectEqual(@as(?u64, 5000), cli.options.timeout);
663	704	}
664	705
	706	test "CLI parses retry repeat and flaky exit policy" {
	707	var cli = CLI.init(std.testing.allocator);
	708	const args = [_][]const u8{
	709	"zig-test",
	710	"--retry",
	711	"2",
	712	"--repeat",
	713	"5",
	714	"--fail-on-flaky",
	715	};
	716	try cli.parse(&args);
	717	try std.testing.expectEqual(@as(usize, 2), cli.options.retries);
	718	try std.testing.expectEqual(@as(usize, 5), cli.options.repeat);
	719	try std.testing.expect(cli.options.fail_on_flaky);
	720
	721	const runner_options = cli.toRunnerOptions();
	722	try std.testing.expectEqual(@as(usize, 2), runner_options.retries);
	723	try std.testing.expectEqual(@as(usize, 5), runner_options.repeat);
	724	try std.testing.expect(runner_options.fail_on_flaky);
	725	}
	726
	727	test "CLI rejects a zero repeat count" {
	728	var cli = CLI.init(std.testing.allocator);
	729	try std.testing.expectError(
	730	CLIError.InvalidArgument,
	731	cli.parse(&.{ "zig-test", "--repeat", "0" }),
	732	);
	733	}
	734
665	735	test "CLI rejects invalid coverage tool" {
666	736	var cli = CLI.init(std.testing.allocator);
667	737	const args = [_][]const u8{ "zig-test", "--coverage-tool", "unknown" };

src/config.zigmodified+14-0

Changes to src/config.zig
		@@ -29,6 +29,9 @@ pub const TestOptions = struct {
29	29	recursive: bool = true,
30	30	filter: ?[]const u8 = null,
31	31	timeout: ?u64 = null,
	32	retries: usize = 0,
	33	repeat: usize = 1,
	34	fail_on_flaky: bool = false,
32	35	};
33	36
34	37	pub const ParallelOptions = struct {
		@@ -86,6 +89,7 @@ pub const ConfigError = error{
86	89	InvalidJobs,
87	90	InvalidPort,
88	91	InvalidTimeout,
	92	InvalidRepeat,
89	93	InvalidDebounce,
90	94	InvalidShard,
91	95	};
		@@ -183,6 +187,7 @@ fn validate(config: TestConfig) !void {
183	187	if (config.test_options.timeout) \|timeout\| {
184	188	if (timeout == 0) return ConfigError.InvalidTimeout;
185	189	}
	190	if (config.test_options.repeat == 0) return ConfigError.InvalidRepeat;
186	191	if (config.watch.debounce_ms == 0) return ConfigError.InvalidDebounce;
187	192	const has_shard_index = config.sharding.index != null;
188	193	const has_shard_count = config.sharding.count != null;
		@@ -199,6 +204,9 @@ test "TestConfig defaults match CLI discovery defaults" {
199	204	try std.testing.expectEqualStrings(".", config.test_options.test_dir);
200	205	try std.testing.expect(config.test_options.recursive);
201	206	try std.testing.expect(config.test_options.timeout == null);
	207	try std.testing.expectEqual(@as(usize, 0), config.test_options.retries);
	208	try std.testing.expectEqual(@as(usize, 1), config.test_options.repeat);
	209	try std.testing.expect(!config.test_options.fail_on_flaky);
202	210	try std.testing.expect(config.sharding.index == null);
203	211	try std.testing.expect(config.sharding.count == null);
204	212	try std.testing.expect(!config.parallel.enabled);
		@@ -219,6 +227,9 @@ test "configuration validation rejects invalid runtime values" {
219	227	invalid.parallel.jobs = 0;
220	228	try std.testing.expectError(ConfigError.InvalidJobs, validate(invalid));
221	229	invalid.parallel.jobs = null;
	230	invalid.test_options.repeat = 0;
	231	try std.testing.expectError(ConfigError.InvalidRepeat, validate(invalid));
	232	invalid.test_options.repeat = 1;
222	233	invalid.sharding.index = 2;
223	234	try std.testing.expectError(ConfigError.InvalidShard, validate(invalid));
224	235	invalid.sharding.count = 1;
		@@ -233,6 +244,9 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" {
233	244	try std.testing.expectEqualStrings("tests/fixtures", config.test_options.test_dir);
234	245	try std.testing.expectEqualStrings("selected test passes", config.test_options.filter.?);
235	246	try std.testing.expect(!config.test_options.recursive);
	247	try std.testing.expectEqual(@as(usize, 1), config.test_options.retries);
	248	try std.testing.expectEqual(@as(usize, 2), config.test_options.repeat);
	249	try std.testing.expect(!config.test_options.fail_on_flaky);
236	250	try std.testing.expectEqual(@as(?usize, 1), config.sharding.index);
237	251	try std.testing.expectEqual(@as(?usize, 1), config.sharding.count);
238	252

src/lib.zigmodified+1-0

Changes to src/lib.zig
		@@ -60,6 +60,7 @@ pub const describeTimeout = suite.describeTimeout;
60	60	pub const TestSuite = suite.TestSuite;
61	61	pub const TestCase = suite.TestCase;
62	62	pub const TestStatus = suite.TestStatus;
	63	pub const TestAttempt = suite.TestAttempt;
63	64	pub const TestRegistry = suite.TestRegistry;
64	65	pub const TestFn = suite.TestFn;
65	66	pub const HookFn = suite.HookFn;

src/main.zigmodified+6-0

Changes to src/main.zig
		@@ -221,6 +221,9 @@ pub fn main(init: std.process.Init.Minimal) !void {
221	221	.reporter_type = cli_parser.options.reporter,
222	222	.junit_output = cli_parser.options.junit_output orelse "test-results.xml",
223	223	.timeout_ms = cli_parser.options.timeout,
	224	.retries = cli_parser.options.retries,
	225	.repeat = cli_parser.options.repeat,
	226	.fail_on_flaky = cli_parser.options.fail_on_flaky,
224	227	.reporter_writer = &stdout_writer.interface,
225	228	.coverage_options = cov_opts,
226	229	.ui_server = if (ui_server) \|*server\| server else null,
		@@ -264,6 +267,9 @@ pub fn main(init: std.process.Init.Minimal) !void {
264	267	.reporter_type = cli_parser.options.reporter,
265	268	.junit_output = cli_parser.options.junit_output orelse "test-results.xml",
266	269	.timeout_ms = cli_parser.options.timeout,
	270	.retries = cli_parser.options.retries,
	271	.repeat = cli_parser.options.repeat,
	272	.fail_on_flaky = cli_parser.options.fail_on_flaky,
267	273	.reporter_writer = &stdout_writer.interface,
268	274	.coverage_options = cov_opts,
269	275	.ui_server = if (ui_server) \|*server\| server else null,

src/reporter.zigmodified+127-5

Changes to src/reporter.zig
		@@ -71,21 +71,25 @@ pub const Reporter = struct {
71	71	pub const TestResults = struct {
72	72	total: usize = 0,
73	73	passed: usize = 0,
	74	flaky: usize = 0,
74	75	failed: usize = 0,
75	76	skipped: usize = 0,
76	77	total_time_ns: u64 = 0,
77	78	failed_tests: std.ArrayList(suite.TestCase),
	79	flaky_tests: std.ArrayList(suite.TestCase),
78	80	allocator: std.mem.Allocator,
79	81
80	82	pub fn init(allocator: std.mem.Allocator) TestResults {
81	83	return TestResults{
82	84	.failed_tests = .empty,
	85	.flaky_tests = .empty,
83	86	.allocator = allocator,
84	87	};
85	88	}
86	89
87	90	pub fn deinit(self: *TestResults) void {
88	91	self.failed_tests.deinit(self.allocator);
	92	self.flaky_tests.deinit(self.allocator);
89	93	}
90	94
91	95	pub fn addTest(self: TestResults, test_case: const suite.TestCase) !void {
		@@ -94,6 +98,10 @@ pub const TestResults = struct {
94	98
95	99	switch (test_case.status) {
96	100	.passed => self.passed += 1,
	101	.flaky => {
	102	self.flaky += 1;
	103	try self.flaky_tests.append(self.allocator, test_case.*);
	104	},
97	105	.failed => {
98	106	self.failed += 1;
99	107	try self.failed_tests.append(self.allocator, test_case.*);
		@@ -188,6 +196,9 @@ pub const SpecReporter = struct {
188	196	try s.output().print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset });
189	197	try s.output().print(" Total: {d}\n", .{results.total});
190	198	try s.output().print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset });
	199	if (results.flaky > 0) {
	200	try s.output().print(" {s}Flaky: {d}{s}\n", .{ Colors.yellow, results.flaky, Colors.reset });
	201	}
191	202	if (results.failed > 0) {
192	203	try s.output().print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset });
193	204	}
		@@ -199,6 +210,9 @@ pub const SpecReporter = struct {
199	210	try s.output().print("Test Summary:\n", .{});
200	211	try s.output().print(" Total: {d}\n", .{results.total});
201	212	try s.output().print(" Passed: {d}\n", .{results.passed});
	213	if (results.flaky > 0) {
	214	try s.output().print(" Flaky: {d}\n", .{results.flaky});
	215	}
202	216	if (results.failed > 0) {
203	217	try s.output().print(" Failed: {d}\n", .{results.failed});
204	218	}
		@@ -265,6 +279,25 @@ pub const SpecReporter = struct {
265	279	try s.output().print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms });
266	280	}
267	281	},
	282	.flaky => {
	283	if (reporter.use_colors) {
	284	try s.output().print("{s}~{s} {s} {s}({d} attempts, {d:.2}ms){s}\n", .{
	285	Colors.yellow,
	286	Colors.reset,
	287	test_case.name,
	288	Colors.gray,
	289	test_case.attempts.items.len,
	290	time_ms,
	291	Colors.reset,
	292	});
	293	} else {
	294	try s.output().print("~ {s} (flaky after {d} attempts, {d:.2}ms)\n", .{
	295	test_case.name,
	296	test_case.attempts.items.len,
	297	time_ms,
	298	});
	299	}
	300	},
268	301	.failed => {
269	302	if (reporter.use_colors) {
270	303	try s.output().print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{
		@@ -353,11 +386,13 @@ pub const DotReporter = struct {
353	386
354	387	if (reporter.use_colors) {
355	388	try s.output().print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset });
	389	try s.output().print("{s}Flaky: {d}{s}, ", .{ Colors.yellow, results.flaky, Colors.reset });
356	390	try s.output().print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset });
357	391	try s.output().print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms });
358	392	} else {
359		try s.output().print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{
	393	try s.output().print("Passed: {d}, Flaky: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{
360	394	results.passed,
	395	results.flaky,
361	396	results.failed,
362	397	results.total,
363	398	total_time_ms,
		@@ -391,6 +426,13 @@ pub const DotReporter = struct {
391	426	try s.output().print(".", .{});
392	427	}
393	428	},
	429	.flaky => {
	430	if (reporter.use_colors) {
	431	try s.output().print("{s}~{s}", .{ Colors.yellow, Colors.reset });
	432	} else {
	433	try s.output().print("~", .{});
	434	}
	435	},
394	436	.failed => {
395	437	if (reporter.use_colors) {
396	438	try s.output().print("{s}F{s}", .{ Colors.red, Colors.reset });
		@@ -422,6 +464,7 @@ pub const JsonReporter = struct {
422	464	writer: std.Io.Writer,
423	465	writer_ref: ?*std.Io.Writer = null,
424	466	suites: std.ArrayList([]const u8),
	467	emitted_tests: bool = false,
425	468
426	469	const Self = @This();
427	470
		@@ -472,9 +515,10 @@ pub const JsonReporter = struct {
472	515	const s = self(reporter);
473	516	const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0;
474	517
475		try s.output().print("],\"summary\":}}\n", .{
	518	try s.output().print("\n],\"summary\":}}\n", .{
476	519	results.total,
477	520	results.passed,
	521	results.flaky,
478	522	results.failed,
479	523	results.skipped,
480	524	total_time_ms,
		@@ -505,11 +549,15 @@ pub const JsonReporter = struct {
505	549
506	550	const status_str = switch (test_case.status) {
507	551	.passed => "passed",
	552	.flaky => "flaky",
508	553	.failed => "failed",
509	554	.skipped => "skipped",
510	555	else => "unknown",
511	556	};
512	557
	558	if (s.emitted_tests) try s.output().print(",\n", .{});
	559	s.emitted_tests = true;
	560
513	561	// Note: In a real implementation, you'd want to properly escape JSON strings
514	562	try s.output().print(" ", .{});
525	592	}
526	593	};
527	594
		@@ -593,6 +660,7 @@ pub const TAPReporter = struct {
593	660
594	661	const status = switch (test_case.status) {
595	662	.passed => "ok",
	663	.flaky => "ok",
596	664	.failed => "not ok",
597	665	.skipped => "ok",
598	666	else => "not ok",
		@@ -600,7 +668,9 @@ pub const TAPReporter = struct {
600	668
601	669	try self.output().print("{s} {d} - {s}", .{ status, self.test_count, test_case.name });
602	670
603		if (test_case.status == .skipped) {
	671	if (test_case.status == .flaky) {
	672	try self.output().print(" # FLAKY attempts={d}\n", .{test_case.attempts.items.len});
	673	} else if (test_case.status == .skipped) {
604	674	try self.output().print(" # SKIP\n", .{});
605	675	} else if (test_case.status == .failed and test_case.error_message != null) {
606	676	try self.output().print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?});
		@@ -631,6 +701,10 @@ pub const JUnitReporter = struct {
631	701	if (test_case.error_message) \|msg\| {
632	702	allocator.free(msg);
633	703	}
	704	for (test_case.attempts) \|attempt\| {
	705	if (attempt.error_message) \|msg\| allocator.free(msg);
	706	}
	707	allocator.free(test_case.attempts);
634	708	}
635	709	self.tests.deinit(allocator);
636	710	allocator.free(self.name);
		@@ -642,6 +716,15 @@ pub const JUnitReporter = struct {
642	716	time: f64,
643	717	status: suite.TestStatus,
644	718	error_message: ?[]const u8,
	719	attempts: []AttemptResult,
	720	};
	721
	722	const AttemptResult = struct {
	723	number: usize,
	724	repetition: usize,
	725	status: suite.TestStatus,
	726	duration_ns: u64,
	727	error_message: ?[]const u8,
645	728	};
646	729
647	730	pub fn init(allocator: std.mem.Allocator, output_file: []const u8) Self {
		@@ -716,11 +799,27 @@ pub const JUnitReporter = struct {
716	799	else
717	800	null;
718	801
	802	const attempts = try self.allocator.alloc(AttemptResult, test_case.attempts.items.len);
	803	errdefer self.allocator.free(attempts);
	804	for (test_case.attempts.items, 0..) \|attempt, index\| {
	805	attempts[index] = .{
	806	.number = attempt.number,
	807	.repetition = attempt.repetition,
	808	.status = attempt.status,
	809	.duration_ns = attempt.duration_ns,
	810	.error_message = if (attempt.error_message) \|message\|
	811	try self.allocator.dupe(u8, message)
	812	else
	813	null,
	814	};
	815	}
	816
719	817	const test_result = TestCaseResult{
720	818	.name = try self.allocator.dupe(u8, test_case.name),
721	819	.time = time_seconds,
722	820	.status = test_case.status,
723	821	.error_message = error_msg,
	822	.attempts = attempts,
724	823	};
725	824
726	825	try test_suite.tests.append(self.allocator, test_result);
		@@ -732,10 +831,11 @@ pub const JUnitReporter = struct {
732	831	defer buffer.deinit(self.allocator);
733	832
734	833	try buffer.appendSlice(self.allocator, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
735		try buffer.print(self.allocator, "<testsuites tests=\"{d}\" failures=\"{d}\" skipped=\"{d}\">\n", .{
	834	try buffer.print(self.allocator, "<testsuites tests=\"{d}\" failures=\"{d}\" skipped=\"{d}\" flaky=\"{d}\">\n", .{
736	835	results.total,
737	836	results.failed,
738	837	results.skipped,
	838	results.flaky,
739	839	});
740	840
741	841	for (self.suites.items) \|suite_result\| {
		@@ -768,11 +868,16 @@ pub const JUnitReporter = struct {
768	868	try buffer.print(self.allocator, " <failure message=\"{s}\"/>\n", .{
769	869	test_result.error_message orelse "Test failed",
770	870	});
	871	try writeAttemptHistory(&buffer, self.allocator, test_result.attempts);
771	872	try buffer.appendSlice(self.allocator, " </testcase>\n");
772	873	} else if (test_result.status == .skipped) {
773	874	try buffer.appendSlice(self.allocator, ">\n");
774	875	try buffer.appendSlice(self.allocator, " <skipped/>\n");
775	876	try buffer.appendSlice(self.allocator, " </testcase>\n");
	877	} else if (test_result.attempts.len > 1) {
	878	try buffer.appendSlice(self.allocator, ">\n");
	879	try writeAttemptHistory(&buffer, self.allocator, test_result.attempts);
	880	try buffer.appendSlice(self.allocator, " </testcase>\n");
776	881	} else {
777	882	try buffer.appendSlice(self.allocator, "/>\n");
778	883	}
		@@ -785,6 +890,23 @@ pub const JUnitReporter = struct {
785	890
786	891	try compat.writeFile(self.allocator, self.output_file, buffer.items);
787	892	}
	893
	894	fn writeAttemptHistory(
	895	buffer: *std.ArrayList(u8),
	896	allocator: std.mem.Allocator,
	897	attempts: []const AttemptResult,
	898	) !void {
	899	try buffer.appendSlice(allocator, " <system-out>");
	900	for (attempts, 0..) \|attempt, index\| {
	901	if (index > 0) try buffer.appendSlice(allocator, " ");
	902	try buffer.print(
	903	allocator,
	904	"attempt={d} repetition={d} status={s} duration_ns={d}",
	905	.{ attempt.number, attempt.repetition, @tagName(attempt.status), attempt.duration_ns },
	906	);
	907	}
	908	try buffer.appendSlice(allocator, "</system-out>\n");
	909	}
788	910	};
789	911
790	912	/// Owns every built-in reporter and selects one through a shared interface.

src/suite.zigmodified+31-0

Changes to src/suite.zig
		@@ -24,10 +24,21 @@ pub const TestStatus = enum {
24	24	pending,
25	25	running,
26	26	passed,
	27	flaky,
27	28	failed,
28	29	skipped,
29	30	};
30	31
	32	/// The outcome of one execution of a test. A logical test may have several
	33	/// attempts when retries or repeated runs are enabled.
	34	pub const TestAttempt = struct {
	35	number: usize,
	36	repetition: usize,
	37	status: TestStatus,
	38	duration_ns: u64,
	39	error_message: ?[]const u8 = null,
	40	};
	41
31	42	/// Individual test case
32	43	pub const TestCase = struct {
33	44	name: []const u8,
		@@ -37,6 +48,9 @@ pub const TestCase = struct {
37	48	status: TestStatus = .pending,
38	49	error_message: ?[]const u8 = null,
39	50	execution_time_ns: u64 = 0,
	51	/// Override the runner retry count for this test when non-null.
	52	retry_count: ?usize = null,
	53	attempts: std.ArrayList(TestAttempt) = .empty,
40	54	skip: bool = false,
41	55	only: bool = false,
42	56	file: []const u8 = "",
		@@ -65,6 +79,20 @@ pub const TestCase = struct {
65	79	.timeout_ms = timeout_ms,
66	80	};
67	81	}
	82
	83	/// Configure an additional-attempt count for this individual test.
	84	pub fn withRetries(self: TestCase, retries: usize) TestCase {
	85	var configured = self;
	86	configured.retry_count = retries;
	87	return configured;
	88	}
	89
	90	pub fn deinit(self: *TestCase, allocator: std.mem.Allocator) void {
	91	for (self.attempts.items) \|attempt\| {
	92	if (attempt.error_message) \|message\| allocator.free(message);
	93	}
	94	self.attempts.deinit(allocator);
	95	}
68	96	};
69	97
70	98	/// Test suite (describe block)
		@@ -103,6 +131,7 @@ pub const TestSuite = struct {
103	131	for (self.suites.items) \|suite\| {
104	132	suite.deinit();
105	133	}
	134	for (self.tests.items) \|*test_case\| test_case.deinit(self.allocator);
106	135	self.tests.deinit(self.allocator);
107	136	self.suites.deinit(self.allocator);
108	137	self.before_each_hooks.deinit(self.allocator);
		@@ -456,12 +485,14 @@ test "TestStatus enum values" {
456	485	const pending_status = TestStatus.pending;
457	486	const running_status = TestStatus.running;
458	487	const passed_status = TestStatus.passed;
	488	const flaky_status = TestStatus.flaky;
459	489	const failed_status = TestStatus.failed;
460	490	const skipped_status = TestStatus.skipped;
461	491
462	492	try std.testing.expect(pending_status == .pending);
463	493	try std.testing.expect(running_status == .running);
464	494	try std.testing.expect(passed_status == .passed);
	495	try std.testing.expect(flaky_status == .flaky);
465	496	try std.testing.expect(failed_status == .failed);
466	497	try std.testing.expect(skipped_status == .skipped);
467	498	}

src/test_loader.zigmodified+63-17

Changes to src/test_loader.zig
		@@ -26,6 +26,12 @@ pub const LoaderOptions = struct {
26	26	junit_output: []const u8 = "test-results.xml",
27	27	/// Global per-file timeout checked by the shared execution policy
28	28	timeout_ms: ?u64 = null,
	29	/// Additional process attempts after a failed test file.
	30	retries: usize = 0,
	31	/// Number of successful repetitions required for every selected file.
	32	repeat: usize = 1,
	33	/// Treat recovered failures as an unsuccessful run.
	34	fail_on_flaky: bool = false,
29	35	/// Optional output supplied by a CLI host. Embedded/test callers default
30	36	/// to stderr so they do not interfere with Zig's stdout test protocol.
31	37	reporter_writer: ?*std.Io.Writer = null,
		@@ -85,6 +91,11 @@ pub fn runDiscoveredTests(
85	91	};
86	92	var results = reporter.TestResults.init(allocator);
87	93	defer results.deinit();
	94	var executed_cases: std.ArrayList(suite.TestCase) = .empty;
	95	defer {
	96	for (executed_cases.items) \|*test_case\| test_case.deinit(allocator);
	97	executed_cases.deinit(allocator);
	98	}
88	99
89	100	try events.emit(.{ .run_started = selected_files });
90	101
		@@ -120,23 +131,49 @@ pub fn runDiscoveredTests(
120	131	try server.broadcast("test_start", json);
121	132	}
122	133
123		const start_time = compat.nanoTimestamp();
124		var result = try runTestFile(allocator, file.path, options);
125		const end_time = compat.nanoTimestamp();
126		const execution_time_ns: u64 = @intCast(end_time - start_time);
127		const timed_out = policy.timedOut(execution_time_ns);
128		if (timed_out) result = false;
129
130	134	files_run += 1;
131	135	var test_case = suite.TestCase.init(file.name, externalTestNoop);
132	136	test_case.file = file.relative_path;
133		test_case.execution_time_ns = execution_time_ns;
134		test_case.status = if (result) .passed else .failed;
135		if (timed_out) {
136		test_case.error_message = "External test file exceeded the configured timeout";
137		} else if (!result) {
138		test_case.error_message = "External zig test process failed";
	137	const repeat_count = @max(options.repeat, 1);
	138	var attempt_number: usize = 0;
	139	var had_failure = false;
	140	var final_failure = false;
	141
	142	repetitions: for (1..repeat_count + 1) \|repetition\| {
	143	for (1..options.retries + 2) \|attempt_in_repetition\| {
	144	attempt_number += 1;
	145	const start_time = compat.nanoTimestamp();
	146	var passed = try runTestFile(allocator, file.path, options);
	147	const duration_ns: u64 = @intCast(compat.nanoTimestamp() - start_time);
	148	const timed_out = policy.timedOut(duration_ns);
	149	if (timed_out) passed = false;
	150	const error_message = if (timed_out)
	151	try allocator.dupe(u8, "External test file exceeded the configured timeout")
	152	else if (!passed)
	153	try allocator.dupe(u8, "External zig test process failed")
	154	else
	155	null;
	156	try test_case.attempts.append(allocator, .{
	157	.number = attempt_number,
	158	.repetition = repetition,
	159	.status = if (passed) .passed else .failed,
	160	.duration_ns = duration_ns,
	161	.error_message = error_message,
	162	});
	163	test_case.execution_time_ns += duration_ns;
	164
	165	if (passed) break;
	166	had_failure = true;
	167	test_case.error_message = error_message;
	168	if (attempt_in_repetition == options.retries + 1) {
	169	final_failure = true;
	170	break :repetitions;
	171	}
	172	}
139	173	}
	174	test_case.status = if (final_failure) .failed else if (had_failure) .flaky else .passed;
	175	if (!final_failure) test_case.error_message = null;
	176	try executed_cases.append(allocator, test_case);
140	177	try results.addTest(&test_case);
141	178	try events.emit(.{ .test_finished = &test_case });
142	179	try events.emit(.{ .suite_finished = file.relative_path });
		@@ -144,14 +181,20 @@ pub fn runDiscoveredTests(
144	181	// Notify UI of test file end
145	182	if (options.ui_server) \|server\| {
146	183	var buffer: [512]u8 = undefined;
147		const status = if (result) "passed" else "failed";
148		const json = try std.fmt.bufPrint(&buffer, "", .{ file.name, status, execution_time_ns });
	184	const json = try std.fmt.bufPrint(&buffer, "", .{
	185	file.name,
	186	@tagName(test_case.status),
	187	test_case.execution_time_ns,
	188	test_case.attempts.items.len,
	189	});
149	190	try server.broadcast("test_end", json);
150	191	try server.broadcast("suite_end", try std.fmt.bufPrint(&buffer, "", .{file.name}));
151	192	}
152	193
153		if (result) {
	194	if (test_case.status == .passed) {
154	195	if (options.verbose) std.debug.print(" ✓ {s} passed\n", .{file.name});
	196	} else if (test_case.status == .flaky) {
	197	std.debug.print(" ~ {s} flaky after {d} attempts\n", .{ file.name, test_case.attempts.items.len });
155	198	} else {
156	199	std.debug.print(" ✗ {s} failed\n", .{file.name});
157	200	if (policy.shouldStop(&results)) {
		@@ -188,7 +231,7 @@ pub fn runDiscoveredTests(
188	231	try events.emit(.{ .run_finished = &results });
189	232	try reporters.flush();
190	233
191		return results.failed == 0;
	234	return results.failed == 0 and (!options.fail_on_flaky or results.flaky == 0);
192	235	}
193	236
194	237	fn externalTestNoop(_: std.mem.Allocator) !void {}
		@@ -292,6 +335,9 @@ test "LoaderOptions default values" {
292	335	try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type);
293	336	try std.testing.expectEqualStrings("test-results.xml", options.junit_output);
294	337	try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
	338	try std.testing.expectEqual(@as(usize, 0), options.retries);
	339	try std.testing.expectEqual(@as(usize, 1), options.repeat);
	340	try std.testing.expect(!options.fail_on_flaky);
295	341	try std.testing.expectEqual(@as(?*std.Io.Writer, null), options.reporter_writer);
296	342	try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options);
297	343	}

src/test_runner.zigmodified+216-23

Changes to src/test_runner.zig
		@@ -19,6 +19,12 @@ pub const RunnerOptions = struct {
19	19	n_jobs: ?usize = null, // Number of parallel jobs
20	20	junit_output: []const u8 = "test-results.xml",
21	21	timeout_ms: ?u64 = null,
	22	/// Additional attempts after a failed execution.
	23	retries: usize = 0,
	24	/// Number of times every selected test is required to pass.
	25	repeat: usize = 1,
	26	/// Treat tests that pass after a retry as an unsuccessful run.
	27	fail_on_flaky: bool = false,
22	28	/// Optional output supplied by a CLI host. Embedded/test callers default
23	29	/// to stderr so they do not interfere with Zig's stdout test protocol.
24	30	reporter_writer: ?*std.Io.Writer = null,
		@@ -75,7 +81,7 @@ pub const TestRunner = struct {
75	81	}
76	82
77	83	// Run tests in parallel or sequential based on options
78		if (self.options.parallel) {
	84	if (self.options.parallel and self.options.retries == 0 and self.options.repeat == 1) {
79	85	const parallel_opts = parallel.ParallelOptions{
80	86	.enabled = true,
81	87	.n_jobs = self.options.n_jobs,
		@@ -95,7 +101,8 @@ pub const TestRunner = struct {
95	101	}
96	102
97	103	try reporters.flush();
98		return all_passed and self.results.failed == 0;
	104	return all_passed and self.results.failed == 0 and
	105	(!self.options.fail_on_flaky or self.results.flaky == 0);
99	106	}
100	107
101	108	// Sequential execution (original behavior)
		@@ -113,7 +120,8 @@ pub const TestRunner = struct {
113	120	// Flush output
114	121	try reporters.flush();
115	122
116		return self.results.failed == 0;
	123	return self.results.failed == 0 and
	124	(!self.options.fail_on_flaky or self.results.flaky == 0);
117	125	}
118	126
119	127	fn addSuiteResults(self: Self, test_suite: suite.TestSuite) !void {
		@@ -189,52 +197,88 @@ pub const TestRunner = struct {
189	197	fn runTest(self: Self, test_case: suite.TestCase, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
190	198	try events.emit(.{ .test_started = test_case.name });
191	199
	200	for (test_case.attempts.items) \|attempt\| {
	201	if (attempt.error_message) \|message\| self.allocator.free(message);
	202	}
	203	test_case.attempts.clearRetainingCapacity();
	204	test_case.execution_time_ns = 0;
	205	test_case.error_message = null;
	206
	207	const retries = test_case.retry_count orelse self.options.retries;
	208	const repeat_count = @max(self.options.repeat, 1);
	209	var attempt_number: usize = 0;
	210	var had_failure = false;
	211	var final_failure = false;
	212
	213	repetitions: for (1..repeat_count + 1) \|repetition\| {
	214	for (1..retries + 2) \|attempt_in_repetition\| {
	215	attempt_number += 1;
	216	const passed = try self.runSingleAttempt(test_case, test_suite, repetition, attempt_number);
	217	if (passed) break;
	218
	219	had_failure = true;
	220	if (attempt_in_repetition == retries + 1) {
	221	final_failure = true;
	222	break :repetitions;
	223	}
	224	}
	225	}
	226
	227	if (!final_failure) {
	228	test_case.status = if (had_failure) .flaky else .passed;
	229	test_case.error_message = null;
	230	}
	231
	232	try self.results.addTest(test_case);
	233	try events.emit(.{ .test_finished = test_case });
	234	}
	235
	236	fn runSingleAttempt(
	237	self: *Self,
	238	test_case: *suite.TestCase,
	239	test_suite: *suite.TestSuite,
	240	repetition: usize,
	241	attempt_number: usize,
	242	) !bool {
192	243	test_case.status = .running;
	244	test_case.error_message = null;
193	245	const start_time = compat.nanoTimestamp();
194	246
195		// Get all beforeEach hooks (including parent hooks)
196	247	var before_hooks = try test_suite.getAllBeforeEachHooks(self.allocator);
197	248	defer before_hooks.deinit(self.allocator);
198	249
199		// Run beforeEach hooks
200	250	var before_failed = false;
201	251	for (before_hooks.items) \|hook\| {
202	252	hook(self.allocator) catch \|err\| {
203	253	test_case.status = .failed;
204		const err_msg = try std.fmt.allocPrint(self.allocator, "beforeEach hook failed: {any}", .{err});
205		test_case.error_message = err_msg;
	254	test_case.error_message = try std.fmt.allocPrint(
	255	self.allocator,
	256	"beforeEach hook failed: {any}",
	257	.{err},
	258	);
206	259	before_failed = true;
207	260	break;
208	261	};
209	262	}
210	263
211		// Run the actual test if beforeEach succeeded
212	264	if (!before_failed) {
213	265	test_case.test_fn(self.allocator) catch \|err\| {
214	266	test_case.status = .failed;
215		const err_msg = try std.fmt.allocPrint(self.allocator, "{any}", .{err});
216		test_case.error_message = err_msg;
	267	test_case.error_message = try std.fmt.allocPrint(self.allocator, "{any}", .{err});
217	268	};
218
219		if (test_case.status == .running) {
220		test_case.status = .passed;
221		}
	269	if (test_case.status == .running) test_case.status = .passed;
222	270	}
223	271
224		// Get all afterEach hooks (including parent hooks)
225	272	var after_hooks = try test_suite.getAllAfterEachHooks(self.allocator);
226	273	defer after_hooks.deinit(self.allocator);
227
228		// Run afterEach hooks (always run, even if test failed)
229	274	for (after_hooks.items) \|hook\| {
230	275	hook(self.allocator) catch \|err\| {
231	276	std.debug.print("afterEach hook failed: {any}\n", .{err});
232	277	};
233	278	}
234	279
235		const end_time = compat.nanoTimestamp();
236		test_case.execution_time_ns = @intCast(end_time - start_time);
237		if (test_case.status == .passed and self.policy().timedOut(test_case.execution_time_ns)) {
	280	const duration_ns: u64 = @intCast(compat.nanoTimestamp() - start_time);
	281	if (test_case.status == .passed and self.policy().timedOut(duration_ns)) {
238	282	test_case.status = .failed;
239	283	test_case.error_message = try std.fmt.allocPrint(
240	284	self.allocator,
		@@ -242,9 +286,15 @@ pub const TestRunner = struct {
242	286	.{self.options.timeout_ms.?},
243	287	);
244	288	}
245
246		try self.results.addTest(test_case);
247		try events.emit(.{ .test_finished = test_case });
	289	test_case.execution_time_ns += duration_ns;
	290	try test_case.attempts.append(self.allocator, .{
	291	.number = attempt_number,
	292	.repetition = repetition,
	293	.status = test_case.status,
	294	.duration_ns = duration_ns,
	295	.error_message = test_case.error_message,
	296	});
	297	return test_case.status == .passed;
248	298	}
249	299
250	300	/// Skip all tests in a suite
		@@ -302,6 +352,9 @@ test "RunnerOptions default values" {
302	352	try std.testing.expectEqual(ReporterType.spec, options.reporter_type);
303	353	try std.testing.expectEqual(true, options.use_colors);
304	354	try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
	355	try std.testing.expectEqual(@as(usize, 0), options.retries);
	356	try std.testing.expectEqual(@as(usize, 1), options.repeat);
	357	try std.testing.expect(!options.fail_on_flaky);
305	358	}
306	359
307	360	test "RunnerOptions custom values" {
		@@ -317,3 +370,143 @@ test "RunnerOptions custom values" {
317	370	try std.testing.expectEqual(ReporterType.json, options.reporter_type);
318	371	try std.testing.expectEqual(false, options.use_colors);
319	372	}
	373
	374	var flaky_fixture_runs: usize = 0;
	375	var failing_fixture_runs: usize = 0;
	376	var skipped_by_bail_runs: usize = 0;
	377	var repeated_fixture_runs: usize = 0;
	378	var timeout_fixture_runs: usize = 0;
	379
	380	fn flakyFixture(_: std.mem.Allocator) !void {
	381	flaky_fixture_runs += 1;
	382	if (flaky_fixture_runs == 1) return error.TransientFailure;
	383	}
	384
	385	fn failingFixture(_: std.mem.Allocator) !void {
	386	failing_fixture_runs += 1;
	387	return error.DeterministicFailure;
	388	}
	389
	390	fn skippedByBailFixture(_: std.mem.Allocator) !void {
	391	skipped_by_bail_runs += 1;
	392	}
	393
	394	fn repeatedFixture(_: std.mem.Allocator) !void {
	395	repeated_fixture_runs += 1;
	396	}
	397
	398	fn timeoutFixture(_: std.mem.Allocator) !void {
	399	timeout_fixture_runs += 1;
	400	compat.sleep(std.time.ns_per_ms);
	401	}
	402
	403	test "retries record deterministic flaky attempt history" {
	404	flaky_fixture_runs = 0;
	405	const allocator = std.testing.allocator;
	406	var registry = suite.TestRegistry.init(allocator);
	407	defer registry.deinit();
	408	const test_suite = try suite.TestSuite.init(allocator, "retry suite");
	409	try test_suite.addTest(suite.TestCase.init("flaky fixture", flakyFixture).withRetries(1));
	410	try registry.registerSuite(test_suite);
	411
	412	var output: [4096]u8 = undefined;
	413	var writer: std.Io.Writer = .fixed(&output);
	414	var runner = TestRunner.init(allocator, &registry, .{
	415	.reporter_writer = &writer,
	416	.use_colors = false,
	417	});
	418	defer runner.deinit();
	419
	420	try std.testing.expect(try runner.run());
	421	const test_case = &test_suite.tests.items[0];
	422	try std.testing.expectEqual(suite.TestStatus.flaky, test_case.status);
	423	try std.testing.expectEqual(@as(usize, 2), test_case.attempts.items.len);
	424	try std.testing.expectEqual(suite.TestStatus.failed, test_case.attempts.items[0].status);
	425	try std.testing.expectEqual(suite.TestStatus.passed, test_case.attempts.items[1].status);
	426	try std.testing.expectEqual(@as(usize, 1), runner.results.flaky);
	427
	428	var strict_runner = TestRunner.init(allocator, &registry, .{
	429	.retries = 1,
	430	.fail_on_flaky = true,
	431	.reporter_writer = &writer,
	432	.use_colors = false,
	433	});
	434	defer strict_runner.deinit();
	435	flaky_fixture_runs = 0;
	436	try std.testing.expect(!try strict_runner.run());
	437	}
	438
	439	test "repeat mode runs every required passing repetition" {
	440	repeated_fixture_runs = 0;
	441	const allocator = std.testing.allocator;
	442	var registry = suite.TestRegistry.init(allocator);
	443	defer registry.deinit();
	444	const test_suite = try suite.TestSuite.init(allocator, "repeat suite");
	445	try test_suite.addTest(suite.TestCase.init("repeated fixture", repeatedFixture));
	446	try registry.registerSuite(test_suite);
	447
	448	var output: [4096]u8 = undefined;
	449	var writer: std.Io.Writer = .fixed(&output);
	450	var runner = TestRunner.init(allocator, &registry, .{
	451	.repeat = 3,
	452	.reporter_writer = &writer,
	453	.use_colors = false,
	454	});
	455	defer runner.deinit();
	456
	457	try std.testing.expect(try runner.run());
	458	try std.testing.expectEqual(@as(usize, 3), repeated_fixture_runs);
	459	try std.testing.expectEqual(@as(usize, 3), test_suite.tests.items[0].attempts.items.len);
	460	}
	461
	462	test "bail waits until retries are exhausted" {
	463	failing_fixture_runs = 0;
	464	skipped_by_bail_runs = 0;
	465	const allocator = std.testing.allocator;
	466	var registry = suite.TestRegistry.init(allocator);
	467	defer registry.deinit();
	468	const test_suite = try suite.TestSuite.init(allocator, "bail suite");
	469	try test_suite.addTest(suite.TestCase.init("always fails", failingFixture));
	470	try test_suite.addTest(suite.TestCase.init("must be skipped", skippedByBailFixture));
	471	try registry.registerSuite(test_suite);
	472
	473	var output: [4096]u8 = undefined;
	474	var writer: std.Io.Writer = .fixed(&output);
	475	var runner = TestRunner.init(allocator, &registry, .{
	476	.bail = true,
	477	.retries = 2,
	478	.reporter_writer = &writer,
	479	.use_colors = false,
	480	});
	481	defer runner.deinit();
	482
	483	try std.testing.expect(!try runner.run());
	484	try std.testing.expectEqual(@as(usize, 3), failing_fixture_runs);
	485	try std.testing.expectEqual(@as(usize, 0), skipped_by_bail_runs);
	486	try std.testing.expectEqual(@as(usize, 3), test_suite.tests.items[0].attempts.items.len);
	487	}
	488
	489	test "timeouts are evaluated for every retry attempt" {
	490	timeout_fixture_runs = 0;
	491	const allocator = std.testing.allocator;
	492	var registry = suite.TestRegistry.init(allocator);
	493	defer registry.deinit();
	494	const test_suite = try suite.TestSuite.init(allocator, "timeout suite");
	495	try test_suite.addTest(suite.TestCase.init("times out", timeoutFixture));
	496	try registry.registerSuite(test_suite);
	497
	498	var output: [4096]u8 = undefined;
	499	var writer: std.Io.Writer = .fixed(&output);
	500	var runner = TestRunner.init(allocator, &registry, .{
	501	.timeout_ms = 0,
	502	.retries = 1,
	503	.reporter_writer = &writer,
	504	.use_colors = false,
	505	});
	506	defer runner.deinit();
	507
	508	try std.testing.expect(!try runner.run());
	509	try std.testing.expectEqual(@as(usize, 2), timeout_fixture_runs);
	510	try std.testing.expectEqual(@as(usize, 2), test_suite.tests.items[0].attempts.items.len);
	511	try std.testing.expectEqual(suite.TestStatus.failed, test_suite.tests.items[0].status);
	512	}

tests/fixtures/zig-test.jsonmodified+4-1

Changes to tests/fixtures/zig-test.json
		@@ -3,7 +3,10 @@
3	3	"test_dir": "tests/fixtures",
4	4	"pattern": "filter_fixture.zig",
5	5	"recursive": false,
6		"filter": "selected test passes"
	6	"filter": "selected test passes",
	7	"retries": 1,
	8	"repeat": 2,
	9	"fail_on_flaky": false
7	10	},
8	11	"sharding": {
9	12	"index": 1,

tests/reporter_test.zigmodified+42-0

Changes to tests/reporter_test.zig
		@@ -68,6 +68,48 @@ test "reporter json output" {
68	68	const output = buffer[0..reporter.writer.end];
69	69	try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null);
70	70	try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null);
	71	var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, output, .{});
	72	defer parsed.deinit();
	73	}
	74
	75	test "reporter json output includes flaky attempt history" {
	76	const allocator = std.testing.allocator;
	77	var buffer: [4096]u8 = undefined;
	78	const writer: std.Io.Writer = .fixed(&buffer);
	79	var reporter = ztf.JsonReporter.init(allocator, writer);
	80	defer reporter.deinit();
	81	var results = ztf.TestResults.init(allocator);
	82	defer results.deinit();
	83	results.total = 1;
	84	results.flaky = 1;
	85	var test_case = ztf.TestCase.init("eventually passes", passingTest);
	86	defer test_case.attempts.deinit(allocator);
	87	test_case.status = .flaky;
	88	try test_case.attempts.append(allocator, .{
	89	.number = 1,
	90	.repetition = 1,
	91	.status = .failed,
	92	.duration_ns = 10,
	93	});
	94	try test_case.attempts.append(allocator, .{
	95	.number = 2,
	96	.repetition = 1,
	97	.status = .passed,
	98	.duration_ns = 5,
	99	});
	100
	101	try reporter.reporter.onRunStart(1);
	102	try reporter.reporter.onSuiteStart("suite");
	103	try reporter.reporter.onTestEnd(&test_case);
	104	try reporter.reporter.onSuiteEnd("suite");
	105	try reporter.reporter.onRunEnd(&results);
	106
	107	const output = buffer[0..reporter.writer.end];
	108	try std.testing.expect(std.mem.indexOf(u8, output, "\"status\":\"flaky\"") != null);
	109	try std.testing.expect(std.mem.indexOf(u8, output, "\"attempts\":[") != null);
	110	try std.testing.expect(std.mem.indexOf(u8, output, "\"durationNs\":10") != null);
	111	var parsed = try std.json.parseFromSlice(std.json.Value, allocator, output, .{});
	112	defer parsed.deinit();
71	113	}
72	114
73	115	test "reporter set selects the same built-ins for every executor" {

Files

Review one file at a time

CHANGELOG.md +6 -0
README.md +21 -1
build.zig +15 -0
docs/execution-pipeline.md +11 -6
docs/retries-and-repeats.md +92 -0
src/cli.zig +70 -0
src/config.zig +14 -0
src/lib.zig +1 -0
src/main.zig +6 -0
src/reporter.zig +127 -5
src/suite.zig +31 -0
src/test_loader.zig +63 -17
src/test_runner.zig +216 -23
tests/fixtures/zig-test.json +4 -1
tests/reporter_test.zig +42 -0

Merge

This pull request was merged.

Reviews

Nobody has reviewed this yet.

0 of 0 required approvals.

Checks

No checks are required on this branch.

Conversations

0 unresolved of 0.