Add retries, repeated runs, and flaky-test reporting

src/cli.zigmodified+70-0

Changes to src/cli.zig

		⋯44 linesShow the 44 lines above this hunk@@ -45,6 +45,10 @@ pub const CLIOptions = struct {
45	45	junit_output: ?[]const u8 = null,
46	46	// Timeout options
47	47	timeout: ?u64 = null, // Global timeout in milliseconds
	48	+ // Retry and repetition options
	49	+ retries: usize = 0,
	50	+ repeat: usize = 1,
	51	+ fail_on_flaky: bool = false,
48	52	};
49	53
50	54	pub const version = build_options.version;
		⋯24 linesShow the 24 lines above this hunk@@ -75,6 +79,9 @@ pub const CLI = struct {
75	79	self.options.no_recursive = !config.test_options.recursive;
76	80	self.options.filter = config.test_options.filter;
77	81	self.options.timeout = config.test_options.timeout;
	82	+ self.options.retries = config.test_options.retries;
	83	+ self.options.repeat = config.test_options.repeat;
	84	+ self.options.fail_on_flaky = config.test_options.fail_on_flaky;
78	85	self.options.shard_index = config.sharding.index;
79	86	self.options.shard_count = config.sharding.count;
80	87	self.options.parallel = config.parallel.enabled;
		⋯26 linesShow the 26 lines above this hunk@@ -107,6 +114,32 @@ pub const CLI = struct {
107	114	self.options.version = true;
108	115	} else if (std.mem.eql(u8, arg, "--bail") or std.mem.eql(u8, arg, "-b")) {
109	116	self.options.bail = true;
	117	+ } else if (std.mem.eql(u8, arg, "--retry") or std.mem.eql(u8, arg, "--retries")) {
	118	+ if (i + 1 >= args.len) {
	119	+ std.debug.print("Error: {s} requires a value\n", .{arg});
	120	+ return CLIError.MissingValue;
	121	+ }
	122	+ i += 1;
	123	+ self.options.retries = std.fmt.parseInt(usize, args[i], 10) catch {
	124	+ std.debug.print("Error: {s} must be a valid number\n", .{arg});
	125	+ return CLIError.InvalidArgument;
	126	+ };
	127	+ } else if (std.mem.eql(u8, arg, "--repeat")) {
	128	+ if (i + 1 >= args.len) {
	129	+ std.debug.print("Error: --repeat requires a value\n", .{});
	130	+ return CLIError.MissingValue;
	131	+ }
	132	+ i += 1;
	133	+ self.options.repeat = std.fmt.parseInt(usize, args[i], 10) catch {
	134	+ std.debug.print("Error: --repeat must be a valid positive number\n", .{});
	135	+ return CLIError.InvalidArgument;
	136	+ };
	137	+ if (self.options.repeat == 0) {
	138	+ std.debug.print("Error: --repeat must be greater than zero\n", .{});
	139	+ return CLIError.InvalidArgument;
	140	+ }
	141	+ } else if (std.mem.eql(u8, arg, "--fail-on-flaky")) {
	142	+ self.options.fail_on_flaky = true;
110	143	} else if (std.mem.eql(u8, arg, "--verbose")) {
111	144	self.options.verbose = true;
112	145	} else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) {
		⋯238 linesShow the 238 lines above this hunk@@ -351,6 +384,9 @@ pub const CLI = struct {
351	384	\\ -h, --help Show this help message
352	385	\\ -v, --version Show version information
353	386	\\ -b, --bail Stop test execution on first failure
	387	+ \\ --retry <N> Retry each failed test up to N times
	388	+ \\ --repeat <N> Require N successful runs of each selected test
	389	+ \\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure
354	390	\\ --filter <pattern> Run only tests matching pattern
355	391	\\ --grep <pattern> Same as --filter (alias)
356	392	\\ --reporter <name> Set reporter type (spec, dot, json, tap, junit)
		⋯57 linesShow the 57 lines above this hunk@@ -414,6 +450,8 @@ pub const CLI = struct {
414	450	\\ zig-test --update-snapshots Update all snapshots
415	451	\\ zig-test --watch Run in watch mode
416	452	\\ zig-test --timeout 10000 Set 10s timeout for all tests
	453	+ \\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests
	454	+ \\ zig-test --repeat 100 --filter parser Stress-run selected tests
417	455	\\ zig-test --profile-memory --fail-on-leak
418	456	\\ zig-test --parallel --jobs 4 Run with 4 parallel workers
419	457	\\ zig-test --config zig-test.json Load config from file
		⋯21 linesShow the 21 lines above this hunk@@ -441,6 +479,9 @@ pub const CLI = struct {
441	479	.n_jobs = self.options.jobs,
442	480	.junit_output = self.options.junit_output orelse "test-results.xml",
443	481	.timeout_ms = self.options.timeout,
	482	+ .retries = self.options.retries,
	483	+ .repeat = self.options.repeat,
	484	+ .fail_on_flaky = self.options.fail_on_flaky,
444	485	};
445	486	}
446	487	};
		⋯215 linesShow the 215 lines above this hunk@@ -662,6 +703,35 @@ test "CLI parse timeout" {
662	703	try std.testing.expectEqual(@as(?u64, 5000), cli.options.timeout);
663	704	}
664	705
	706	+test "CLI parses retry repeat and flaky exit policy" {
	707	+ var cli = CLI.init(std.testing.allocator);
	708	+ const args = [_][]const u8{
	709	+ "zig-test",
	710	+ "--retry",
	711	+ "2",
	712	+ "--repeat",
	713	+ "5",
	714	+ "--fail-on-flaky",
	715	+ };
	716	+ try cli.parse(&args);
	717	+ try std.testing.expectEqual(@as(usize, 2), cli.options.retries);
	718	+ try std.testing.expectEqual(@as(usize, 5), cli.options.repeat);
	719	+ try std.testing.expect(cli.options.fail_on_flaky);
	720	+
	721	+ const runner_options = cli.toRunnerOptions();
	722	+ try std.testing.expectEqual(@as(usize, 2), runner_options.retries);
	723	+ try std.testing.expectEqual(@as(usize, 5), runner_options.repeat);
	724	+ try std.testing.expect(runner_options.fail_on_flaky);
	725	+}
	726	+
	727	+test "CLI rejects a zero repeat count" {
	728	+ var cli = CLI.init(std.testing.allocator);
	729	+ try std.testing.expectError(
	730	+ CLIError.InvalidArgument,
	731	+ cli.parse(&.{ "zig-test", "--repeat", "0" }),
	732	+ );
	733	+}
	734	+
665	735	test "CLI rejects invalid coverage tool" {
666	736	var cli = CLI.init(std.testing.allocator);
667	737	const args = [_][]const u8{ "zig-test", "--coverage-tool", "unknown" };