Add retries, repeated runs, and flaky-test reporting

src/test_loader.zigmodified+63-17

Changes to src/test_loader.zig

		⋯25 linesShow the 25 lines above this hunk@@ -26,6 +26,12 @@ pub const LoaderOptions = struct {
26	26	junit_output: []const u8 = "test-results.xml",
27	27	/// Global per-file timeout checked by the shared execution policy
28	28	timeout_ms: ?u64 = null,
	29	+ /// Additional process attempts after a failed test file.
	30	+ retries: usize = 0,
	31	+ /// Number of successful repetitions required for every selected file.
	32	+ repeat: usize = 1,
	33	+ /// Treat recovered failures as an unsuccessful run.
	34	+ fail_on_flaky: bool = false,
29	35	/// Optional output supplied by a CLI host. Embedded/test callers default
30	36	/// to stderr so they do not interfere with Zig's stdout test protocol.
31	37	reporter_writer: ?*std.Io.Writer = null,
		⋯53 linesShow the 53 lines above this hunk@@ -85,6 +91,11 @@ pub fn runDiscoveredTests(
85	91	};
86	92	var results = reporter.TestResults.init(allocator);
87	93	defer results.deinit();
	94	+ var executed_cases: std.ArrayList(suite.TestCase) = .empty;
	95	+ defer {
	96	+ for (executed_cases.items) |*test_case| test_case.deinit(allocator);
	97	+ executed_cases.deinit(allocator);
	98	+ }
88	99
89	100	try events.emit(.{ .run_started = selected_files });
90	101
		⋯29 linesShow the 29 lines above this hunk@@ -120,23 +131,49 @@ pub fn runDiscoveredTests(
120	131	try server.broadcast("test_start", json);
121	132	}
122	133
123		- const start_time = compat.nanoTimestamp();
124		- var result = try runTestFile(allocator, file.path, options);
125		- const end_time = compat.nanoTimestamp();
126		- const execution_time_ns: u64 = @intCast(end_time - start_time);
127		- const timed_out = policy.timedOut(execution_time_ns);
128		- if (timed_out) result = false;
129		-
130	134	files_run += 1;
131	135	var test_case = suite.TestCase.init(file.name, externalTestNoop);
132	136	test_case.file = file.relative_path;
133		- test_case.execution_time_ns = execution_time_ns;
134		- test_case.status = if (result) .passed else .failed;
135		- if (timed_out) {
136		- test_case.error_message = "External test file exceeded the configured timeout";
137		- } else if (!result) {
138		- test_case.error_message = "External zig test process failed";
	137	+ const repeat_count = @max(options.repeat, 1);
	138	+ var attempt_number: usize = 0;
	139	+ var had_failure = false;
	140	+ var final_failure = false;
	141	+
	142	+ repetitions: for (1..repeat_count + 1) |repetition| {
	143	+ for (1..options.retries + 2) |attempt_in_repetition| {
	144	+ attempt_number += 1;
	145	+ const start_time = compat.nanoTimestamp();
	146	+ var passed = try runTestFile(allocator, file.path, options);
	147	+ const duration_ns: u64 = @intCast(compat.nanoTimestamp() - start_time);
	148	+ const timed_out = policy.timedOut(duration_ns);
	149	+ if (timed_out) passed = false;
	150	+ const error_message = if (timed_out)
	151	+ try allocator.dupe(u8, "External test file exceeded the configured timeout")
	152	+ else if (!passed)
	153	+ try allocator.dupe(u8, "External zig test process failed")
	154	+ else
	155	+ null;
	156	+ try test_case.attempts.append(allocator, .{
	157	+ .number = attempt_number,
	158	+ .repetition = repetition,
	159	+ .status = if (passed) .passed else .failed,
	160	+ .duration_ns = duration_ns,
	161	+ .error_message = error_message,
	162	+ });
	163	+ test_case.execution_time_ns += duration_ns;
	164	+
	165	+ if (passed) break;
	166	+ had_failure = true;
	167	+ test_case.error_message = error_message;
	168	+ if (attempt_in_repetition == options.retries + 1) {
	169	+ final_failure = true;
	170	+ break :repetitions;
	171	+ }
	172	+ }
139	173	}
	174	+ test_case.status = if (final_failure) .failed else if (had_failure) .flaky else .passed;
	175	+ if (!final_failure) test_case.error_message = null;
	176	+ try executed_cases.append(allocator, test_case);
140	177	try results.addTest(&test_case);
141	178	try events.emit(.{ .test_finished = &test_case });
142	179	try events.emit(.{ .suite_finished = file.relative_path });
		⋯1 lineShow the 1 line above this hunk@@ -144,14 +181,20 @@ pub fn runDiscoveredTests(
144	181	// Notify UI of test file end
145	182	if (options.ui_server) |server| {
146	183	var buffer: [512]u8 = undefined;
147		- const status = if (result) "passed" else "failed";
148		- const json = try std.fmt.bufPrint(&buffer, "", .{ file.name, status, execution_time_ns });
	184	+ const json = try std.fmt.bufPrint(&buffer, "", .{
	185	+ file.name,
	186	+ @tagName(test_case.status),
	187	+ test_case.execution_time_ns,
	188	+ test_case.attempts.items.len,
	189	+ });
149	190	try server.broadcast("test_end", json);
150	191	try server.broadcast("suite_end", try std.fmt.bufPrint(&buffer, "", .{file.name}));
151	192	}
152	193
153		- if (result) {
	194	+ if (test_case.status == .passed) {
154	195	if (options.verbose) std.debug.print(" ✓ {s} passed\n", .{file.name});
	196	+ } else if (test_case.status == .flaky) {
	197	+ std.debug.print(" ~ {s} flaky after {d} attempts\n", .{ file.name, test_case.attempts.items.len });
155	198	} else {
156	199	std.debug.print(" ✗ {s} failed\n", .{file.name});
157	200	if (policy.shouldStop(&results)) {
		⋯30 linesShow the 30 lines above this hunk@@ -188,7 +231,7 @@ pub fn runDiscoveredTests(
188	231	try events.emit(.{ .run_finished = &results });
189	232	try reporters.flush();
190	233
191		- return results.failed == 0;
	234	+ return results.failed == 0 and (!options.fail_on_flaky or results.flaky == 0);
192	235	}
193	236
194	237	fn externalTestNoop(_: std.mem.Allocator) !void {}
		⋯97 linesShow the 97 lines above this hunk@@ -292,6 +335,9 @@ test "LoaderOptions default values" {
292	335	try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type);
293	336	try std.testing.expectEqualStrings("test-results.xml", options.junit_output);
294	337	try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
	338	+ try std.testing.expectEqual(@as(usize, 0), options.retries);
	339	+ try std.testing.expectEqual(@as(usize, 1), options.repeat);
	340	+ try std.testing.expect(!options.fail_on_flaky);
295	341	try std.testing.expectEqual(@as(?*std.Io.Writer, null), options.reporter_writer);
296	342	try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options);
297	343	}