Add retries, repeated runs, and flaky-test reporting

src/reporter.zigmodified+127-5

Changes to src/reporter.zig

		⋯70 linesShow the 70 lines above this hunk@@ -71,21 +71,25 @@ pub const Reporter = struct {
71	71	pub const TestResults = struct {
72	72	total: usize = 0,
73	73	passed: usize = 0,
	74	+ flaky: usize = 0,
74	75	failed: usize = 0,
75	76	skipped: usize = 0,
76	77	total_time_ns: u64 = 0,
77	78	failed_tests: std.ArrayList(suite.TestCase),
	79	+ flaky_tests: std.ArrayList(suite.TestCase),
78	80	allocator: std.mem.Allocator,
79	81
80	82	pub fn init(allocator: std.mem.Allocator) TestResults {
81	83	return TestResults{
82	84	.failed_tests = .empty,
	85	+ .flaky_tests = .empty,
83	86	.allocator = allocator,
84	87	};
85	88	}
86	89
87	90	pub fn deinit(self: *TestResults) void {
88	91	self.failed_tests.deinit(self.allocator);
	92	+ self.flaky_tests.deinit(self.allocator);
89	93	}
90	94
91	95	pub fn addTest(self: *TestResults, test_case: *const suite.TestCase) !void {
		⋯2 linesShow the 2 lines above this hunk@@ -94,6 +98,10 @@ pub const TestResults = struct {
94	98
95	99	switch (test_case.status) {
96	100	.passed => self.passed += 1,
	101	+ .flaky => {
	102	+ self.flaky += 1;
	103	+ try self.flaky_tests.append(self.allocator, test_case.*);
	104	+ },
97	105	.failed => {
98	106	self.failed += 1;
99	107	try self.failed_tests.append(self.allocator, test_case.*);
		⋯88 linesShow the 88 lines above this hunk@@ -188,6 +196,9 @@ pub const SpecReporter = struct {
188	196	try s.output().print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset });
189	197	try s.output().print(" Total: {d}\n", .{results.total});
190	198	try s.output().print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset });
	199	+ if (results.flaky > 0) {
	200	+ try s.output().print(" {s}Flaky: {d}{s}\n", .{ Colors.yellow, results.flaky, Colors.reset });
	201	+ }
191	202	if (results.failed > 0) {
192	203	try s.output().print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset });
193	204	}
		⋯5 linesShow the 5 lines above this hunk@@ -199,6 +210,9 @@ pub const SpecReporter = struct {
199	210	try s.output().print("Test Summary:\n", .{});
200	211	try s.output().print(" Total: {d}\n", .{results.total});
201	212	try s.output().print(" Passed: {d}\n", .{results.passed});
	213	+ if (results.flaky > 0) {
	214	+ try s.output().print(" Flaky: {d}\n", .{results.flaky});
	215	+ }
202	216	if (results.failed > 0) {
203	217	try s.output().print(" Failed: {d}\n", .{results.failed});
204	218	}
		⋯60 linesShow the 60 lines above this hunk@@ -265,6 +279,25 @@ pub const SpecReporter = struct {
265	279	try s.output().print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms });
266	280	}
267	281	},
	282	+ .flaky => {
	283	+ if (reporter.use_colors) {
	284	+ try s.output().print("{s}~{s} {s} {s}({d} attempts, {d:.2}ms){s}\n", .{
	285	+ Colors.yellow,
	286	+ Colors.reset,
	287	+ test_case.name,
	288	+ Colors.gray,
	289	+ test_case.attempts.items.len,
	290	+ time_ms,
	291	+ Colors.reset,
	292	+ });
	293	+ } else {
	294	+ try s.output().print("~ {s} (flaky after {d} attempts, {d:.2}ms)\n", .{
	295	+ test_case.name,
	296	+ test_case.attempts.items.len,
	297	+ time_ms,
	298	+ });
	299	+ }
	300	+ },
268	301	.failed => {
269	302	if (reporter.use_colors) {
270	303	try s.output().print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{
		⋯82 linesShow the 82 lines above this hunk@@ -353,11 +386,13 @@ pub const DotReporter = struct {
353	386
354	387	if (reporter.use_colors) {
355	388	try s.output().print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset });
	389	+ try s.output().print("{s}Flaky: {d}{s}, ", .{ Colors.yellow, results.flaky, Colors.reset });
356	390	try s.output().print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset });
357	391	try s.output().print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms });
358	392	} else {
359		- try s.output().print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{
	393	+ try s.output().print("Passed: {d}, Flaky: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{
360	394	results.passed,
	395	+ results.flaky,
361	396	results.failed,
362	397	results.total,
363	398	total_time_ms,
		⋯27 linesShow the 27 lines above this hunk@@ -391,6 +426,13 @@ pub const DotReporter = struct {
391	426	try s.output().print(".", .{});
392	427	}
393	428	},
	429	+ .flaky => {
	430	+ if (reporter.use_colors) {
	431	+ try s.output().print("{s}~{s}", .{ Colors.yellow, Colors.reset });
	432	+ } else {
	433	+ try s.output().print("~", .{});
	434	+ }
	435	+ },
394	436	.failed => {
395	437	if (reporter.use_colors) {
396	438	try s.output().print("{s}F{s}", .{ Colors.red, Colors.reset });
		⋯25 linesShow the 25 lines above this hunk@@ -422,6 +464,7 @@ pub const JsonReporter = struct {
422	464	writer: std.Io.Writer,
423	465	writer_ref: ?*std.Io.Writer = null,
424	466	suites: std.ArrayList([]const u8),
	467	+ emitted_tests: bool = false,
425	468
426	469	const Self = @This();
427	470
		⋯44 linesShow the 44 lines above this hunk@@ -472,9 +515,10 @@ pub const JsonReporter = struct {
472	515	const s = self(reporter);
473	516	const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0;
474	517
475		- try s.output().print("],\"summary\":}}\n", .{
	518	+ try s.output().print("\n],\"summary\":}}\n", .{
476	519	results.total,
477	520	results.passed,
	521	+ results.flaky,
478	522	results.failed,
479	523	results.skipped,
480	524	total_time_ms,
		⋯24 linesShow the 24 lines above this hunk@@ -505,11 +549,15 @@ pub const JsonReporter = struct {
505	549
506	550	const status_str = switch (test_case.status) {
507	551	.passed => "passed",
	552	+ .flaky => "flaky",
508	553	.failed => "failed",
509	554	.skipped => "skipped",
510	555	else => "unknown",
511	556	};
512	557
	558	+ if (s.emitted_tests) try s.output().print(",\n", .{});
	559	+ s.emitted_tests = true;
	560	+
513	561	// Note: In a real implementation, you'd want to properly escape JSON strings
514	562	try s.output().print(" ", .{});
525	592	}
526	593	};
527	594
		⋯65 linesShow the 65 lines above this hunk@@ -593,6 +660,7 @@ pub const TAPReporter = struct {
593	660
594	661	const status = switch (test_case.status) {
595	662	.passed => "ok",
	663	+ .flaky => "ok",
596	664	.failed => "not ok",
597	665	.skipped => "ok",
598	666	else => "not ok",
		⋯1 lineShow the 1 line above this hunk@@ -600,7 +668,9 @@ pub const TAPReporter = struct {
600	668
601	669	try self.output().print("{s} {d} - {s}", .{ status, self.test_count, test_case.name });
602	670
603		- if (test_case.status == .skipped) {
	671	+ if (test_case.status == .flaky) {
	672	+ try self.output().print(" # FLAKY attempts={d}\n", .{test_case.attempts.items.len});
	673	+ } else if (test_case.status == .skipped) {
604	674	try self.output().print(" # SKIP\n", .{});
605	675	} else if (test_case.status == .failed and test_case.error_message != null) {
606	676	try self.output().print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?});
		⋯24 linesShow the 24 lines above this hunk@@ -631,6 +701,10 @@ pub const JUnitReporter = struct {
631	701	if (test_case.error_message) |msg| {
632	702	allocator.free(msg);
633	703	}
	704	+ for (test_case.attempts) |attempt| {
	705	+ if (attempt.error_message) |msg| allocator.free(msg);
	706	+ }
	707	+ allocator.free(test_case.attempts);
634	708	}
635	709	self.tests.deinit(allocator);
636	710	allocator.free(self.name);
		⋯5 linesShow the 5 lines above this hunk@@ -642,6 +716,15 @@ pub const JUnitReporter = struct {
642	716	time: f64,
643	717	status: suite.TestStatus,
644	718	error_message: ?[]const u8,
	719	+ attempts: []AttemptResult,
	720	+ };
	721	+
	722	+ const AttemptResult = struct {
	723	+ number: usize,
	724	+ repetition: usize,
	725	+ status: suite.TestStatus,
	726	+ duration_ns: u64,
	727	+ error_message: ?[]const u8,
645	728	};
646	729
647	730	pub fn init(allocator: std.mem.Allocator, output_file: []const u8) Self {
		⋯68 linesShow the 68 lines above this hunk@@ -716,11 +799,27 @@ pub const JUnitReporter = struct {
716	799	else
717	800	null;
718	801
	802	+ const attempts = try self.allocator.alloc(AttemptResult, test_case.attempts.items.len);
	803	+ errdefer self.allocator.free(attempts);
	804	+ for (test_case.attempts.items, 0..) |attempt, index| {
	805	+ attempts[index] = .{
	806	+ .number = attempt.number,
	807	+ .repetition = attempt.repetition,
	808	+ .status = attempt.status,
	809	+ .duration_ns = attempt.duration_ns,
	810	+ .error_message = if (attempt.error_message) |message|
	811	+ try self.allocator.dupe(u8, message)
	812	+ else
	813	+ null,
	814	+ };
	815	+ }
	816	+
719	817	const test_result = TestCaseResult{
720	818	.name = try self.allocator.dupe(u8, test_case.name),
721	819	.time = time_seconds,
722	820	.status = test_case.status,
723	821	.error_message = error_msg,
	822	+ .attempts = attempts,
724	823	};
725	824
726	825	try test_suite.tests.append(self.allocator, test_result);
		⋯5 linesShow the 5 lines above this hunk@@ -732,10 +831,11 @@ pub const JUnitReporter = struct {
732	831	defer buffer.deinit(self.allocator);
733	832
734	833	try buffer.appendSlice(self.allocator, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
735		- try buffer.print(self.allocator, "<testsuites tests=\"{d}\" failures=\"{d}\" skipped=\"{d}\">\n", .{
	834	+ try buffer.print(self.allocator, "<testsuites tests=\"{d}\" failures=\"{d}\" skipped=\"{d}\" flaky=\"{d}\">\n", .{
736	835	results.total,
737	836	results.failed,
738	837	results.skipped,
	838	+ results.flaky,
739	839	});
740	840
741	841	for (self.suites.items) |suite_result| {
		⋯26 linesShow the 26 lines above this hunk@@ -768,11 +868,16 @@ pub const JUnitReporter = struct {
768	868	try buffer.print(self.allocator, " <failure message=\"{s}\"/>\n", .{
769	869	test_result.error_message orelse "Test failed",
770	870	});
	871	+ try writeAttemptHistory(&buffer, self.allocator, test_result.attempts);
771	872	try buffer.appendSlice(self.allocator, " </testcase>\n");
772	873	} else if (test_result.status == .skipped) {
773	874	try buffer.appendSlice(self.allocator, ">\n");
774	875	try buffer.appendSlice(self.allocator, " <skipped/>\n");
775	876	try buffer.appendSlice(self.allocator, " </testcase>\n");
	877	+ } else if (test_result.attempts.len > 1) {
	878	+ try buffer.appendSlice(self.allocator, ">\n");
	879	+ try writeAttemptHistory(&buffer, self.allocator, test_result.attempts);
	880	+ try buffer.appendSlice(self.allocator, " </testcase>\n");
776	881	} else {
777	882	try buffer.appendSlice(self.allocator, "/>\n");
778	883	}
		⋯6 linesShow the 6 lines above this hunk@@ -785,6 +890,23 @@ pub const JUnitReporter = struct {
785	890
786	891	try compat.writeFile(self.allocator, self.output_file, buffer.items);
787	892	}
	893	+
	894	+ fn writeAttemptHistory(
	895	+ buffer: *std.ArrayList(u8),
	896	+ allocator: std.mem.Allocator,
	897	+ attempts: []const AttemptResult,
	898	+ ) !void {
	899	+ try buffer.appendSlice(allocator, " <system-out>");
	900	+ for (attempts, 0..) |attempt, index| {
	901	+ if (index > 0) try buffer.appendSlice(allocator, "&#10;");
	902	+ try buffer.print(
	903	+ allocator,
	904	+ "attempt={d} repetition={d} status={s} duration_ns={d}",
	905	+ .{ attempt.number, attempt.repetition, @tagName(attempt.status), attempt.duration_ns },
	906	+ );
	907	+ }
	908	+ try buffer.appendSlice(allocator, "</system-out>\n");
	909	+ }
788	910	};
789	911
790	912	/// Owns every built-in reporter and selects one through a shared interface.