Add retries, repeated runs, and flaky-test reporting

tests/reporter_test.zigmodified+42-0

Changes to tests/reporter_test.zig

		⋯67 linesShow the 67 lines above this hunk@@ -68,6 +68,48 @@ test "reporter json output" {
68	68	const output = buffer[0..reporter.writer.end];
69	69	try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null);
70	70	try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null);
	71	+ var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, output, .{});
	72	+ defer parsed.deinit();
	73	+}
	74	+
	75	+test "reporter json output includes flaky attempt history" {
	76	+ const allocator = std.testing.allocator;
	77	+ var buffer: [4096]u8 = undefined;
	78	+ const writer: std.Io.Writer = .fixed(&buffer);
	79	+ var reporter = ztf.JsonReporter.init(allocator, writer);
	80	+ defer reporter.deinit();
	81	+ var results = ztf.TestResults.init(allocator);
	82	+ defer results.deinit();
	83	+ results.total = 1;
	84	+ results.flaky = 1;
	85	+ var test_case = ztf.TestCase.init("eventually passes", passingTest);
	86	+ defer test_case.attempts.deinit(allocator);
	87	+ test_case.status = .flaky;
	88	+ try test_case.attempts.append(allocator, .{
	89	+ .number = 1,
	90	+ .repetition = 1,
	91	+ .status = .failed,
	92	+ .duration_ns = 10,
	93	+ });
	94	+ try test_case.attempts.append(allocator, .{
	95	+ .number = 2,
	96	+ .repetition = 1,
	97	+ .status = .passed,
	98	+ .duration_ns = 5,
	99	+ });
	100	+
	101	+ try reporter.reporter.onRunStart(1);
	102	+ try reporter.reporter.onSuiteStart("suite");
	103	+ try reporter.reporter.onTestEnd(&test_case);
	104	+ try reporter.reporter.onSuiteEnd("suite");
	105	+ try reporter.reporter.onRunEnd(&results);
	106	+
	107	+ const output = buffer[0..reporter.writer.end];
	108	+ try std.testing.expect(std.mem.indexOf(u8, output, "\"status\":\"flaky\"") != null);
	109	+ try std.testing.expect(std.mem.indexOf(u8, output, "\"attempts\":[") != null);
	110	+ try std.testing.expect(std.mem.indexOf(u8, output, "\"durationNs\":10") != null);
	111	+ var parsed = try std.json.parseFromSlice(std.json.Value, allocator, output, .{});
	112	+ defer parsed.deinit();
71	113	}
72	114
73	115	test "reporter set selects the same built-ins for every executor" {