ReviewOS
Reload

also looking at this

zig-utils/zig-test-framework

Add retries, repeated runs, and flaky-test reporting

#25
Merged glennmichael123 wants to merge codex/issue-7-retries-repeat-flaky into main
15 files +719 -53
src/test_runner.zigmodified+216-23
@@ -19,6 +19,12 @@ pub const RunnerOptions = struct {
1919 n_jobs: ?usize = null, // Number of parallel jobs
2020 junit_output: []const u8 = "test-results.xml",
2121 timeout_ms: ?u64 = null,
22 /// Additional attempts after a failed execution.
23 retries: usize = 0,
24 /// Number of times every selected test is required to pass.
25 repeat: usize = 1,
26 /// Treat tests that pass after a retry as an unsuccessful run.
27 fail_on_flaky: bool = false,
2228 /// Optional output supplied by a CLI host. Embedded/test callers default
2329 /// to stderr so they do not interfere with Zig's stdout test protocol.
2430 reporter_writer: ?*std.Io.Writer = null,
@@ -75,7 +81,7 @@ pub const TestRunner = struct {
7581 }
7682
7783 // Run tests in parallel or sequential based on options
78 if (self.options.parallel) {
84 if (self.options.parallel and self.options.retries == 0 and self.options.repeat == 1) {
7985 const parallel_opts = parallel.ParallelOptions{
8086 .enabled = true,
8187 .n_jobs = self.options.n_jobs,
@@ -95,7 +101,8 @@ pub const TestRunner = struct {
95101 }
96102
97103 try reporters.flush();
98 return all_passed and self.results.failed == 0;
104 return all_passed and self.results.failed == 0 and
105 (!self.options.fail_on_flaky or self.results.flaky == 0);
99106 }
100107
101108 // Sequential execution (original behavior)
@@ -113,7 +120,8 @@ pub const TestRunner = struct {
113120 // Flush output
114121 try reporters.flush();
115122
116 return self.results.failed == 0;
123 return self.results.failed == 0 and
124 (!self.options.fail_on_flaky or self.results.flaky == 0);
117125 }
118126
119127 fn addSuiteResults(self: *Self, test_suite: *suite.TestSuite) !void {
@@ -189,52 +197,88 @@ pub const TestRunner = struct {
189197 fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
190198 try events.emit(.{ .test_started = test_case.name });
191199
200 for (test_case.attempts.items) |attempt| {
201 if (attempt.error_message) |message| self.allocator.free(message);
202 }
203 test_case.attempts.clearRetainingCapacity();
204 test_case.execution_time_ns = 0;
205 test_case.error_message = null;
206
207 const retries = test_case.retry_count orelse self.options.retries;
208 const repeat_count = @max(self.options.repeat, 1);
209 var attempt_number: usize = 0;
210 var had_failure = false;
211 var final_failure = false;
212
213 repetitions: for (1..repeat_count + 1) |repetition| {
214 for (1..retries + 2) |attempt_in_repetition| {
215 attempt_number += 1;
216 const passed = try self.runSingleAttempt(test_case, test_suite, repetition, attempt_number);
217 if (passed) break;
218
219 had_failure = true;
220 if (attempt_in_repetition == retries + 1) {
221 final_failure = true;
222 break :repetitions;
223 }
224 }
225 }
226
227 if (!final_failure) {
228 test_case.status = if (had_failure) .flaky else .passed;
229 test_case.error_message = null;
230 }
231
232 try self.results.addTest(test_case);
233 try events.emit(.{ .test_finished = test_case });
234 }
235
236 fn runSingleAttempt(
237 self: *Self,
238 test_case: *suite.TestCase,
239 test_suite: *suite.TestSuite,
240 repetition: usize,
241 attempt_number: usize,
242 ) !bool {
192243 test_case.status = .running;
244 test_case.error_message = null;
193245 const start_time = compat.nanoTimestamp();
194246
195 // Get all beforeEach hooks (including parent hooks)
196247 var before_hooks = try test_suite.getAllBeforeEachHooks(self.allocator);
197248 defer before_hooks.deinit(self.allocator);
198249
199 // Run beforeEach hooks
200250 var before_failed = false;
201251 for (before_hooks.items) |hook| {
202252 hook(self.allocator) catch |err| {
203253 test_case.status = .failed;
204 const err_msg = try std.fmt.allocPrint(self.allocator, "beforeEach hook failed: {any}", .{err});
205 test_case.error_message = err_msg;
254 test_case.error_message = try std.fmt.allocPrint(
255 self.allocator,
256 "beforeEach hook failed: {any}",
257 .{err},
258 );
206259 before_failed = true;
207260 break;
208261 };
209262 }
210263
211 // Run the actual test if beforeEach succeeded
212264 if (!before_failed) {
213265 test_case.test_fn(self.allocator) catch |err| {
214266 test_case.status = .failed;
215 const err_msg = try std.fmt.allocPrint(self.allocator, "{any}", .{err});
216 test_case.error_message = err_msg;
267 test_case.error_message = try std.fmt.allocPrint(self.allocator, "{any}", .{err});
217268 };
218
219 if (test_case.status == .running) {
220 test_case.status = .passed;
221 }
269 if (test_case.status == .running) test_case.status = .passed;
222270 }
223271
224 // Get all afterEach hooks (including parent hooks)
225272 var after_hooks = try test_suite.getAllAfterEachHooks(self.allocator);
226273 defer after_hooks.deinit(self.allocator);
227
228 // Run afterEach hooks (always run, even if test failed)
229274 for (after_hooks.items) |hook| {
230275 hook(self.allocator) catch |err| {
231276 std.debug.print("afterEach hook failed: {any}\n", .{err});
232277 };
233278 }
234279
235 const end_time = compat.nanoTimestamp();
236 test_case.execution_time_ns = @intCast(end_time - start_time);
237 if (test_case.status == .passed and self.policy().timedOut(test_case.execution_time_ns)) {
280 const duration_ns: u64 = @intCast(compat.nanoTimestamp() - start_time);
281 if (test_case.status == .passed and self.policy().timedOut(duration_ns)) {
238282 test_case.status = .failed;
239283 test_case.error_message = try std.fmt.allocPrint(
240284 self.allocator,
@@ -242,9 +286,15 @@ pub const TestRunner = struct {
242286 .{self.options.timeout_ms.?},
243287 );
244288 }
245
246 try self.results.addTest(test_case);
247 try events.emit(.{ .test_finished = test_case });
289 test_case.execution_time_ns += duration_ns;
290 try test_case.attempts.append(self.allocator, .{
291 .number = attempt_number,
292 .repetition = repetition,
293 .status = test_case.status,
294 .duration_ns = duration_ns,
295 .error_message = test_case.error_message,
296 });
297 return test_case.status == .passed;
248298 }
249299
250300 /// Skip all tests in a suite
@@ -302,6 +352,9 @@ test "RunnerOptions default values" {
302352 try std.testing.expectEqual(ReporterType.spec, options.reporter_type);
303353 try std.testing.expectEqual(true, options.use_colors);
304354 try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
355 try std.testing.expectEqual(@as(usize, 0), options.retries);
356 try std.testing.expectEqual(@as(usize, 1), options.repeat);
357 try std.testing.expect(!options.fail_on_flaky);
305358}
306359
307360test "RunnerOptions custom values" {
@@ -317,3 +370,143 @@ test "RunnerOptions custom values" {
317370 try std.testing.expectEqual(ReporterType.json, options.reporter_type);
318371 try std.testing.expectEqual(false, options.use_colors);
319372}
373
374var flaky_fixture_runs: usize = 0;
375var failing_fixture_runs: usize = 0;
376var skipped_by_bail_runs: usize = 0;
377var repeated_fixture_runs: usize = 0;
378var timeout_fixture_runs: usize = 0;
379
380fn flakyFixture(_: std.mem.Allocator) !void {
381 flaky_fixture_runs += 1;
382 if (flaky_fixture_runs == 1) return error.TransientFailure;
383}
384
385fn failingFixture(_: std.mem.Allocator) !void {
386 failing_fixture_runs += 1;
387 return error.DeterministicFailure;
388}
389
390fn skippedByBailFixture(_: std.mem.Allocator) !void {
391 skipped_by_bail_runs += 1;
392}
393
394fn repeatedFixture(_: std.mem.Allocator) !void {
395 repeated_fixture_runs += 1;
396}
397
398fn timeoutFixture(_: std.mem.Allocator) !void {
399 timeout_fixture_runs += 1;
400 compat.sleep(std.time.ns_per_ms);
401}
402
403test "retries record deterministic flaky attempt history" {
404 flaky_fixture_runs = 0;
405 const allocator = std.testing.allocator;
406 var registry = suite.TestRegistry.init(allocator);
407 defer registry.deinit();
408 const test_suite = try suite.TestSuite.init(allocator, "retry suite");
409 try test_suite.addTest(suite.TestCase.init("flaky fixture", flakyFixture).withRetries(1));
410 try registry.registerSuite(test_suite);
411
412 var output: [4096]u8 = undefined;
413 var writer: std.Io.Writer = .fixed(&output);
414 var runner = TestRunner.init(allocator, &registry, .{
415 .reporter_writer = &writer,
416 .use_colors = false,
417 });
418 defer runner.deinit();
419
420 try std.testing.expect(try runner.run());
421 const test_case = &test_suite.tests.items[0];
422 try std.testing.expectEqual(suite.TestStatus.flaky, test_case.status);
423 try std.testing.expectEqual(@as(usize, 2), test_case.attempts.items.len);
424 try std.testing.expectEqual(suite.TestStatus.failed, test_case.attempts.items[0].status);
425 try std.testing.expectEqual(suite.TestStatus.passed, test_case.attempts.items[1].status);
426 try std.testing.expectEqual(@as(usize, 1), runner.results.flaky);
427
428 var strict_runner = TestRunner.init(allocator, &registry, .{
429 .retries = 1,
430 .fail_on_flaky = true,
431 .reporter_writer = &writer,
432 .use_colors = false,
433 });
434 defer strict_runner.deinit();
435 flaky_fixture_runs = 0;
436 try std.testing.expect(!try strict_runner.run());
437}
438
439test "repeat mode runs every required passing repetition" {
440 repeated_fixture_runs = 0;
441 const allocator = std.testing.allocator;
442 var registry = suite.TestRegistry.init(allocator);
443 defer registry.deinit();
444 const test_suite = try suite.TestSuite.init(allocator, "repeat suite");
445 try test_suite.addTest(suite.TestCase.init("repeated fixture", repeatedFixture));
446 try registry.registerSuite(test_suite);
447
448 var output: [4096]u8 = undefined;
449 var writer: std.Io.Writer = .fixed(&output);
450 var runner = TestRunner.init(allocator, &registry, .{
451 .repeat = 3,
452 .reporter_writer = &writer,
453 .use_colors = false,
454 });
455 defer runner.deinit();
456
457 try std.testing.expect(try runner.run());
458 try std.testing.expectEqual(@as(usize, 3), repeated_fixture_runs);
459 try std.testing.expectEqual(@as(usize, 3), test_suite.tests.items[0].attempts.items.len);
460}
461
462test "bail waits until retries are exhausted" {
463 failing_fixture_runs = 0;
464 skipped_by_bail_runs = 0;
465 const allocator = std.testing.allocator;
466 var registry = suite.TestRegistry.init(allocator);
467 defer registry.deinit();
468 const test_suite = try suite.TestSuite.init(allocator, "bail suite");
469 try test_suite.addTest(suite.TestCase.init("always fails", failingFixture));
470 try test_suite.addTest(suite.TestCase.init("must be skipped", skippedByBailFixture));
471 try registry.registerSuite(test_suite);
472
473 var output: [4096]u8 = undefined;
474 var writer: std.Io.Writer = .fixed(&output);
475 var runner = TestRunner.init(allocator, &registry, .{
476 .bail = true,
477 .retries = 2,
478 .reporter_writer = &writer,
479 .use_colors = false,
480 });
481 defer runner.deinit();
482
483 try std.testing.expect(!try runner.run());
484 try std.testing.expectEqual(@as(usize, 3), failing_fixture_runs);
485 try std.testing.expectEqual(@as(usize, 0), skipped_by_bail_runs);
486 try std.testing.expectEqual(@as(usize, 3), test_suite.tests.items[0].attempts.items.len);
487}
488
489test "timeouts are evaluated for every retry attempt" {
490 timeout_fixture_runs = 0;
491 const allocator = std.testing.allocator;
492 var registry = suite.TestRegistry.init(allocator);
493 defer registry.deinit();
494 const test_suite = try suite.TestSuite.init(allocator, "timeout suite");
495 try test_suite.addTest(suite.TestCase.init("times out", timeoutFixture));
496 try registry.registerSuite(test_suite);
497
498 var output: [4096]u8 = undefined;
499 var writer: std.Io.Writer = .fixed(&output);
500 var runner = TestRunner.init(allocator, &registry, .{
501 .timeout_ms = 0,
502 .retries = 1,
503 .reporter_writer = &writer,
504 .use_colors = false,
505 });
506 defer runner.deinit();
507
508 try std.testing.expect(!try runner.run());
509 try std.testing.expectEqual(@as(usize, 2), timeout_fixture_runs);
510 try std.testing.expectEqual(@as(usize, 2), test_suite.tests.items[0].attempts.items.len);
511 try std.testing.expectEqual(suite.TestStatus.failed, test_suite.tests.items[0].status);
512}