also looking at this
Add reproducible randomized test order
#29
18 files
+487
-42
| @@ -49,6 +49,8 @@ pub const CLIOptions = struct { | ||
| 49 | 49 | retries: usize = 0, |
| 50 | 50 | repeat: usize = 1, |
| 51 | 51 | fail_on_flaky: bool = false, |
| 52 | shuffle: bool = false, | |
| 53 | seed: ?u64 = null, | |
| 52 | 54 | }; |
| 53 | 55 | |
| 54 | 56 | pub const version = build_options.version; |
| @@ -82,6 +84,8 @@ pub const CLI = struct { | ||
| 82 | 84 | self.options.retries = config.test_options.retries; |
| 83 | 85 | self.options.repeat = config.test_options.repeat; |
| 84 | 86 | self.options.fail_on_flaky = config.test_options.fail_on_flaky; |
| 87 | self.options.shuffle = config.test_options.shuffle or config.test_options.seed != null; | |
| 88 | self.options.seed = config.test_options.seed; | |
| 85 | 89 | self.options.shard_index = config.sharding.index; |
| 86 | 90 | self.options.shard_count = config.sharding.count; |
| 87 | 91 | self.options.parallel = config.parallel.enabled; |
| @@ -140,6 +144,19 @@ pub const CLI = struct { | ||
| 140 | 144 | } |
| 141 | 145 | } else if (std.mem.eql(u8, arg, "--fail-on-flaky")) { |
| 142 | 146 | self.options.fail_on_flaky = true; |
| 147 | } else if (std.mem.eql(u8, arg, "--shuffle")) { | |
| 148 | self.options.shuffle = true; | |
| 149 | } else if (std.mem.eql(u8, arg, "--seed")) { | |
| 150 | if (i + 1 >= args.len) { | |
| 151 | std.debug.print("Error: --seed requires a value\n", .{}); | |
| 152 | return CLIError.MissingValue; | |
| 153 | } | |
| 154 | i += 1; | |
| 155 | self.options.seed = std.fmt.parseInt(u64, args[i], 10) catch { | |
| 156 | std.debug.print("Error: --seed must be a valid unsigned integer\n", .{}); | |
| 157 | return CLIError.InvalidArgument; | |
| 158 | }; | |
| 159 | self.options.shuffle = true; | |
| 143 | 160 | } else if (std.mem.eql(u8, arg, "--verbose")) { |
| 144 | 161 | self.options.verbose = true; |
| 145 | 162 | } else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) { |
| @@ -386,6 +403,8 @@ pub const CLI = struct { | ||
| 386 | 403 | \\ --retry <N> Retry each failed test up to N times |
| 387 | 404 | \\ --repeat <N> Require N successful runs of each selected test |
| 388 | 405 | \\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure |
| 406 | \\ --shuffle Randomize suite and test order | |
| 407 | \\ --seed <N> Reproduce a randomized order (implies --shuffle) | |
| 389 | 408 | \\ --filter <pattern> Run only tests matching pattern |
| 390 | 409 | \\ --grep <pattern> Same as --filter (alias) |
| 391 | 410 | \\ --reporter <name> Set reporter type (spec, dot, json, tap, junit) |
| @@ -452,6 +471,8 @@ pub const CLI = struct { | ||
| 452 | 471 | \\ zig-test --timeout 10000 Set 10s timeout for all tests |
| 453 | 472 | \\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests |
| 454 | 473 | \\ zig-test --repeat 100 --filter parser Stress-run selected tests |
| 474 | \\ zig-test --shuffle Run in a generated random order | |
| 475 | \\ zig-test --seed 12345 Replay a randomized order | |
| 455 | 476 | \\ zig-test --profile-memory --fail-on-leak |
| 456 | 477 | \\ zig-test --parallel --jobs 4 Run with 4 parallel workers |
| 457 | 478 | \\ zig-test --config zig-test.json Load config from file |
| @@ -482,6 +503,8 @@ pub const CLI = struct { | ||
| 482 | 503 | .retries = self.options.retries, |
| 483 | 504 | .repeat = self.options.repeat, |
| 484 | 505 | .fail_on_flaky = self.options.fail_on_flaky, |
| 506 | .shuffle = self.options.shuffle, | |
| 507 | .seed = self.options.seed, | |
| 485 | 508 | }; |
| 486 | 509 | } |
| 487 | 510 | }; |
| @@ -724,6 +747,17 @@ test "CLI parses retry repeat and flaky exit policy" { | ||
| 724 | 747 | try std.testing.expect(runner_options.fail_on_flaky); |
| 725 | 748 | } |
| 726 | 749 | |
| 750 | test "CLI parses shuffle and a seed implies it" { | |
| 751 | var cli = CLI.init(std.testing.allocator); | |
| 752 | try cli.parse(&.{ "zig-test", "--seed", "12345" }); | |
| 753 | try std.testing.expect(cli.options.shuffle); | |
| 754 | try std.testing.expectEqual(@as(?u64, 12345), cli.options.seed); | |
| 755 | ||
| 756 | const runner_options = cli.toRunnerOptions(); | |
| 757 | try std.testing.expect(runner_options.shuffle); | |
| 758 | try std.testing.expectEqual(@as(?u64, 12345), runner_options.seed); | |
| 759 | } | |
| 760 | ||
| 727 | 761 | test "CLI rejects a zero repeat count" { |
| 728 | 762 | var cli = CLI.init(std.testing.allocator); |
| 729 | 763 | try std.testing.expectError( |