ReviewOS

also looking at this

zig-utils/zig-test-framework

Add deterministic test sharding for distributed CI

#22
Merged glennmichael123 wants to merge codex/issue-4-deterministic-sharding into main
10 files +322 -5
src/config.zigmodified+25-0
Changes to src/config.zig
@@ -8,6 +8,7 @@ pub const ConfigFormat = enum { json };
88/// actually honor. Defaults intentionally match `CLIOptions`.
99pub const TestConfig = struct {
1010 test_options: TestOptions = .{},
11 sharding: ShardingOptions = .{},
1112 parallel: ParallelOptions = .{},
1213 reporter: ReporterOptions = .{},
1314 snapshot: SnapshotOptions = .{},
@@ -17,6 +18,11 @@ pub const TestConfig = struct {
1718 coverage: CoverageOptions = .{},
1819};
1920
21pub const ShardingOptions = struct {
22 index: ?usize = null,
23 count: ?usize = null,
24};
25
2026pub const TestOptions = struct {
2127 pattern: []const u8 = "*.test.zig",
2228 test_dir: []const u8 = ".",
@@ -64,6 +70,7 @@ pub const CoverageOptions = struct {
6470
6571const FileConfig = struct {
6672 @"test": TestOptions = .{},
73 sharding: ShardingOptions = .{},
6774 parallel: ParallelOptions = .{},
6875 reporter: ReporterOptions = .{},
6976 snapshot: SnapshotOptions = .{},
@@ -80,6 +87,7 @@ pub const ConfigError = error{
8087 InvalidPort,
8188 InvalidTimeout,
8289 InvalidDebounce,
90 InvalidShard,
8391};
8492
8593/// Owns strings parsed from configuration files until `deinit` is called.
@@ -116,6 +124,7 @@ pub const ConfigLoader = struct {
116124 );
117125 const config = TestConfig{
118126 .test_options = file_config.@"test",
127 .sharding = file_config.sharding,
119128 .parallel = file_config.parallel,
120129 .reporter = file_config.reporter,
121130 .snapshot = file_config.snapshot,
@@ -175,6 +184,13 @@ fn validate(config: TestConfig) !void {
175184 if (timeout == 0) return ConfigError.InvalidTimeout;
176185 }
177186 if (config.watch.debounce_ms == 0) return ConfigError.InvalidDebounce;
187 const has_shard_index = config.sharding.index != null;
188 const has_shard_count = config.sharding.count != null;
189 if (has_shard_index != has_shard_count) return ConfigError.InvalidShard;
190 if (config.sharding.count) |count| {
191 const index = config.sharding.index.?;
192 if (count == 0 or index == 0 or index > count) return ConfigError.InvalidShard;
193 }
178194}
179195
180196test "TestConfig defaults match CLI discovery defaults" {
@@ -183,6 +199,8 @@ test "TestConfig defaults match CLI discovery defaults" {
183199 try std.testing.expectEqualStrings(".", config.test_options.test_dir);
184200 try std.testing.expect(config.test_options.recursive);
185201 try std.testing.expect(config.test_options.timeout == null);
202 try std.testing.expect(config.sharding.index == null);
203 try std.testing.expect(config.sharding.count == null);
186204 try std.testing.expect(!config.parallel.enabled);
187205 try std.testing.expectEqualStrings("spec", config.reporter.reporter);
188206}
@@ -200,6 +218,11 @@ test "configuration validation rejects invalid runtime values" {
200218 invalid.reporter.reporter = "spec";
201219 invalid.parallel.jobs = 0;
202220 try std.testing.expectError(ConfigError.InvalidJobs, validate(invalid));
221 invalid.parallel.jobs = null;
222 invalid.sharding.index = 2;
223 try std.testing.expectError(ConfigError.InvalidShard, validate(invalid));
224 invalid.sharding.count = 1;
225 try std.testing.expectError(ConfigError.InvalidShard, validate(invalid));
203226}
204227
205228test "ConfigLoader loads supported JSON and rejects unknown keys" {
@@ -210,6 +233,8 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" {
210233 try std.testing.expectEqualStrings("tests/fixtures", config.test_options.test_dir);
211234 try std.testing.expectEqualStrings("selected test passes", config.test_options.filter.?);
212235 try std.testing.expect(!config.test_options.recursive);
236 try std.testing.expectEqual(@as(?usize, 1), config.sharding.index);
237 try std.testing.expectEqual(@as(?usize, 1), config.sharding.count);
213238
214239 try std.testing.expectError(
215240 error.UnknownField,