also looking at this
Add deterministic test sharding for distributed CI
#22
10 files
+322
-5
| @@ -1,5 +1,6 @@ | ||
| 1 | 1 | const std = ; |
| 2 | 2 | const discovery = ; |
| 3 | const sharding = ; | |
| 3 | 4 | const coverage = ; |
| 4 | 5 | const ui_server = ; |
| 5 | 6 | const compat = ; |
| @@ -14,6 +15,8 @@ pub const LoaderOptions = struct { | ||
| 14 | 15 | verbose: bool = false, |
| 15 | 16 | /// Whether child Zig test processes may emit colors |
| 16 | 17 | use_colors: bool = true, |
| 18 | /// Optional one-based file shard to execute | |
| 19 | shard: ?sharding.ShardOptions = null, | |
| 17 | 20 | /// Coverage options |
| 18 | 21 | coverage_options: ?coverage.CoverageOptions = null, |
| 19 | 22 | /// UI server for real-time updates |
| @@ -31,8 +34,17 @@ pub fn runDiscoveredTests( | ||
| 31 | 34 | return false; |
| 32 | 35 | } |
| 33 | 36 | |
| 34 | std.debug.print("Found {} test file(s):\n", .{discovered.files.items.len}); | |
| 37 | const selected_files = try selectedFileCount(discovered, options.shard); | |
| 38 | if (options.shard) |shard| { | |
| 39 | std.debug.print( | |
| 40 | "Shard {}/{}: selected {} of {} test file(s).\n", | |
| 41 | .{ shard.index, shard.count, selected_files, discovered.files.items.len }, | |
| 42 | ); | |
| 43 | } | |
| 44 | ||
| 45 | std.debug.print("Found {} test file(s):\n", .{selected_files}); | |
| 35 | 46 | for (discovered.files.items) |file| { |
| 47 | if (!try fileSelected(file.relative_path, options.shard)) continue; | |
| 36 | 48 | std.debug.print(" - {s}\n", .{file.relative_path}); |
| 37 | 49 | } |
| 38 | 50 | std.debug.print("\n", .{}); |
| @@ -40,7 +52,7 @@ pub fn runDiscoveredTests( | ||
| 40 | 52 | // Notify UI of run start |
| 41 | 53 | if (options.ui_server) |server| { |
| 42 | 54 | var buffer: [256]u8 = undefined; |
| 43 | const json = try std.fmt.bufPrint(&buffer, "", .{discovered.files.items.len}); | |
| 55 | const json = try std.fmt.bufPrint(&buffer, "", .{selected_files}); | |
| 44 | 56 | try server.broadcast("run_start", json); |
| 45 | 57 | } |
| 46 | 58 | |
| @@ -57,6 +69,8 @@ pub fn runDiscoveredTests( | ||
| 57 | 69 | } |
| 58 | 70 | |
| 59 | 71 | for (discovered.files.items) |file| { |
| 72 | if (!try fileSelected(file.relative_path, options.shard)) continue; | |
| 73 | ||
| 60 | 74 | // Run each test file using zig test command |
| 61 | 75 | std.debug.print("Running {s}...\n", .{file.relative_path}); |
| 62 | 76 | |
| @@ -102,7 +116,7 @@ pub fn runDiscoveredTests( | ||
| 102 | 116 | |
| 103 | 117 | // Print coverage summary if enabled |
| 104 | 118 | if (options.coverage_options) |cov_opts| { |
| 105 | if (cov_opts.enabled) { | |
| 119 | if (cov_opts.enabled and files_run > 0) { | |
| 106 | 120 | std.debug.print("\n", .{}); |
| 107 | 121 | const cov_result = coverage.parseCoverageReport(allocator, cov_opts.output_dir) catch |err| { |
| 108 | 122 | std.debug.print("Warning: Could not parse coverage report: {any}\n", .{err}); |
| @@ -122,6 +136,10 @@ pub fn runDiscoveredTests( | ||
| 122 | 136 | |
| 123 | 137 | std.debug.print("\n", .{}); |
| 124 | 138 | std.debug.print("Test Summary:\n", .{}); |
| 139 | if (options.shard) |shard| { | |
| 140 | std.debug.print(" Shard: {}/{}\n", .{ shard.index, shard.count }); | |
| 141 | std.debug.print(" Selected: {} of {} files\n", .{ selected_files, discovered.files.items.len }); | |
| 142 | } | |
| 125 | 143 | std.debug.print(" Files run: {}\n", .{files_run}); |
| 126 | 144 | std.debug.print(" Passed: {}\n", .{total_passed}); |
| 127 | 145 | std.debug.print(" Failed: {}\n", .{total_failed}); |
| @@ -129,6 +147,21 @@ pub fn runDiscoveredTests( | ||
| 129 | 147 | return total_failed == 0; |
| 130 | 148 | } |
| 131 | 149 | |
| 150 | fn fileSelected(relative_path: []const u8, shard: ?sharding.ShardOptions) !bool { | |
| 151 | return if (shard) |selection| selection.includes(relative_path) else true; | |
| 152 | } | |
| 153 | ||
| 154 | fn selectedFileCount( | |
| 155 | discovered: *const discovery.DiscoveryResult, | |
| 156 | shard: ?sharding.ShardOptions, | |
| 157 | ) !usize { | |
| 158 | var count: usize = 0; | |
| 159 | for (discovered.files.items) |file| { | |
| 160 | if (try fileSelected(file.relative_path, shard)) count += 1; | |
| 161 | } | |
| 162 | return count; | |
| 163 | } | |
| 164 | ||
| 132 | 165 | /// Run a single test file using `zig test` |
| 133 | 166 | fn runTestFile( |
| 134 | 167 | allocator: std.mem.Allocator, |
| @@ -207,6 +240,7 @@ test "LoaderOptions default values" { | ||
| 207 | 240 | try std.testing.expectEqual(@as(?[]const u8, null), options.filter); |
| 208 | 241 | try std.testing.expectEqual(false, options.verbose); |
| 209 | 242 | try std.testing.expectEqual(true, options.use_colors); |
| 243 | try std.testing.expectEqual(@as(?sharding.ShardOptions, null), options.shard); | |
| 210 | 244 | try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options); |
| 211 | 245 | } |
| 212 | 246 | |
| @@ -256,3 +290,24 @@ test "LoaderOptions with coverage disabled" { | ||
| 256 | 290 | |
| 257 | 291 | try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options); |
| 258 | 292 | } |
| 293 | ||
| 294 | test "file sharding partitions discovered files exactly once" { | |
| 295 | const allocator = std.testing.allocator; | |
| 296 | var discovered = discovery.DiscoveryResult.init(allocator); | |
| 297 | defer discovered.deinit(); | |
| 298 | ||
| 299 | try discovered.addFile("tests/api.test.zig", "api.test.zig", "api.test.zig"); | |
| 300 | try discovered.addFile("tests/db.test.zig", "db.test.zig", "db.test.zig"); | |
| 301 | try discovered.addFile("tests/math.test.zig", "math.test.zig", "math.test.zig"); | |
| 302 | try discovered.addFile("tests/string.test.zig", "string.test.zig", "string.test.zig"); | |
| 303 | ||
| 304 | var total_selected: usize = 0; | |
| 305 | for (1..4) |index| { | |
| 306 | total_selected += try selectedFileCount( | |
| 307 | &discovered, | |
| 308 | .{ .index = index, .count = 3 }, | |
| 309 | ); | |
| 310 | } | |
| 311 | ||
| 312 | try std.testing.expectEqual(discovered.files.items.len, total_selected); | |
| 313 | } | |