| @@ -3,205 +3,505 @@ const suite = ; |
| 3 | 3 | const reporter = ; |
| 4 | 4 | const compat = ; |
| 5 | 5 | |
| 6 | | -/// Options for parallel test execution |
| 6 | +/// Options for bounded parallel test execution. |
| 7 | 7 | pub const ParallelOptions = struct { |
| 8 | | - /// Number of worker threads (null = auto-detect CPU count) |
| 8 | + /// Exact maximum worker count. Null uses the logical CPU count, falling |
| 9 | + /// back to one worker when the platform cannot report it. |
| 9 | 10 | n_jobs: ?usize = null, |
| 10 | | - /// Whether parallel execution is enabled |
| 11 | 11 | enabled: bool = false, |
| 12 | + filter: ?[]const u8 = null, |
| 12 | 13 | }; |
| 13 | 14 | |
| 14 | | -/// Result from running a test in parallel |
| 15 | | -pub const TestResult = struct { |
| 16 | | - test_case: *suite.TestCase, |
| 17 | | - suite_name: []const u8, |
| 18 | | - success: bool, |
| 19 | | - mutex: compat.Mutex = .{}, |
| 15 | +pub const ParallelError = error{ |
| 16 | + ParallelNotEnabled, |
| 17 | + InvalidJobCount, |
| 20 | 18 | }; |
| 21 | 19 | |
| 22 | | -/// Context for parallel test execution |
| 23 | | -pub const ParallelContext = struct { |
| 24 | | - allocator: std.mem.Allocator, |
| 25 | | - reporter: *reporter.Reporter, |
| 26 | | - results: std.ArrayList(TestResult), |
| 27 | | - mutex: compat.Mutex = .{}, |
| 28 | | - |
| 29 | | - pub fn init(allocator: std.mem.Allocator, rep: *reporter.Reporter) ParallelContext { |
| 30 | | - return .{ |
| 31 | | - .allocator = allocator, |
| 32 | | - .reporter = rep, |
| 33 | | - .results = .empty, |
| 34 | | - }; |
| 20 | +pub fn resolveWorkerCount(n_jobs: ?usize) !usize { |
| 21 | + if (n_jobs) |jobs| { |
| 22 | + if (jobs == 0) return ParallelError.InvalidJobCount; |
| 23 | + return jobs; |
| 35 | 24 | } |
| 25 | + return @max(std.Thread.getCpuCount() catch 1, 1); |
| 26 | +} |
| 36 | 27 | |
| 37 | | - pub fn deinit(self: *ParallelContext) void { |
| 38 | | - self.results.deinit(self.allocator); |
| 28 | +const WorkerContext = struct { |
| 29 | + allocator: std.mem.Allocator, |
| 30 | + test_suite: *suite.TestSuite, |
| 31 | + tests: []const *suite.TestCase, |
| 32 | + next_index: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), |
| 33 | + error_mutex: compat.Mutex = .{}, |
| 34 | + |
| 35 | + fn worker(self: *WorkerContext) void { |
| 36 | + while (true) { |
| 37 | + const index = self.next_index.fetchAdd(1, .monotonic); |
| 38 | + if (index >= self.tests.len) return; |
| 39 | + self.runTest(self.tests[index]); |
| 40 | + } |
| 39 | 41 | } |
| 40 | | -}; |
| 41 | 42 | |
| 42 | | -/// Task for executing a single test |
| 43 | | -pub const TestTask = struct { |
| 44 | | - context: *ParallelContext, |
| 45 | | - test_case: *suite.TestCase, |
| 46 | | - suite_name: []const u8, |
| 47 | | - |
| 48 | | - pub fn run(self: *TestTask) void { |
| 49 | | - // Execute the test |
| 50 | | - const success = self.executeTest(); |
| 51 | | - |
| 52 | | - // Store result in thread-safe manner |
| 53 | | - self.context.mutex.lock(); |
| 54 | | - defer self.context.mutex.unlock(); |
| 55 | | - |
| 56 | | - self.context.results.append(self.context.allocator, .{ |
| 57 | | - .test_case = self.test_case, |
| 58 | | - .suite_name = self.suite_name, |
| 59 | | - .success = success, |
| 60 | | - }) catch |err| { |
| 61 | | - std.debug.print("Error appending result: {any}\n", .{err}); |
| 43 | + fn runTest(self: *WorkerContext, test_case: *suite.TestCase) void { |
| 44 | + test_case.status = .running; |
| 45 | + const start_time = compat.nanoTimestamp(); |
| 46 | + |
| 47 | + var before_hooks = self.test_suite.getAllBeforeEachHooks(self.allocator) catch { |
| 48 | + self.fail(test_case, "could not collect beforeEach hooks", .{}); |
| 49 | + self.finishTiming(test_case, start_time); |
| 50 | + return; |
| 62 | 51 | }; |
| 63 | | - } |
| 52 | + defer before_hooks.deinit(self.allocator); |
| 53 | + |
| 54 | + var before_failed = false; |
| 55 | + for (before_hooks.items) |hook| { |
| 56 | + hook(self.allocator) catch |err| { |
| 57 | + self.fail(test_case, "beforeEach hook failed: {any}", .{err}); |
| 58 | + before_failed = true; |
| 59 | + break; |
| 60 | + }; |
| 61 | + } |
| 64 | 62 | |
| 65 | | - fn executeTest(self: *TestTask) bool { |
| 66 | | - self.test_case.status = .running; |
| 63 | + if (!before_failed) { |
| 64 | + test_case.test_fn(self.allocator) catch |err| { |
| 65 | + self.fail(test_case, "{any}", .{err}); |
| 66 | + }; |
| 67 | + if (test_case.status == .running) test_case.status = .passed; |
| 68 | + } |
| 67 | 69 | |
| 68 | | - // Call the test function |
| 69 | | - self.test_case.test_fn(self.context.allocator) catch |err| { |
| 70 | | - self.test_case.status = .failed; |
| 71 | | - const err_msg = std.fmt.allocPrint( |
| 72 | | - self.context.allocator, |
| 73 | | - "Test failed: {any}", |
| 74 | | - .{err}, |
| 75 | | - ) catch "Out of memory"; |
| 76 | | - self.test_case.error_message = err_msg; |
| 77 | | - return false; |
| 70 | + var after_hooks = self.test_suite.getAllAfterEachHooks(self.allocator) catch { |
| 71 | + self.fail(test_case, "could not collect afterEach hooks", .{}); |
| 72 | + self.finishTiming(test_case, start_time); |
| 73 | + return; |
| 78 | 74 | }; |
| 75 | + defer after_hooks.deinit(self.allocator); |
| 76 | + for (after_hooks.items) |hook| { |
| 77 | + hook(self.allocator) catch |err| { |
| 78 | + self.fail(test_case, "afterEach hook failed: {any}", .{err}); |
| 79 | + break; |
| 80 | + }; |
| 81 | + } |
| 79 | 82 | |
| 80 | | - self.test_case.status = .passed; |
| 81 | | - return true; |
| 83 | + self.finishTiming(test_case, start_time); |
| 84 | + } |
| 85 | + |
| 86 | + fn fail(self: *WorkerContext, test_case: *suite.TestCase, comptime format: []const u8, args: anytype) void { |
| 87 | + test_case.status = .failed; |
| 88 | + self.error_mutex.lock(); |
| 89 | + defer self.error_mutex.unlock(); |
| 90 | + test_case.error_message = std.fmt.allocPrint(self.allocator, format, args) catch "Out of memory"; |
| 91 | + } |
| 92 | + |
| 93 | + fn finishTiming(_: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void { |
| 94 | + test_case.execution_time_ns = @intCast(compat.nanoTimestamp() - start_time); |
| 82 | 95 | } |
| 83 | 96 | }; |
| 84 | 97 | |
| 85 | | -/// Run tests in parallel using thread pool |
| 86 | | -pub fn runTestsParallel( |
| 98 | +const RunContext = struct { |
| 87 | 99 | allocator: std.mem.Allocator, |
| 88 | | - test_registry: *suite.TestRegistry, |
| 100 | + registry: *suite.TestRegistry, |
| 89 | 101 | rep: *reporter.Reporter, |
| 90 | 102 | options: ParallelOptions, |
| 91 | | -) !bool { |
| 92 | | - if (!options.enabled) { |
| 93 | | - return error.ParallelNotEnabled; |
| 94 | | - } |
| 103 | + worker_limit: usize, |
| 95 | 104 | |
| 96 | | - // Create parallel context |
| 97 | | - var context = ParallelContext.init(allocator, rep); |
| 98 | | - defer context.deinit(); |
| 105 | + fn runSuite(self: *RunContext, test_suite: *suite.TestSuite) !bool { |
| 106 | + try self.rep.onSuiteStart(test_suite.name); |
| 99 | 107 | |
| 100 | | - // Count total tests |
| 101 | | - var total_tests: usize = 0; |
| 102 | | - for (test_registry.root_suites.items) |test_suite| { |
| 103 | | - total_tests += test_suite.tests.items.len; |
| 104 | | - } |
| 105 | | - |
| 106 | | - try rep.onRunStart(total_tests); |
| 108 | + if (test_suite.shouldSkip()) { |
| 109 | + try self.reportSkippedSuite(test_suite); |
| 110 | + try self.rep.onSuiteEnd(test_suite.name); |
| 111 | + return true; |
| 112 | + } |
| 107 | 113 | |
| 108 | | - // Collect tasks and spawn threads |
| 109 | | - var threads = std.ArrayList(std.Thread).empty; |
| 110 | | - defer threads.deinit(allocator); |
| 114 | + test_suite.runBeforeAllHooks(self.allocator) catch |err| { |
| 115 | + std.debug.print("beforeAll hook failed: {any}\n", .{err}); |
| 116 | + try self.reportSkippedSuite(test_suite); |
| 117 | + try self.rep.onSuiteEnd(test_suite.name); |
| 118 | + return true; |
| 119 | + }; |
| 111 | 120 | |
| 112 | | - for (test_registry.root_suites.items) |test_suite| { |
| 113 | | - try rep.onSuiteStart(test_suite.name); |
| 121 | + var scheduled: std.ArrayList(*suite.TestCase) = .empty; |
| 122 | + defer scheduled.deinit(self.allocator); |
| 114 | 123 | |
| 115 | 124 | for (test_suite.tests.items) |*test_case| { |
| 116 | | - if (test_case.skip) { |
| 125 | + if (!self.shouldRun(test_suite, test_case)) { |
| 117 | 126 | test_case.status = .skipped; |
| 118 | 127 | continue; |
| 119 | 128 | } |
| 129 | + try self.rep.onTestStart(test_case.name); |
| 130 | + try scheduled.append(self.allocator, test_case); |
| 131 | + } |
| 120 | 132 | |
| 121 | | - // Create task |
| 122 | | - const task = try allocator.create(TestTask); |
| 123 | | - task.* = .{ |
| 124 | | - .context = &context, |
| 125 | | - .test_case = test_case, |
| 126 | | - .suite_name = test_suite.name, |
| 127 | | - }; |
| 133 | + try self.runBatch(test_suite, scheduled.items); |
| 134 | + |
| 135 | + var all_passed = true; |
| 136 | + for (test_suite.tests.items) |*test_case| { |
| 137 | + try self.rep.onTestEnd(test_case); |
| 138 | + if (test_case.status == .failed) all_passed = false; |
| 139 | + } |
| 128 | 140 | |
| 129 | | - // Spawn thread for task |
| 130 | | - const thread = try std.Thread.spawn(.{}, runTestTask, .{task}); |
| 131 | | - try threads.append(allocator, thread); |
| 141 | + for (test_suite.suites.items) |nested_suite| { |
| 142 | + if (!try self.runSuite(nested_suite)) all_passed = false; |
| 132 | 143 | } |
| 144 | + |
| 145 | + test_suite.runAfterAllHooks(self.allocator) catch |err| { |
| 146 | + std.debug.print("afterAll hook failed: {any}\n", .{err}); |
| 147 | + all_passed = false; |
| 148 | + }; |
| 149 | + |
| 150 | + try self.rep.onSuiteEnd(test_suite.name); |
| 151 | + return all_passed; |
| 133 | 152 | } |
| 134 | 153 | |
| 135 | | - // Wait for all threads to complete |
| 136 | | - for (threads.items) |thread| { |
| 137 | | - thread.join(); |
| 154 | + fn shouldRun(self: *RunContext, test_suite: *suite.TestSuite, test_case: *suite.TestCase) bool { |
| 155 | + if (test_case.skip) return false; |
| 156 | + if (self.registry.has_only and !test_case.only and !test_suite.hasOnly()) return false; |
| 157 | + if (self.options.filter) |filter| { |
| 158 | + if (std.mem.indexOf(u8, test_case.name, filter) == null) return false; |
| 159 | + } |
| 160 | + return true; |
| 138 | 161 | } |
| 139 | 162 | |
| 140 | | - // Report results (now that all tests are done) |
| 141 | | - var all_passed = true; |
| 142 | | - for (test_registry.root_suites.items) |test_suite| { |
| 163 | + fn runBatch(self: *RunContext, test_suite: *suite.TestSuite, tests: []const *suite.TestCase) !void { |
| 164 | + if (tests.len == 0) return; |
| 165 | + |
| 166 | + var worker_context = WorkerContext{ |
| 167 | + .allocator = self.allocator, |
| 168 | + .test_suite = test_suite, |
| 169 | + .tests = tests, |
| 170 | + }; |
| 171 | + const count = @min(self.worker_limit, tests.len); |
| 172 | + var threads: std.ArrayList(std.Thread) = .empty; |
| 173 | + defer threads.deinit(self.allocator); |
| 174 | + |
| 175 | + for (0..count) |_| { |
| 176 | + const thread = std.Thread.spawn(.{}, WorkerContext.worker, .{&worker_context}) catch |err| { |
| 177 | + for (threads.items) |started| started.join(); |
| 178 | + return err; |
| 179 | + }; |
| 180 | + threads.append(self.allocator, thread) catch |err| { |
| 181 | + thread.join(); |
| 182 | + for (threads.items) |started| started.join(); |
| 183 | + return err; |
| 184 | + }; |
| 185 | + } |
| 186 | + for (threads.items) |thread| thread.join(); |
| 187 | + } |
| 188 | + |
| 189 | + fn reportSkippedSuite(self: *RunContext, test_suite: *suite.TestSuite) !void { |
| 143 | 190 | for (test_suite.tests.items) |*test_case| { |
| 144 | | - try rep.onTestEnd(test_case); |
| 145 | | - if (test_case.status == .failed) { |
| 146 | | - all_passed = false; |
| 147 | | - } |
| 191 | + test_case.status = .skipped; |
| 192 | + try self.rep.onTestEnd(test_case); |
| 193 | + } |
| 194 | + for (test_suite.suites.items) |nested| { |
| 195 | + try self.rep.onSuiteStart(nested.name); |
| 196 | + try self.reportSkippedSuite(nested); |
| 197 | + try self.rep.onSuiteEnd(nested.name); |
| 148 | 198 | } |
| 149 | | - try rep.onSuiteEnd(test_suite.name); |
| 150 | 199 | } |
| 200 | +}; |
| 151 | 201 | |
| 152 | | - // Create results summary |
| 153 | | - var results = reporter.TestResults.init(allocator); |
| 154 | | - defer results.deinit(); |
| 202 | +/// Run suites in declaration order while executing each suite's direct tests |
| 203 | +/// with a bounded worker set. Reporter callbacks are serialized and emitted in |
| 204 | +/// declaration order after each batch completes. |
| 205 | +pub fn runTestsParallel( |
| 206 | + allocator: std.mem.Allocator, |
| 207 | + test_registry: *suite.TestRegistry, |
| 208 | + rep: *reporter.Reporter, |
| 209 | + options: ParallelOptions, |
| 210 | +) !bool { |
| 211 | + if (!options.enabled) return ParallelError.ParallelNotEnabled; |
| 212 | + const worker_limit = try resolveWorkerCount(options.n_jobs); |
| 213 | + |
| 214 | + const total_tests = test_registry.countAllTests(); |
| 215 | + try rep.onRunStart(total_tests); |
| 155 | 216 | |
| 217 | + var context = RunContext{ |
| 218 | + .allocator = allocator, |
| 219 | + .registry = test_registry, |
| 220 | + .rep = rep, |
| 221 | + .options = options, |
| 222 | + .worker_limit = worker_limit, |
| 223 | + }; |
| 224 | + |
| 225 | + var all_passed = true; |
| 156 | 226 | for (test_registry.root_suites.items) |test_suite| { |
| 157 | | - for (test_suite.tests.items) |test_case| { |
| 158 | | - results.total += 1; |
| 159 | | - switch (test_case.status) { |
| 160 | | - .passed => results.passed += 1, |
| 161 | | - .failed => results.failed += 1, |
| 162 | | - .skipped => results.skipped += 1, |
| 163 | | - else => {}, |
| 164 | | - } |
| 165 | | - } |
| 227 | + if (!try context.runSuite(test_suite)) all_passed = false; |
| 166 | 228 | } |
| 167 | 229 | |
| 230 | + var results = reporter.TestResults.init(allocator); |
| 231 | + defer results.deinit(); |
| 232 | + for (test_registry.root_suites.items) |test_suite| { |
| 233 | + try addSuiteResults(&results, test_suite); |
| 234 | + } |
| 168 | 235 | try rep.onRunEnd(&results); |
| 169 | | - |
| 170 | | - return all_passed; |
| 236 | + return all_passed and results.failed == 0; |
| 171 | 237 | } |
| 172 | 238 | |
| 173 | | -fn runTestTask(task: *TestTask) void { |
| 174 | | - task.run(); |
| 239 | +fn addSuiteResults(results: *reporter.TestResults, test_suite: *suite.TestSuite) !void { |
| 240 | + for (test_suite.tests.items) |*test_case| try results.addTest(test_case); |
| 241 | + for (test_suite.suites.items) |nested| try addSuiteResults(results, nested); |
| 175 | 242 | } |
| 176 | 243 | |
| 177 | | -// Tests |
| 178 | | -test "ParallelOptions default values" { |
| 244 | +test "ParallelOptions defaults to CPU-based workers" { |
| 179 | 245 | const options = ParallelOptions{}; |
| 246 | + try std.testing.expect(options.n_jobs == null); |
| 247 | + try std.testing.expect(!options.enabled); |
| 248 | + try std.testing.expectEqual( |
| 249 | + @max(std.Thread.getCpuCount() catch 1, 1), |
| 250 | + try resolveWorkerCount(null), |
| 251 | + ); |
| 252 | +} |
| 253 | + |
| 254 | +test "configured worker count rejects zero" { |
| 255 | + try std.testing.expectEqual(@as(usize, 2), try resolveWorkerCount(2)); |
| 256 | + try std.testing.expectError(ParallelError.InvalidJobCount, resolveWorkerCount(0)); |
| 257 | +} |
| 258 | + |
| 259 | +var active_tests = std.atomic.Value(usize).init(0); |
| 260 | +var peak_tests = std.atomic.Value(usize).init(0); |
| 261 | + |
| 262 | +fn boundedTest(_: std.mem.Allocator) !void { |
| 263 | + const active = active_tests.fetchAdd(1, .monotonic) + 1; |
| 264 | + var peak = peak_tests.load(.monotonic); |
| 265 | + while (active > peak) { |
| 266 | + if (peak_tests.cmpxchgWeak(peak, active, .monotonic, .monotonic)) |observed| { |
| 267 | + peak = observed; |
| 268 | + } else break; |
| 269 | + } |
| 270 | + compat.sleep(20 * std.time.ns_per_ms); |
| 271 | + _ = active_tests.fetchSub(1, .monotonic); |
| 272 | +} |
| 273 | + |
| 274 | +fn passingTest(_: std.mem.Allocator) !void {} |
| 275 | + |
| 276 | +fn failingTest(_: std.mem.Allocator) !void { |
| 277 | + return error.ExpectedFailure; |
| 278 | +} |
| 279 | + |
| 280 | +var parent_before_each = std.atomic.Value(usize).init(0); |
| 281 | +var parent_after_each = std.atomic.Value(usize).init(0); |
| 282 | +var child_before_each = std.atomic.Value(usize).init(0); |
| 283 | +var child_after_each = std.atomic.Value(usize).init(0); |
| 284 | +var parent_before_all = std.atomic.Value(usize).init(0); |
| 285 | +var child_before_all = std.atomic.Value(usize).init(0); |
| 286 | +var parent_after_all = std.atomic.Value(usize).init(0); |
| 287 | +var child_after_all = std.atomic.Value(usize).init(0); |
| 288 | + |
| 289 | +fn parentBeforeEach(_: std.mem.Allocator) !void { |
| 290 | + _ = parent_before_each.fetchAdd(1, .monotonic); |
| 291 | +} |
| 292 | + |
| 293 | +fn parentAfterEach(_: std.mem.Allocator) !void { |
| 294 | + _ = parent_after_each.fetchAdd(1, .monotonic); |
| 295 | +} |
| 296 | + |
| 297 | +fn childBeforeEach(_: std.mem.Allocator) !void { |
| 298 | + _ = child_before_each.fetchAdd(1, .monotonic); |
| 299 | +} |
| 300 | + |
| 301 | +fn childAfterEach(_: std.mem.Allocator) !void { |
| 302 | + _ = child_after_each.fetchAdd(1, .monotonic); |
| 303 | +} |
| 304 | + |
| 305 | +fn parentBeforeAll(_: std.mem.Allocator) !void { |
| 306 | + _ = parent_before_all.fetchAdd(1, .monotonic); |
| 307 | +} |
| 308 | + |
| 309 | +fn childBeforeAll(_: std.mem.Allocator) !void { |
| 310 | + if (parent_before_all.load(.monotonic) != 1) return error.ParentBeforeAllMissing; |
| 311 | + _ = child_before_all.fetchAdd(1, .monotonic); |
| 312 | +} |
| 180 | 313 | |
| 181 | | - try std.testing.expectEqual(@as(?usize, null), options.n_jobs); |
| 182 | | - try std.testing.expectEqual(false, options.enabled); |
| 314 | +fn childAfterAll(_: std.mem.Allocator) !void { |
| 315 | + _ = child_after_all.fetchAdd(1, .monotonic); |
| 183 | 316 | } |
| 184 | 317 | |
| 185 | | -test "ParallelOptions with custom values" { |
| 186 | | - const options = ParallelOptions{ |
| 187 | | - .n_jobs = 4, |
| 318 | +fn parentAfterAll(_: std.mem.Allocator) !void { |
| 319 | + if (child_after_all.load(.monotonic) != 1) return error.ChildAfterAllMissing; |
| 320 | + _ = parent_after_all.fetchAdd(1, .monotonic); |
| 321 | +} |
| 322 | + |
| 323 | +const RecordingReporter = struct { |
| 324 | + reporter: reporter.Reporter, |
| 325 | + run_starts: usize = 0, |
| 326 | + run_ends: usize = 0, |
| 327 | + suite_starts: usize = 0, |
| 328 | + suite_ends: usize = 0, |
| 329 | + test_starts: usize = 0, |
| 330 | + ended_count: usize = 0, |
| 331 | + ended_names: [16][]const u8 = undefined, |
| 332 | + total: usize = 0, |
| 333 | + passed: usize = 0, |
| 334 | + failed: usize = 0, |
| 335 | + skipped: usize = 0, |
| 336 | + |
| 337 | + fn init(allocator: std.mem.Allocator) RecordingReporter { |
| 338 | + return .{ |
| 339 | + .reporter = .{ |
| 340 | + .vtable = &.{ |
| 341 | + .onRunStart = onRunStart, |
| 342 | + .onRunEnd = onRunEnd, |
| 343 | + .onSuiteStart = onSuiteStart, |
| 344 | + .onSuiteEnd = onSuiteEnd, |
| 345 | + .onTestStart = onTestStart, |
| 346 | + .onTestEnd = onTestEnd, |
| 347 | + }, |
| 348 | + .allocator = allocator, |
| 349 | + }, |
| 350 | + }; |
| 351 | + } |
| 352 | + |
| 353 | + fn self(rep: *reporter.Reporter) *RecordingReporter { |
| 354 | + return @fieldParentPtr("reporter", rep); |
| 355 | + } |
| 356 | + |
| 357 | + fn onRunStart(rep: *reporter.Reporter, _: usize) !void { |
| 358 | + self(rep).run_starts += 1; |
| 359 | + } |
| 360 | + |
| 361 | + fn onRunEnd(rep: *reporter.Reporter, results: *reporter.TestResults) !void { |
| 362 | + const recording = self(rep); |
| 363 | + recording.run_ends += 1; |
| 364 | + recording.total = results.total; |
| 365 | + recording.passed = results.passed; |
| 366 | + recording.failed = results.failed; |
| 367 | + recording.skipped = results.skipped; |
| 368 | + } |
| 369 | + |
| 370 | + fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void { |
| 371 | + self(rep).suite_starts += 1; |
| 372 | + } |
| 373 | + |
| 374 | + fn onSuiteEnd(rep: *reporter.Reporter, _: []const u8) !void { |
| 375 | + self(rep).suite_ends += 1; |
| 376 | + } |
| 377 | + |
| 378 | + fn onTestStart(rep: *reporter.Reporter, _: []const u8) !void { |
| 379 | + self(rep).test_starts += 1; |
| 380 | + } |
| 381 | + |
| 382 | + fn onTestEnd(rep: *reporter.Reporter, test_case: *const suite.TestCase) !void { |
| 383 | + const recording = self(rep); |
| 384 | + recording.ended_names[recording.ended_count] = test_case.name; |
| 385 | + recording.ended_count += 1; |
| 386 | + } |
| 387 | +}; |
| 388 | + |
| 389 | +fn verifyConcurrencyLimit(limit: ?usize) !void { |
| 390 | + const allocator = std.heap.smp_allocator; |
| 391 | + var registry = suite.TestRegistry.init(allocator); |
| 392 | + defer registry.deinit(); |
| 393 | + const test_suite = try suite.TestSuite.init(allocator, "bounded"); |
| 394 | + for (0..8) |_| try test_suite.addTest(suite.TestCase.init("work", boundedTest)); |
| 395 | + try registry.registerSuite(test_suite); |
| 396 | + |
| 397 | + active_tests.store(0, .monotonic); |
| 398 | + peak_tests.store(0, .monotonic); |
| 399 | + var recording = RecordingReporter.init(allocator); |
| 400 | + try std.testing.expect(try runTestsParallel(allocator, ®istry, &recording.reporter, .{ |
| 188 | 401 | .enabled = true, |
| 189 | | - }; |
| 402 | + .n_jobs = limit, |
| 403 | + })); |
| 190 | 404 | |
| 191 | | - try std.testing.expectEqual(@as(?usize, 4), options.n_jobs); |
| 192 | | - try std.testing.expectEqual(true, options.enabled); |
| 405 | + const resolved = try resolveWorkerCount(limit); |
| 406 | + try std.testing.expect(peak_tests.load(.monotonic) > 0); |
| 407 | + try std.testing.expect(peak_tests.load(.monotonic) <= @min(resolved, 8)); |
| 193 | 408 | } |
| 194 | 409 | |
| 195 | | -test "ParallelContext initialization" { |
| 196 | | - const allocator = std.testing.allocator; |
| 410 | +test "parallel executor bounds one worker" { |
| 411 | + try verifyConcurrencyLimit(1); |
| 412 | + try std.testing.expectEqual(@as(usize, 1), peak_tests.load(.monotonic)); |
| 413 | +} |
| 197 | 414 | |
| 198 | | - var rep = reporter.Reporter{ |
| 199 | | - .vtable = undefined, |
| 200 | | - .allocator = allocator, |
| 201 | | - }; |
| 415 | +test "parallel executor bounds two workers" { |
| 416 | + try verifyConcurrencyLimit(2); |
| 417 | + try std.testing.expectEqual(@as(usize, 2), peak_tests.load(.monotonic)); |
| 418 | +} |
| 419 | + |
| 420 | +test "parallel executor bounds default CPU workers" { |
| 421 | + try verifyConcurrencyLimit(null); |
| 422 | +} |
| 202 | 423 | |
| 203 | | - var context = ParallelContext.init(allocator, &rep); |
| 204 | | - defer context.deinit(); |
| 424 | +test "parallel reporting is serialized, ordered, and complete" { |
| 425 | + const allocator = std.heap.smp_allocator; |
| 426 | + var registry = suite.TestRegistry.init(allocator); |
| 427 | + defer registry.deinit(); |
| 428 | + const test_suite = try suite.TestSuite.init(allocator, "ordered"); |
| 429 | + try test_suite.addTest(suite.TestCase.init("first", passingTest)); |
| 430 | + try test_suite.addTest(suite.TestCase.init("second", failingTest)); |
| 431 | + var skipped = suite.TestCase.init("third", passingTest); |
| 432 | + skipped.skip = true; |
| 433 | + try test_suite.addTest(skipped); |
| 434 | + try registry.registerSuite(test_suite); |
| 435 | + |
| 436 | + var recording = RecordingReporter.init(allocator); |
| 437 | + try std.testing.expect(!try runTestsParallel(allocator, ®istry, &recording.reporter, .{ |
| 438 | + .enabled = true, |
| 439 | + .n_jobs = 2, |
| 440 | + })); |
| 441 | + |
| 442 | + try std.testing.expectEqual(@as(usize, 1), recording.run_starts); |
| 443 | + try std.testing.expectEqual(@as(usize, 1), recording.run_ends); |
| 444 | + try std.testing.expectEqual(@as(usize, 1), recording.suite_starts); |
| 445 | + try std.testing.expectEqual(@as(usize, 1), recording.suite_ends); |
| 446 | + try std.testing.expectEqual(@as(usize, 2), recording.test_starts); |
| 447 | + try std.testing.expectEqual(@as(usize, 3), recording.ended_count); |
| 448 | + try std.testing.expectEqualStrings("first", recording.ended_names[0]); |
| 449 | + try std.testing.expectEqualStrings("second", recording.ended_names[1]); |
| 450 | + try std.testing.expectEqualStrings("third", recording.ended_names[2]); |
| 451 | + try std.testing.expectEqual(@as(usize, 3), recording.total); |
| 452 | + try std.testing.expectEqual(@as(usize, 1), recording.passed); |
| 453 | + try std.testing.expectEqual(@as(usize, 1), recording.failed); |
| 454 | + try std.testing.expectEqual(@as(usize, 1), recording.skipped); |
| 455 | +} |
| 205 | 456 | |
| 206 | | - try std.testing.expectEqual(@as(usize, 0), context.results.items.len); |
| 457 | +test "parallel executor preserves nested hook lifecycles" { |
| 458 | + const counters = [_]*std.atomic.Value(usize){ |
| 459 | + &parent_before_each, |
| 460 | + &parent_after_each, |
| 461 | + &child_before_each, |
| 462 | + &child_after_each, |
| 463 | + &parent_before_all, |
| 464 | + &child_before_all, |
| 465 | + &parent_after_all, |
| 466 | + &child_after_all, |
| 467 | + }; |
| 468 | + for (counters) |counter| counter.store(0, .monotonic); |
| 469 | + |
| 470 | + const allocator = std.heap.smp_allocator; |
| 471 | + var registry = suite.TestRegistry.init(allocator); |
| 472 | + defer registry.deinit(); |
| 473 | + const parent = try suite.TestSuite.init(allocator, "parent"); |
| 474 | + const child = try suite.TestSuite.init(allocator, "child"); |
| 475 | + try parent.addBeforeAll(parentBeforeAll); |
| 476 | + try parent.addAfterAll(parentAfterAll); |
| 477 | + try parent.addBeforeEach(parentBeforeEach); |
| 478 | + try parent.addAfterEach(parentAfterEach); |
| 479 | + try parent.addTest(suite.TestCase.init("parent test", passingTest)); |
| 480 | + try child.addBeforeAll(childBeforeAll); |
| 481 | + try child.addAfterAll(childAfterAll); |
| 482 | + try child.addBeforeEach(childBeforeEach); |
| 483 | + try child.addAfterEach(childAfterEach); |
| 484 | + try child.addTest(suite.TestCase.init("child one", passingTest)); |
| 485 | + try child.addTest(suite.TestCase.init("child two", passingTest)); |
| 486 | + try parent.addSuite(child); |
| 487 | + try registry.registerSuite(parent); |
| 488 | + |
| 489 | + var recording = RecordingReporter.init(allocator); |
| 490 | + try std.testing.expect(try runTestsParallel(allocator, ®istry, &recording.reporter, .{ |
| 491 | + .enabled = true, |
| 492 | + .n_jobs = 2, |
| 493 | + })); |
| 494 | + |
| 495 | + try std.testing.expectEqual(@as(usize, 3), parent_before_each.load(.monotonic)); |
| 496 | + try std.testing.expectEqual(@as(usize, 3), parent_after_each.load(.monotonic)); |
| 497 | + try std.testing.expectEqual(@as(usize, 2), child_before_each.load(.monotonic)); |
| 498 | + try std.testing.expectEqual(@as(usize, 2), child_after_each.load(.monotonic)); |
| 499 | + try std.testing.expectEqual(@as(usize, 1), parent_before_all.load(.monotonic)); |
| 500 | + try std.testing.expectEqual(@as(usize, 1), child_before_all.load(.monotonic)); |
| 501 | + try std.testing.expectEqual(@as(usize, 1), parent_after_all.load(.monotonic)); |
| 502 | + try std.testing.expectEqual(@as(usize, 1), child_after_all.load(.monotonic)); |
| 503 | + try std.testing.expectEqual(@as(usize, 2), recording.suite_starts); |
| 504 | + try std.testing.expectEqual(@as(usize, 2), recording.suite_ends); |
| 505 | + try std.testing.expectEqual(@as(usize, 3), recording.total); |
| 506 | + try std.testing.expectEqual(@as(usize, 3), recording.passed); |
| 207 | 507 | } |