ReviewOS
Reload

also looking at this

zig-utils/zig-test-framework

fix: enforce bounded parallel test execution

#21
Merged glennmichael123 wants to merge codex/issue-15-bounded-parallel into main
3 files +478 -176

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

src/parallel.zigmodified+439-139
@@ -3,205 +3,505 @@ const suite = ;
33const reporter = ;
44const compat = ;
55
6/// Options for parallel test execution
6/// Options for bounded parallel test execution.
77pub const ParallelOptions = struct {
8 /// Number of worker threads (null = auto-detect CPU count)
8 /// Exact maximum worker count. Null uses the logical CPU count, falling
9 /// back to one worker when the platform cannot report it.
910 n_jobs: ?usize = null,
10 /// Whether parallel execution is enabled
1111 enabled: bool = false,
12 filter: ?[]const u8 = null,
1213};
1314
14/// Result from running a test in parallel
15pub const TestResult = struct {
16 test_case: *suite.TestCase,
17 suite_name: []const u8,
18 success: bool,
19 mutex: compat.Mutex = .{},
15pub const ParallelError = error{
16 ParallelNotEnabled,
17 InvalidJobCount,
2018};
2119
22/// Context for parallel test execution
23pub const ParallelContext = struct {
24 allocator: std.mem.Allocator,
25 reporter: *reporter.Reporter,
26 results: std.ArrayList(TestResult),
27 mutex: compat.Mutex = .{},
28
29 pub fn init(allocator: std.mem.Allocator, rep: *reporter.Reporter) ParallelContext {
30 return .{
31 .allocator = allocator,
32 .reporter = rep,
33 .results = .empty,
34 };
20pub fn resolveWorkerCount(n_jobs: ?usize) !usize {
21 if (n_jobs) |jobs| {
22 if (jobs == 0) return ParallelError.InvalidJobCount;
23 return jobs;
3524 }
25 return @max(std.Thread.getCpuCount() catch 1, 1);
26}
3627
37 pub fn deinit(self: *ParallelContext) void {
38 self.results.deinit(self.allocator);
28const WorkerContext = struct {
29 allocator: std.mem.Allocator,
30 test_suite: *suite.TestSuite,
31 tests: []const *suite.TestCase,
32 next_index: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
33 error_mutex: compat.Mutex = .{},
34
35 fn worker(self: *WorkerContext) void {
36 while (true) {
37 const index = self.next_index.fetchAdd(1, .monotonic);
38 if (index >= self.tests.len) return;
39 self.runTest(self.tests[index]);
40 }
3941 }
40};
4142
42/// Task for executing a single test
43pub const TestTask = struct {
44 context: *ParallelContext,
45 test_case: *suite.TestCase,
46 suite_name: []const u8,
47
48 pub fn run(self: *TestTask) void {
49 // Execute the test
50 const success = self.executeTest();
51
52 // Store result in thread-safe manner
53 self.context.mutex.lock();
54 defer self.context.mutex.unlock();
55
56 self.context.results.append(self.context.allocator, .{
57 .test_case = self.test_case,
58 .suite_name = self.suite_name,
59 .success = success,
60 }) catch |err| {
61 std.debug.print("Error appending result: {any}\n", .{err});
43 fn runTest(self: *WorkerContext, test_case: *suite.TestCase) void {
44 test_case.status = .running;
45 const start_time = compat.nanoTimestamp();
46
47 var before_hooks = self.test_suite.getAllBeforeEachHooks(self.allocator) catch {
48 self.fail(test_case, "could not collect beforeEach hooks", .{});
49 self.finishTiming(test_case, start_time);
50 return;
6251 };
63 }
52 defer before_hooks.deinit(self.allocator);
53
54 var before_failed = false;
55 for (before_hooks.items) |hook| {
56 hook(self.allocator) catch |err| {
57 self.fail(test_case, "beforeEach hook failed: {any}", .{err});
58 before_failed = true;
59 break;
60 };
61 }
6462
65 fn executeTest(self: *TestTask) bool {
66 self.test_case.status = .running;
63 if (!before_failed) {
64 test_case.test_fn(self.allocator) catch |err| {
65 self.fail(test_case, "{any}", .{err});
66 };
67 if (test_case.status == .running) test_case.status = .passed;
68 }
6769
68 // Call the test function
69 self.test_case.test_fn(self.context.allocator) catch |err| {
70 self.test_case.status = .failed;
71 const err_msg = std.fmt.allocPrint(
72 self.context.allocator,
73 "Test failed: {any}",
74 .{err},
75 ) catch "Out of memory";
76 self.test_case.error_message = err_msg;
77 return false;
70 var after_hooks = self.test_suite.getAllAfterEachHooks(self.allocator) catch {
71 self.fail(test_case, "could not collect afterEach hooks", .{});
72 self.finishTiming(test_case, start_time);
73 return;
7874 };
75 defer after_hooks.deinit(self.allocator);
76 for (after_hooks.items) |hook| {
77 hook(self.allocator) catch |err| {
78 self.fail(test_case, "afterEach hook failed: {any}", .{err});
79 break;
80 };
81 }
7982
80 self.test_case.status = .passed;
81 return true;
83 self.finishTiming(test_case, start_time);
84 }
85
86 fn fail(self: *WorkerContext, test_case: *suite.TestCase, comptime format: []const u8, args: anytype) void {
87 test_case.status = .failed;
88 self.error_mutex.lock();
89 defer self.error_mutex.unlock();
90 test_case.error_message = std.fmt.allocPrint(self.allocator, format, args) catch "Out of memory";
91 }
92
93 fn finishTiming(_: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void {
94 test_case.execution_time_ns = @intCast(compat.nanoTimestamp() - start_time);
8295 }
8396};
8497
85/// Run tests in parallel using thread pool
86pub fn runTestsParallel(
98const RunContext = struct {
8799 allocator: std.mem.Allocator,
88 test_registry: *suite.TestRegistry,
100 registry: *suite.TestRegistry,
89101 rep: *reporter.Reporter,
90102 options: ParallelOptions,
91) !bool {
92 if (!options.enabled) {
93 return error.ParallelNotEnabled;
94 }
103 worker_limit: usize,
95104
96 // Create parallel context
97 var context = ParallelContext.init(allocator, rep);
98 defer context.deinit();
105 fn runSuite(self: *RunContext, test_suite: *suite.TestSuite) !bool {
106 try self.rep.onSuiteStart(test_suite.name);
99107
100 // Count total tests
101 var total_tests: usize = 0;
102 for (test_registry.root_suites.items) |test_suite| {
103 total_tests += test_suite.tests.items.len;
104 }
105
106 try rep.onRunStart(total_tests);
108 if (test_suite.shouldSkip()) {
109 try self.reportSkippedSuite(test_suite);
110 try self.rep.onSuiteEnd(test_suite.name);
111 return true;
112 }
107113
108 // Collect tasks and spawn threads
109 var threads = std.ArrayList(std.Thread).empty;
110 defer threads.deinit(allocator);
114 test_suite.runBeforeAllHooks(self.allocator) catch |err| {
115 std.debug.print("beforeAll hook failed: {any}\n", .{err});
116 try self.reportSkippedSuite(test_suite);
117 try self.rep.onSuiteEnd(test_suite.name);
118 return true;
119 };
111120
112 for (test_registry.root_suites.items) |test_suite| {
113 try rep.onSuiteStart(test_suite.name);
121 var scheduled: std.ArrayList(*suite.TestCase) = .empty;
122 defer scheduled.deinit(self.allocator);
114123
115124 for (test_suite.tests.items) |*test_case| {
116 if (test_case.skip) {
125 if (!self.shouldRun(test_suite, test_case)) {
117126 test_case.status = .skipped;
118127 continue;
119128 }
129 try self.rep.onTestStart(test_case.name);
130 try scheduled.append(self.allocator, test_case);
131 }
120132
121 // Create task
122 const task = try allocator.create(TestTask);
123 task.* = .{
124 .context = &context,
125 .test_case = test_case,
126 .suite_name = test_suite.name,
127 };
133 try self.runBatch(test_suite, scheduled.items);
134
135 var all_passed = true;
136 for (test_suite.tests.items) |*test_case| {
137 try self.rep.onTestEnd(test_case);
138 if (test_case.status == .failed) all_passed = false;
139 }
128140
129 // Spawn thread for task
130 const thread = try std.Thread.spawn(.{}, runTestTask, .{task});
131 try threads.append(allocator, thread);
141 for (test_suite.suites.items) |nested_suite| {
142 if (!try self.runSuite(nested_suite)) all_passed = false;
132143 }
144
145 test_suite.runAfterAllHooks(self.allocator) catch |err| {
146 std.debug.print("afterAll hook failed: {any}\n", .{err});
147 all_passed = false;
148 };
149
150 try self.rep.onSuiteEnd(test_suite.name);
151 return all_passed;
133152 }
134153
135 // Wait for all threads to complete
136 for (threads.items) |thread| {
137 thread.join();
154 fn shouldRun(self: *RunContext, test_suite: *suite.TestSuite, test_case: *suite.TestCase) bool {
155 if (test_case.skip) return false;
156 if (self.registry.has_only and !test_case.only and !test_suite.hasOnly()) return false;
157 if (self.options.filter) |filter| {
158 if (std.mem.indexOf(u8, test_case.name, filter) == null) return false;
159 }
160 return true;
138161 }
139162
140 // Report results (now that all tests are done)
141 var all_passed = true;
142 for (test_registry.root_suites.items) |test_suite| {
163 fn runBatch(self: *RunContext, test_suite: *suite.TestSuite, tests: []const *suite.TestCase) !void {
164 if (tests.len == 0) return;
165
166 var worker_context = WorkerContext{
167 .allocator = self.allocator,
168 .test_suite = test_suite,
169 .tests = tests,
170 };
171 const count = @min(self.worker_limit, tests.len);
172 var threads: std.ArrayList(std.Thread) = .empty;
173 defer threads.deinit(self.allocator);
174
175 for (0..count) |_| {
176 const thread = std.Thread.spawn(.{}, WorkerContext.worker, .{&worker_context}) catch |err| {
177 for (threads.items) |started| started.join();
178 return err;
179 };
180 threads.append(self.allocator, thread) catch |err| {
181 thread.join();
182 for (threads.items) |started| started.join();
183 return err;
184 };
185 }
186 for (threads.items) |thread| thread.join();
187 }
188
189 fn reportSkippedSuite(self: *RunContext, test_suite: *suite.TestSuite) !void {
143190 for (test_suite.tests.items) |*test_case| {
144 try rep.onTestEnd(test_case);
145 if (test_case.status == .failed) {
146 all_passed = false;
147 }
191 test_case.status = .skipped;
192 try self.rep.onTestEnd(test_case);
193 }
194 for (test_suite.suites.items) |nested| {
195 try self.rep.onSuiteStart(nested.name);
196 try self.reportSkippedSuite(nested);
197 try self.rep.onSuiteEnd(nested.name);
148198 }
149 try rep.onSuiteEnd(test_suite.name);
150199 }
200};
151201
152 // Create results summary
153 var results = reporter.TestResults.init(allocator);
154 defer results.deinit();
202/// Run suites in declaration order while executing each suite's direct tests
203/// with a bounded worker set. Reporter callbacks are serialized and emitted in
204/// declaration order after each batch completes.
205pub fn runTestsParallel(
206 allocator: std.mem.Allocator,
207 test_registry: *suite.TestRegistry,
208 rep: *reporter.Reporter,
209 options: ParallelOptions,
210) !bool {
211 if (!options.enabled) return ParallelError.ParallelNotEnabled;
212 const worker_limit = try resolveWorkerCount(options.n_jobs);
213
214 const total_tests = test_registry.countAllTests();
215 try rep.onRunStart(total_tests);
155216
217 var context = RunContext{
218 .allocator = allocator,
219 .registry = test_registry,
220 .rep = rep,
221 .options = options,
222 .worker_limit = worker_limit,
223 };
224
225 var all_passed = true;
156226 for (test_registry.root_suites.items) |test_suite| {
157 for (test_suite.tests.items) |test_case| {
158 results.total += 1;
159 switch (test_case.status) {
160 .passed => results.passed += 1,
161 .failed => results.failed += 1,
162 .skipped => results.skipped += 1,
163 else => {},
164 }
165 }
227 if (!try context.runSuite(test_suite)) all_passed = false;
166228 }
167229
230 var results = reporter.TestResults.init(allocator);
231 defer results.deinit();
232 for (test_registry.root_suites.items) |test_suite| {
233 try addSuiteResults(&results, test_suite);
234 }
168235 try rep.onRunEnd(&results);
169
170 return all_passed;
236 return all_passed and results.failed == 0;
171237}
172238
173fn runTestTask(task: *TestTask) void {
174 task.run();
239fn addSuiteResults(results: *reporter.TestResults, test_suite: *suite.TestSuite) !void {
240 for (test_suite.tests.items) |*test_case| try results.addTest(test_case);
241 for (test_suite.suites.items) |nested| try addSuiteResults(results, nested);
175242}
176243
177// Tests
178test "ParallelOptions default values" {
244test "ParallelOptions defaults to CPU-based workers" {
179245 const options = ParallelOptions{};
246 try std.testing.expect(options.n_jobs == null);
247 try std.testing.expect(!options.enabled);
248 try std.testing.expectEqual(
249 @max(std.Thread.getCpuCount() catch 1, 1),
250 try resolveWorkerCount(null),
251 );
252}
253
254test "configured worker count rejects zero" {
255 try std.testing.expectEqual(@as(usize, 2), try resolveWorkerCount(2));
256 try std.testing.expectError(ParallelError.InvalidJobCount, resolveWorkerCount(0));
257}
258
259var active_tests = std.atomic.Value(usize).init(0);
260var peak_tests = std.atomic.Value(usize).init(0);
261
262fn boundedTest(_: std.mem.Allocator) !void {
263 const active = active_tests.fetchAdd(1, .monotonic) + 1;
264 var peak = peak_tests.load(.monotonic);
265 while (active > peak) {
266 if (peak_tests.cmpxchgWeak(peak, active, .monotonic, .monotonic)) |observed| {
267 peak = observed;
268 } else break;
269 }
270 compat.sleep(20 * std.time.ns_per_ms);
271 _ = active_tests.fetchSub(1, .monotonic);
272}
273
274fn passingTest(_: std.mem.Allocator) !void {}
275
276fn failingTest(_: std.mem.Allocator) !void {
277 return error.ExpectedFailure;
278}
279
280var parent_before_each = std.atomic.Value(usize).init(0);
281var parent_after_each = std.atomic.Value(usize).init(0);
282var child_before_each = std.atomic.Value(usize).init(0);
283var child_after_each = std.atomic.Value(usize).init(0);
284var parent_before_all = std.atomic.Value(usize).init(0);
285var child_before_all = std.atomic.Value(usize).init(0);
286var parent_after_all = std.atomic.Value(usize).init(0);
287var child_after_all = std.atomic.Value(usize).init(0);
288
289fn parentBeforeEach(_: std.mem.Allocator) !void {
290 _ = parent_before_each.fetchAdd(1, .monotonic);
291}
292
293fn parentAfterEach(_: std.mem.Allocator) !void {
294 _ = parent_after_each.fetchAdd(1, .monotonic);
295}
296
297fn childBeforeEach(_: std.mem.Allocator) !void {
298 _ = child_before_each.fetchAdd(1, .monotonic);
299}
300
301fn childAfterEach(_: std.mem.Allocator) !void {
302 _ = child_after_each.fetchAdd(1, .monotonic);
303}
304
305fn parentBeforeAll(_: std.mem.Allocator) !void {
306 _ = parent_before_all.fetchAdd(1, .monotonic);
307}
308
309fn childBeforeAll(_: std.mem.Allocator) !void {
310 if (parent_before_all.load(.monotonic) != 1) return error.ParentBeforeAllMissing;
311 _ = child_before_all.fetchAdd(1, .monotonic);
312}
180313
181 try std.testing.expectEqual(@as(?usize, null), options.n_jobs);
182 try std.testing.expectEqual(false, options.enabled);
314fn childAfterAll(_: std.mem.Allocator) !void {
315 _ = child_after_all.fetchAdd(1, .monotonic);
183316}
184317
185test "ParallelOptions with custom values" {
186 const options = ParallelOptions{
187 .n_jobs = 4,
318fn parentAfterAll(_: std.mem.Allocator) !void {
319 if (child_after_all.load(.monotonic) != 1) return error.ChildAfterAllMissing;
320 _ = parent_after_all.fetchAdd(1, .monotonic);
321}
322
323const RecordingReporter = struct {
324 reporter: reporter.Reporter,
325 run_starts: usize = 0,
326 run_ends: usize = 0,
327 suite_starts: usize = 0,
328 suite_ends: usize = 0,
329 test_starts: usize = 0,
330 ended_count: usize = 0,
331 ended_names: [16][]const u8 = undefined,
332 total: usize = 0,
333 passed: usize = 0,
334 failed: usize = 0,
335 skipped: usize = 0,
336
337 fn init(allocator: std.mem.Allocator) RecordingReporter {
338 return .{
339 .reporter = .{
340 .vtable = &.{
341 .onRunStart = onRunStart,
342 .onRunEnd = onRunEnd,
343 .onSuiteStart = onSuiteStart,
344 .onSuiteEnd = onSuiteEnd,
345 .onTestStart = onTestStart,
346 .onTestEnd = onTestEnd,
347 },
348 .allocator = allocator,
349 },
350 };
351 }
352
353 fn self(rep: *reporter.Reporter) *RecordingReporter {
354 return @fieldParentPtr("reporter", rep);
355 }
356
357 fn onRunStart(rep: *reporter.Reporter, _: usize) !void {
358 self(rep).run_starts += 1;
359 }
360
361 fn onRunEnd(rep: *reporter.Reporter, results: *reporter.TestResults) !void {
362 const recording = self(rep);
363 recording.run_ends += 1;
364 recording.total = results.total;
365 recording.passed = results.passed;
366 recording.failed = results.failed;
367 recording.skipped = results.skipped;
368 }
369
370 fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void {
371 self(rep).suite_starts += 1;
372 }
373
374 fn onSuiteEnd(rep: *reporter.Reporter, _: []const u8) !void {
375 self(rep).suite_ends += 1;
376 }
377
378 fn onTestStart(rep: *reporter.Reporter, _: []const u8) !void {
379 self(rep).test_starts += 1;
380 }
381
382 fn onTestEnd(rep: *reporter.Reporter, test_case: *const suite.TestCase) !void {
383 const recording = self(rep);
384 recording.ended_names[recording.ended_count] = test_case.name;
385 recording.ended_count += 1;
386 }
387};
388
389fn verifyConcurrencyLimit(limit: ?usize) !void {
390 const allocator = std.heap.smp_allocator;
391 var registry = suite.TestRegistry.init(allocator);
392 defer registry.deinit();
393 const test_suite = try suite.TestSuite.init(allocator, "bounded");
394 for (0..8) |_| try test_suite.addTest(suite.TestCase.init("work", boundedTest));
395 try registry.registerSuite(test_suite);
396
397 active_tests.store(0, .monotonic);
398 peak_tests.store(0, .monotonic);
399 var recording = RecordingReporter.init(allocator);
400 try std.testing.expect(try runTestsParallel(allocator, &registry, &recording.reporter, .{
188401 .enabled = true,
189 };
402 .n_jobs = limit,
403 }));
190404
191 try std.testing.expectEqual(@as(?usize, 4), options.n_jobs);
192 try std.testing.expectEqual(true, options.enabled);
405 const resolved = try resolveWorkerCount(limit);
406 try std.testing.expect(peak_tests.load(.monotonic) > 0);
407 try std.testing.expect(peak_tests.load(.monotonic) <= @min(resolved, 8));
193408}
194409
195test "ParallelContext initialization" {
196 const allocator = std.testing.allocator;
410test "parallel executor bounds one worker" {
411 try verifyConcurrencyLimit(1);
412 try std.testing.expectEqual(@as(usize, 1), peak_tests.load(.monotonic));
413}
197414
198 var rep = reporter.Reporter{
199 .vtable = undefined,
200 .allocator = allocator,
201 };
415test "parallel executor bounds two workers" {
416 try verifyConcurrencyLimit(2);
417 try std.testing.expectEqual(@as(usize, 2), peak_tests.load(.monotonic));
418}
419
420test "parallel executor bounds default CPU workers" {
421 try verifyConcurrencyLimit(null);
422}
202423
203 var context = ParallelContext.init(allocator, &rep);
204 defer context.deinit();
424test "parallel reporting is serialized, ordered, and complete" {
425 const allocator = std.heap.smp_allocator;
426 var registry = suite.TestRegistry.init(allocator);
427 defer registry.deinit();
428 const test_suite = try suite.TestSuite.init(allocator, "ordered");
429 try test_suite.addTest(suite.TestCase.init("first", passingTest));
430 try test_suite.addTest(suite.TestCase.init("second", failingTest));
431 var skipped = suite.TestCase.init("third", passingTest);
432 skipped.skip = true;
433 try test_suite.addTest(skipped);
434 try registry.registerSuite(test_suite);
435
436 var recording = RecordingReporter.init(allocator);
437 try std.testing.expect(!try runTestsParallel(allocator, &registry, &recording.reporter, .{
438 .enabled = true,
439 .n_jobs = 2,
440 }));
441
442 try std.testing.expectEqual(@as(usize, 1), recording.run_starts);
443 try std.testing.expectEqual(@as(usize, 1), recording.run_ends);
444 try std.testing.expectEqual(@as(usize, 1), recording.suite_starts);
445 try std.testing.expectEqual(@as(usize, 1), recording.suite_ends);
446 try std.testing.expectEqual(@as(usize, 2), recording.test_starts);
447 try std.testing.expectEqual(@as(usize, 3), recording.ended_count);
448 try std.testing.expectEqualStrings("first", recording.ended_names[0]);
449 try std.testing.expectEqualStrings("second", recording.ended_names[1]);
450 try std.testing.expectEqualStrings("third", recording.ended_names[2]);
451 try std.testing.expectEqual(@as(usize, 3), recording.total);
452 try std.testing.expectEqual(@as(usize, 1), recording.passed);
453 try std.testing.expectEqual(@as(usize, 1), recording.failed);
454 try std.testing.expectEqual(@as(usize, 1), recording.skipped);
455}
205456
206 try std.testing.expectEqual(@as(usize, 0), context.results.items.len);
457test "parallel executor preserves nested hook lifecycles" {
458 const counters = [_]*std.atomic.Value(usize){
459 &parent_before_each,
460 &parent_after_each,
461 &child_before_each,
462 &child_after_each,
463 &parent_before_all,
464 &child_before_all,
465 &parent_after_all,
466 &child_after_all,
467 };
468 for (counters) |counter| counter.store(0, .monotonic);
469
470 const allocator = std.heap.smp_allocator;
471 var registry = suite.TestRegistry.init(allocator);
472 defer registry.deinit();
473 const parent = try suite.TestSuite.init(allocator, "parent");
474 const child = try suite.TestSuite.init(allocator, "child");
475 try parent.addBeforeAll(parentBeforeAll);
476 try parent.addAfterAll(parentAfterAll);
477 try parent.addBeforeEach(parentBeforeEach);
478 try parent.addAfterEach(parentAfterEach);
479 try parent.addTest(suite.TestCase.init("parent test", passingTest));
480 try child.addBeforeAll(childBeforeAll);
481 try child.addAfterAll(childAfterAll);
482 try child.addBeforeEach(childBeforeEach);
483 try child.addAfterEach(childAfterEach);
484 try child.addTest(suite.TestCase.init("child one", passingTest));
485 try child.addTest(suite.TestCase.init("child two", passingTest));
486 try parent.addSuite(child);
487 try registry.registerSuite(parent);
488
489 var recording = RecordingReporter.init(allocator);
490 try std.testing.expect(try runTestsParallel(allocator, &registry, &recording.reporter, .{
491 .enabled = true,
492 .n_jobs = 2,
493 }));
494
495 try std.testing.expectEqual(@as(usize, 3), parent_before_each.load(.monotonic));
496 try std.testing.expectEqual(@as(usize, 3), parent_after_each.load(.monotonic));
497 try std.testing.expectEqual(@as(usize, 2), child_before_each.load(.monotonic));
498 try std.testing.expectEqual(@as(usize, 2), child_after_each.load(.monotonic));
499 try std.testing.expectEqual(@as(usize, 1), parent_before_all.load(.monotonic));
500 try std.testing.expectEqual(@as(usize, 1), child_before_all.load(.monotonic));
501 try std.testing.expectEqual(@as(usize, 1), parent_after_all.load(.monotonic));
502 try std.testing.expectEqual(@as(usize, 1), child_after_all.load(.monotonic));
503 try std.testing.expectEqual(@as(usize, 2), recording.suite_starts);
504 try std.testing.expectEqual(@as(usize, 2), recording.suite_ends);
505 try std.testing.expectEqual(@as(usize, 3), recording.total);
506 try std.testing.expectEqual(@as(usize, 3), recording.passed);
207507}
src/test_runner.zigmodified+20-37
@@ -79,57 +79,35 @@ pub const TestRunner = struct {
7979 return RunnerError.NoTestsFound;
8080 }
8181
82 // Notify reporter of run start
83 try current_reporter.onRunStart(total_tests);
84
8582 // Run tests in parallel or sequential based on options
8683 if (self.options.parallel) {
8784 const parallel_opts = parallel.ParallelOptions{
8885 .enabled = true,
8986 .n_jobs = self.options.n_jobs,
87 .filter = self.options.filter,
9088 };
9189
92 _ = parallel.runTestsParallel(
90 const all_passed = try parallel.runTestsParallel(
9391 self.allocator,
9492 self.registry,
9593 current_reporter,
9694 parallel_opts,
97 ) catch |err| {
98 std.debug.print("Parallel execution failed: {any}, falling back to sequential\n", .{err});
99 // Fall back to sequential execution
100 for (self.registry.root_suites.items) |test_suite| {
101 try self.runSuite(test_suite, current_reporter);
102 if (self.options.bail and self.results.failed > 0) {
103 break;
104 }
105 }
106 return false;
107 };
108
109 // Update results from parallel execution
110 self.results.total = 0;
111 self.results.passed = 0;
112 self.results.failed = 0;
113 self.results.skipped = 0;
95 );
11496
11597 for (self.registry.root_suites.items) |test_suite| {
116 for (test_suite.tests.items) |test_case| {
117 self.results.total += 1;
118 switch (test_case.status) {
119 .passed => self.results.passed += 1,
120 .failed => self.results.failed += 1,
121 .skipped => self.results.skipped += 1,
122 else => {},
123 }
124 }
98 try self.addSuiteResults(test_suite);
12599 }
126 } else {
127 // Sequential execution (original behavior)
128 for (self.registry.root_suites.items) |test_suite| {
129 try self.runSuite(test_suite, current_reporter);
130 if (self.options.bail and self.results.failed > 0) {
131 break;
132 }
100
101 try stdout_writer.interface.flush();
102 return all_passed and self.results.failed == 0;
103 }
104
105 // Sequential execution (original behavior)
106 try current_reporter.onRunStart(total_tests);
107 for (self.registry.root_suites.items) |test_suite| {
108 try self.runSuite(test_suite, current_reporter);
109 if (self.options.bail and self.results.failed > 0) {
110 break;
133111 }
134112 }
135113
@@ -142,6 +120,11 @@ pub const TestRunner = struct {
142120 return self.results.failed == 0;
143121 }
144122
123 fn addSuiteResults(self: *Self, test_suite: *suite.TestSuite) !void {
124 for (test_suite.tests.items) |*test_case| try self.results.addTest(test_case);
125 for (test_suite.suites.items) |nested| try self.addSuiteResults(nested);
126 }
127
145128 /// Run a single test suite
146129 fn runSuite(self: *Self, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void {
147130 // Skip if marked as skip or if has_only and this isn't marked as only