also looking at this
Unify discovery and programmatic test execution
#23
12 files
+656
-169
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | const std = ; |
| 2 | 2 | const suite = ; |
| 3 | 3 | const reporter_mod = ; |
| 4 | const pipeline = ; | |
| 4 | 5 | const parallel = ; |
| 5 | 6 | const compat = ; |
| 6 | 7 | |
| @@ -16,15 +17,14 @@ pub const RunnerOptions = struct { | ||
| 16 | 17 | use_colors: bool = true, |
| 17 | 18 | parallel: bool = false, // Enable parallel execution |
| 18 | 19 | n_jobs: ?usize = null, // Number of parallel jobs |
| 20 | junit_output: []const u8 = "test-results.xml", | |
| 21 | timeout_ms: ?u64 = null, | |
| 22 | /// Optional output supplied by a CLI host. Embedded/test callers default | |
| 23 | /// to stderr so they do not interfere with Zig's stdout test protocol. | |
| 24 | reporter_writer: ?*std.Io.Writer = null, | |
| 19 | 25 | }; |
| 20 | 26 | |
| 21 | pub const ReporterType = enum { | |
| 22 | spec, | |
| 23 | dot, | |
| 24 | json, | |
| 25 | tap, | |
| 26 | junit, | |
| 27 | }; | |
| 27 | pub const ReporterType = reporter_mod.ReporterType; | |
| 28 | 28 | |
| 29 | 29 | pub const TestRunner = struct { |
| 30 | 30 | allocator: std.mem.Allocator, |
| @@ -49,32 +49,27 @@ pub const TestRunner = struct { | ||
| 49 | 49 | |
| 50 | 50 | /// Run all registered tests |
| 51 | 51 | pub fn run(self: *Self) !bool { |
| 52 | const stdout_file = std.Io.File.stdout(); | |
| 53 | var stdout_buffer: [4096]u8 = undefined; | |
| 52 | const stderr_file = std.Io.File.stderr(); | |
| 53 | var stderr_buffer: [4096]u8 = undefined; | |
| 54 | 54 | var threaded_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty }); |
| 55 | 55 | defer threaded_io.deinit(); |
| 56 | var stdout_writer = stdout_file.writer(threaded_io.io(), &stdout_buffer); | |
| 57 | ||
| 58 | // Create reporter | |
| 59 | var spec_reporter = reporter_mod.SpecReporter.init(self.allocator, stdout_writer.interface); | |
| 60 | var dot_reporter = reporter_mod.DotReporter.init(self.allocator, stdout_writer.interface); | |
| 61 | var json_reporter = reporter_mod.JsonReporter.init(self.allocator, stdout_writer.interface); | |
| 62 | defer json_reporter.deinit(); | |
| 63 | var tap_reporter = reporter_mod.TAPReporter.init(self.allocator, stdout_writer.interface); | |
| 64 | var junit_reporter = reporter_mod.JUnitReporter.init(self.allocator, "test-results.xml"); | |
| 65 | defer junit_reporter.deinit(); | |
| 66 | ||
| 67 | var current_reporter: *reporter_mod.Reporter = switch (self.options.reporter_type) { | |
| 68 | .spec => &spec_reporter.reporter, | |
| 69 | .dot => &dot_reporter.reporter, | |
| 70 | .json => &json_reporter.reporter, | |
| 71 | .tap => &tap_reporter.reporter, | |
| 72 | .junit => &junit_reporter.reporter, | |
| 73 | }; | |
| 74 | ||
| 75 | current_reporter.use_colors = self.options.use_colors; | |
| 76 | ||
| 77 | const total_tests = self.registry.countAllTests(); | |
| 56 | var stderr_writer = stderr_file.writer(threaded_io.io(), &stderr_buffer); | |
| 57 | const reporter_writer = self.options.reporter_writer orelse &stderr_writer.interface; | |
| 58 | ||
| 59 | var reporters = reporter_mod.ReporterSet.initRef( | |
| 60 | self.allocator, | |
| 61 | reporter_writer, | |
| 62 | self.options.reporter_type, | |
| 63 | self.options.junit_output, | |
| 64 | self.options.use_colors, | |
| 65 | ); | |
| 66 | defer reporters.deinit(); | |
| 67 | const current_reporter = reporters.selected(); | |
| 68 | const events = pipeline.EventStream{ .rep = current_reporter }; | |
| 69 | ||
| 70 | var plan = try pipeline.TestPlan.fromRegistry(self.allocator, self.registry); | |
| 71 | defer plan.deinit(); | |
| 72 | const total_tests = plan.items.items.len; | |
| 78 | 73 | if (total_tests == 0) { |
| 79 | 74 | return RunnerError.NoTestsFound; |
| 80 | 75 | } |
| @@ -85,6 +80,7 @@ pub const TestRunner = struct { | ||
| 85 | 80 | .enabled = true, |
| 86 | 81 | .n_jobs = self.options.n_jobs, |
| 87 | 82 | .filter = self.options.filter, |
| 83 | .timeout_ms = self.options.timeout_ms, | |
| 88 | 84 | }; |
| 89 | 85 | |
| 90 | 86 | const all_passed = try parallel.runTestsParallel( |
| @@ -98,24 +94,24 @@ pub const TestRunner = struct { | ||
| 98 | 94 | try self.addSuiteResults(test_suite); |
| 99 | 95 | } |
| 100 | 96 | |
| 101 | try stdout_writer.interface.flush(); | |
| 97 | try reporters.flush(); | |
| 102 | 98 | return all_passed and self.results.failed == 0; |
| 103 | 99 | } |
| 104 | 100 | |
| 105 | 101 | // Sequential execution (original behavior) |
| 106 | try current_reporter.onRunStart(total_tests); | |
| 102 | try events.emit(.{ .run_started = total_tests }); | |
| 107 | 103 | for (self.registry.root_suites.items) |test_suite| { |
| 108 | try self.runSuite(test_suite, current_reporter); | |
| 109 | if (self.options.bail and self.results.failed > 0) { | |
| 104 | try self.runSuite(test_suite, events); | |
| 105 | if (self.policy().shouldStop(&self.results)) { | |
| 110 | 106 | break; |
| 111 | 107 | } |
| 112 | 108 | } |
| 113 | 109 | |
| 114 | 110 | // Notify reporter of run end |
| 115 | try current_reporter.onRunEnd(&self.results); | |
| 111 | try events.emit(.{ .run_finished = &self.results }); | |
| 116 | 112 | |
| 117 | 113 | // Flush output |
| 118 | try stdout_writer.interface.flush(); | |
| 114 | try reporters.flush(); | |
| 119 | 115 | |
| 120 | 116 | return self.results.failed == 0; |
| 121 | 117 | } |
| @@ -126,26 +122,26 @@ pub const TestRunner = struct { | ||
| 126 | 122 | } |
| 127 | 123 | |
| 128 | 124 | /// Run a single test suite |
| 129 | fn runSuite(self: *Self, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void { | |
| 125 | fn runSuite(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { | |
| 130 | 126 | // Skip if marked as skip or if has_only and this isn't marked as only |
| 131 | 127 | if (test_suite.shouldSkip()) { |
| 132 | try self.skipAllTests(test_suite); | |
| 128 | try self.skipAllTests(test_suite, events); | |
| 133 | 129 | return; |
| 134 | 130 | } |
| 135 | 131 | |
| 136 | 132 | if (self.registry.has_only and !test_suite.hasOnly()) { |
| 137 | try self.skipAllTests(test_suite); | |
| 133 | try self.skipAllTests(test_suite, events); | |
| 138 | 134 | return; |
| 139 | 135 | } |
| 140 | 136 | |
| 141 | 137 | // Notify reporter |
| 142 | try rep.onSuiteStart(test_suite.name); | |
| 138 | try events.emit(.{ .suite_started = test_suite.name }); | |
| 143 | 139 | |
| 144 | 140 | // Run beforeAll hooks |
| 145 | 141 | test_suite.runBeforeAllHooks(self.allocator) catch |err| { |
| 146 | 142 | std.debug.print("beforeAll hook failed: {any}\n", .{err}); |
| 147 | try self.skipAllTests(test_suite); | |
| 148 | try rep.onSuiteEnd(test_suite.name); | |
| 143 | try self.skipAllTests(test_suite, events); | |
| 144 | try events.emit(.{ .suite_finished = test_suite.name }); | |
| 149 | 145 | return; |
| 150 | 146 | }; |
| 151 | 147 | |
| @@ -153,32 +149,30 @@ pub const TestRunner = struct { | ||
| 153 | 149 | for (test_suite.tests.items) |*test_case| { |
| 154 | 150 | if (test_case.skip or (self.registry.has_only and !test_case.only)) { |
| 155 | 151 | test_case.status = .skipped; |
| 156 | try rep.onTestEnd(test_case); | |
| 152 | try events.emit(.{ .test_finished = test_case }); | |
| 157 | 153 | try self.results.addTest(test_case); |
| 158 | 154 | continue; |
| 159 | 155 | } |
| 160 | 156 | |
| 161 | 157 | // Check filter |
| 162 | if (self.options.filter) |filter| { | |
| 163 | if (std.mem.indexOf(u8, test_case.name, filter) == null) { | |
| 164 | test_case.status = .skipped; | |
| 165 | try rep.onTestEnd(test_case); | |
| 166 | try self.results.addTest(test_case); | |
| 167 | continue; | |
| 168 | } | |
| 158 | if (!self.policy().matches(test_case.name)) { | |
| 159 | test_case.status = .skipped; | |
| 160 | try events.emit(.{ .test_finished = test_case }); | |
| 161 | try self.results.addTest(test_case); | |
| 162 | continue; | |
| 169 | 163 | } |
| 170 | 164 | |
| 171 | try self.runTest(test_case, test_suite, rep); | |
| 165 | try self.runTest(test_case, test_suite, events); | |
| 172 | 166 | |
| 173 | if (self.options.bail and test_case.status == .failed) { | |
| 167 | if (self.policy().shouldStop(&self.results)) { | |
| 174 | 168 | break; |
| 175 | 169 | } |
| 176 | 170 | } |
| 177 | 171 | |
| 178 | 172 | // Run nested suites |
| 179 | 173 | for (test_suite.suites.items) |nested_suite| { |
| 180 | try self.runSuite(nested_suite, rep); | |
| 181 | if (self.options.bail and self.results.failed > 0) { | |
| 174 | try self.runSuite(nested_suite, events); | |
| 175 | if (self.policy().shouldStop(&self.results)) { | |
| 182 | 176 | break; |
| 183 | 177 | } |
| 184 | 178 | } |
| @@ -188,12 +182,12 @@ pub const TestRunner = struct { | ||
| 188 | 182 | std.debug.print("afterAll hook failed: {any}\n", .{err}); |
| 189 | 183 | }; |
| 190 | 184 | |
| 191 | try rep.onSuiteEnd(test_suite.name); | |
| 185 | try events.emit(.{ .suite_finished = test_suite.name }); | |
| 192 | 186 | } |
| 193 | 187 | |
| 194 | 188 | /// Run a single test |
| 195 | fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void { | |
| 196 | try rep.onTestStart(test_case.name); | |
| 189 | fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { | |
| 190 | try events.emit(.{ .test_started = test_case.name }); | |
| 197 | 191 | |
| 198 | 192 | test_case.status = .running; |
| 199 | 193 | const start_time = compat.nanoTimestamp(); |
| @@ -240,22 +234,39 @@ pub const TestRunner = struct { | ||
| 240 | 234 | |
| 241 | 235 | const end_time = compat.nanoTimestamp(); |
| 242 | 236 | test_case.execution_time_ns = @intCast(end_time - start_time); |
| 237 | if (test_case.status == .passed and self.policy().timedOut(test_case.execution_time_ns)) { | |
| 238 | test_case.status = .failed; | |
| 239 | test_case.error_message = try std.fmt.allocPrint( | |
| 240 | self.allocator, | |
| 241 | "Test exceeded timeout of {d}ms", | |
| 242 | .{self.options.timeout_ms.?}, | |
| 243 | ); | |
| 244 | } | |
| 243 | 245 | |
| 244 | 246 | try self.results.addTest(test_case); |
| 245 | try rep.onTestEnd(test_case); | |
| 247 | try events.emit(.{ .test_finished = test_case }); | |
| 246 | 248 | } |
| 247 | 249 | |
| 248 | 250 | /// Skip all tests in a suite |
| 249 | fn skipAllTests(self: *Self, test_suite: *suite.TestSuite) !void { | |
| 251 | fn skipAllTests(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { | |
| 250 | 252 | for (test_suite.tests.items) |*test_case| { |
| 251 | 253 | test_case.status = .skipped; |
| 252 | 254 | try self.results.addTest(test_case); |
| 255 | try events.emit(.{ .test_finished = test_case }); | |
| 253 | 256 | } |
| 254 | 257 | |
| 255 | 258 | for (test_suite.suites.items) |nested_suite| { |
| 256 | try self.skipAllTests(nested_suite); | |
| 259 | try self.skipAllTests(nested_suite, events); | |
| 257 | 260 | } |
| 258 | 261 | } |
| 262 | ||
| 263 | fn policy(self: *const Self) pipeline.ExecutionPolicy { | |
| 264 | return .{ | |
| 265 | .bail = self.options.bail, | |
| 266 | .filter = self.options.filter, | |
| 267 | .timeout_ms = self.options.timeout_ms, | |
| 268 | }; | |
| 269 | } | |
| 259 | 270 | }; |
| 260 | 271 | |
| 261 | 272 | /// Helper function to run tests with default options |
| @@ -290,6 +301,7 @@ test "RunnerOptions default values" { | ||
| 290 | 301 | try std.testing.expectEqual(@as(?[]const u8, null), options.filter); |
| 291 | 302 | try std.testing.expectEqual(ReporterType.spec, options.reporter_type); |
| 292 | 303 | try std.testing.expectEqual(true, options.use_colors); |
| 304 | try std.testing.expectEqual(@as(?u64, null), options.timeout_ms); | |
| 293 | 305 | } |
| 294 | 306 | |
| 295 | 307 | test "RunnerOptions custom values" { |