Unify discovery and programmatic test execution

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

build.zigmodified+7-0

Changes to build.zig

		⋯285 linesShow the 285 lines above this hunk@@ -286,6 +286,13 @@ pub fn build(b: *std.Build) void {
286	286	run_shard_two.addArgs(&.{ "--test-dir", "tests", "--pattern", "sample.test.zig", "--shard-index", "2", "--shard-count", "2", "--no-color" });
287	287	test_step.dependOn(&run_shard_two.step);
288	288
	289	+ const run_discovery_tap = b.addRunArtifact(exe);
	290	+ run_discovery_tap.addArgs(&.{ "--test-dir", "tests", "--pattern", "sample.test.zig", "--reporter", "tap", "--no-color" });
	291	+ run_discovery_tap.expectStdOutEqual(
	292	+ "TAP version 14\n1..1\n# Subtest: sample.test.zig\nok 1 - sample.test.zig\n",
	293	+ );
	294	+ test_step.dependOn(&run_discovery_tap.step);
	295	+
289	296	const run_version = b.addRunArtifact(exe);
290	297	run_version.addArg("--version");
291	298	run_version.expectStdErrEqual(b.fmt("Zig Test Framework v{s}\n", .{package_version}));

src/cli.zigmodified+6-7

Changes to src/cli.zig

		⋯325 linesShow the 325 lines above this hunk@@ -326,7 +326,6 @@ pub const CLI = struct {
326	326	if (!self.options.watch and self.options.watch_debounce != 300) return "--watch-debounce";
327	327	if (!self.options.ui and self.options.ui_port != 8080) return "--ui-port";
328	328	if (!self.options.ui and !std.mem.eql(u8, self.options.ui_host, "127.0.0.1")) return "--ui-host";
329		- if (self.options.reporter != .spec) return "--reporter";
330	329	if (self.options.quiet) return "--quiet";
331	330	if (self.options.parallel) return "--parallel";
332	331	if (self.options.jobs != null) return "--jobs";
		⋯2 linesShow the 2 lines above this hunk@@ -335,8 +334,6 @@ pub const CLI = struct {
335	334	if (self.options.profile_memory) return "--profile-memory";
336	335	if (self.options.memory_threshold != 0) return "--memory-threshold";
337	336	if (self.options.fail_on_leak) return "--fail-on-leak";
338		- if (self.options.junit_output != null) return "--junit-output";
339		- if (self.options.timeout != null) return "--timeout";
340	337	if (self.options.ui) return "--ui";
341	338	return null;
342	339	}
		⋯99 linesShow the 99 lines above this hunk@@ -442,6 +439,8 @@ pub const CLI = struct {
442	439	.use_colors = !self.options.no_color,
443	440	.parallel = self.options.parallel,
444	441	.n_jobs = self.options.jobs,
	442	+ .junit_output = self.options.junit_output orelse "test-results.xml",
	443	+ .timeout_ms = self.options.timeout,
445	444	};
446	445	}
447	446	};
		⋯243 linesShow the 243 lines above this hunk@@ -691,18 +690,18 @@ test "discovery accepts implemented options" {
691	690	try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption());
692	691	}
693	692
694		-test "discovery identifies unsupported options" {
	693	+test "discovery accepts reporter selection" {
695	694	var cli = CLI.init(std.testing.allocator);
696	695	const args = [_][]const u8{ "zig-test", "--test-dir", "tests", "--reporter", "json" };
697	696	try cli.parse(&args);
698		- try std.testing.expectEqualStrings("--reporter", cli.unsupportedDiscoveryOption().?);
	697	+ try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption());
699	698	}
700	699
701		-test "discovery reports timeout and parallel options as unsupported" {
	700	+test "discovery accepts timeout and reports parallel as unsupported" {
702	701	var cli = CLI.init(std.testing.allocator);
703	702
704	703	cli.options.timeout = 1000;
705		- try std.testing.expectEqualStrings("--timeout", cli.unsupportedDiscoveryOption().?);
	704	+ try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption());
706	705
707	706	cli.options.timeout = null;
708	707	cli.options.parallel = true;

src/lib.zigmodified+10-0

Changes to src/lib.zig

		⋯4 linesShow the 4 lines above this hunk@@ -5,6 +5,7 @@ pub const assertions = ;
5	5	pub const suite = ;
6	6	pub const test_runner = ;
7	7	pub const reporter = ;
	8	+pub const pipeline = ;
8	9	pub const matchers = ;
9	10	pub const mock = ;
10	11	pub const cli = ;
		⋯68 linesShow the 68 lines above this hunk@@ -79,6 +80,15 @@ pub const JsonReporter = reporter.JsonReporter;
79	80	pub const TAPReporter = reporter.TAPReporter;
80	81	pub const JUnitReporter = reporter.JUnitReporter;
81	82	pub const Colors = reporter.Colors;
	83	+pub const ReporterSet = reporter.ReporterSet;
	84	+
	85	+// Unified execution pipeline
	86	+pub const TestPlan = pipeline.TestPlan;
	87	+pub const PlanItem = pipeline.PlanItem;
	88	+pub const PlanOrigin = pipeline.PlanOrigin;
	89	+pub const ExecutionPolicy = pipeline.ExecutionPolicy;
	90	+pub const LifecycleEvent = pipeline.LifecycleEvent;
	91	+pub const EventStream = pipeline.EventStream;
82	92
83	93	// Matchers
84	94	pub const Matchers = matchers.Matchers;

src/main.zigmodified+15-1

Changes to src/main.zig

		⋯160 linesShow the 160 lines above this hunk@@ -161,6 +161,11 @@ pub fn main(init: std.process.Init.Minimal) !void {
161	161	}
162	162
163	163	var all_passed: bool = undefined;
	164	+ const stdout_file = std.Io.File.stdout();
	165	+ var stdout_buffer: [4096]u8 = undefined;
	166	+ var reporter_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty });
	167	+ defer reporter_io.deinit();
	168	+ var stdout_writer = stdout_file.writer(reporter_io.io(), &stdout_buffer);
164	169	const shard_options: ?lib.ShardOptions = if (cli_parser.options.shard_index) |index|
165	170	.{ .index = index, .count = cli_parser.options.shard_count.? }
166	171	else
		⋯39 linesShow the 39 lines above this hunk@@ -206,6 +211,10 @@ pub fn main(init: std.process.Init.Minimal) !void {
206	211	.verbose = cli_parser.options.verbose,
207	212	.use_colors = !cli_parser.options.no_color,
208	213	.shard = shard_options,
	214	+ .reporter_type = cli_parser.options.reporter,
	215	+ .junit_output = cli_parser.options.junit_output orelse "test-results.xml",
	216	+ .timeout_ms = cli_parser.options.timeout,
	217	+ .reporter_writer = &stdout_writer.interface,
209	218	.coverage_options = cov_opts,
210	219	.ui_server = if (ui_server) |*server| server else null,
211	220	};
		⋯33 linesShow the 33 lines above this hunk@@ -245,6 +254,10 @@ pub fn main(init: std.process.Init.Minimal) !void {
245	254	.verbose = cli_parser.options.verbose,
246	255	.use_colors = !cli_parser.options.no_color,
247	256	.shard = shard_options,
	257	+ .reporter_type = cli_parser.options.reporter,
	258	+ .junit_output = cli_parser.options.junit_output orelse "test-results.xml",
	259	+ .timeout_ms = cli_parser.options.timeout,
	260	+ .reporter_writer = &stdout_writer.interface,
248	261	.coverage_options = cov_opts,
249	262	.ui_server = if (ui_server) |*server| server else null,
250	263	};
		⋯5 linesShow the 5 lines above this hunk@@ -256,7 +269,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
256	269	defer lib.cleanupRegistry();
257	270
258	271	// Create test runner with CLI options
259		- const runner_options = cli_parser.toRunnerOptions();
	272	+ var runner_options = cli_parser.toRunnerOptions();
	273	+ runner_options.reporter_writer = &stdout_writer.interface;
260	274	var runner = lib.TestRunner.init(allocator, registry, runner_options);
261	275	defer runner.deinit();
262	276

src/parallel.zigmodified+15-1

Changes to src/parallel.zig

		@@ -1,6 +1,7 @@
1	1	const std = ;
2	2	const suite = ;
3	3	const reporter = ;
	4	+const pipeline = ;
4	5	const compat = ;
5	6
6	7	/// Options for bounded parallel test execution.
		⋯3 linesShow the 3 lines above this hunk@@ -10,6 +11,7 @@ pub const ParallelOptions = struct {
10	11	n_jobs: ?usize = null,
11	12	enabled: bool = false,
12	13	filter: ?[]const u8 = null,
	14	+ timeout_ms: ?u64 = null,
13	15	};
14	16
15	17	pub const ParallelError = error{
		⋯15 linesShow the 15 lines above this hunk@@ -31,6 +33,7 @@ const WorkerContext = struct {
31	33	tests: []const *suite.TestCase,
32	34	next_index: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
33	35	error_mutex: compat.Mutex = .{},
	36	+ policy: pipeline.ExecutionPolicy,
34	37
35	38	fn worker(self: *WorkerContext) void {
36	39	while (true) {
		⋯53 linesShow the 53 lines above this hunk@@ -90,8 +93,15 @@ const WorkerContext = struct {
90	93	test_case.error_message = std.fmt.allocPrint(self.allocator, format, args) catch "Out of memory";
91	94	}
92	95
93		- fn finishTiming(_: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void {
	96	+ fn finishTiming(self: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void {
94	97	test_case.execution_time_ns = @intCast(compat.nanoTimestamp() - start_time);
	98	+ if (test_case.status == .passed and self.policy.timedOut(test_case.execution_time_ns)) {
	99	+ self.fail(
	100	+ test_case,
	101	+ "Test exceeded timeout of {d}ms",
	102	+ .{self.policy.timeout_ms.?},
	103	+ );
	104	+ }
95	105	}
96	106	};
97	107
		⋯69 linesShow the 69 lines above this hunk@@ -167,6 +177,10 @@ const RunContext = struct {
167	177	.allocator = self.allocator,
168	178	.test_suite = test_suite,
169	179	.tests = tests,
	180	+ .policy = .{
	181	+ .filter = self.options.filter,
	182	+ .timeout_ms = self.options.timeout_ms,
	183	+ },
170	184	};
171	185	const count = @min(self.worker_limit, tests.len);
172	186	var threads: std.ArrayList(std.Thread) = .empty;

src/pipeline.zigadded+169-0

Changes to src/pipeline.zig

		@@ -0,0 +1,169 @@
	1	+const std = ;
	2	+const discovery = ;
	3	+const reporter = ;
	4	+const suite = ;
	5	+
	6	+pub const PlanOrigin = enum {
	7	+ registered,
	8	+ discovered,
	9	+};
	10	+
	11	+pub const RegisteredTest = struct {
	12	+ test_suite: *suite.TestSuite,
	13	+ test_case: *suite.TestCase,
	14	+};
	15	+
	16	+pub const PlanItem = union(PlanOrigin) {
	17	+ registered: RegisteredTest,
	18	+ discovered: *const discovery.TestFile,
	19	+
	20	+ pub fn identity(self: PlanItem) []const u8 {
	21	+ return switch (self) {
	22	+ .registered => |item| item.test_case.name,
	23	+ .discovered => |file| file.relative_path,
	24	+ };
	25	+ }
	26	+};
	27	+
	28	+/// Executor-neutral plan produced by registration or file discovery.
	29	+pub const TestPlan = struct {
	30	+ allocator: std.mem.Allocator,
	31	+ items: std.ArrayList(PlanItem) = .empty,
	32	+
	33	+ pub fn init(allocator: std.mem.Allocator) TestPlan {
	34	+ return .{ .allocator = allocator };
	35	+ }
	36	+
	37	+ pub fn deinit(self: *TestPlan) void {
	38	+ self.items.deinit(self.allocator);
	39	+ }
	40	+
	41	+ pub fn fromRegistry(
	42	+ allocator: std.mem.Allocator,
	43	+ registry: *suite.TestRegistry,
	44	+ ) !TestPlan {
	45	+ var plan = TestPlan.init(allocator);
	46	+ errdefer plan.deinit();
	47	+ for (registry.root_suites.items) |test_suite| {
	48	+ try plan.appendSuite(test_suite);
	49	+ }
	50	+ return plan;
	51	+ }
	52	+
	53	+ pub fn fromDiscovery(
	54	+ allocator: std.mem.Allocator,
	55	+ discovered: *const discovery.DiscoveryResult,
	56	+ ) !TestPlan {
	57	+ var plan = TestPlan.init(allocator);
	58	+ errdefer plan.deinit();
	59	+ for (discovered.files.items) |*file| {
	60	+ try plan.appendDiscovered(file);
	61	+ }
	62	+ return plan;
	63	+ }
	64	+
	65	+ pub fn appendDiscovered(self: *TestPlan, file: *const discovery.TestFile) !void {
	66	+ try self.items.append(self.allocator, .{ .discovered = file });
	67	+ }
	68	+
	69	+ fn appendSuite(self: *TestPlan, test_suite: *suite.TestSuite) !void {
	70	+ for (test_suite.tests.items) |*test_case| {
	71	+ try self.items.append(self.allocator, .{ .registered = .{
	72	+ .test_suite = test_suite,
	73	+ .test_case = test_case,
	74	+ } });
	75	+ }
	76	+ for (test_suite.suites.items) |nested| {
	77	+ try self.appendSuite(nested);
	78	+ }
	79	+ }
	80	+};
	81	+
	82	+/// Shared execution policy. Executors apply it at their natural granularity:
	83	+/// registered test names or filters forwarded to an external Zig test process.
	84	+pub const ExecutionPolicy = struct {
	85	+ bail: bool = false,
	86	+ filter: ?[]const u8 = null,
	87	+ timeout_ms: ?u64 = null,
	88	+
	89	+ pub fn matches(self: ExecutionPolicy, name: []const u8) bool {
	90	+ const filter = self.filter orelse return true;
	91	+ return std.mem.indexOf(u8, name, filter) != null;
	92	+ }
	93	+
	94	+ pub fn shouldStop(self: ExecutionPolicy, results: *const reporter.TestResults) bool {
	95	+ return self.bail and results.failed > 0;
	96	+ }
	97	+
	98	+ pub fn timedOut(self: ExecutionPolicy, execution_time_ns: u64) bool {
	99	+ const timeout_ms = self.timeout_ms orelse return false;
	100	+ return execution_time_ns > timeout_ms * std.time.ns_per_ms;
	101	+ }
	102	+};
	103	+
	104	+pub const LifecycleEvent = union(enum) {
	105	+ run_started: usize,
	106	+ run_finished: *reporter.TestResults,
	107	+ suite_started: []const u8,
	108	+ suite_finished: []const u8,
	109	+ test_started: []const u8,
	110	+ test_finished: *const suite.TestCase,
	111	+};
	112	+
	113	+/// Routes the typed lifecycle shared by every executor into existing reporter
	114	+/// callbacks. Additional consumers can be added without changing executors.
	115	+pub const EventStream = struct {
	116	+ rep: *reporter.Reporter,
	117	+
	118	+ pub fn emit(self: EventStream, event: LifecycleEvent) !void {
	119	+ switch (event) {
	120	+ .run_started => |total| try self.rep.onRunStart(total),
	121	+ .run_finished => |results| try self.rep.onRunEnd(results),
	122	+ .suite_started => |name| try self.rep.onSuiteStart(name),
	123	+ .suite_finished => |name| try self.rep.onSuiteEnd(name),
	124	+ .test_started => |name| try self.rep.onTestStart(name),
	125	+ .test_finished => |test_case| try self.rep.onTestEnd(test_case),
	126	+ }
	127	+ }
	128	+};
	129	+
	130	+fn passingTest(_: std.mem.Allocator) !void {}
	131	+
	132	+test "registered and discovered inputs produce one plan model" {
	133	+ const allocator = std.testing.allocator;
	134	+
	135	+ var registry = suite.TestRegistry.init(allocator);
	136	+ defer registry.deinit();
	137	+ const test_suite = try suite.TestSuite.init(allocator, "unit");
	138	+ try test_suite.addTest(suite.TestCase.init("passes", passingTest));
	139	+ try registry.registerSuite(test_suite);
	140	+
	141	+ var registered = try TestPlan.fromRegistry(allocator, &registry);
	142	+ defer registered.deinit();
	143	+ try std.testing.expectEqual(@as(usize, 1), registered.items.items.len);
	144	+ try std.testing.expectEqualStrings("passes", registered.items.items[0].identity());
	145	+
	146	+ var discovered = discovery.DiscoveryResult.init(allocator);
	147	+ defer discovered.deinit();
	148	+ try discovered.addFile("tests/sample.test.zig", "sample.test.zig", "sample.test.zig");
	149	+
	150	+ var external = try TestPlan.fromDiscovery(allocator, &discovered);
	151	+ defer external.deinit();
	152	+ try std.testing.expectEqual(@as(usize, 1), external.items.items.len);
	153	+ try std.testing.expectEqualStrings("sample.test.zig", external.items.items[0].identity());
	154	+}
	155	+
	156	+test "execution policy shares matching and bail semantics" {
	157	+ const policy = ExecutionPolicy{ .bail = true, .filter = "selected", .timeout_ms = 500 };
	158	+ try std.testing.expect(policy.matches("selected test"));
	159	+ try std.testing.expect(!policy.matches("other test"));
	160	+ try std.testing.expectEqual(@as(?u64, 500), policy.timeout_ms);
	161	+ try std.testing.expect(!policy.timedOut(499 * std.time.ns_per_ms));
	162	+ try std.testing.expect(policy.timedOut(501 * std.time.ns_per_ms));
	163	+
	164	+ var results = reporter.TestResults.init(std.testing.allocator);
	165	+ defer results.deinit();
	166	+ try std.testing.expect(!policy.shouldStop(&results));
	167	+ results.failed = 1;
	168	+ try std.testing.expect(policy.shouldStop(&results));
	169	+}

src/reporter.zigmodified+182-56

Changes to src/reporter.zig

		⋯1 lineShow the 1 line above this hunk@@ -2,6 +2,14 @@ const std = ;
2	2	const suite = ;
3	3	const compat = ;
4	4
	5	+pub const ReporterType = enum {
	6	+ spec,
	7	+ dot,
	8	+ json,
	9	+ tap,
	10	+ junit,
	11	+};
	12	+
5	13	/// ANSI color codes
6	14	pub const Colors = struct {
7	15	pub const reset = "\x1b[0m";
		⋯93 linesShow the 93 lines above this hunk@@ -101,6 +109,7 @@ pub const SpecReporter = struct {
101	109	reporter: Reporter,
102	110	indent_level: usize = 0,
103	111	writer: std.Io.Writer,
	112	+ writer_ref: ?*std.Io.Writer = null,
104	113
105	114	const Self = @This();
106	115
		⋯7 linesShow the 7 lines above this hunk@@ -114,6 +123,16 @@ pub const SpecReporter = struct {
114	123	};
115	124	}
116	125
	126	+ pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self {
	127	+ var result = init(allocator, writer.*);
	128	+ result.writer_ref = writer;
	129	+ return result;
	130	+ }
	131	+
	132	+ fn output(instance: *Self) *std.Io.Writer {
	133	+ return instance.writer_ref orelse &instance.writer;
	134	+ }
	135	+
117	136	const vtable = Reporter.VTable{
118	137	.onRunStart = onRunStart,
119	138	.onRunEnd = onRunEnd,
		⋯9 linesShow the 9 lines above this hunk@@ -129,64 +148,64 @@ pub const SpecReporter = struct {
129	148
130	149	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
131	150	const s = self(reporter);
132		- try s.writer.print("\n", .{});
	151	+ try s.output().print("\n", .{});
133	152	if (reporter.use_colors) {
134		- try s.writer.print("{s}Running {d} test(s)...{s}\n\n", .{ Colors.bold, total_tests, Colors.reset });
	153	+ try s.output().print("{s}Running {d} test(s)...{s}\n\n", .{ Colors.bold, total_tests, Colors.reset });
135	154	} else {
136		- try s.writer.print("Running {d} test(s)...\n\n", .{total_tests});
	155	+ try s.output().print("Running {d} test(s)...\n\n", .{total_tests});
137	156	}
138	157	}
139	158
140	159	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
141	160	const s = self(reporter);
142		- try s.writer.print("\n", .{});
	161	+ try s.output().print("\n", .{});
143	162
144	163	// Print failed tests details
145	164	if (results.failed > 0) {
146	165	if (reporter.use_colors) {
147		- try s.writer.print("{s}Failed Tests:{s}\n\n", .{ Colors.bold ++ Colors.red, Colors.reset });
	166	+ try s.output().print("{s}Failed Tests:{s}\n\n", .{ Colors.bold ++ Colors.red, Colors.reset });
148	167	} else {
149		- try s.writer.print("Failed Tests:\n\n", .{});
	168	+ try s.output().print("Failed Tests:\n\n", .{});
150	169	}
151	170
152	171	for (results.failed_tests.items) |test_case| {
153	172	if (reporter.use_colors) {
154		- try s.writer.print(" {s}✗{s} {s}\n", .{ Colors.red, Colors.reset, test_case.name });
	173	+ try s.output().print(" {s}✗{s} {s}\n", .{ Colors.red, Colors.reset, test_case.name });
155	174	} else {
156		- try s.writer.print(" ✗ {s}\n", .{test_case.name});
	175	+ try s.output().print(" ✗ {s}\n", .{test_case.name});
157	176	}
158	177	if (test_case.error_message) |msg| {
159		- try s.writer.print(" {s}\n", .{msg});
	178	+ try s.output().print(" {s}\n", .{msg});
160	179	}
161	180	}
162		- try s.writer.print("\n", .{});
	181	+ try s.output().print("\n", .{});
163	182	}
164	183
165	184	// Print summary
166	185	const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0;
167	186
168	187	if (reporter.use_colors) {
169		- try s.writer.print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset });
170		- try s.writer.print(" Total: {d}\n", .{results.total});
171		- try s.writer.print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset });
	188	+ try s.output().print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset });
	189	+ try s.output().print(" Total: {d}\n", .{results.total});
	190	+ try s.output().print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset });
172	191	if (results.failed > 0) {
173		- try s.writer.print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset });
	192	+ try s.output().print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset });
174	193	}
175	194	if (results.skipped > 0) {
176		- try s.writer.print(" {s}Skipped: {d}{s}\n", .{ Colors.yellow, results.skipped, Colors.reset });
	195	+ try s.output().print(" {s}Skipped: {d}{s}\n", .{ Colors.yellow, results.skipped, Colors.reset });
177	196	}
178		- try s.writer.print(" Time: {d:.2}ms\n", .{total_time_ms});
	197	+ try s.output().print(" Time: {d:.2}ms\n", .{total_time_ms});
179	198	} else {
180		- try s.writer.print("Test Summary:\n", .{});
181		- try s.writer.print(" Total: {d}\n", .{results.total});
182		- try s.writer.print(" Passed: {d}\n", .{results.passed});
	199	+ try s.output().print("Test Summary:\n", .{});
	200	+ try s.output().print(" Total: {d}\n", .{results.total});
	201	+ try s.output().print(" Passed: {d}\n", .{results.passed});
183	202	if (results.failed > 0) {
184		- try s.writer.print(" Failed: {d}\n", .{results.failed});
	203	+ try s.output().print(" Failed: {d}\n", .{results.failed});
185	204	}
186	205	if (results.skipped > 0) {
187		- try s.writer.print(" Skipped: {d}\n", .{results.skipped});
	206	+ try s.output().print(" Skipped: {d}\n", .{results.skipped});
188	207	}
189		- try s.writer.print(" Time: {d:.2}ms\n", .{total_time_ms});
	208	+ try s.output().print(" Time: {d:.2}ms\n", .{total_time_ms});
190	209	}
191	210	}
192	211
		⋯3 linesShow the 3 lines above this hunk@@ -196,13 +215,13 @@ pub const SpecReporter = struct {
196	215	// Print indent
197	216	var i: usize = 0;
198	217	while (i < s.indent_level) : (i += 1) {
199		- try s.writer.print(" ", .{});
	218	+ try s.output().print(" ", .{});
200	219	}
201	220
202	221	if (reporter.use_colors) {
203		- try s.writer.print("{s}{s}{s}\n", .{ Colors.bold, suite_name, Colors.reset });
	222	+ try s.output().print("{s}{s}{s}\n", .{ Colors.bold, suite_name, Colors.reset });
204	223	} else {
205		- try s.writer.print("{s}\n", .{suite_name});
	224	+ try s.output().print("{s}\n", .{suite_name});
206	225	}
207	226	s.indent_level += 1;
208	227	}
		⋯17 linesShow the 17 lines above this hunk@@ -226,7 +245,7 @@ pub const SpecReporter = struct {
226	245	// Print indent
227	246	var i: usize = 0;
228	247	while (i < s.indent_level) : (i += 1) {
229		- try s.writer.print(" ", .{});
	248	+ try s.output().print(" ", .{});
230	249	}
231	250
232	251	const time_ms = @as(f64, @floatFromInt(test_case.execution_time_ns)) / 1_000_000.0;
		⋯1 lineShow the 1 line above this hunk@@ -234,7 +253,7 @@ pub const SpecReporter = struct {
234	253	switch (test_case.status) {
235	254	.passed => {
236	255	if (reporter.use_colors) {
237		- try s.writer.print("{s}✓{s} {s} {s}({d:.2}ms){s}\n", .{
	256	+ try s.output().print("{s}✓{s} {s} {s}({d:.2}ms){s}\n", .{
238	257	Colors.green,
239	258	Colors.reset,
240	259	test_case.name,
		⋯2 linesShow the 2 lines above this hunk@@ -243,12 +262,12 @@ pub const SpecReporter = struct {
243	262	Colors.reset,
244	263	});
245	264	} else {
246		- try s.writer.print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms });
	265	+ try s.output().print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms });
247	266	}
248	267	},
249	268	.failed => {
250	269	if (reporter.use_colors) {
251		- try s.writer.print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{
	270	+ try s.output().print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{
252	271	Colors.red,
253	272	Colors.reset,
254	273	test_case.name,
		⋯2 linesShow the 2 lines above this hunk@@ -257,12 +276,12 @@ pub const SpecReporter = struct {
257	276	Colors.reset,
258	277	});
259	278	} else {
260		- try s.writer.print("✗ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms });
	279	+ try s.output().print("✗ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms });
261	280	}
262	281	},
263	282	.skipped => {
264	283	if (reporter.use_colors) {
265		- try s.writer.print("{s}⊘{s} {s} {s}(skipped){s}\n", .{
	284	+ try s.output().print("{s}⊘{s} {s} {s}(skipped){s}\n", .{
266	285	Colors.yellow,
267	286	Colors.reset,
268	287	test_case.name,
		⋯1 lineShow the 1 line above this hunk@@ -270,7 +289,7 @@ pub const SpecReporter = struct {
270	289	Colors.reset,
271	290	});
272	291	} else {
273		- try s.writer.print("⊘ {s} (skipped)\n", .{test_case.name});
	292	+ try s.output().print("⊘ {s} (skipped)\n", .{test_case.name});
274	293	}
275	294	},
276	295	else => {},
		⋯5 linesShow the 5 lines above this hunk@@ -282,6 +301,7 @@ pub const SpecReporter = struct {
282	301	pub const DotReporter = struct {
283	302	reporter: Reporter,
284	303	writer: std.Io.Writer,
	304	+ writer_ref: ?*std.Io.Writer = null,
285	305	tests_per_line: usize = 80,
286	306	current_line_count: usize = 0,
287	307
		⋯9 linesShow the 9 lines above this hunk@@ -297,6 +317,16 @@ pub const DotReporter = struct {
297	317	};
298	318	}
299	319
	320	+ pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self {
	321	+ var result = init(allocator, writer.*);
	322	+ result.writer_ref = writer;
	323	+ return result;
	324	+ }
	325	+
	326	+ fn output(instance: *Self) *std.Io.Writer {
	327	+ return instance.writer_ref orelse &instance.writer;
	328	+ }
	329	+
300	330	const vtable = Reporter.VTable{
301	331	.onRunStart = onRunStart,
302	332	.onRunEnd = onRunEnd,
		⋯9 linesShow the 9 lines above this hunk@@ -312,21 +342,21 @@ pub const DotReporter = struct {
312	342
313	343	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
314	344	const s = self(reporter);
315		- try s.writer.print("\nRunning {d} tests:\n", .{total_tests});
	345	+ try s.output().print("\nRunning {d} tests:\n", .{total_tests});
316	346	}
317	347
318	348	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
319	349	const s = self(reporter);
320		- try s.writer.print("\n\n", .{});
	350	+ try s.output().print("\n\n", .{});
321	351
322	352	const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0;
323	353
324	354	if (reporter.use_colors) {
325		- try s.writer.print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset });
326		- try s.writer.print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset });
327		- try s.writer.print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms });
	355	+ try s.output().print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset });
	356	+ try s.output().print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset });
	357	+ try s.output().print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms });
328	358	} else {
329		- try s.writer.print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{
	359	+ try s.output().print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{
330	360	results.passed,
331	361	results.failed,
332	362	results.total,
		⋯23 linesShow the 23 lines above this hunk@@ -356,23 +386,23 @@ pub const DotReporter = struct {
356	386	switch (test_case.status) {
357	387	.passed => {
358	388	if (reporter.use_colors) {
359		- try s.writer.print("{s}.{s}", .{ Colors.green, Colors.reset });
	389	+ try s.output().print("{s}.{s}", .{ Colors.green, Colors.reset });
360	390	} else {
361		- try s.writer.print(".", .{});
	391	+ try s.output().print(".", .{});
362	392	}
363	393	},
364	394	.failed => {
365	395	if (reporter.use_colors) {
366		- try s.writer.print("{s}F{s}", .{ Colors.red, Colors.reset });
	396	+ try s.output().print("{s}F{s}", .{ Colors.red, Colors.reset });
367	397	} else {
368		- try s.writer.print("F", .{});
	398	+ try s.output().print("F", .{});
369	399	}
370	400	},
371	401	.skipped => {
372	402	if (reporter.use_colors) {
373		- try s.writer.print("{s}S{s}", .{ Colors.yellow, Colors.reset });
	403	+ try s.output().print("{s}S{s}", .{ Colors.yellow, Colors.reset });
374	404	} else {
375		- try s.writer.print("S", .{});
	405	+ try s.output().print("S", .{});
376	406	}
377	407	},
378	408	else => {},
		⋯1 lineShow the 1 line above this hunk@@ -380,7 +410,7 @@ pub const DotReporter = struct {
380	410
381	411	s.current_line_count += 1;
382	412	if (s.current_line_count >= s.tests_per_line) {
383		- try s.writer.print("\n", .{});
	413	+ try s.output().print("\n", .{});
384	414	s.current_line_count = 0;
385	415	}
386	416	}
		⋯3 linesShow the 3 lines above this hunk@@ -390,6 +420,7 @@ pub const DotReporter = struct {
390	420	pub const JsonReporter = struct {
391	421	reporter: Reporter,
392	422	writer: std.Io.Writer,
	423	+ writer_ref: ?*std.Io.Writer = null,
393	424	suites: std.ArrayList([]const u8),
394	425
395	426	const Self = @This();
		⋯9 linesShow the 9 lines above this hunk@@ -405,6 +436,16 @@ pub const JsonReporter = struct {
405	436	};
406	437	}
407	438
	439	+ pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self {
	440	+ var result = init(allocator, writer.*);
	441	+ result.writer_ref = writer;
	442	+ return result;
	443	+ }
	444	+
	445	+ fn output(instance: *Self) *std.Io.Writer {
	446	+ return instance.writer_ref orelse &instance.writer;
	447	+ }
	448	+
408	449	pub fn deinit(s: *Self) void {
409	450	s.suites.deinit(s.reporter.allocator);
410	451	}
		⋯13 linesShow the 13 lines above this hunk@@ -424,14 +465,14 @@ pub const JsonReporter = struct {
424	465
425	466	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
426	467	const s = self(reporter);
427		- try s.writer.print("{{\"totalTests\":{d},\"tests\":[\n", .{total_tests});
	468	+ try s.output().print("{{\"totalTests\":{d},\"tests\":[\n", .{total_tests});
428	469	}
429	470
430	471	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
431	472	const s = self(reporter);
432	473	const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0;
433	474
434		- try s.writer.print("],\"summary\":{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{
	475	+ try s.output().print("],\"summary\":{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{
435	476	results.total,
436	477	results.passed,
437	478	results.failed,
		⋯32 linesShow the 32 lines above this hunk@@ -470,17 +511,17 @@ pub const JsonReporter = struct {
470	511	};
471	512
472	513	// Note: In a real implementation, you'd want to properly escape JSON strings
473		- try s.writer.print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{
	514	+ try s.output().print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{
474	515	test_case.name,
475	516	status_str,
476	517	time_ms,
477	518	});
478	519
479	520	if (test_case.error_message) |msg| {
480		- try s.writer.print(",\"error\":\"{s}\"", .{msg});
	521	+ try s.output().print(",\"error\":\"{s}\"", .{msg});
481	522	}
482	523
483		- try s.writer.print("}},\n", .{});
	524	+ try s.output().print("}},\n", .{});
484	525	}
485	526	};
486	527
		⋯1 lineShow the 1 line above this hunk@@ -488,6 +529,7 @@ pub const JsonReporter = struct {
488	529	pub const TAPReporter = struct {
489	530	reporter: Reporter,
490	531	writer: std.Io.Writer,
	532	+ writer_ref: ?*std.Io.Writer = null,
491	533	test_count: usize = 0,
492	534
493	535	const Self = @This();
		⋯16 linesShow the 16 lines above this hunk@@ -510,9 +552,19 @@ pub const TAPReporter = struct {
510	552	};
511	553	}
512	554
	555	+ pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self {
	556	+ var result = init(allocator, writer.*);
	557	+ result.writer_ref = writer;
	558	+ return result;
	559	+ }
	560	+
	561	+ fn output(instance: *Self) *std.Io.Writer {
	562	+ return instance.writer_ref orelse &instance.writer;
	563	+ }
	564	+
513	565	fn onRunStart(reporter: *Reporter, total: usize) !void {
514	566	const self: *Self = @fieldParentPtr("reporter", reporter);
515		- try self.writer.print("TAP version 14\n1..{d}\n", .{total});
	567	+ try self.output().print("TAP version 14\n1..{d}\n", .{total});
516	568	}
517	569
518	570	fn onRunEnd(reporter: *Reporter, results: *TestResults) !void {
		⋯3 linesShow the 3 lines above this hunk@@ -522,7 +574,7 @@ pub const TAPReporter = struct {
522	574
523	575	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
524	576	const self: *Self = @fieldParentPtr("reporter", reporter);
525		- try self.writer.print("# Subtest: {s}\n", .{suite_name});
	577	+ try self.output().print("# Subtest: {s}\n", .{suite_name});
526	578	}
527	579
528	580	fn onSuiteEnd(reporter: *Reporter, suite_name: []const u8) !void {
		⋯17 linesShow the 17 lines above this hunk@@ -546,14 +598,14 @@ pub const TAPReporter = struct {
546	598	else => "not ok",
547	599	};
548	600
549		- try self.writer.print("{s} {d} - {s}", .{ status, self.test_count, test_case.name });
	601	+ try self.output().print("{s} {d} - {s}", .{ status, self.test_count, test_case.name });
550	602
551	603	if (test_case.status == .skipped) {
552		- try self.writer.print(" # SKIP\n", .{});
	604	+ try self.output().print(" # SKIP\n", .{});
553	605	} else if (test_case.status == .failed and test_case.error_message != null) {
554		- try self.writer.print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?});
	606	+ try self.output().print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?});
555	607	} else {
556		- try self.writer.print("\n", .{});
	608	+ try self.output().print("\n", .{});
557	609	}
558	610	}
559	611	};
		⋯175 linesShow the 175 lines above this hunk@@ -735,6 +787,80 @@ pub const JUnitReporter = struct {
735	787	}
736	788	};
737	789
	790	+/// Owns every built-in reporter and selects one through a shared interface.
	791	+/// Keeping construction here lets every executor expose identical reporters.
	792	+pub const ReporterSet = struct {
	793	+ kind: ReporterType,
	794	+ spec: SpecReporter,
	795	+ dot: DotReporter,
	796	+ json: JsonReporter,
	797	+ tap: TAPReporter,
	798	+ junit: JUnitReporter,
	799	+
	800	+ pub fn init(
	801	+ allocator: std.mem.Allocator,
	802	+ writer: std.Io.Writer,
	803	+ kind: ReporterType,
	804	+ junit_output: []const u8,
	805	+ use_colors: bool,
	806	+ ) ReporterSet {
	807	+ var reporters = ReporterSet{
	808	+ .kind = kind,
	809	+ .spec = SpecReporter.init(allocator, writer),
	810	+ .dot = DotReporter.init(allocator, writer),
	811	+ .json = JsonReporter.init(allocator, writer),
	812	+ .tap = TAPReporter.init(allocator, writer),
	813	+ .junit = JUnitReporter.init(allocator, junit_output),
	814	+ };
	815	+ reporters.selected().use_colors = use_colors;
	816	+ return reporters;
	817	+ }
	818	+
	819	+ pub fn initRef(
	820	+ allocator: std.mem.Allocator,
	821	+ writer: *std.Io.Writer,
	822	+ kind: ReporterType,
	823	+ junit_output: []const u8,
	824	+ use_colors: bool,
	825	+ ) ReporterSet {
	826	+ var reporters = ReporterSet{
	827	+ .kind = kind,
	828	+ .spec = SpecReporter.initRef(allocator, writer),
	829	+ .dot = DotReporter.initRef(allocator, writer),
	830	+ .json = JsonReporter.initRef(allocator, writer),
	831	+ .tap = TAPReporter.initRef(allocator, writer),
	832	+ .junit = JUnitReporter.init(allocator, junit_output),
	833	+ };
	834	+ reporters.selected().use_colors = use_colors;
	835	+ return reporters;
	836	+ }
	837	+
	838	+ pub fn deinit(self: *ReporterSet) void {
	839	+ self.json.deinit();
	840	+ self.junit.deinit();
	841	+ }
	842	+
	843	+ pub fn selected(self: *ReporterSet) *Reporter {
	844	+ return switch (self.kind) {
	845	+ .spec => &self.spec.reporter,
	846	+ .dot => &self.dot.reporter,
	847	+ .json => &self.json.reporter,
	848	+ .tap => &self.tap.reporter,
	849	+ .junit => &self.junit.reporter,
	850	+ };
	851	+ }
	852	+
	853	+ pub fn flush(self: *ReporterSet) !void {
	854	+ switch (self.kind) {
	855	+ .spec => try self.spec.output().flush(),
	856	+ .dot => try self.dot.output().flush(),
	857	+ .json => try self.json.output().flush(),
	858	+ .tap => try self.tap.output().flush(),
	859	+ .junit => {},
	860	+ }
	861	+ }
	862	+};
	863	+
738	864	// Tests
739	865	test "Colors constants exist" {
740	866	// Just verify the constants are defined

src/test_loader.zigmodified+89-38

Changes to src/test_loader.zig

		@@ -1,6 +1,9 @@
1	1	const std = ;
2	2	const discovery = ;
3	3	const sharding = ;
	4	+const pipeline = ;
	5	+const reporter = ;
	6	+const suite = ;
4	7	const coverage = ;
5	8	const ui_server = ;
6	9	const compat = ;
		⋯10 linesShow the 10 lines above this hunk@@ -17,6 +20,15 @@ pub const LoaderOptions = struct {
17	20	use_colors: bool = true,
18	21	/// Optional one-based file shard to execute
19	22	shard: ?sharding.ShardOptions = null,
	23	+ /// Reporter shared with programmatic execution
	24	+ reporter_type: reporter.ReporterType = .spec,
	25	+ /// Output path used by the JUnit reporter
	26	+ junit_output: []const u8 = "test-results.xml",
	27	+ /// Global per-file timeout checked by the shared execution policy
	28	+ timeout_ms: ?u64 = null,
	29	+ /// Optional output supplied by a CLI host. Embedded/test callers default
	30	+ /// to stderr so they do not interfere with Zig's stdout test protocol.
	31	+ reporter_writer: ?*std.Io.Writer = null,
20	32	/// Coverage options
21	33	coverage_options: ?coverage.CoverageOptions = null,
22	34	/// UI server for real-time updates
		⋯11 linesShow the 11 lines above this hunk@@ -34,7 +46,9 @@ pub fn runDiscoveredTests(
34	46	return false;
35	47	}
36	48
37		- const selected_files = try selectedFileCount(discovered, options.shard);
	49	+ var plan = try discoveryPlan(allocator, discovered, options.shard);
	50	+ defer plan.deinit();
	51	+ const selected_files = plan.items.items.len;
38	52	if (options.shard) |shard| {
39	53	std.debug.print(
40	54	"Shard {}/{}: selected {} of {} test file(s).\n",
		⋯2 linesShow the 2 lines above this hunk@@ -43,12 +57,37 @@ pub fn runDiscoveredTests(
43	57	}
44	58
45	59	std.debug.print("Found {} test file(s):\n", .{selected_files});
46		- for (discovered.files.items) |file| {
47		- if (!try fileSelected(file.relative_path, options.shard)) continue;
	60	+ for (plan.items.items) |item| {
	61	+ const file = item.discovered;
48	62	std.debug.print(" - {s}\n", .{file.relative_path});
49	63	}
50	64	std.debug.print("\n", .{});
51	65
	66	+ const stderr_file = std.Io.File.stderr();
	67	+ var stderr_buffer: [4096]u8 = undefined;
	68	+ var threaded_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty });
	69	+ defer threaded_io.deinit();
	70	+ var stderr_writer = stderr_file.writer(threaded_io.io(), &stderr_buffer);
	71	+ const reporter_writer = options.reporter_writer orelse &stderr_writer.interface;
	72	+ var reporters = reporter.ReporterSet.initRef(
	73	+ allocator,
	74	+ reporter_writer,
	75	+ options.reporter_type,
	76	+ options.junit_output,
	77	+ options.use_colors,
	78	+ );
	79	+ defer reporters.deinit();
	80	+ const events = pipeline.EventStream{ .rep = reporters.selected() };
	81	+ const policy = pipeline.ExecutionPolicy{
	82	+ .bail = options.bail,
	83	+ .filter = options.filter,
	84	+ .timeout_ms = options.timeout_ms,
	85	+ };
	86	+ var results = reporter.TestResults.init(allocator);
	87	+ defer results.deinit();
	88	+
	89	+ try events.emit(.{ .run_started = selected_files });
	90	+
52	91	// Notify UI of run start
53	92	if (options.ui_server) |server| {
54	93	var buffer: [256]u8 = undefined;
		⋯1 lineShow the 1 line above this hunk@@ -56,8 +95,6 @@ pub fn runDiscoveredTests(
56	95	try server.broadcast("run_start", json);
57	96	}
58	97
59		- var total_passed: usize = 0;
60		- var total_failed: usize = 0;
61	98	var files_run: usize = 0;
62	99
63	100	// Clean coverage directory once before all tests if coverage is enabled
		⋯4 linesShow the 4 lines above this hunk@@ -68,11 +105,12 @@ pub fn runDiscoveredTests(
68	105	}
69	106	}
70	107
71		- for (discovered.files.items) |file| {
72		- if (!try fileSelected(file.relative_path, options.shard)) continue;
73		-
	108	+ for (plan.items.items) |item| {
	109	+ const file = item.discovered;
74	110	// Run each test file using zig test command
75	111	std.debug.print("Running {s}...\n", .{file.relative_path});
	112	+ try events.emit(.{ .suite_started = file.relative_path });
	113	+ try events.emit(.{ .test_started = file.name });
76	114
77	115	// Notify UI of test file start
78	116	if (options.ui_server) |server| {
		⋯4 linesShow the 4 lines above this hunk@@ -83,11 +121,25 @@ pub fn runDiscoveredTests(
83	121	}
84	122
85	123	const start_time = compat.nanoTimestamp();
86		- const result = try runTestFile(allocator, file.path, options);
	124	+ var result = try runTestFile(allocator, file.path, options);
87	125	const end_time = compat.nanoTimestamp();
88	126	const execution_time_ns: u64 = @intCast(end_time - start_time);
	127	+ const timed_out = policy.timedOut(execution_time_ns);
	128	+ if (timed_out) result = false;
89	129
90	130	files_run += 1;
	131	+ var test_case = suite.TestCase.init(file.name, externalTestNoop);
	132	+ test_case.file = file.relative_path;
	133	+ test_case.execution_time_ns = execution_time_ns;
	134	+ test_case.status = if (result) .passed else .failed;
	135	+ if (timed_out) {
	136	+ test_case.error_message = "External test file exceeded the configured timeout";
	137	+ } else if (!result) {
	138	+ test_case.error_message = "External zig test process failed";
	139	+ }
	140	+ try results.addTest(&test_case);
	141	+ try events.emit(.{ .test_finished = &test_case });
	142	+ try events.emit(.{ .suite_finished = file.relative_path });
91	143
92	144	// Notify UI of test file end
93	145	if (options.ui_server) |server| {
		⋯5 linesShow the 5 lines above this hunk@@ -99,15 +151,10 @@ pub fn runDiscoveredTests(
99	151	}
100	152
101	153	if (result) {
102		- total_passed += 1;
103		- if (options.verbose) {
104		- std.debug.print(" ✓ {s} passed\n", .{file.name});
105		- }
	154	+ if (options.verbose) std.debug.print(" ✓ {s} passed\n", .{file.name});
106	155	} else {
107		- total_failed += 1;
108	156	std.debug.print(" ✗ {s} failed\n", .{file.name});
109		-
110		- if (options.bail) {
	157	+ if (policy.shouldStop(&results)) {
111	158	std.debug.print("\nStopping on first failure (--bail)\n", .{});
112	159	break;
113	160	}
		⋯4 linesShow the 4 lines above this hunk@@ -118,48 +165,49 @@ pub fn runDiscoveredTests(
118	165	if (options.coverage_options) |cov_opts| {
119	166	if (cov_opts.enabled and files_run > 0) {
120	167	std.debug.print("\n", .{});
121		- const cov_result = coverage.parseCoverageReport(allocator, cov_opts.output_dir) catch |err| {
	168	+ const cov_result: ?coverage.CoverageResult = coverage.parseCoverageReport(allocator, cov_opts.output_dir) catch |err| result: {
122	169	std.debug.print("Warning: Could not parse coverage report: {any}\n", .{err});
123		- return total_failed == 0;
	170	+ break :result null;
124	171	};
125		-
126		- coverage.printCoverageSummary(cov_result);
	172	+ if (cov_result) |summary| coverage.printCoverageSummary(summary);
127	173	}
128	174	}
129	175
130	176	// Notify UI of run end
131	177	if (options.ui_server) |server| {
132	178	var buffer: [512]u8 = undefined;
133		- const json = try std.fmt.bufPrint(&buffer, "", .{ files_run, total_passed, total_failed });
	179	+ const json = try std.fmt.bufPrint(&buffer, "{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d}}}", .{ results.total, results.passed, results.failed, results.skipped });
134	180	try server.broadcast("run_end", json);
135	181	}
136	182
137		- std.debug.print("\n", .{});
138		- std.debug.print("Test Summary:\n", .{});
139	183	if (options.shard) |shard| {
	184	+ std.debug.print("\nShard Summary:\n", .{});
140	185	std.debug.print(" Shard: {}/{}\n", .{ shard.index, shard.count });
141	186	std.debug.print(" Selected: {} of {} files\n", .{ selected_files, discovered.files.items.len });
142	187	}
143		- std.debug.print(" Files run: {}\n", .{files_run});
144		- std.debug.print(" Passed: {}\n", .{total_passed});
145		- std.debug.print(" Failed: {}\n", .{total_failed});
	188	+ try events.emit(.{ .run_finished = &results });
	189	+ try reporters.flush();
146	190
147		- return total_failed == 0;
	191	+ return results.failed == 0;
148	192	}
149	193
	194	+fn externalTestNoop(_: std.mem.Allocator) !void {}
	195	+
150	196	fn fileSelected(relative_path: []const u8, shard: ?sharding.ShardOptions) !bool {
151	197	return if (shard) |selection| selection.includes(relative_path) else true;
152	198	}
153	199
154		-fn selectedFileCount(
155		- discovered: *const discovery.DiscoveryResult,
	200	+fn discoveryPlan(
	201	+ allocator: std.mem.Allocator,
	202	+ discovered: *discovery.DiscoveryResult,
156	203	shard: ?sharding.ShardOptions,
157		-) !usize {
158		- var count: usize = 0;
159		- for (discovered.files.items) |file| {
160		- if (try fileSelected(file.relative_path, shard)) count += 1;
	204	+) !pipeline.TestPlan {
	205	+ var plan = pipeline.TestPlan.init(allocator);
	206	+ errdefer plan.deinit();
	207	+ for (discovered.files.items) |*file| {
	208	+ if (try fileSelected(file.relative_path, shard)) try plan.appendDiscovered(file);
161	209	}
162		- return count;
	210	+ return plan;
163	211	}
164	212
165	213	/// Run a single test file using `zig test`
		⋯75 linesShow the 75 lines above this hunk@@ -241,6 +289,10 @@ test "LoaderOptions default values" {
241	289	try std.testing.expectEqual(false, options.verbose);
242	290	try std.testing.expectEqual(true, options.use_colors);
243	291	try std.testing.expectEqual(@as(?sharding.ShardOptions, null), options.shard);
	292	+ try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type);
	293	+ try std.testing.expectEqualStrings("test-results.xml", options.junit_output);
	294	+ try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
	295	+ try std.testing.expectEqual(@as(?*std.Io.Writer, null), options.reporter_writer);
244	296	try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options);
245	297	}
246	298
		⋯56 linesShow the 56 lines above this hunk@@ -303,10 +355,9 @@ test "file sharding partitions discovered files exactly once" {
303	355
304	356	var total_selected: usize = 0;
305	357	for (1..4) |index| {
306		- total_selected += try selectedFileCount(
307		- &discovered,
308		- .{ .index = index, .count = 3 },
309		- );
	358	+ var plan = try discoveryPlan(allocator, &discovered, .{ .index = index, .count = 3 });
	359	+ defer plan.deinit();
	360	+ total_selected += plan.items.items.len;
310	361	}
311	362
312	363	try std.testing.expectEqual(discovered.files.items.len, total_selected);

src/test_runner.zigmodified+73-61

Changes to src/test_runner.zig

		@@ -1,6 +1,7 @@
1	1	const std = ;
2	2	const suite = ;
3	3	const reporter_mod = ;
	4	+const pipeline = ;
4	5	const parallel = ;
5	6	const compat = ;
6	7
		⋯9 linesShow the 9 lines above this hunk@@ -16,15 +17,14 @@ pub const RunnerOptions = struct {
16	17	use_colors: bool = true,
17	18	parallel: bool = false, // Enable parallel execution
18	19	n_jobs: ?usize = null, // Number of parallel jobs
	20	+ junit_output: []const u8 = "test-results.xml",
	21	+ timeout_ms: ?u64 = null,
	22	+ /// Optional output supplied by a CLI host. Embedded/test callers default
	23	+ /// to stderr so they do not interfere with Zig's stdout test protocol.
	24	+ reporter_writer: ?*std.Io.Writer = null,
19	25	};
20	26
21		-pub const ReporterType = enum {
22		- spec,
23		- dot,
24		- json,
25		- tap,
26		- junit,
27		-};
	27	+pub const ReporterType = reporter_mod.ReporterType;
28	28
29	29	pub const TestRunner = struct {
30	30	allocator: std.mem.Allocator,
		⋯18 linesShow the 18 lines above this hunk@@ -49,32 +49,27 @@ pub const TestRunner = struct {
49	49
50	50	/// Run all registered tests
51	51	pub fn run(self: *Self) !bool {
52		- const stdout_file = std.Io.File.stdout();
53		- var stdout_buffer: [4096]u8 = undefined;
	52	+ const stderr_file = std.Io.File.stderr();
	53	+ var stderr_buffer: [4096]u8 = undefined;
54	54	var threaded_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty });
55	55	defer threaded_io.deinit();
56		- var stdout_writer = stdout_file.writer(threaded_io.io(), &stdout_buffer);
57		-
58		- // Create reporter
59		- var spec_reporter = reporter_mod.SpecReporter.init(self.allocator, stdout_writer.interface);
60		- var dot_reporter = reporter_mod.DotReporter.init(self.allocator, stdout_writer.interface);
61		- var json_reporter = reporter_mod.JsonReporter.init(self.allocator, stdout_writer.interface);
62		- defer json_reporter.deinit();
63		- var tap_reporter = reporter_mod.TAPReporter.init(self.allocator, stdout_writer.interface);
64		- var junit_reporter = reporter_mod.JUnitReporter.init(self.allocator, "test-results.xml");
65		- defer junit_reporter.deinit();
66		-
67		- var current_reporter: *reporter_mod.Reporter = switch (self.options.reporter_type) {
68		- .spec => &spec_reporter.reporter,
69		- .dot => &dot_reporter.reporter,
70		- .json => &json_reporter.reporter,
71		- .tap => &tap_reporter.reporter,
72		- .junit => &junit_reporter.reporter,
73		- };
74		-
75		- current_reporter.use_colors = self.options.use_colors;
76		-
77		- const total_tests = self.registry.countAllTests();
	56	+ var stderr_writer = stderr_file.writer(threaded_io.io(), &stderr_buffer);
	57	+ const reporter_writer = self.options.reporter_writer orelse &stderr_writer.interface;
	58	+
	59	+ var reporters = reporter_mod.ReporterSet.initRef(
	60	+ self.allocator,
	61	+ reporter_writer,
	62	+ self.options.reporter_type,
	63	+ self.options.junit_output,
	64	+ self.options.use_colors,
	65	+ );
	66	+ defer reporters.deinit();
	67	+ const current_reporter = reporters.selected();
	68	+ const events = pipeline.EventStream{ .rep = current_reporter };
	69	+
	70	+ var plan = try pipeline.TestPlan.fromRegistry(self.allocator, self.registry);
	71	+ defer plan.deinit();
	72	+ const total_tests = plan.items.items.len;
78	73	if (total_tests == 0) {
79	74	return RunnerError.NoTestsFound;
80	75	}
		⋯4 linesShow the 4 lines above this hunk@@ -85,6 +80,7 @@ pub const TestRunner = struct {
85	80	.enabled = true,
86	81	.n_jobs = self.options.n_jobs,
87	82	.filter = self.options.filter,
	83	+ .timeout_ms = self.options.timeout_ms,
88	84	};
89	85
90	86	const all_passed = try parallel.runTestsParallel(
		⋯7 linesShow the 7 lines above this hunk@@ -98,24 +94,24 @@ pub const TestRunner = struct {
98	94	try self.addSuiteResults(test_suite);
99	95	}
100	96
101		- try stdout_writer.interface.flush();
	97	+ try reporters.flush();
102	98	return all_passed and self.results.failed == 0;
103	99	}
104	100
105	101	// Sequential execution (original behavior)
106		- try current_reporter.onRunStart(total_tests);
	102	+ try events.emit(.{ .run_started = total_tests });
107	103	for (self.registry.root_suites.items) |test_suite| {
108		- try self.runSuite(test_suite, current_reporter);
109		- if (self.options.bail and self.results.failed > 0) {
	104	+ try self.runSuite(test_suite, events);
	105	+ if (self.policy().shouldStop(&self.results)) {
110	106	break;
111	107	}
112	108	}
113	109
114	110	// Notify reporter of run end
115		- try current_reporter.onRunEnd(&self.results);
	111	+ try events.emit(.{ .run_finished = &self.results });
116	112
117	113	// Flush output
118		- try stdout_writer.interface.flush();
	114	+ try reporters.flush();
119	115
120	116	return self.results.failed == 0;
121	117	}
		⋯4 linesShow the 4 lines above this hunk@@ -126,26 +122,26 @@ pub const TestRunner = struct {
126	122	}
127	123
128	124	/// Run a single test suite
129		- fn runSuite(self: *Self, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void {
	125	+ fn runSuite(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
130	126	// Skip if marked as skip or if has_only and this isn't marked as only
131	127	if (test_suite.shouldSkip()) {
132		- try self.skipAllTests(test_suite);
	128	+ try self.skipAllTests(test_suite, events);
133	129	return;
134	130	}
135	131
136	132	if (self.registry.has_only and !test_suite.hasOnly()) {
137		- try self.skipAllTests(test_suite);
	133	+ try self.skipAllTests(test_suite, events);
138	134	return;
139	135	}
140	136
141	137	// Notify reporter
142		- try rep.onSuiteStart(test_suite.name);
	138	+ try events.emit(.{ .suite_started = test_suite.name });
143	139
144	140	// Run beforeAll hooks
145	141	test_suite.runBeforeAllHooks(self.allocator) catch |err| {
146	142	std.debug.print("beforeAll hook failed: {any}\n", .{err});
147		- try self.skipAllTests(test_suite);
148		- try rep.onSuiteEnd(test_suite.name);
	143	+ try self.skipAllTests(test_suite, events);
	144	+ try events.emit(.{ .suite_finished = test_suite.name });
149	145	return;
150	146	};
151	147
		⋯1 lineShow the 1 line above this hunk@@ -153,32 +149,30 @@ pub const TestRunner = struct {
153	149	for (test_suite.tests.items) |*test_case| {
154	150	if (test_case.skip or (self.registry.has_only and !test_case.only)) {
155	151	test_case.status = .skipped;
156		- try rep.onTestEnd(test_case);
	152	+ try events.emit(.{ .test_finished = test_case });
157	153	try self.results.addTest(test_case);
158	154	continue;
159	155	}
160	156
161	157	// Check filter
162		- if (self.options.filter) |filter| {
163		- if (std.mem.indexOf(u8, test_case.name, filter) == null) {
164		- test_case.status = .skipped;
165		- try rep.onTestEnd(test_case);
166		- try self.results.addTest(test_case);
167		- continue;
168		- }
	158	+ if (!self.policy().matches(test_case.name)) {
	159	+ test_case.status = .skipped;
	160	+ try events.emit(.{ .test_finished = test_case });
	161	+ try self.results.addTest(test_case);
	162	+ continue;
169	163	}
170	164
171		- try self.runTest(test_case, test_suite, rep);
	165	+ try self.runTest(test_case, test_suite, events);
172	166
173		- if (self.options.bail and test_case.status == .failed) {
	167	+ if (self.policy().shouldStop(&self.results)) {
174	168	break;
175	169	}
176	170	}
177	171
178	172	// Run nested suites
179	173	for (test_suite.suites.items) |nested_suite| {
180		- try self.runSuite(nested_suite, rep);
181		- if (self.options.bail and self.results.failed > 0) {
	174	+ try self.runSuite(nested_suite, events);
	175	+ if (self.policy().shouldStop(&self.results)) {
182	176	break;
183	177	}
184	178	}
		⋯3 linesShow the 3 lines above this hunk@@ -188,12 +182,12 @@ pub const TestRunner = struct {
188	182	std.debug.print("afterAll hook failed: {any}\n", .{err});
189	183	};
190	184
191		- try rep.onSuiteEnd(test_suite.name);
	185	+ try events.emit(.{ .suite_finished = test_suite.name });
192	186	}
193	187
194	188	/// Run a single test
195		- fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void {
196		- try rep.onTestStart(test_case.name);
	189	+ fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
	190	+ try events.emit(.{ .test_started = test_case.name });
197	191
198	192	test_case.status = .running;
199	193	const start_time = compat.nanoTimestamp();
		⋯40 linesShow the 40 lines above this hunk@@ -240,22 +234,39 @@ pub const TestRunner = struct {
240	234
241	235	const end_time = compat.nanoTimestamp();
242	236	test_case.execution_time_ns = @intCast(end_time - start_time);
	237	+ if (test_case.status == .passed and self.policy().timedOut(test_case.execution_time_ns)) {
	238	+ test_case.status = .failed;
	239	+ test_case.error_message = try std.fmt.allocPrint(
	240	+ self.allocator,
	241	+ "Test exceeded timeout of {d}ms",
	242	+ .{self.options.timeout_ms.?},
	243	+ );
	244	+ }
243	245
244	246	try self.results.addTest(test_case);
245		- try rep.onTestEnd(test_case);
	247	+ try events.emit(.{ .test_finished = test_case });
246	248	}
247	249
248	250	/// Skip all tests in a suite
249		- fn skipAllTests(self: *Self, test_suite: *suite.TestSuite) !void {
	251	+ fn skipAllTests(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void {
250	252	for (test_suite.tests.items) |*test_case| {
251	253	test_case.status = .skipped;
252	254	try self.results.addTest(test_case);
	255	+ try events.emit(.{ .test_finished = test_case });
253	256	}
254	257
255	258	for (test_suite.suites.items) |nested_suite| {
256		- try self.skipAllTests(nested_suite);
	259	+ try self.skipAllTests(nested_suite, events);
257	260	}
258	261	}
	262	+
	263	+ fn policy(self: *const Self) pipeline.ExecutionPolicy {
	264	+ return .{
	265	+ .bail = self.options.bail,
	266	+ .filter = self.options.filter,
	267	+ .timeout_ms = self.options.timeout_ms,
	268	+ };
	269	+ }
259	270	};
260	271
261	272	/// Helper function to run tests with default options
		⋯28 linesShow the 28 lines above this hunk@@ -290,6 +301,7 @@ test "RunnerOptions default values" {
290	301	try std.testing.expectEqual(@as(?[]const u8, null), options.filter);
291	302	try std.testing.expectEqual(ReporterType.spec, options.reporter_type);
292	303	try std.testing.expectEqual(true, options.use_colors);
	304	+ try std.testing.expectEqual(@as(?u64, null), options.timeout_ms);
293	305	}
294	306
295	307	test "RunnerOptions custom values" {

tests/reporter_test.zigmodified+16-0

Changes to tests/reporter_test.zig

		⋯68 linesShow the 68 lines above this hunk@@ -69,3 +69,19 @@ test "reporter json output" {
69	69	try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null);
70	70	try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null);
71	71	}
	72	+
	73	+test "reporter set selects the same built-ins for every executor" {
	74	+ var buffer: [1024]u8 = undefined;
	75	+ const writer: std.Io.Writer = .fixed(&buffer);
	76	+ var reporters = ztf.ReporterSet.init(
	77	+ std.testing.allocator,
	78	+ writer,
	79	+ .tap,
	80	+ "test-results.xml",
	81	+ false,
	82	+ );
	83	+ defer reporters.deinit();
	84	+
	85	+ try std.testing.expect(reporters.selected() == &reporters.tap.reporter);
	86	+ try std.testing.expect(!reporters.selected().use_colors);
	87	+}