also looking at this

Define stable JSON event protocol

#27

Merged glennmichael123 wants to merge codex/issue-10-json-events into main

11 files +442 -177

README.mdmodified+19-14

Changes to README.md
		@@ -820,28 +820,33 @@ Passed: 12, Failed: 0, Total: 12 (1.23ms)
820	820
821	821	### JSON Reporter
822	822
823		Machine-readable output for tooling:
	823	Machine-readable output uses the versioned event protocol shared with the live
	824	UI and test history:
824	825
825	826	```json
826		{
827		"totalTests": 12,
828		"tests": [
829		{
	827	[
	828	{
	829	"protocol_version": 1,
	830	"type": "run_start",
	831	"data": { "total": 12 }
	832	},
	833	{
	834	"protocol_version": 1,
	835	"type": "test_end",
	836	"data": {
830	837	"name": "should add two numbers",
	838	"suite": "Math operations",
831	839	"status": "passed",
832		"time": 0.05
	840	"duration_ns": 50000,
	841	"error_message": null
833	842	}
834		],
835		"summary": {
836		"total": 12,
837		"passed": 12,
838		"failed": 0,
839		"skipped": 0,
840		"time": 1.23
841	843	}
842		}
	844	]
843	845	```
844	846
	847	See [docs/json-event-protocol.md](docs/json-event-protocol.md) for every event
	848	type and the compatibility policy.
	849
845	850	## API Reference
846	851
847	852	See [docs/api.md](docs/api.md) for complete API documentation.

docs/execution-pipeline.mdmodified+4-3

Changes to docs/execution-pipeline.md
		@@ -43,9 +43,10 @@ New plan producers should append `PlanItem` values. New executors should emit
43	43	implement `Reporter.VTable`; `EventStream` adapts the typed lifecycle to that
44	44	stable callback interface.
45	45
46		The event types are intentionally internal Zig structures. The separately
47		versioned JSON wire protocol is built on this layer rather than making this API
48		depend on one serialization format.
	46	The lifecycle structures remain format-independent. The JSON reporter, live UI,
	47	and test history adapt them to the public, versioned
	48	[JSON event protocol](json-event-protocol.md), so integrations share one wire
	49	model without coupling the execution API to JSON.
49	50
50	51	## Migration
51	52

docs/json-event-protocol.mdadded+71-0

Changes to docs/json-event-protocol.md
		@@ -0,0 +1,71 @@
	1	# JSON event protocol
	2
	3	The framework exposes one versioned event model for the JSON reporter, live UI,
	4	and saved test history. Protocol version 1 uses this envelope for every event:
	5
	6	```json
	7	{
	8	"protocol_version": 1,
	9	"type": "test_end",
	10	"data": {
	11	"name": "adds two numbers",
	12	"suite": "math",
	13	"status": "passed",
	14	"duration_ns": 42000,
	15	"error_message": null
	16	}
	17	}
	18	```
	19
	20	All strings are encoded by Zig's standard JSON serializer. Consumers should
	21	decode JSON rather than depending on whitespace or object-field order.
	22
	23	## Transports
	24
	25	- The JSON reporter writes an ordered JSON array of event envelopes.
	26	- The live UI sends each envelope as a server-sent event. The SSE event name is
	27	the same as the envelope's `type`.
	28	- Test-history files contain an ordered JSON array using the same envelopes.
	29
	30	Events preserve execution order. A run begins with `run_start` and finishes
	31	with `run_end`; suite and test start/end events are nested in between. Retry
	32	events are emitted in attempt order immediately before their `test_end`.
	33
	34	## Version 1 events
	35
	36	\| Type \| Data fields \|
	37	\| --- \| --- \|
	38	\| `run_start` \| `total` \|
	39	\| `run_end` \| `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns` \|
	40	\| `suite_start`, `suite_end` \| `name` \|
	41	\| `test_start` \| `name`, optional `suite` \|
	42	\| `test_end` \| `name`, optional `suite`, `status`, `duration_ns`, optional `error_message` \|
	43	\| `hook` \| `kind`, optional `suite`, optional `test_name`, `status`, `duration_ns`, optional `error_message` \|
	44	\| `output` \| `stream`, `text`, optional `test_name` \|
	45	\| `retry` \| `name`, `attempt`, `repetition`, `status`, `duration_ns`, optional `error_message` \|
	46	\| `coverage` \| optional `line_percent`, `function_percent`, `branch_percent`, `report_path` \|
	47
	48	Statuses are `pending`, `running`, `passed`, `flaky`, `failed`, or `skipped`.
	49	Hook kinds are `before_all`, `before_each`, `after_each`, or `after_all`.
	50	Output streams are `stdout` or `stderr`.
	51
	52	Optional fields are serialized as `null` when no value is available. Durations
	53	are unsigned integer nanoseconds. Percentages are numbers from 0 through 100.
	54
	55	## Compatibility policy
	56
	57	`protocol_version` is the major wire-format version. Version 1 may gain new
	58	event types, enum values, and optional data fields without changing the version.
	59	Consumers must ignore event types and fields they do not understand. Existing
	60	fields will not change meaning or type within a major version.
	61
	62	Removing or renaming an event or field, changing a field's type or semantics,
	63	or changing the envelope shape requires a new major version. Producers emit one
	64	major version per stream. Consumers should reject an unsupported major version
	65	with a clear error instead of interpreting it as an older version.
	66
	67	## Zig API
	68
	69	`event_protocol.Event` is the tagged union for all event types, and
	70	`event_protocol.encodeAlloc` returns a standards-compliant encoded envelope.
	71	The root module also exports these as `ProtocolEvent` and `protocol_version`.

src/event_protocol.zigadded+136-0

Changes to src/event_protocol.zig
		@@ -0,0 +1,136 @@
	1	const std = ;
	2
	3	/// Major version of the public JSON event protocol.
	4	pub const version: u16 = 1;
	5
	6	pub const Status = enum { pending, running, passed, flaky, failed, skipped };
	7	pub const HookKind = enum { before_all, before_each, after_each, after_all };
	8	pub const OutputStream = enum { stdout, stderr };
	9
	10	pub const RunStart = struct { total: usize };
	11	pub const RunEnd = struct {
	12	total: usize,
	13	passed: usize,
	14	flaky: usize,
	15	failed: usize,
	16	skipped: usize,
	17	duration_ns: u64 = 0,
	18	};
	19	pub const Suite = struct { name: []const u8 };
	20	pub const TestStart = struct { name: []const u8, suite: ?[]const u8 = null };
	21	pub const TestEnd = struct {
	22	name: []const u8,
	23	suite: ?[]const u8 = null,
	24	status: Status,
	25	duration_ns: u64,
	26	error_message: ?[]const u8 = null,
	27	};
	28	pub const Hook = struct {
	29	kind: HookKind,
	30	suite: ?[]const u8 = null,
	31	test_name: ?[]const u8 = null,
	32	status: Status,
	33	duration_ns: u64 = 0,
	34	error_message: ?[]const u8 = null,
	35	};
	36	pub const Output = struct {
	37	stream: OutputStream,
	38	text: []const u8,
	39	test_name: ?[]const u8 = null,
	40	};
	41	pub const Retry = struct {
	42	name: []const u8,
	43	attempt: usize,
	44	repetition: usize,
	45	status: Status,
	46	duration_ns: u64,
	47	error_message: ?[]const u8 = null,
	48	};
	49	pub const Coverage = struct {
	50	line_percent: ?f64 = null,
	51	function_percent: ?f64 = null,
	52	branch_percent: ?f64 = null,
	53	report_path: ?[]const u8 = null,
	54	};
	55
	56	/// Every serialized value is an envelope with the same version/type/data
	57	/// shape. Unknown fields and event types are intentionally safe to ignore.
	58	pub const Event = union(enum) {
	59	run_start: RunStart,
	60	run_end: RunEnd,
	61	suite_start: Suite,
	62	suite_end: Suite,
	63	test_start: TestStart,
	64	test_end: TestEnd,
	65	hook: Hook,
	66	output: Output,
	67	retry: Retry,
	68	coverage: Coverage,
	69
	70	pub fn eventName(self: Event) []const u8 {
	71	return @tagName(std.meta.activeTag(self));
	72	}
	73
	74	pub fn jsonStringify(self: Event, json: anytype) !void {
	75	try json.beginObject();
	76	try json.objectField("protocol_version");
	77	try json.write(version);
	78	try json.objectField("type");
	79	try json.write(self.eventName());
	80	try json.objectField("data");
	81	switch (self) {
	82	inline else => \|payload\| try json.write(payload),
	83	}
	84	try json.endObject();
	85	}
	86	};
	87
	88	pub fn status(value: anytype) Status {
	89	return std.meta.stringToEnum(Status, @tagName(value)) orelse unreachable;
	90	}
	91
	92	pub fn encodeAlloc(allocator: std.mem.Allocator, event: Event) ![]u8 {
	93	return std.json.Stringify.valueAlloc(allocator, event, .{});
	94	}
	95
	96	test "every event type follows the versioned envelope contract" {
	97	const events = [_]Event{
	98	.{ .run_start = .{ .total = 2 } },
	99	.{ .run_end = .{ .total = 2, .passed = 1, .flaky = 0, .failed = 1, .skipped = 0 } },
	100	.{ .suite_start = .{ .name = "suite \"one\"" } },
	101	.{ .suite_end = .{ .name = "suite \"one\"" } },
	102	.{ .test_start = .{ .name = "line\nbreak", .suite = "suite" } },
	103	.{ .test_end = .{ .name = "line\nbreak", .status = .failed, .duration_ns = 42, .error_message = "bad \"value\"" } },
	104	.{ .hook = .{ .kind = .before_each, .test_name = "case", .status = .passed } },
	105	.{ .output = .{ .stream = .stderr, .text = "hello\nworld", .test_name = "case" } },
	106	.{ .retry = .{ .name = "case", .attempt = 2, .repetition = 1, .status = .passed, .duration_ns = 10 } },
	107	.{ .coverage = .{ .line_percent = 90.5, .report_path = "coverage/index.html" } },
	108	};
	109
	110	inline for (events) \|event\| {
	111	const encoded = try encodeAlloc(std.testing.allocator, event);
	112	defer std.testing.allocator.free(encoded);
	113	var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, encoded, .{});
	114	defer parsed.deinit();
	115	const object = parsed.value.object;
	116	try std.testing.expectEqual(@as(i64, version), object.get("protocol_version").?.integer);
	117	try std.testing.expectEqualStrings(event.eventName(), object.get("type").?.string);
	118	try std.testing.expect(object.get("data") != null);
	119	}
	120	}
	121
	122	test "event payload strings use standard JSON escaping" {
	123	const encoded = try encodeAlloc(std.testing.allocator, .{ .test_end = .{
	124	.name = "case\tname",
	125	.status = .failed,
	126	.duration_ns = 1,
	127	.error_message = "line one\n\"quoted\" and \\escaped",
	128	} });
	129	defer std.testing.allocator.free(encoded);
	130
	131	var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, encoded, .{});
	132	defer parsed.deinit();
	133	const data = parsed.value.object.get("data").?.object;
	134	try std.testing.expectEqualStrings("case\tname", data.get("name").?.string);
	135	try std.testing.expectEqualStrings("line one\n\"quoted\" and \\escaped", data.get("error_message").?.string);
	136	}

src/lib.zigmodified+3-0

Changes to src/lib.zig
		@@ -6,6 +6,7 @@ pub const suite = ;
6	6	pub const test_runner = ;
7	7	pub const reporter = ;
8	8	pub const pipeline = ;
	9	pub const event_protocol = ;
9	10	pub const matchers = ;
10	11	pub const mock = ;
11	12	pub const cli = ;
		@@ -90,6 +91,8 @@ pub const PlanOrigin = pipeline.PlanOrigin;
90	91	pub const ExecutionPolicy = pipeline.ExecutionPolicy;
91	92	pub const LifecycleEvent = pipeline.LifecycleEvent;
92	93	pub const EventStream = pipeline.EventStream;
	94	pub const ProtocolEvent = event_protocol.Event;
	95	pub const protocol_version = event_protocol.version;
93	96
94	97	// Matchers
95	98	pub const Matchers = matchers.Matchers;

src/reporter.zigmodified+44-58

Changes to src/reporter.zig
		@@ -1,6 +1,7 @@
1	1	const std = ;
2	2	const suite = ;
3	3	const compat = ;
	4	const protocol = ;
4	5
5	6	pub const ReporterType = enum {
6	7	spec,
		@@ -464,7 +465,7 @@ pub const JsonReporter = struct {
464	465	writer: std.Io.Writer,
465	466	writer_ref: ?*std.Io.Writer = null,
466	467	suites: std.ArrayList([]const u8),
467		emitted_tests: bool = false,
	468	emitted_events: bool = false,
468	469
469	470	const Self = @This();
470	471
		@@ -508,87 +509,72 @@ pub const JsonReporter = struct {
508	509
509	510	fn onRunStart(reporter: *Reporter, total_tests: usize) !void {
510	511	const s = self(reporter);
511		try s.output().print("{{\"totalTests\":{d},\"tests\":[\n", .{total_tests});
	512	s.emitted_events = false;
	513	s.suites.clearRetainingCapacity();
	514	try s.output().writeAll("[\n");
	515	try s.emit(.{ .run_start = .{ .total = total_tests } });
512	516	}
513	517
514	518	fn onRunEnd(reporter: Reporter, results: TestResults) !void {
515	519	const s = self(reporter);
516		const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0;
517
518		try s.output().print("\n],\"summary\":{{\"total\":{d},\"passed\":{d},\"flaky\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{
519		results.total,
520		results.passed,
521		results.flaky,
522		results.failed,
523		results.skipped,
524		total_time_ms,
525		});
	520	try s.emit(.{ .run_end = .{
	521	.total = results.total,
	522	.passed = results.passed,
	523	.flaky = results.flaky,
	524	.failed = results.failed,
	525	.skipped = results.skipped,
	526	.duration_ns = results.total_time_ns,
	527	} });
	528	try s.output().writeAll("\n]\n");
526	529	}
527	530
528	531	fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void {
529	532	const s = self(reporter);
530	533	try s.suites.append(reporter.allocator, suite_name);
	534	try s.emit(.{ .suite_start = .{ .name = suite_name } });
531	535	}
532	536
533	537	fn onSuiteEnd(reporter: *Reporter, suite_name: []const u8) !void {
534		_ = suite_name;
535	538	const s = self(reporter);
	539	try s.emit(.{ .suite_end = .{ .name = suite_name } });
536	540	if (s.suites.items.len > 0) {
537	541	_ = s.suites.pop();
538	542	}
539	543	}
540	544
541	545	fn onTestStart(reporter: *Reporter, test_name: []const u8) !void {
542		_ = reporter;
543		_ = test_name;
	546	const s = self(reporter);
	547	try s.emit(.{ .test_start = .{
	548	.name = test_name,
	549	.suite = if (s.suites.getLastOrNull()) \|name\| name else null,
	550	} });
544	551	}
545	552
546	553	fn onTestEnd(reporter: Reporter, test_case: const suite.TestCase) !void {
547	554	const s = self(reporter);
548		const time_ms = @as(f64, @floatFromInt(test_case.execution_time_ns)) / 1_000_000.0;
549
550		const status_str = switch (test_case.status) {
551		.passed => "passed",
552		.flaky => "flaky",
553		.failed => "failed",
554		.skipped => "skipped",
555		else => "unknown",
556		};
557
558		if (s.emitted_tests) try s.output().print(",\n", .{});
559		s.emitted_tests = true;
560
561		// Note: In a real implementation, you'd want to properly escape JSON strings
562		try s.output().print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{
563		test_case.name,
564		status_str,
565		time_ms,
566		});
567
568		if (test_case.error_message) \|msg\| {
569		try s.output().print(",\"error\":\"{s}\"", .{msg});
570		}
571
572		try s.output().print(",\"attempts\":[", .{});
573		for (test_case.attempts.items, 0..) \|attempt, index\| {
574		if (index > 0) try s.output().print(",", .{});
575		const attempt_status = switch (attempt.status) {
576		.passed => "passed",
577		.flaky => "flaky",
578		.failed => "failed",
579		.skipped => "skipped",
580		else => "unknown",
581		};
582		try s.output().print(
583		"{{\"number\":{d},\"repetition\":{d},\"status\":\"{s}\",\"durationNs\":{d}",
584		.{ attempt.number, attempt.repetition, attempt_status, attempt.duration_ns },
585		);
586		if (attempt.error_message) \|message\| {
587		try s.output().print(",\"error\":\"{s}\"", .{message});
588		}
589		try s.output().print("}}", .{});
	555	for (test_case.attempts.items) \|attempt\| {
	556	try s.emit(.{ .retry = .{
	557	.name = test_case.name,
	558	.attempt = attempt.number,
	559	.repetition = attempt.repetition,
	560	.status = protocol.status(attempt.status),
	561	.duration_ns = attempt.duration_ns,
	562	.error_message = attempt.error_message,
	563	} });
590	564	}
591		try s.output().print("]}}", .{});
	565	try s.emit(.{ .test_end = .{
	566	.name = test_case.name,
	567	.suite = if (s.suites.getLastOrNull()) \|name\| name else null,
	568	.status = protocol.status(test_case.status),
	569	.duration_ns = test_case.execution_time_ns,
	570	.error_message = test_case.error_message,
	571	} });
	572	}
	573
	574	fn emit(s: *Self, event: protocol.Event) !void {
	575	if (s.emitted_events) try s.output().writeAll(",\n");
	576	s.emitted_events = true;
	577	try std.json.Stringify.value(event, .{}, s.output());
592	578	}
593	579	};
594	580

src/test_history.zigmodified+68-33

Changes to src/test_history.zig
		@@ -2,12 +2,14 @@ const std = ;
2	2	const reporter_mod = ;
3	3	const suite = ;
4	4	const compat = ;
	5	const protocol = ;
5	6
6	7	/// Test history entry
7	8	pub const HistoryEntry = struct {
8	9	timestamp: i64,
9	10	total: usize,
10	11	passed: usize,
	12	flaky: usize = 0,
11	13	failed: usize,
12	14	skipped: usize,
13	15	duration_ns: u64,
		@@ -69,6 +71,7 @@ pub const TestHistory = struct {
69	71	.timestamp = self.start_time,
70	72	.total = total,
71	73	.passed = 0,
	74	.flaky = 0,
72	75	.failed = 0,
73	76	.skipped = 0,
74	77	.duration_ns = 0,
		@@ -111,6 +114,7 @@ pub const TestHistory = struct {
111	114	const end_time = compat.milliTimestamp();
112	115	self.current_entry.?.duration_ns = @intCast((end_time - self.start_time) * std.time.ns_per_ms);
113	116	self.current_entry.?.passed = results.passed;
	117	self.current_entry.?.flaky = results.flaky;
114	118	self.current_entry.?.failed = results.failed;
115	119	self.current_entry.?.skipped = results.skipped;
116	120
		@@ -128,42 +132,36 @@ pub const TestHistory = struct {
128	132	var filename_buf: [256]u8 = undefined;
129	133	const filename = try std.fmt.bufPrint(&filename_buf, "{s}/test-run-{d}.json", .{ self.history_dir, self.current_entry.?.timestamp });
130	134
131		// Write JSON
	135	// Store the same versioned events consumed by reporters and the UI.
132	136	var buffer = std.ArrayList(u8).empty;
133	137	defer buffer.deinit(self.allocator);
134
135		try buffer.appendSlice(self.allocator, "{\n");
136		try buffer.print(self.allocator, " \"timestamp\": {d},\n", .{self.current_entry.?.timestamp});
137		try buffer.print(self.allocator, " \"total\": {d},\n", .{self.current_entry.?.total});
138		try buffer.print(self.allocator, " \"passed\": {d},\n", .{self.current_entry.?.passed});
139		try buffer.print(self.allocator, " \"failed\": {d},\n", .{self.current_entry.?.failed});
140		try buffer.print(self.allocator, " \"skipped\": {d},\n", .{self.current_entry.?.skipped});
141		try buffer.print(self.allocator, " \"duration_ns\": {d},\n", .{self.current_entry.?.duration_ns});
142		try buffer.appendSlice(self.allocator, " \"tests\": [\n");
143
144		for (self.current_entry.?.tests.items, 0..) \|test_record, i\| {
145		try buffer.appendSlice(self.allocator, " {\n");
146		try buffer.print(self.allocator, " \"name\": \"{s}\",\n", .{test_record.name});
147		try buffer.print(self.allocator, " \"suite_name\": \"{s}\",\n", .{test_record.suite_name});
148		try buffer.print(self.allocator, " \"status\": \"{s}\",\n", .{test_record.status});
149		try buffer.print(self.allocator, " \"execution_time_ns\": {d}", .{test_record.execution_time_ns});
150
151		if (test_record.error_message) \|msg\| {
152		try buffer.appendSlice(self.allocator, ",\n");
153		try buffer.print(self.allocator, " \"error_message\": \"{s}\"\n", .{msg});
154		} else {
155		try buffer.appendSlice(self.allocator, "\n");
156		}
157
158		if (i < self.current_entry.?.tests.items.len - 1) {
159		try buffer.appendSlice(self.allocator, " },\n");
160		} else {
161		try buffer.appendSlice(self.allocator, " }\n");
162		}
	138	var allocating: std.Io.Writer.Allocating = .fromArrayList(self.allocator, &buffer);
	139	errdefer buffer = allocating.toArrayList();
	140	const writer = &allocating.writer;
	141	try writer.writeAll("[\n");
	142	try std.json.Stringify.value(protocol.Event{ .run_start = .{ .total = self.current_entry.?.total } }, .{}, writer);
	143	for (self.current_entry.?.tests.items) \|test_record\| {
	144	try writer.writeAll(",\n");
	145	try std.json.Stringify.value(protocol.Event{ .test_end = .{
	146	.name = test_record.name,
	147	.suite = test_record.suite_name,
	148	.status = std.meta.stringToEnum(protocol.Status, test_record.status) orelse .failed,
	149	.duration_ns = test_record.execution_time_ns,
	150	.error_message = test_record.error_message,
	151	} }, .{}, writer);
163	152	}
164
165		try buffer.appendSlice(self.allocator, " ]\n");
166		try buffer.appendSlice(self.allocator, "}\n");
	153	try writer.writeAll(",\n");
	154	try std.json.Stringify.value(protocol.Event{ .run_end = .{
	155	.total = self.current_entry.?.total,
	156	.passed = self.current_entry.?.passed,
	157	.flaky = self.current_entry.?.flaky,
	158	.failed = self.current_entry.?.failed,
	159	.skipped = self.current_entry.?.skipped,
	160	.duration_ns = self.current_entry.?.duration_ns,
	161	} }, .{}, writer);
	162	try writer.writeAll("\n]\n");
	163	try writer.flush();
	164	buffer = allocating.toArrayList();
167	165
168	166	try compat.writeFile(self.allocator, filename, buffer.items);
169	167	}
		@@ -190,6 +188,7 @@ pub const TestHistory = struct {
190	188	.timestamp = 0,
191	189	.total = 0,
192	190	.passed = 0,
	191	.flaky = 0,
193	192	.failed = 0,
194	193	.skipped = 0,
195	194	.duration_ns = 0,
		@@ -249,3 +248,39 @@ test "TestHistory start and record" {
249	248	try history.recordTest("test1", "suite1", .passed, 1000000, null);
250	249	try std.testing.expectEqual(@as(usize, 1), history.current_entry.?.tests.items.len);
251	250	}
	251
	252	test "TestHistory saves versioned protocol events" {
	253	const allocator = std.testing.allocator;
	254	var tmp = std.testing.tmpDir(.{});
	255	defer tmp.cleanup();
	256	const history_dir = try std.fmt.allocPrint(allocator, ".zig-cache/tmp/{s}", .{tmp.sub_path});
	257	defer allocator.free(history_dir);
	258
	259	var history = TestHistory.init(allocator, history_dir);
	260	defer history.deinit();
	261	try history.startRun(1);
	262	try history.recordTest("quoted \"test\"", "suite\nname", .passed, 42, null);
	263	var results = reporter_mod.TestResults.init(allocator);
	264	defer results.deinit();
	265	results.total = 1;
	266	results.passed = 1;
	267	results.total_time_ns = 42;
	268	try history.finishRun(&results);
	269
	270	const filename = try std.fmt.allocPrint(allocator, "{s}/test-run-{d}.json", .{ history_dir, history.current_entry.?.timestamp });
	271	defer allocator.free(filename);
	272	const content = try compat.readFileAlloc(allocator, filename);
	273	defer allocator.free(content);
	274	var parsed = try std.json.parseFromSlice(std.json.Value, allocator, content, .{});
	275	defer parsed.deinit();
	276
	277	const events = parsed.value.array.items;
	278	try std.testing.expectEqual(@as(usize, 3), events.len);
	279	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	280	try std.testing.expectEqualStrings("test_end", events[1].object.get("type").?.string);
	281	try std.testing.expectEqualStrings("quoted \"test\"", events[1].object.get("data").?.object.get("name").?.string);
	282	try std.testing.expectEqualStrings("run_end", events[2].object.get("type").?.string);
	283	for (events) \|event\| {
	284	try std.testing.expectEqual(@as(i64, protocol.version), event.object.get("protocol_version").?.integer);
	285	}
	286	}

src/test_loader.zigmodified+35-19

Changes to src/test_loader.zig
		@@ -7,6 +7,7 @@ const suite = ;
7	7	const coverage = ;
8	8	const ui_server = ;
9	9	const compat = ;
	10	const protocol = ;
10	11
11	12	/// Options for running discovered tests
12	13	pub const LoaderOptions = struct {
		@@ -41,12 +42,6 @@ pub const LoaderOptions = struct {
41	42	ui_server: ?*ui_server.UIServer = null,
42	43	};
43	44
44		fn broadcastJson(server: *ui_server.UIServer, allocator: std.mem.Allocator, event: []const u8, value: anytype) !void {
45		const json = try std.json.Stringify.valueAlloc(allocator, value, .{});
46		defer allocator.free(json);
47		try server.broadcast(event, json);
48		}
49
50	45	/// Run all discovered test files
51	46	pub fn runDiscoveredTests(
52	47	allocator: std.mem.Allocator,
		@@ -107,7 +102,7 @@ pub fn runDiscoveredTests(
107	102
108	103	// Notify UI of run start
109	104	if (options.ui_server) \|server\| {
110		try broadcastJson(server, allocator, "run_start", .{ .total = selected_files });
	105	try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files } });
111	106	}
112	107
113	108	var files_run: usize = 0;
		@@ -129,8 +124,8 @@ pub fn runDiscoveredTests(
129	124
130	125	// Notify UI of test file start
131	126	if (options.ui_server) \|server\| {
132		try broadcastJson(server, allocator, "suite_start", .{ .name = file.name });
133		try broadcastJson(server, allocator, "test_start", .{ .name = file.name });
	127	try server.broadcastEvent(allocator, .{ .suite_start = .{ .name = file.name } });
	128	try server.broadcastEvent(allocator, .{ .test_start = .{ .name = file.name, .suite = file.name } });
134	129	}
135	130
136	131	files_run += 1;
		@@ -182,14 +177,24 @@ pub fn runDiscoveredTests(
182	177
183	178	// Notify UI of test file end
184	179	if (options.ui_server) \|server\| {
185		try broadcastJson(server, allocator, "test_end", .{
	180	for (test_case.attempts.items) \|attempt\| {
	181	try server.broadcastEvent(allocator, .{ .retry = .{
	182	.name = file.name,
	183	.attempt = attempt.number,
	184	.repetition = attempt.repetition,
	185	.status = protocol.status(attempt.status),
	186	.duration_ns = attempt.duration_ns,
	187	.error_message = attempt.error_message,
	188	} });
	189	}
	190	try server.broadcastEvent(allocator, .{ .test_end = .{
186	191	.name = file.name,
187		.status = @tagName(test_case.status),
188		.execution_time_ns = test_case.execution_time_ns,
189		.attempts = test_case.attempts.items.len,
190		.error_message = "",
191		});
192		try broadcastJson(server, allocator, "suite_end", .{ .name = file.name });
	192	.suite = file.name,
	193	.status = protocol.status(test_case.status),
	194	.duration_ns = test_case.execution_time_ns,
	195	.error_message = test_case.error_message,
	196	} });
	197	try server.broadcastEvent(allocator, .{ .suite_end = .{ .name = file.name } });
193	198	}
194	199
195	200	if (test_case.status == .passed) {
		@@ -213,19 +218,30 @@ pub fn runDiscoveredTests(
213	218	std.debug.print("Warning: Could not parse coverage report: {any}\n", .{err});
214	219	break :result null;
215	220	};
216		if (cov_result) \|summary\| coverage.printCoverageSummary(summary);
	221	if (cov_result) \|summary\| {
	222	coverage.printCoverageSummary(summary);
	223	if (options.ui_server) \|server\| {
	224	try server.broadcastEvent(allocator, .{ .coverage = .{
	225	.line_percent = summary.linePercentage(),
	226	.function_percent = summary.functionPercentage(),
	227	.branch_percent = summary.branchPercentage(),
	228	.report_path = summary.report_dir,
	229	} });
	230	}
	231	}
217	232	}
218	233	}
219	234
220	235	// Notify UI of run end
221	236	if (options.ui_server) \|server\| {
222		try broadcastJson(server, allocator, "run_end", .{
	237	try server.broadcastEvent(allocator, .{ .run_end = .{
223	238	.total = results.total,
224	239	.passed = results.passed,
225	240	.flaky = results.flaky,
226	241	.failed = results.failed,
227	242	.skipped = results.skipped,
228		});
	243	.duration_ns = results.total_time_ns,
	244	} });
229	245	}
230	246
231	247	if (options.shard) \|shard\| {

src/ui/index.htmlmodified+7-7

Changes to src/ui/index.html
		@@ -332,26 +332,26 @@
332	332	});
333	333
334	334	eventSource.addEventListener('run_start', (e) => {
335		const data = JSON.parse(e.data);
	335	const data = JSON.parse(e.data).data;
336	336	clearTests();
337	337	state.total = data.total;
338	338	updateStats();
339	339	});
340	340
341	341	eventSource.addEventListener('suite_start', (e) => {
342		const data = JSON.parse(e.data);
	342	const data = JSON.parse(e.data).data;
343	343	state.currentSuite = data.name;
344	344	addSuite(data.name);
345	345	});
346	346
347	347	eventSource.addEventListener('test_start', (e) => {
348		const data = JSON.parse(e.data);
349		addTest(state.currentSuite, data.name, 'running');
	348	const data = JSON.parse(e.data).data;
	349	addTest(data.suite \|\| state.currentSuite, data.name, 'running');
350	350	});
351	351
352	352	eventSource.addEventListener('test_end', (e) => {
353		const data = JSON.parse(e.data);
354		updateTest(state.currentSuite, data.name, data.status, data.execution_time_ns, data.error_message);
	353	const data = JSON.parse(e.data).data;
	354	updateTest(data.suite \|\| state.currentSuite, data.name, data.status, data.duration_ns, data.error_message \|\| '');
355	355
356	356	if (data.status === 'passed') state.passed++;
357	357	else if (data.status === 'flaky') state.flaky++;
		@@ -362,7 +362,7 @@
362	362	});
363	363
364	364	eventSource.addEventListener('run_end', (e) => {
365		const data = JSON.parse(e.data);
	365	const data = JSON.parse(e.data).data;
366	366	state.total = data.total;
367	367	state.passed = data.passed;
368	368	state.flaky = data.flaky;

src/ui_server.zigmodified+33-34

Changes to src/ui_server.zig
		@@ -3,6 +3,7 @@ const reporter_mod = ;
3	3	const suite = ;
4	4	const test_history = ;
5	5	const compat = ;
	6	const protocol = ;
6	7
7	8	/// Options for the UI server
8	9	pub const UIServerOptions = struct {
		@@ -282,13 +283,19 @@ pub const UIServer = struct {
282	283	index += 1;
283	284	}
284	285	}
	286
	287	/// Serialize and broadcast one versioned protocol event.
	288	pub fn broadcastEvent(self: *Self, allocator: std.mem.Allocator, event: protocol.Event) !void {
	289	const encoded = try protocol.encodeAlloc(allocator, event);
	290	defer allocator.free(encoded);
	291	try self.broadcast(event.eventName(), encoded);
	292	}
285	293	};
286	294
287	295	/// UI Reporter - sends test events to the UI server
288	296	pub const UIReporter = struct {
289	297	reporter: reporter_mod.Reporter,
290	298	server: *UIServer,
291		buffer: std.ArrayList(u8),
292	299
293	300	const Self = @This();
294	301
		@@ -307,71 +314,66 @@ pub const UIReporter = struct {
307	314	.use_colors = false,
308	315	},
309	316	.server = server,
310		.buffer = .empty,
311	317	};
312	318	}
313	319
314		pub fn deinit(self: *Self) void {
315		self.buffer.deinit(self.reporter.allocator);
316		}
317
318		fn encode(self: *Self, value: anytype) !void {
319		self.buffer.clearRetainingCapacity();
320		var writer: std.Io.Writer.Allocating = .fromArrayList(self.reporter.allocator, &self.buffer);
321		defer self.buffer = writer.toArrayList();
322		try std.json.Stringify.value(value, .{}, &writer.writer);
323		}
	320	pub fn deinit(_: *Self) void {}
324	321
325	322	fn onRunStart(reporter: *reporter_mod.Reporter, total: usize) !void {
326	323	const self: *Self = @fieldParentPtr("reporter", reporter);
327		try self.encode(.{ .total = total });
328		try self.server.broadcast("run_start", self.buffer.items);
	324	try self.server.broadcastEvent(reporter.allocator, .{ .run_start = .{ .total = total } });
329	325	}
330	326
331	327	fn onRunEnd(reporter: reporter_mod.Reporter, results: reporter_mod.TestResults) !void {
332	328	const self: *Self = @fieldParentPtr("reporter", reporter);
333	329
334		try self.encode(.{
	330	try self.server.broadcastEvent(reporter.allocator, .{ .run_end = .{
335	331	.total = results.total,
336	332	.passed = results.passed,
337	333	.flaky = results.flaky,
338	334	.failed = results.failed,
339	335	.skipped = results.skipped,
340		});
341		try self.server.broadcast("run_end", self.buffer.items);
	336	.duration_ns = results.total_time_ns,
	337	} });
342	338	}
343	339
344	340	fn onSuiteStart(reporter: *reporter_mod.Reporter, suite_name: []const u8) !void {
345	341	const self: *Self = @fieldParentPtr("reporter", reporter);
346	342
347		try self.encode(.{ .name = suite_name });
348		try self.server.broadcast("suite_start", self.buffer.items);
	343	try self.server.broadcastEvent(reporter.allocator, .{ .suite_start = .{ .name = suite_name } });
349	344	}
350	345
351	346	fn onSuiteEnd(reporter: *reporter_mod.Reporter, suite_name: []const u8) !void {
352	347	const self: *Self = @fieldParentPtr("reporter", reporter);
353	348
354		try self.encode(.{ .name = suite_name });
355		try self.server.broadcast("suite_end", self.buffer.items);
	349	try self.server.broadcastEvent(reporter.allocator, .{ .suite_end = .{ .name = suite_name } });
356	350	}
357	351
358	352	fn onTestStart(reporter: *reporter_mod.Reporter, test_name: []const u8) !void {
359	353	const self: *Self = @fieldParentPtr("reporter", reporter);
360	354
361		try self.encode(.{ .name = test_name });
362		try self.server.broadcast("test_start", self.buffer.items);
	355	try self.server.broadcastEvent(reporter.allocator, .{ .test_start = .{ .name = test_name } });
363	356	}
364	357
365	358	fn onTestEnd(reporter: reporter_mod.Reporter, test_case: const suite.TestCase) !void {
366	359	const self: *Self = @fieldParentPtr("reporter", reporter);
367	360
368		try self.encode(.{
	361	for (test_case.attempts.items) \|attempt\| {
	362	try self.server.broadcastEvent(reporter.allocator, .{ .retry = .{
	363	.name = test_case.name,
	364	.attempt = attempt.number,
	365	.repetition = attempt.repetition,
	366	.status = protocol.status(attempt.status),
	367	.duration_ns = attempt.duration_ns,
	368	.error_message = attempt.error_message,
	369	} });
	370	}
	371	try self.server.broadcastEvent(reporter.allocator, .{ .test_end = .{
369	372	.name = test_case.name,
370		.status = @tagName(test_case.status),
371		.execution_time_ns = test_case.execution_time_ns,
372		.error_message = test_case.error_message orelse "",
373		});
374		try self.server.broadcast("test_end", self.buffer.items);
	373	.status = protocol.status(test_case.status),
	374	.duration_ns = test_case.execution_time_ns,
	375	.error_message = test_case.error_message,
	376	} });
375	377	}
376	378	};
377	379
		@@ -471,10 +473,7 @@ test "UIReporter initialization" {
471	473	defer ui_reporter.deinit();
472	474
473	475	try std.testing.expectEqual(false, ui_reporter.reporter.use_colors);
474		try std.testing.expectEqual(@as(usize, 0), ui_reporter.buffer.items.len);
475
476	476	try ui_reporter.reporter.onSuiteStart("quoted \"suite\"\nname");
477		try std.testing.expectEqualStrings("{\"name\":\"quoted \\\"suite\\\"\\nname\"}", ui_reporter.buffer.items);
478	477	}
479	478
480	479	fn writeTestRequest(io: std.Io, stream: std.Io.net.Stream, target: []const u8) !void {
		@@ -535,9 +534,9 @@ test "UI server streams broadcast events over SSE" {
535	534	try expectLine(&stream_reader.interface, "data: {}");
536	535	try expectLine(&stream_reader.interface, "");
537	536
538		try server.broadcast("run_start", "{\"total\":2}");
	537	try server.broadcastEvent(std.testing.allocator, .{ .run_start = .{ .total = 2 } });
539	538	try expectLine(&stream_reader.interface, "event: run_start");
540		try expectLine(&stream_reader.interface, "data: {\"total\":2}");
	539	try expectLine(&stream_reader.interface, "data: {\"protocol_version\":1,\"type\":\"run_start\",\"data\":{\"total\":2}}");
541	540	try expectLine(&stream_reader.interface, "");
542	541	}
543	542

tests/reporter_test.zigmodified+22-9

Changes to tests/reporter_test.zig
		@@ -47,8 +47,8 @@ test "reporter dot output" {
47	47	try std.testing.expect(std.mem.indexOf(u8, output, "Passed: 1") != null);
48	48	}
49	49
50		test "reporter json output" {
51		var buffer: [2048]u8 = undefined;
	50	test "reporter json output uses versioned event envelopes" {
	51	var buffer: [4096]u8 = undefined;
52	52	const writer: std.Io.Writer = .fixed(&buffer);
53	53	var reporter = ztf.JsonReporter.init(std.testing.allocator, writer);
54	54	defer reporter.deinit();
		@@ -56,20 +56,29 @@ test "reporter json output" {
56	56	defer results.deinit();
57	57	results.total = 1;
58	58	results.passed = 1;
59		var test_case = ztf.TestCase.init("passes", passingTest);
	59	var test_case = ztf.TestCase.init("passes \"quoted\"\nname", passingTest);
60	60	test_case.status = .passed;
61	61
62	62	try reporter.reporter.onRunStart(1);
63		try reporter.reporter.onSuiteStart("suite");
	63	try reporter.reporter.onSuiteStart("suite\tone");
	64	try reporter.reporter.onTestStart(test_case.name);
64	65	try reporter.reporter.onTestEnd(&test_case);
65		try reporter.reporter.onSuiteEnd("suite");
	66	try reporter.reporter.onSuiteEnd("suite\tone");
66	67	try reporter.reporter.onRunEnd(&results);
67	68
68	69	const output = buffer[0..reporter.writer.end];
69		try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null);
70		try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null);
71	70	var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, output, .{});
72	71	defer parsed.deinit();
	72	const events = parsed.value.array.items;
	73	try std.testing.expectEqual(@as(usize, 6), events.len);
	74	try std.testing.expectEqual(@as(i64, ztf.protocol_version), events[0].object.get("protocol_version").?.integer);
	75	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	76	try std.testing.expectEqualStrings("test_start", events[2].object.get("type").?.string);
	77	const test_data = events[3].object.get("data").?.object;
	78	try std.testing.expectEqualStrings(test_case.name, test_data.get("name").?.string);
	79	try std.testing.expectEqualStrings("suite\tone", test_data.get("suite").?.string);
	80	try std.testing.expectEqualStrings("run_end", events[5].object.get("type").?.string);
	81	try std.testing.expectEqual(@as(i64, 1), events[5].object.get("data").?.object.get("passed").?.integer);
73	82	}
74	83
75	84	test "reporter json output includes flaky attempt history" {
		@@ -106,10 +115,14 @@ test "reporter json output includes flaky attempt history" {
106	115
107	116	const output = buffer[0..reporter.writer.end];
108	117	try std.testing.expect(std.mem.indexOf(u8, output, "\"status\":\"flaky\"") != null);
109		try std.testing.expect(std.mem.indexOf(u8, output, "\"attempts\":[") != null);
110		try std.testing.expect(std.mem.indexOf(u8, output, "\"durationNs\":10") != null);
	118	try std.testing.expect(std.mem.indexOf(u8, output, "\"type\":\"retry\"") != null);
	119	try std.testing.expect(std.mem.indexOf(u8, output, "\"duration_ns\":10") != null);
111	120	var parsed = try std.json.parseFromSlice(std.json.Value, allocator, output, .{});
112	121	defer parsed.deinit();
	122	const events = parsed.value.array.items;
	123	try std.testing.expectEqualStrings("retry", events[2].object.get("type").?.string);
	124	try std.testing.expectEqualStrings("retry", events[3].object.get("type").?.string);
	125	try std.testing.expectEqualStrings("test_end", events[4].object.get("type").?.string);
113	126	}
114	127
115	128	test "reporter set selects the same built-ins for every executor" {

Files

Review one file at a time

README.md +19 -14
docs/execution-pipeline.md +4 -3
docs/json-event-protocol.md +71 -0
src/event_protocol.zig +136 -0
src/lib.zig +3 -0
src/reporter.zig +44 -58
src/test_history.zig +68 -33
src/test_loader.zig +35 -19
src/ui/index.html +7 -7
src/ui_server.zig +33 -34
tests/reporter_test.zig +22 -9

Merge

This pull request was merged.

Reviews

Nobody has reviewed this yet.

0 of 0 required approvals.

Checks

No checks are required on this branch.

Conversations

{{ unresolvedThreads }} unresolved of {{ threads.length }}.