also looking at this
Define stable JSON event protocol
#27
11 files
+442
-177
| @@ -2,12 +2,14 @@ const std = ; | ||
| 2 | 2 | const reporter_mod = ; |
| 3 | 3 | const suite = ; |
| 4 | 4 | const compat = ; |
| 5 | const protocol = ; | |
| 5 | 6 | |
| 6 | 7 | /// Test history entry |
| 7 | 8 | pub const HistoryEntry = struct { |
| 8 | 9 | timestamp: i64, |
| 9 | 10 | total: usize, |
| 10 | 11 | passed: usize, |
| 12 | flaky: usize = 0, | |
| 11 | 13 | failed: usize, |
| 12 | 14 | skipped: usize, |
| 13 | 15 | duration_ns: u64, |
| @@ -69,6 +71,7 @@ pub const TestHistory = struct { | ||
| 69 | 71 | .timestamp = self.start_time, |
| 70 | 72 | .total = total, |
| 71 | 73 | .passed = 0, |
| 74 | .flaky = 0, | |
| 72 | 75 | .failed = 0, |
| 73 | 76 | .skipped = 0, |
| 74 | 77 | .duration_ns = 0, |
| @@ -111,6 +114,7 @@ pub const TestHistory = struct { | ||
| 111 | 114 | const end_time = compat.milliTimestamp(); |
| 112 | 115 | self.current_entry.?.duration_ns = @intCast((end_time - self.start_time) * std.time.ns_per_ms); |
| 113 | 116 | self.current_entry.?.passed = results.passed; |
| 117 | self.current_entry.?.flaky = results.flaky; | |
| 114 | 118 | self.current_entry.?.failed = results.failed; |
| 115 | 119 | self.current_entry.?.skipped = results.skipped; |
| 116 | 120 | |
| @@ -128,42 +132,36 @@ pub const TestHistory = struct { | ||
| 128 | 132 | var filename_buf: [256]u8 = undefined; |
| 129 | 133 | const filename = try std.fmt.bufPrint(&filename_buf, "{s}/test-run-{d}.json", .{ self.history_dir, self.current_entry.?.timestamp }); |
| 130 | 134 | |
| 131 | // Write JSON | |
| 135 | // Store the same versioned events consumed by reporters and the UI. | |
| 132 | 136 | var buffer = std.ArrayList(u8).empty; |
| 133 | 137 | defer buffer.deinit(self.allocator); |
| 134 | ||
| 135 | try buffer.appendSlice(self.allocator, "{\n"); | |
| 136 | try buffer.print(self.allocator, " \"timestamp\": {d},\n", .{self.current_entry.?.timestamp}); | |
| 137 | try buffer.print(self.allocator, " \"total\": {d},\n", .{self.current_entry.?.total}); | |
| 138 | try buffer.print(self.allocator, " \"passed\": {d},\n", .{self.current_entry.?.passed}); | |
| 139 | try buffer.print(self.allocator, " \"failed\": {d},\n", .{self.current_entry.?.failed}); | |
| 140 | try buffer.print(self.allocator, " \"skipped\": {d},\n", .{self.current_entry.?.skipped}); | |
| 141 | try buffer.print(self.allocator, " \"duration_ns\": {d},\n", .{self.current_entry.?.duration_ns}); | |
| 142 | try buffer.appendSlice(self.allocator, " \"tests\": [\n"); | |
| 143 | ||
| 144 | for (self.current_entry.?.tests.items, 0..) |test_record, i| { | |
| 145 | try buffer.appendSlice(self.allocator, " {\n"); | |
| 146 | try buffer.print(self.allocator, " \"name\": \"{s}\",\n", .{test_record.name}); | |
| 147 | try buffer.print(self.allocator, " \"suite_name\": \"{s}\",\n", .{test_record.suite_name}); | |
| 148 | try buffer.print(self.allocator, " \"status\": \"{s}\",\n", .{test_record.status}); | |
| 149 | try buffer.print(self.allocator, " \"execution_time_ns\": {d}", .{test_record.execution_time_ns}); | |
| 150 | ||
| 151 | if (test_record.error_message) |msg| { | |
| 152 | try buffer.appendSlice(self.allocator, ",\n"); | |
| 153 | try buffer.print(self.allocator, " \"error_message\": \"{s}\"\n", .{msg}); | |
| 154 | } else { | |
| 155 | try buffer.appendSlice(self.allocator, "\n"); | |
| 156 | } | |
| 157 | ||
| 158 | if (i < self.current_entry.?.tests.items.len - 1) { | |
| 159 | try buffer.appendSlice(self.allocator, " },\n"); | |
| 160 | } else { | |
| 161 | try buffer.appendSlice(self.allocator, " }\n"); | |
| 162 | } | |
| 138 | var allocating: std.Io.Writer.Allocating = .fromArrayList(self.allocator, &buffer); | |
| 139 | errdefer buffer = allocating.toArrayList(); | |
| 140 | const writer = &allocating.writer; | |
| 141 | try writer.writeAll("[\n"); | |
| 142 | try std.json.Stringify.value(protocol.Event{ .run_start = .{ .total = self.current_entry.?.total } }, .{}, writer); | |
| 143 | for (self.current_entry.?.tests.items) |test_record| { | |
| 144 | try writer.writeAll(",\n"); | |
| 145 | try std.json.Stringify.value(protocol.Event{ .test_end = .{ | |
| 146 | .name = test_record.name, | |
| 147 | .suite = test_record.suite_name, | |
| 148 | .status = std.meta.stringToEnum(protocol.Status, test_record.status) orelse .failed, | |
| 149 | .duration_ns = test_record.execution_time_ns, | |
| 150 | .error_message = test_record.error_message, | |
| 151 | } }, .{}, writer); | |
| 163 | 152 | } |
| 164 | ||
| 165 | try buffer.appendSlice(self.allocator, " ]\n"); | |
| 166 | try buffer.appendSlice(self.allocator, "}\n"); | |
| 153 | try writer.writeAll(",\n"); | |
| 154 | try std.json.Stringify.value(protocol.Event{ .run_end = .{ | |
| 155 | .total = self.current_entry.?.total, | |
| 156 | .passed = self.current_entry.?.passed, | |
| 157 | .flaky = self.current_entry.?.flaky, | |
| 158 | .failed = self.current_entry.?.failed, | |
| 159 | .skipped = self.current_entry.?.skipped, | |
| 160 | .duration_ns = self.current_entry.?.duration_ns, | |
| 161 | } }, .{}, writer); | |
| 162 | try writer.writeAll("\n]\n"); | |
| 163 | try writer.flush(); | |
| 164 | buffer = allocating.toArrayList(); | |
| 167 | 165 | |
| 168 | 166 | try compat.writeFile(self.allocator, filename, buffer.items); |
| 169 | 167 | } |
| @@ -190,6 +188,7 @@ pub const TestHistory = struct { | ||
| 190 | 188 | .timestamp = 0, |
| 191 | 189 | .total = 0, |
| 192 | 190 | .passed = 0, |
| 191 | .flaky = 0, | |
| 193 | 192 | .failed = 0, |
| 194 | 193 | .skipped = 0, |
| 195 | 194 | .duration_ns = 0, |
| @@ -249,3 +248,39 @@ test "TestHistory start and record" { | ||
| 249 | 248 | try history.recordTest("test1", "suite1", .passed, 1000000, null); |
| 250 | 249 | try std.testing.expectEqual(@as(usize, 1), history.current_entry.?.tests.items.len); |
| 251 | 250 | } |
| 251 | ||
| 252 | test "TestHistory saves versioned protocol events" { | |
| 253 | const allocator = std.testing.allocator; | |
| 254 | var tmp = std.testing.tmpDir(.{}); | |
| 255 | defer tmp.cleanup(); | |
| 256 | const history_dir = try std.fmt.allocPrint(allocator, ".zig-cache/tmp/{s}", .{tmp.sub_path}); | |
| 257 | defer allocator.free(history_dir); | |
| 258 | ||
| 259 | var history = TestHistory.init(allocator, history_dir); | |
| 260 | defer history.deinit(); | |
| 261 | try history.startRun(1); | |
| 262 | try history.recordTest("quoted \"test\"", "suite\nname", .passed, 42, null); | |
| 263 | var results = reporter_mod.TestResults.init(allocator); | |
| 264 | defer results.deinit(); | |
| 265 | results.total = 1; | |
| 266 | results.passed = 1; | |
| 267 | results.total_time_ns = 42; | |
| 268 | try history.finishRun(&results); | |
| 269 | ||
| 270 | const filename = try std.fmt.allocPrint(allocator, "{s}/test-run-{d}.json", .{ history_dir, history.current_entry.?.timestamp }); | |
| 271 | defer allocator.free(filename); | |
| 272 | const content = try compat.readFileAlloc(allocator, filename); | |
| 273 | defer allocator.free(content); | |
| 274 | var parsed = try std.json.parseFromSlice(std.json.Value, allocator, content, .{}); | |
| 275 | defer parsed.deinit(); | |
| 276 | ||
| 277 | const events = parsed.value.array.items; | |
| 278 | try std.testing.expectEqual(@as(usize, 3), events.len); | |
| 279 | try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string); | |
| 280 | try std.testing.expectEqualStrings("test_end", events[1].object.get("type").?.string); | |
| 281 | try std.testing.expectEqualStrings("quoted \"test\"", events[1].object.get("data").?.object.get("name").?.string); | |
| 282 | try std.testing.expectEqualStrings("run_end", events[2].object.get("type").?.string); | |
| 283 | for (events) |event| { | |
| 284 | try std.testing.expectEqual(@as(i64, protocol.version), event.object.get("protocol_version").?.integer); | |
| 285 | } | |
| 286 | } | |