Add reproducible randomized test order

tests/reporter_test.zigmodified+33-0

Changes to tests/reporter_test.zig

		⋯55 linesShow the 55 lines above this hunk@@ -56,6 +56,8 @@ test "reporter json output uses versioned event envelopes" {
56	56	defer results.deinit();
57	57	results.total = 1;
58	58	results.passed = 1;
	59	+ results.random_seed = 4242;
	60	+ reporter.reporter.random_seed = 4242;
59	61	var test_case = ztf.TestCase.init("passes \"quoted\"\nname", passingTest);
60	62	test_case.status = .passed;
61	63
		⋯11 linesShow the 11 lines above this hunk@@ -73,12 +75,14 @@ test "reporter json output uses versioned event envelopes" {
73	75	try std.testing.expectEqual(@as(usize, 6), events.len);
74	76	try std.testing.expectEqual(@as(i64, ztf.protocol_version), events[0].object.get("protocol_version").?.integer);
75	77	try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string);
	78	+ try std.testing.expectEqual(@as(i64, 4242), events[0].object.get("data").?.object.get("random_seed").?.integer);
76	79	try std.testing.expectEqualStrings("test_start", events[2].object.get("type").?.string);
77	80	const test_data = events[3].object.get("data").?.object;
78	81	try std.testing.expectEqualStrings(test_case.name, test_data.get("name").?.string);
79	82	try std.testing.expectEqualStrings("suite\tone", test_data.get("suite").?.string);
80	83	try std.testing.expectEqualStrings("run_end", events[5].object.get("type").?.string);
81	84	try std.testing.expectEqual(@as(i64, 1), events[5].object.get("data").?.object.get("passed").?.integer);
	85	+ try std.testing.expectEqual(@as(i64, 4242), events[5].object.get("data").?.object.get("random_seed").?.integer);
82	86	}
83	87
84	88	test "reporter json output includes flaky attempt history" {
		⋯55 linesShow the 55 lines above this hunk@@ -140,3 +144,32 @@ test "reporter set selects the same built-ins for every executor" {
140	144	try std.testing.expect(reporters.selected() == &reporters.tap.reporter);
141	145	try std.testing.expect(!reporters.selected().use_colors);
142	146	}
	147	+
	148	+test "JUnit reporter records the random seed as a suite property" {
	149	+ const allocator = std.testing.allocator;
	150	+ var tmp = std.testing.tmpDir(.{});
	151	+ defer tmp.cleanup();
	152	+ const output_path = try std.fmt.allocPrint(allocator, ".zig-cache/tmp/{s}/results.xml", .{tmp.sub_path});
	153	+ defer allocator.free(output_path);
	154	+
	155	+ var reporter = ztf.JUnitReporter.init(allocator, output_path);
	156	+ defer reporter.deinit();
	157	+ reporter.reporter.random_seed = 73;
	158	+ var results = ztf.TestResults.init(allocator);
	159	+ defer results.deinit();
	160	+ results.total = 1;
	161	+ results.passed = 1;
	162	+ results.random_seed = 73;
	163	+ var test_case = ztf.TestCase.init("passes", passingTest);
	164	+ test_case.status = .passed;
	165	+
	166	+ try reporter.reporter.onRunStart(1);
	167	+ try reporter.reporter.onSuiteStart("suite");
	168	+ try reporter.reporter.onTestEnd(&test_case);
	169	+ try reporter.reporter.onSuiteEnd("suite");
	170	+ try reporter.reporter.onRunEnd(&results);
	171	+
	172	+ const output = try ztf.compat.readFileAlloc(allocator, output_path);
	173	+ defer allocator.free(output);
	174	+ try std.testing.expect(std.mem.indexOf(u8, output, "<property name=\"random_seed\" value=\"73\"/>") != null);
	175	+}