feat(video): add native AVC/HEVC and synchronized AAC MP4 streaming

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

packages/video/NATIVE_MEDIA.mdmodified+30-0

Changes to packages/video/NATIVE_MEDIA.md

		⋯56 linesShow the 56 lines above this hunk@@ -57,3 +57,33 @@ tests, not Pi hardware FPS, long-run reliability or compression-quality benchmar
57	57	References: [x265 API](https://x265.readthedocs.io/en/master/api.html),
58	58	[x264 public header](https://code.videolan.org/videolan/x264/-/blob/master/x264.h),
59	59	[ISO BMFF streaming requirements](https://www.w3.org/TR/mse-byte-stream-format-isobmff/).
	60	+
	61	+## AAC-LC with synchronized fragmented MP4
	62	+
	63	+`mp4_stream.Writer.initAv` extends the same bounded video muxer with an optional
	64	+AAC track. Its `AudioConfig` consumes the encoder's real AudioSpecificConfig,
	65	+rate/channels/bitrate and codec delay. Use the native encoder in
	66	+[Home #918](https://github.com/home-lang/home/pull/918); the muxer itself has no
	67	+codec dependency and remains independently testable.
	68	+
	69	+Audio raw access units use the sample-rate timebase and `mp4a`/`esds`; video
	70	+retains microsecond capture timestamps. Pass a sink with both write and patch
	71	+callbacks. One AAC packet is staged so the final duration can omit padding.
	72	+After draining AAC, `finish(start_us, input_frames)` emits the final packet and
	73	+patches only fixed header fields: 64-bit movie/track/media durations and edit
	74	+lengths. Memory does not grow with recording duration. A seekable sink and clean
	75	+finalization are required; pipe output and crash recovery are not provided.
	76	+
	77	+The audio edit list represents the initial offset and actual encoder delay.
	78	+Global `sgpd` and per-fragment `sbgp` roll associations mark explicit decoder
	79	+pre-roll. Without those associations Apple readers can apply an extra implicit
	80	+priming trim. The final short fragment duration prevents padded tail playback.
	81	+Both are checked by FAAD2/AVFoundation camera integration, including sub-frame
	82	+input, 44.1/48kHz, stereo, video gaps and independently decodable split files.
	83	+Do not infer AAC time from callback scheduling or assume a fixed codec delay.
	84	+The caller must provide continuous PCM to the encoder and preserve sample counts.
	85	+Microsecond movie/edit times can round by less than one PCM sample.
	86	+
	87	+Four focused muxer tests pass on Zig 0.15.2 and the Home development toolchain,
	88	+including sink failure poisoning and fixed edit/duration patch values. No
	89	+FFmpeg/Libav or complete Home media-graph validation is represented here.

packages/video/src/containers/mp4_stream.zigmodified+270-19

Changes to packages/video/src/containers/mp4_stream.zig

		@@ -1,9 +1,16 @@
1		-//! Home single-track streaming fragmented MP4, adapted from mp4_muxer.zig.
	1	+//! Home streaming fragmented MP4 with optional AAC-LC audio, adapted from mp4_muxer.zig.
2	2	//! Complete sample description + mvex; exact sizes/offsets; no file-sized buffer.
3	3	//! AVC/HEVC are length-prefixed, with out-of-band parameter sets (avc1/hvc1).
4	4	const std = ;
5	5	pub const Codec = enum { h264, hevc };
6		-pub const Sink = struct { context: *anyopaque, write: *const fn (*anyopaque, []const u8) anyerror!void };
	6	+pub const Sink = struct {
	7	+ context: *anyopaque,
	8	+ write: *const fn (*anyopaque, []const u8) anyerror!void,
	9	+ patch: ?*const fn (*anyopaque, u64, []const u8) anyerror!void = null,
	10	+};
	11	+pub const AudioConfig = struct { sample_rate: u32, channels: u32, bitrate: u32, delay: u32, config: [2]u8 };
	12	+const AudioOffsets = struct { track: usize = 0, media: usize = 0, empty: usize = 0, edit: usize = 0 };
	13	+
7	14	const Buffer = struct {
8	15	data: [16384]u8 = undefined,
9	16	len: usize = 0,
		⋯109 linesShow the 109 lines above this hunk@@ -119,13 +126,144 @@ fn config(b: *Buffer, codec: Codec, headers: []const u8) !void {
119	126	}
120	127	b.end(at);
121	128	}
	129	+fn audioTrack(b: *Buffer, cfg: AudioConfig) !AudioOffsets {
	130	+ const rates = [_]u32{ 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000 };
	131	+ var index: ?u16 = null;
	132	+ for (rates, 0..) |rate, i| if (rate == cfg.sample_rate) {
	133	+ index = @intCast(i);
	134	+ };
	135	+ if (index == null or cfg.sample_rate > 48000 or cfg.channels < 1 or cfg.channels > 2 or cfg.bitrate < 16000 or cfg.bitrate > 320000 or cfg.delay > 16384) return error.InvalidConfiguration;
	136	+ const expected = (@as(u16, 2) << 11) | (index.? << 7) | (@as(u16, @intCast(cfg.channels)) << 3);
	137	+ if (std.mem.readInt(u16, &cfg.config, .big) != expected) return error.InvalidConfiguration;
	138	+ var offsets: AudioOffsets = .{};
	139	+ const trak = try b.box("trak");
	140	+ const tkhd = try b.full("tkhd", 0x1000007);
	141	+ try b.zero(16);
	142	+ try b.int32(2);
	143	+ try b.zero(4);
	144	+ offsets.track = b.len;
	145	+ try b.int64(0);
	146	+ try b.zero(8);
	147	+ try b.zero(4);
	148	+ try b.int16(0x100);
	149	+ try b.int16(0);
	150	+ try b.matrix();
	151	+ try b.zero(8);
	152	+ b.end(tkhd);
	153	+ const edts = try b.box("edts");
	154	+ const elst = try b.full("elst", 0x1000000);
	155	+ try b.int32(2);
	156	+ offsets.empty = b.len;
	157	+ try b.int64(0);
	158	+ try b.int64(std.math.maxInt(u64));
	159	+ try b.int16(1);
	160	+ try b.int16(0);
	161	+ offsets.edit = b.len;
	162	+ try b.int64(0);
	163	+ try b.int64(cfg.delay);
	164	+ try b.int16(1);
	165	+ try b.int16(0);
	166	+ b.end(elst);
	167	+ b.end(edts);
	168	+ const mdia = try b.box("mdia");
	169	+ const mdhd = try b.full("mdhd", 0x1000000);
	170	+ try b.zero(16);
	171	+ try b.int32(cfg.sample_rate);
	172	+ offsets.media = b.len;
	173	+ try b.int64(0);
	174	+ try b.int16(0x55c4);
	175	+ try b.int16(0);
	176	+ b.end(mdhd);
	177	+ const hdlr = try b.full("hdlr", 0);
	178	+ try b.int32(0);
	179	+ try b.bytes("soun");
	180	+ try b.zero(12);
	181	+ try b.bytes("Home AAC\x00");
	182	+ b.end(hdlr);
	183	+ const minf = try b.box("minf");
	184	+ const smhd = try b.full("smhd", 0);
	185	+ try b.zero(4);
	186	+ b.end(smhd);
	187	+ const dinf = try b.box("dinf");
	188	+ const dref = try b.full("dref", 0);
	189	+ try b.int32(1);
	190	+ const url = try b.full("url ", 1);
	191	+ b.end(url);
	192	+ b.end(dref);
	193	+ b.end(dinf);
	194	+ const stbl = try b.box("stbl");
	195	+ const stsd = try b.full("stsd", 0);
	196	+ try b.int32(1);
	197	+ const entry = try b.box("mp4a");
	198	+ try b.zero(6);
	199	+ try b.int16(1);
	200	+ try b.zero(8);
	201	+ try b.int16(@intCast(cfg.channels));
	202	+ try b.int16(16);
	203	+ try b.zero(4);
	204	+ try b.int32(cfg.sample_rate << 16);
	205	+ const esds = try b.full("esds", 0);
	206	+ try b.bytes(&.{ 3, 25 });
	207	+ try b.int16(2);
	208	+ try b.bytes(&.{ 0, 4, 17, 0x40, 0x15, 0, 0x20, 0 });
	209	+ try b.int32(cfg.bitrate);
	210	+ try b.int32(cfg.bitrate);
	211	+ try b.bytes(&.{ 5, 2 });
	212	+ try b.bytes(&cfg.config);
	213	+ try b.bytes(&.{ 6, 1, 2 });
	214	+ b.end(esds);
	215	+ b.end(entry);
	216	+ b.end(stsd);
	217	+ for ([_]*const [4]u8{ "stts", "stsc", "stco" }) |name| {
	218	+ const at = try b.full(name, 0);
	219	+ try b.int32(0);
	220	+ b.end(at);
	221	+ }
	222	+ const stsz = try b.full("stsz", 0);
	223	+ try b.zero(8);
	224	+ b.end(stsz);
	225	+ // Explicit decoder pre-roll makes the edit list authoritative. Without
	226	+ // roll sample groups Apple assumes an additional legacy 2112-sample delay.
	227	+ const sgpd = try b.full("sgpd", 0x1000000);
	228	+ try b.bytes("roll");
	229	+ try b.int32(2);
	230	+ try b.int32(1);
	231	+ try b.int16(65535);
	232	+ b.end(sgpd);
	233	+ const sbgp = try b.full("sbgp", 0);
	234	+ try b.bytes("roll");
	235	+ try b.int32(0);
	236	+ b.end(sbgp);
	237	+ b.end(stbl);
	238	+ b.end(minf);
	239	+ b.end(mdia);
	240	+ b.end(trak);
	241	+ return offsets;
	242	+}
122	243	pub const Writer = struct {
123	244	sink: Sink,
124	245	sequence: u32 = 1,
125	246	last_dts: u64 = 0,
126	247	frames: u64 = 0,
127	248	failed: bool = false,
	249	+ finished: bool = false,
	250	+ audio_config: ?AudioConfig = null,
	251	+ audio_packets: u64 = 0,
	252	+ audio_pending: [8192]u8 = undefined,
	253	+ audio_pending_len: usize = 0,
	254	+ video_end_us: u64 = 0,
	255	+ movie_duration: usize = 0,
	256	+ video_duration: usize = 0,
	257	+ video_media_duration: usize = 0,
	258	+ audio_offsets: AudioOffsets = .{},
	259	+ pub fn initAv(sink: Sink, width: u32, height: u32, codec: Codec, headers: []const u8, audio_cfg: AudioConfig) !Writer {
	260	+ if (sink.patch == null) return error.InvalidConfiguration;
	261	+ return initInternal(sink, width, height, codec, headers, audio_cfg);
	262	+ }
128	263	pub fn init(sink: Sink, width: u32, height: u32, codec: Codec, headers: []const u8) !Writer {
	264	+ return initInternal(sink, width, height, codec, headers, null);
	265	+ }
	266	+ fn initInternal(sink: Sink, width: u32, height: u32, codec: Codec, headers: []const u8, audio_cfg: ?AudioConfig) !Writer {
129	267	if (width == 0 or height == 0 or width > 8192 or height > 8192) return error.InvalidConfiguration;
130	268	var b: Buffer = .{};
131	269	const ftyp = try b.box("ftyp");
		⋯2 linesShow the 2 lines above this hunk@@ -134,32 +272,37 @@ pub const Writer = struct {
134	272	try b.bytes("iso6isommp41");
135	273	b.end(ftyp);
136	274	const moov = try b.box("moov");
137		- const mvhd = try b.full("mvhd", 0);
138		- try b.zero(8);
	275	+ const mvhd = try b.full("mvhd", 0x1000000);
	276	+ try b.zero(16);
139	277	try b.int32(1000000);
140		- try b.int32(0);
	278	+ const movie_duration = b.len;
	279	+ try b.int64(0);
141	280	try b.int32(0x10000);
142	281	try b.int16(0x100);
143	282	try b.zero(10);
144	283	try b.matrix();
145	284	try b.zero(24);
146		- try b.int32(2);
	285	+ try b.int32(if (audio_cfg != null) 3 else 2);
147	286	b.end(mvhd);
148	287	const trak = try b.box("trak");
149		- const tkhd = try b.full("tkhd", 7);
150		- try b.zero(8);
151		- try b.int32(1);
	288	+ const tkhd = try b.full("tkhd", 0x1000007);
152	289	try b.zero(16);
	290	+ try b.int32(1);
	291	+ try b.zero(4);
	292	+ const video_duration = b.len;
	293	+ try b.int64(0);
	294	+ try b.zero(8);
153	295	try b.zero(8);
154	296	try b.matrix();
155	297	try b.int32(width << 16);
156	298	try b.int32(height << 16);
157	299	b.end(tkhd);
158	300	const mdia = try b.box("mdia");
159		- const mdhd = try b.full("mdhd", 0);
160		- try b.zero(8);
	301	+ const mdhd = try b.full("mdhd", 0x1000000);
	302	+ try b.zero(16);
161	303	try b.int32(1000000);
162		- try b.int32(0);
	304	+ const video_media_duration = b.len;
	305	+ try b.int64(0);
163	306	try b.int16(0x55c4);
164	307	try b.int16(0);
165	308	b.end(mdhd);
		⋯52 linesShow the 52 lines above this hunk@@ -218,24 +361,88 @@ pub const Writer = struct {
218	361	b.end(minf);
219	362	b.end(mdia);
220	363	b.end(trak);
	364	+ const offsets = if (audio_cfg) |cfg| try audioTrack(&b, cfg) else AudioOffsets{};
221	365	const mvex = try b.box("mvex");
222	366	const trex = try b.full("trex", 0);
223	367	try b.int32(1);
224	368	try b.int32(1);
225	369	try b.zero(12);
226	370	b.end(trex);
	371	+ if (audio_cfg != null) {
	372	+ const audio_trex = try b.full("trex", 0);
	373	+ try b.int32(2);
	374	+ try b.int32(1);
	375	+ try b.int32(1024);
	376	+ try b.int32(0);
	377	+ try b.int32(0x02000000);
	378	+ b.end(audio_trex);
	379	+ }
227	380	b.end(mvex);
228	381	b.end(moov);
229	382	try sink.write(sink.context, b.slice());
230		- return .{ .sink = sink };
	383	+ return .{ .sink = sink, .audio_config = audio_cfg, .audio_offsets = offsets, .movie_duration = movie_duration, .video_duration = video_duration, .video_media_duration = video_media_duration };
231	384	}
232	385	/// One complete sample. Caller supplies its actual duration, using the next
233	386	/// capture time; the last sample uses nominal frame duration. PTS == DTS.
234	387	pub fn frame(s: *Writer, sample: []const u8, dts_us: u64, duration_us: u32, key: bool) !void {
235		- if (s.failed) return error.WriterFailed;
	388	+ if (s.failed or s.finished) return error.WriterFailed;
236	389	if (sample.len == 0 or sample.len > std.math.maxInt(u32) - 8 or duration_us == 0 or (s.frames != 0 and dts_us <= s.last_dts) or s.sequence == std.math.maxInt(u32)) return error.InvalidSample;
237	390	var nals: Nals = .{ .data = sample };
238	391	while (try nals.next()) |_| {}
	392	+ const end = try std.math.add(u64, dts_us, duration_us);
	393	+ try s.fragment(1, sample, dts_us, duration_us, key);
	394	+ s.frames += 1;
	395	+ s.last_dts = dts_us;
	396	+ s.video_end_us = end;
	397	+ }
	398	+ pub fn audio(s: *Writer, sample: []const u8) !void {
	399	+ if (s.audio_config == null or s.failed or s.finished) return error.WriterFailed;
	400	+ if (sample.len == 0 or sample.len > 8192 or s.sequence == std.math.maxInt(u32)) return error.InvalidSample;
	401	+ if (s.audio_pending_len != 0) try s.flushAudio(1024);
	402	+ @memcpy(s.audio_pending[0..sample.len], sample);
	403	+ s.audio_pending_len = sample.len;
	404	+ }
	405	+ fn flushAudio(s: *Writer, duration: u32) !void {
	406	+ if (s.audio_pending_len == 0) return;
	407	+ const time = try std.math.mul(u64, s.audio_packets, 1024);
	408	+ try s.fragment(2, s.audio_pending[0..s.audio_pending_len], time, duration, true);
	409	+ s.audio_pending_len = 0;
	410	+ s.audio_packets += 1;
	411	+ }
	412	+ /// Patch fixed metadata only after draining AAC. Decoder delay and tail
	413	+ /// padding stay in media samples; edit lists expose exactly the input PCM.
	414	+ pub fn finish(s: *Writer, audio_start_us: u64, presented_frames: u64) !void {
	415	+ if (s.failed or s.finished) return error.WriterFailed;
	416	+ const cfg = s.audio_config orelse return error.InvalidConfiguration;
	417	+ const encoded = try std.math.mul(u64, s.audio_packets + @intFromBool(s.audio_pending_len != 0), 1024);
	418	+ const valid_media = try std.math.add(u64, presented_frames, cfg.delay);
	419	+ if (encoded < valid_media or encoded - valid_media >= 1024) return error.InvalidSample;
	420	+ const padding = encoded - valid_media;
	421	+ const scaled = @as(u128, presented_frames) * 1000000 / cfg.sample_rate;
	422	+ if (scaled > std.math.maxInt(u64)) return error.InvalidSample;
	423	+ const duration: u64 = @intCast(scaled);
	424	+ const audio_end = try std.math.add(u64, audio_start_us, duration);
	425	+ errdefer s.failed = true;
	426	+ // A short final duration also signals tail trim to fragmented readers
	427	+ // that do not apply the finite edit duration to the last fragment.
	428	+ try s.flushAudio(@intCast(1024 - padding));
	429	+ try s.patch64(s.movie_duration, @max(s.video_end_us, audio_end));
	430	+ try s.patch64(s.video_duration, s.video_end_us);
	431	+ try s.patch64(s.video_media_duration, s.video_end_us);
	432	+ try s.patch64(s.audio_offsets.track, audio_end);
	433	+ try s.patch64(s.audio_offsets.media, valid_media);
	434	+ try s.patch64(s.audio_offsets.empty, audio_start_us);
	435	+ try s.patch64(s.audio_offsets.edit, duration);
	436	+ s.finished = true;
	437	+ }
	438	+ fn patch64(s: *Writer, offset: usize, value: u64) !void {
	439	+ var bytes: [8]u8 = undefined;
	440	+ std.mem.writeInt(u64, &bytes, value, .big);
	441	+ try s.sink.patch.?(s.sink.context, offset, &bytes);
	442	+ }
	443	+ fn fragment(s: *Writer, track: u32, sample: []const u8, time: u64, duration: u32, key: bool) !void {
	444	+ if (s.failed or s.finished) return error.WriterFailed;
	445	+ if (s.sequence == std.math.maxInt(u32)) return error.InvalidSample;
239	446	var b: Buffer = .{};
240	447	const moof = try b.box("moof");
241	448	const mfhd = try b.full("mfhd", 0);
		⋯1 lineShow the 1 line above this hunk@@ -243,19 +450,27 @@ pub const Writer = struct {
243	450	b.end(mfhd);
244	451	const traf = try b.box("traf");
245	452	const tfhd = try b.full("tfhd", 0x20000);
246		- try b.int32(1);
	453	+ try b.int32(track);
247	454	b.end(tfhd);
248	455	const tfdt = try b.full("tfdt", 0x1000000);
249		- try b.int64(dts_us);
	456	+ try b.int64(time);
250	457	b.end(tfdt);
251	458	const trun = try b.full("trun", 0x701);
252	459	try b.int32(1);
253	460	const offset = b.len;
254	461	try b.int32(0);
255		- try b.int32(duration_us);
	462	+ try b.int32(duration);
256	463	try b.int32(@intCast(sample.len));
257	464	try b.int32(if (key) 0x02000000 else 0x01010000);
258	465	b.end(trun);
	466	+ if (track == 2) {
	467	+ const sbgp = try b.full("sbgp", 0);
	468	+ try b.bytes("roll");
	469	+ try b.int32(1);
	470	+ try b.int32(1);
	471	+ try b.int32(1);
	472	+ b.end(sbgp);
	473	+ }
259	474	b.end(traf);
260	475	b.end(moof);
261	476	std.mem.writeInt(u32, b.data[offset..][0..4], @intCast(b.len + 8), .big);
		⋯3 linesShow the 3 lines above this hunk@@ -265,8 +480,6 @@ pub const Writer = struct {
265	480	try s.sink.write(s.sink.context, b.slice());
266	481	try s.sink.write(s.sink.context, sample);
267	482	s.sequence += 1;
268		- s.frames += 1;
269		- s.last_dts = dts_us;
270	483	}
271	484	};
272	485	test "NAL lengths reject truncation and empty units" {
		⋯45 linesShow the 45 lines above this hunk@@ -318,3 +531,41 @@ test "Matroska CodecPrivate reuses checked config and respects destination bound
318	531	try std.testing.expectError(error.HeaderTooLarge, codecPrivate(.h264, &headers, destination[0..1]));
319	532	try std.testing.expectError(error.MissingParameterSets, codecPrivate(.h264, &.{}, &destination));
320	533	}
	534	+test "AAC track sample clock, edit delay/tail, fixed patches and sink failure" {
	535	+ const Out = struct {
	536	+ bytes: std.ArrayList(u8) = .empty,
	537	+ fail: bool = false,
	538	+ fn write(ctx: *anyopaque, data: []const u8) anyerror!void {
	539	+ const s: *@This() = @ptrCast(@alignCast(ctx));
	540	+ try s.bytes.appendSlice(std.testing.allocator, data);
	541	+ }
	542	+ fn patch(ctx: *anyopaque, offset: u64, data: []const u8) anyerror!void {
	543	+ const s: *@This() = @ptrCast(@alignCast(ctx));
	544	+ if (s.fail) return error.DiskFull;
	545	+ if (offset + data.len > s.bytes.items.len) return error.InvalidOffset;
	546	+ @memcpy(s.bytes.items[@intCast(offset)..][0..data.len], data);
	547	+ }
	548	+ };
	549	+ var out: Out = .{};
	550	+ defer out.bytes.deinit(std.testing.allocator);
	551	+ const headers = [_]u8{ 0, 0, 0, 4, 0x67, 100, 0, 40, 0, 0, 0, 2, 0x68, 0x80 };
	552	+ const cfg: AudioConfig = .{ .sample_rate = 48000, .channels = 2, .bitrate = 128000, .delay = 2048, .config = .{ 0x11, 0x90 } };
	553	+ const sink: Sink = .{ .context = &out, .write = Out.write, .patch = Out.patch };
	554	+ var writer = try Writer.initAv(sink, 128, 96, .h264, &headers, cfg);
	555	+ try std.testing.expect(std.mem.indexOf(u8, out.bytes.items, "mp4a") != null);
	556	+ try writer.frame(&.{ 0, 0, 0, 2, 0x65, 0x80 }, 0, 41667, true);
	557	+ try std.testing.expectError(error.InvalidSample, writer.finish(12345, 1000));
	558	+ for (0..3) |_| try writer.audio(&.{ 0x21, 0x10 });
	559	+ try writer.finish(12345, 1000);
	560	+ try std.testing.expectEqual(@as(u64, 12345), std.mem.readInt(u64, out.bytes.items[writer.audio_offsets.empty..][0..8], .big));
	561	+ try std.testing.expectEqual(@as(u64, 20833), std.mem.readInt(u64, out.bytes.items[writer.audio_offsets.edit..][0..8], .big));
	562	+ try std.testing.expectEqual(@as(u64, 2048), std.mem.readInt(u64, out.bytes.items[writer.audio_offsets.edit + 8 ..][0..8], .big));
	563	+ try std.testing.expectEqual(@as(u64, 3048), std.mem.readInt(u64, out.bytes.items[writer.audio_offsets.media..][0..8], .big));
	564	+ try std.testing.expectError(error.WriterFailed, writer.audio(&.{1}));
	565	+ var failure = try Writer.initAv(sink, 128, 96, .h264, &headers, cfg);
	566	+ try failure.audio(&.{ 0x21, 0x10 });
	567	+ try failure.audio(&.{ 0x21, 0x10 });
	568	+ out.fail = true;
	569	+ try std.testing.expectError(error.DiskFull, failure.finish(0, 0));
	570	+ try std.testing.expectError(error.WriterFailed, failure.audio(&.{1}));
	571	+}