ReviewOS

also looking at this

home-lang/home

feat(video): add native AVC/HEVC and synchronized AAC MP4 streaming

#913
Merged chris wants to merge feat/avc-hevc-mp4 into main
14 files +1119 -0

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

packages/video/docs/native-video.mdadded+25-0
Changes to packages/video/docs/native-video.md
@@ -0,0 +1,25 @@
1# Header-checked native video adapter
2
3`codecs/video/native_encoder.{c,h,zig}` compiles the x264/x265 boundary against
4installed vendor headers. Only Home-owned scalar configuration and borrowed
5length-prefixed packet data cross the interface. The Zig wrapper validates input
6geometry, timestamps and encode/drain state. Link the C source and libx264/libx265.
7
8`Encoder.requestKeyframe()` / `home_encoder_request_keyframe` requests an IDR for
9the next submitted input. Requests coalesce. This supports independently decodable
10recording splits without waiting for the regular GOP interval. Inspect emitted
11packet timestamps and key flags before switching outputs: buffered AVC output
12can precede the newly requested IDR. Keep headers outside packet history and
13remux each segment with a fresh sample description and a rebased timeline.
14
15Use requests only on an active encoder before draining, serialize every operation,
16and consume borrowed output before another adapter call. The Zig wrapper rejects
17requests after drain/failure. A request neither drains nor resets the encoder.
18
19The standalone `tests/native_encoder_idr.zig` imports this module as `encoder` and
20links its compiled C boundary. Its regression encodes eight frames under GOP100,
21forces IDRs at inputs 3 and 6, checks actual AVC/HEVC IDR NAL types and packet keys,
22counts every delayed/flushed frame, and rejects requests after drain. It passes
23on Zig 0.15.2 and Home's installed 0.17 development toolchain with x264/x265.
24zig-camera additionally remuxes split files and pre-roll snapshots and validates
25them with independent decoders. No FFmpeg or Libav is used in this path.
packages/video/src/codecs/video/native_encoder.cmodified+6-0
Changes to packages/video/src/codecs/video/native_encoder.c
@@ -14,6 +14,7 @@ struct home_encoder {
1414 uint8_t *buffer;
1515 size_t capacity;
1616 uint32_t width, height;
17 int force_key;
1718};
1819void home_encoder_close(home_encoder *e) {
1920 if (!e) return;
@@ -113,6 +114,8 @@ int home_encoder_encode(home_encoder *e, const uint8_t *const planes[3], const i
113114 if (!planes[0] || !planes[1] || !planes[2] || strides[0]<(int)e->width || strides[1]<(int)e->width/2 || strides[2]<(int)e->width/2) return -1;
114115 if (e->avc) {
115116 x264_picture_t in, decoded; x264_picture_init(&in); x264_picture_init(&decoded);
117 in.i_type=e->force_key ? X264_TYPE_IDR : X264_TYPE_AUTO;
118 e->force_key=0;
116119 in.img.i_csp=X264_CSP_I420; in.img.i_plane=3; in.i_pts=pts;
117120 for (int i=0;i<3;i++) { in.img.plane[i]=(uint8_t *)planes[i]; in.img.i_stride[i]=strides[i]; }
118121 x264_nal_t *nal=NULL; int n=0;
@@ -124,6 +127,7 @@ int home_encoder_encode(home_encoder *e, const uint8_t *const planes[3], const i
124127 }
125128 for (int i=0;i<3;i++) { e->in->planes[i]=(void *)planes[i]; e->in->stride[i]=strides[i]; }
126129 e->in->pts=pts;
130 e->in->sliceType=e->force_key ? X265_TYPE_IDR : X265_TYPE_AUTO;e->force_key=0;
127131 x265_nal *nal=NULL; uint32_t n=0;
128132 if (x265_encoder_encode(e->hevc,&nal,&n,e->in,e->out)<=0 || !n) return -3;
129133 out->pts=e->out->pts; out->key=IS_X265_TYPE_I(e->out->sliceType);
@@ -149,3 +153,5 @@ int home_encoder_flush(home_encoder *e, home_packet *out) {
149153 out->pts=e->out->pts;out->key=IS_X265_TYPE_I(e->out->sliceType);
150154 return collect(e,NULL,nal,n,out);
151155}
156
157int home_encoder_request_keyframe(home_encoder *e) { if(!e)return -1;e->force_key=1;return 0; }
packages/video/src/codecs/video/native_encoder.hmodified+1-0
Changes to packages/video/src/codecs/video/native_encoder.h
@@ -12,6 +12,7 @@ typedef struct { const uint8_t *data; size_t size; int64_t pts; int key; } home_
1212int home_encoder_open(const home_encoder_config *, home_encoder **);
1313int home_encoder_headers(home_encoder *, home_packet *);
1414int home_encoder_encode(home_encoder *, const uint8_t *const planes[3], const int strides[3], int64_t pts, home_packet *);
15int home_encoder_request_keyframe(home_encoder *);
1516int home_encoder_flush(home_encoder *, home_packet *);
1617void home_encoder_close(home_encoder *);
1718#endif
packages/video/src/codecs/video/native_encoder.zigmodified+5-0
Changes to packages/video/src/codecs/video/native_encoder.zig
@@ -8,6 +8,7 @@ const RawPacket = extern struct { data: [*]const u8, size: usize, pts: i64, key:
88extern fn home_encoder_open(*const Config, *?*Handle) c_int;
99extern fn home_encoder_headers(*Handle, *RawPacket) c_int;
1010extern fn home_encoder_encode(*Handle, *const [3][*]const u8, *const [3]c_int, i64, *RawPacket) c_int;
11extern fn home_encoder_request_keyframe(*Handle) c_int;
1112extern fn home_encoder_flush(*Handle, *RawPacket) c_int;
1213extern fn home_encoder_close(*Handle) void;
1314pub const Packet = struct { data: []const u8, pts: i64, key: bool };
@@ -31,6 +32,10 @@ pub const Encoder = struct {
3132 try check(home_encoder_headers(self.handle, &packet));
3233 return packet.data[0..packet.size];
3334 }
35 pub fn requestKeyframe(self: *Encoder) !void {
36 if (self.failed or self.draining) return error.EncoderFailed;
37 try check(home_encoder_request_keyframe(self.handle));
38 }
3439 pub fn flush(self: *Encoder) !?Packet {
3540 if (self.failed) return error.EncoderFailed;
3641 self.draining = true;
packages/video/tests/native_encoder_idr.zigadded+38-0
Changes to packages/video/tests/native_encoder_idr.zig
@@ -0,0 +1,38 @@
1//! Standalone adapter test; link native_encoder.c with system x264/x265.
2const std = ;
3const native = ;
4fn observe(packet: native.Packet, codec: u32, seen: *[8]bool) !void {
5 const index: usize = @intCast(@divTrunc(packet.pts, 41667));
6 try std.testing.expect(index < 8 and !seen[index]);
7 seen[index] = true;
8 const key = index == 0 or index == 3 or index == 6;
9 try std.testing.expectEqual(key, packet.key);
10 var at: usize = 0;
11 var idr = false;
12 while (at < packet.data.len) {
13 try std.testing.expect(packet.data.len - at >= 4);
14 const n = std.mem.readInt(u32, packet.data[at..][0..4], .big);
15 at += 4;
16 try std.testing.expect(n > 0 and n <= packet.data.len - at);
17 const kind = if (codec == 1) packet.data[at] & 31 else (packet.data[at] >> 1) & 63;
18 idr = idr or (if (codec == 1) kind == 5 else kind == 19 or kind == 20);
19 at += n;
20 }
21 try std.testing.expectEqual(key, idr);
22}
23test "requested IDRs override long GOP without losing delayed AVC frames" {
24 const y: [128 * 96]u8 = @splat(64);
25 const uv: [64 * 48]u8 = @splat(128);
26 for ([_]u32{ 1, 2 }) |codec| {
27 var encoder = try native.Encoder.init(.{ .codec = codec, .width = 128, .height = 96, .fps_num = 24, .fps_den = 1, .bitrate = 0, .quality = 90, .gop = 100 });
28 defer encoder.deinit();
29 var seen: [8]bool = @splat(false);
30 for (0..8) |i| {
31 if (i == 3 or i == 6) try encoder.requestKeyframe();
32 if (try encoder.encode(.{ &y, &uv, &uv }, .{ 128, 64, 64 }, 128, 96, @intCast(i * 41667))) |packet| try observe(packet, codec, &seen);
33 }
34 while (try encoder.flush()) |packet| try observe(packet, codec, &seen);
35 for (seen) |value| try std.testing.expect(value);
36 try std.testing.expectError(error.EncoderFailed, encoder.requestKeyframe());
37 }
38}