From b98c2faeb7efcfddb1abd188c9d239e7fc07c22a Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Thu, 26 Jun 2025 16:57:13 +0200 Subject: [PATCH 01/12] wip, baseline of depayloader --- lib/ex_webrtc/rtp/h264/depayloader.ex | 99 ++++++++++++ lib/ex_webrtc/rtp/h264/nal_formats/fu.ex | 90 +++++++++++ .../rtp/h264/nal_formats/fu/header.ex | 65 ++++++++ lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex | 54 +++++++ lib/ex_webrtc/rtp/h264/nal_header.ex | 109 +++++++++++++ lib/ex_webrtc/rtp/h264/payload.ex | 150 ++++++++++++++++++ 6 files changed, 567 insertions(+) create mode 100644 lib/ex_webrtc/rtp/h264/depayloader.ex create mode 100644 lib/ex_webrtc/rtp/h264/nal_formats/fu.ex create mode 100644 lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex create mode 100644 lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex create mode 100644 lib/ex_webrtc/rtp/h264/nal_header.ex create mode 100644 lib/ex_webrtc/rtp/h264/payload.ex diff --git a/lib/ex_webrtc/rtp/h264/depayloader.ex b/lib/ex_webrtc/rtp/h264/depayloader.ex new file mode 100644 index 00000000..d23b0015 --- /dev/null +++ b/lib/ex_webrtc/rtp/h264/depayloader.ex @@ -0,0 +1,99 @@ +defmodule ExWebRTC.RTP.Depayloader.H264 do + @moduledoc """ + Depayloads H264 RTP payloads into H264 NAL Units. + + Based on [RFC 6184](https://tools.ietf.org/html/rfc6184). + + Supported types: Single NALU, FU-A, STAP-A. + """ + @behaviour ExWebRTC.RTP.Depayloader.Behaviour + + require Logger + + alias ExWebRTC.RTP.H264.{FU, NAL, StapA} + + @frame_prefix <<1::32>> + @annexb_prefix <<1::4>> + + defmodule State do + @moduledoc false + defstruct parser_acc: nil + end + + @type t() :: %__MODULE__{ + current_nal: nil, + current_timestamp: nil + } + + defstruct [:current_nal, :current_timestamp] + + @impl true + def new() do + %__MODULE__{} + end + + # TODO: handle timestamps + @impl true + def depayload(depayloader, packet) + + def depayload(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {nil, depayloader} + + def depayload(depayloader, packet) do + with {:ok, {header, _payload} = nal} <- NAL.Header.parse_unit_header(packet.payload), + unit_type = NAL.Header.decode_type(header), + {:ok, {nalu, depayloader}} <- handle_unit_type(unit_type, depayloader, packet, nal) do + {nalu, depayloader} + else + {:error, reason} -> + Logger.warning(""" + Couldn't parse payload, reason: #{reason}. \ + Resetting depayloader state. Payload: #{inspect(packet.payload)}.\ + """) + + {:ok, %{depayloader | current_nal: nil, current_timestamp: nil}} + end + end + + defp handle_unit_type(:single_nalu, _depayloader, _packet, nal) do + {header, payload} = nal + {:ok, {prefix_annexb(payload), depayloader}} + end + + defp handle_unit_type( + :fu_a, + {current_nal, current_timestamp} = depayloader, + packet, + {header, payload} = nal + ) do + if current_nal != nil and current_timestamp != packet.timestamp do + {:error, "fu-a colliding rtp timestamps"} + + Logger.debug(""" + Received packet with timestamp from a new frame that is not a beginning of this frame \ + and without finishing the previous frame. Dropping both.\ + """) + end + + case FU.parse(payload, current_nal) do + {:ok, {payload, type}} -> + {:ok, result} + + {:incomplete, tmp} -> + {:ok, {nil, %{depayloader | current_nal: curent_nal <> tmp}}} + + {:error, _reason} = error -> + error + end + end + + defp handle_unit_type(:stap_a, depayloader, {_header, data}, buffer, state) do + with {:ok, result} <- StapA.parse(data) do + nals = Enum.reduce(result, <<>>, fn nal, acc -> acc <> prefix_annexb(nal) end) + {:ok, {nals, depayloader}} + end + end + + defp prefix_annexb(nal) do + @annexb_prefix <> nal + end +end diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex new file mode 100644 index 00000000..4c437288 --- /dev/null +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex @@ -0,0 +1,90 @@ +defmodule ExWebRTC.RTP.H264.FU do + @moduledoc """ + Module responsible for parsing H264 Fragmentation Unit. + """ + use Bunch + alias __MODULE__ + alias Membrane.RTP.H264.NAL + + defstruct data: [] + + @type t :: %__MODULE__{data: [binary()]} + + @doc """ + Parses H264 Fragmentation Unit + + If a packet that is being parsed is not considered last then a `{:incomplete, t()}` + tuple will be returned. + In case of last packet `{:ok, {type, data}}` tuple will be returned, where data + is `NAL Unit` created by concatenating subsequent Fragmentation Units. + """ + @spec parse(binary(), t) :: + {:ok, {binary(), NAL.Header.type()}} + | {:error, :packet_malformed | :invalid_first_packet} + | {:incomplete, t()} + def parse(packet, acc) do + with {:ok, {header, value}} <- FU.Header.parse(packet) do + do_parse(header, value, acc) + end + end + + @doc """ + Serialize H264 unit into list of FU-A payloads + """ + @spec serialize(binary(), pos_integer()) :: list(binary()) | {:error, :unit_too_small} + def serialize(data, preferred_size) do + case data do + <> -> + <> = header + + payload = + head + |> FU.Header.add_header(1, 0, type) + |> NAL.Header.add_header(r, nri, NAL.Header.encode_type(:fu_a)) + + [payload | do_serialize(rest, r, nri, type, preferred_size)] + + _data -> + {:error, :unit_too_small} + end + end + + defp do_serialize(data, r, nri, type, preferred_size) do + case data do + <> when byte_size(rest) > 0 -> + payload = + head + |> FU.Header.add_header(0, 0, type) + |> NAL.Header.add_header(r, nri, NAL.Header.encode_type(:fu_a)) + + [payload] ++ do_serialize(rest, r, nri, type, preferred_size) + + rest -> + [ + rest + |> FU.Header.add_header(0, 1, type) + |> NAL.Header.add_header(r, nri, NAL.Header.encode_type(:fu_a)) + ] + end + end + + defp do_parse(header, packet, acc) + + defp do_parse(%FU.Header{start_bit: true}, packet, acc), + do: {:incomplete, %__MODULE__{acc | data: [packet]}} + + defp do_parse(%FU.Header{start_bit: false}, _data, %__MODULE__{data: []}), + do: {:error, :invalid_first_packet} + + defp do_parse(%FU.Header{end_bit: true, type: type}, packet, %__MODULE__{data: acc_data}) do + result = + [packet | acc_data] + |> Enum.reverse() + |> Enum.join() + + {:ok, {result, type}} + end + + defp do_parse(_header, packet, %__MODULE__{data: acc_data} = fu), + do: {:incomplete, %__MODULE__{fu | data: [packet | acc_data]}} +end diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex new file mode 100644 index 00000000..6b256e2a --- /dev/null +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex @@ -0,0 +1,65 @@ +defmodule ExWebRTC.RTP.H264.FU.Header do + @moduledoc """ + Defines a structure representing Fragmentation Unit (FU) header + which is defined in [RFC6184](https://tools.ietf.org/html/rfc6184#page-31) + + ``` + +---------------+ + |0|1|2|3|4|5|6|7| + +-+-+-+-+-+-+-+-+ + |S|E|R| Type | + +---------------+ + ``` + """ + + alias Membrane.RTP.H264.NAL + + @typedoc """ + MUST be set to true only in the first packet in a sequence. + """ + @type start_flag :: boolean() + + @typedoc """ + MUST be set to true only in the last packet in a sequence. + """ + @type end_flag :: boolean() + + @enforce_keys [:type] + defstruct start_bit: false, end_bit: false, type: 0 + + @type t :: %__MODULE__{ + start_bit: start_flag(), + end_bit: end_flag(), + type: NAL.Header.type() + } + + defguardp valid_frame_boundary(start, finish) when start != 1 or finish != 1 + + @doc """ + Parses Fragmentation Unit Header + + It will fail if the Start bit and End bit are both set to one in the + same Fragmentation Unit Header, because a fragmented NAL unit + MUST NOT be transmitted in one FU. + """ + @spec parse(data :: binary()) :: {:error, :packet_malformed} | {:ok, {t(), nal :: binary()}} + def parse(<>) + when nal_type in 1..23 and valid_frame_boundary(start, finish) do + header = %__MODULE__{ + start_bit: start == 1, + end_bit: finish == 1, + type: nal_type + } + + {:ok, {header, rest}} + end + + def parse(_binary), do: {:error, :packet_malformed} + + @doc """ + Adds FU header + """ + @spec add_header(binary(), 0 | 1, 0 | 1, NAL.Header.type()) :: binary() + def add_header(payload, start_bit, end_bit, type), + do: <> <> payload +end diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex b/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex new file mode 100644 index 00000000..8970b94e --- /dev/null +++ b/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex @@ -0,0 +1,54 @@ +defmodule ExWebRTC.RTP.H264.StapA do + @moduledoc """ + Module responsible for parsing Single Time Agregation Packets type A. + + Documented in [RFC6184](https://tools.ietf.org/html/rfc6184#page-22) + + ``` + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | RTP Header | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |STAP-A NAL HDR | NALU 1 Size | NALU 1 HDR | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | NALU 1 Data | + : : + + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | | NALU 2 Size | NALU 2 HDR | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | NALU 2 Data | + : : + | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | :...OPTIONAL RTP padding | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ``` + """ + use Bunch + + alias ExWebRTC.RTP.H264.NAL + + @spec parse(binary()) :: {:ok, [binary()]} | {:error, :packet_malformed} + def parse(data) do + do_parse(data, []) + end + + defp do_parse(<<>>, acc), do: {:ok, Enum.reverse(acc)} + + defp do_parse(<>, acc), + do: do_parse(rest, [nalu | acc]) + + defp do_parse(_data, _acc), do: {:error, :packet_malformed} + + @spec aggregation_unit_size(binary()) :: pos_integer() + def aggregation_unit_size(nalu), do: byte_size(nalu) + 2 + + @spec serialize([binary], 0..1, 0..3) :: binary + def serialize(payloads, f, nri) do + payloads + |> Enum.reverse() + |> Enum.map(&<>) + |> IO.iodata_to_binary() + |> NAL.Header.add_header(f, nri, NAL.Header.encode_type(:stap_a)) + end +end diff --git a/lib/ex_webrtc/rtp/h264/nal_header.ex b/lib/ex_webrtc/rtp/h264/nal_header.ex new file mode 100644 index 00000000..7c1c22ea --- /dev/null +++ b/lib/ex_webrtc/rtp/h264/nal_header.ex @@ -0,0 +1,109 @@ +defmodule ExWebRTC.RTP.H264.NALHeader do + @moduledoc """ + Defines a structure representing Network Abstraction Layer Unit Header + + Defined in [RFC 6184](https://tools.ietf.org/html/rfc6184#section-5.3) + + ``` + +---------------+ + |0|1|2|3|4|5|6|7| + +-+-+-+-+-+-+-+-+ + |F|NRI| Type | + +---------------+ + ``` + """ + + @typedoc """ + NRI stands for nal_ref_idc. This value represents importance of + frame that is being parsed. + + The higher the value the more important frame is (for example key + frames have nri value of 3) and a value of 00 indicates that the + content of the NAL unit is not used to reconstruct reference pictures + for inter picture prediction. NAL units with NRI equal 0 can be discarded + without risking the integrity of the reference pictures, although these + payloads might contain metadata. + """ + @type nri :: 0..3 + + @typedoc """ + Specifies the type of RBSP (Raw Byte Sequence Payload) data structure contained in the NAL unit. + + Types are defined as follows. + + | ID | RBSP Type | + |----------|----------------| + | 0 | Unspecified | + | 1-23 | NAL unit types | + | 24 | STAP-A | + | 25 | STAP-B | + | 26 | MTAP-16 | + | 27 | MTAP-24 | + | 28 | FU-A | + | 29 | FU-B | + | Reserved | 30-31 | + + """ + @type type :: 1..31 + @type supported_types :: :stap_a | :fu_a | :single_nalu + @type unsupported_types :: :stap_b | :mtap_16 | :mtap_24 | :fu_b + @type types :: supported_types | unsupported_types | :reserved + + defstruct [:nal_ref_idc, :type] + + @type t :: %__MODULE__{ + nal_ref_idc: nri(), + type: type() + } + + @spec parse_unit_header(binary()) :: {:error, :malformed_data} | {:ok, {t(), binary()}} + def parse_unit_header(raw_nal) + + def parse_unit_header(<<0::1, nri::2, type::5, rest::binary>>) do + nal = %__MODULE__{ + nal_ref_idc: nri, + type: type + } + + {:ok, {nal, rest}} + end + + # If first bit is not set to 0 packet is flagged as malformed + def parse_unit_header(_binary), do: {:error, :malformed_data} + + @doc """ + Adds NAL header to payload + """ + @spec add_header(binary(), 0 | 1, nri(), type()) :: binary() + def add_header(payload, f, nri, type), + do: <> <> payload + + @doc """ + Parses type stored in NAL Header + """ + @spec decode_type(t) :: types() + def decode_type(%__MODULE__{type: type}), do: do_decode_type(type) + + defp do_decode_type(number) when number in 1..21, do: :single_nalu + defp do_decode_type(number) when number in [22, 23], do: :reserved + defp do_decode_type(24), do: :stap_a + defp do_decode_type(25), do: :stap_b + defp do_decode_type(26), do: :mtap_16 + defp do_decode_type(27), do: :mtap_24 + defp do_decode_type(28), do: :fu_a + defp do_decode_type(29), do: :fu_b + defp do_decode_type(number) when number in [30, 31], do: :reserved + + @doc """ + Encodes given NAL type + """ + @spec encode_type(types()) :: type() + def encode_type(:single_nalu), do: 1 + def encode_type(:stap_a), do: 24 + def encode_type(:stap_b), do: 25 + def encode_type(:mtap_16), do: 26 + def encode_type(:mtap_24), do: 27 + def encode_type(:fu_a), do: 28 + def encode_type(:fu_b), do: 29 + def encode_type(:reserved), do: 30 +end diff --git a/lib/ex_webrtc/rtp/h264/payload.ex b/lib/ex_webrtc/rtp/h264/payload.ex new file mode 100644 index 00000000..60a6b557 --- /dev/null +++ b/lib/ex_webrtc/rtp/h264/payload.ex @@ -0,0 +1,150 @@ +defmodule ExWebRTC.RTP.H264.Payload do + @moduledoc false + # Defines VP8 payload structure stored in RTP packet payload. + # + # Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). + + @type t() :: %__MODULE__{ + n: 0 | 1, + s: 0 | 1, + pid: non_neg_integer(), + picture_id: non_neg_integer() | nil, + tl0picidx: non_neg_integer() | nil, + tid: non_neg_integer() | nil, + y: 0 | 1 | nil, + keyidx: non_neg_integer() | nil, + payload: binary() + } + + @enforce_keys [:n, :s, :pid, :payload] + defstruct @enforce_keys ++ [:picture_id, :tl0picidx, :tid, :y, :keyidx] + + @doc """ + Parses RTP payload as H264 payload. + """ + @spec parse(binary()) :: {:ok, t()} | {:error, :invalid_packet} + def parse(rtp_payload) + + def parse(<<>>), do: {:error, :invalid_packet} + + def parse(<<0::1, 0::1, n::1, s::1, 0::1, pid::3, payload::binary>>) do + if payload == <<>> do + {:error, :invalid_packet} + else + {:ok, + %__MODULE__{ + n: n, + s: s, + pid: pid, + payload: payload + }} + end + end + + def parse(<<1::1, 0::1, n::1, s::1, 0::1, pid::3, i::1, l::1, t::1, k::1, 0::4, rest::binary>>) do + with {:ok, picture_id, rest} <- parse_picture_id(i, rest), + {:ok, tl0picidx, rest} <- parse_tl0picidx(l, rest), + {:ok, tid, y, keyidx, rest} <- parse_tidykeyidx(t, k, rest) do + if rest == <<>> do + {:error, :invalid_packet} + else + {:ok, + %__MODULE__{ + n: n, + s: s, + pid: pid, + picture_id: picture_id, + tl0picidx: tl0picidx, + tid: tid, + y: y, + keyidx: keyidx, + payload: rest + }} + end + end + end + + def parse(_), do: {:error, :invalid_packet} + + defp parse_picture_id(0, rest), + do: {:ok, nil, rest} + + defp parse_picture_id(1, <<0::1, picture_id::7, rest::binary>>), do: {:ok, picture_id, rest} + defp parse_picture_id(1, <<1::1, picture_id::15, rest::binary>>), do: {:ok, picture_id, rest} + defp parse_picture_id(_, _), do: {:error, :invalid_packet} + + defp parse_tl0picidx(0, rest), do: {:ok, nil, rest} + defp parse_tl0picidx(1, <>), do: {:ok, tl0picidx, rest} + defp parse_tl0picidx(_, _), do: {:error, :invalid_packet} + + defp parse_tidykeyidx(0, 0, rest), do: {:ok, nil, nil, nil, rest} + + defp parse_tidykeyidx(1, 0, <>), + do: {:ok, tid, y, nil, rest} + + # note that both pion and web browser always set y bit to 0 in this case + # but RFC 7741, sec. 4.2 (definition for Y bit) explicitly states that Y bit + # can be set when T is 0 and K is 1 + defp parse_tidykeyidx(0, 1, <<_tid::2, y::1, keyidx::5, rest::binary>>), + do: {:ok, nil, y, keyidx, rest} + + defp parse_tidykeyidx(1, 1, <>), + do: {:ok, tid, y, keyidx, rest} + + defp parse_tidykeyidx(_, _, _), do: {:error, :invalid_packet} + + @spec serialize(t()) :: binary() + def serialize( + %__MODULE__{ + picture_id: nil, + tl0picidx: nil, + tid: nil, + y: nil, + keyidx: nil + } = vp8_payload + ) do + p = vp8_payload + <<0::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, p.payload::binary>> + end + + def serialize(vp8_payload) do + p = vp8_payload + i = if p.picture_id, do: 1, else: 0 + l = if p.tl0picidx, do: 1, else: 0 + t = if p.tid, do: 1, else: 0 + k = if p.keyidx, do: 1, else: 0 + + payload = + <<1::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, i::1, l::1, t::1, k::1, 0::4>> + |> add_picture_id(p.picture_id) + |> add_tl0picidx(p.tl0picidx) + |> add_tidykeyidx(p.tid, p.y, p.keyidx) + + <> + end + + defp add_picture_id(payload, nil), do: payload + + defp add_picture_id(payload, picture_id) when picture_id in 0..127 do + <> + end + + defp add_picture_id(payload, picture_id) when picture_id in 128..32_767 do + <> + end + + defp add_tl0picidx(payload, nil), do: payload + + defp add_tl0picidx(payload, tl0picidx) do + <> + end + + defp add_tidykeyidx(payload, nil, nil, nil), do: payload + + defp add_tidykeyidx(_payload, tid, nil, _keyidx) when tid != nil, + do: raise("VP8 Y bit has to be set when TID is set") + + defp add_tidykeyidx(payload, tid, y, keyidx) do + <> + end +end From b8b7d16c9b7858a1dba09c19b634ee61e8031880 Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Thu, 3 Jul 2025 14:42:48 +0200 Subject: [PATCH 02/12] first working --- lib/ex_webrtc/rtp/depayloader.ex | 1 + lib/ex_webrtc/rtp/h264/depayloader.ex | 60 ++++++++++--------- lib/ex_webrtc/rtp/h264/nal_formats/fu.ex | 2 +- .../rtp/h264/nal_formats/fu/header.ex | 2 +- lib/ex_webrtc/rtp/h264/nal_header.ex | 2 +- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/lib/ex_webrtc/rtp/depayloader.ex b/lib/ex_webrtc/rtp/depayloader.ex index 79b067cd..de92e4dc 100644 --- a/lib/ex_webrtc/rtp/depayloader.ex +++ b/lib/ex_webrtc/rtp/depayloader.ex @@ -36,6 +36,7 @@ defmodule ExWebRTC.RTP.Depayloader do defp to_depayloader_module(mime_type) do case String.downcase(mime_type) do "video/vp8" -> {:ok, ExWebRTC.RTP.Depayloader.VP8} + "video/h264" -> {:ok, ExWebRTC.RTP.Depayloader.H264} "audio/opus" -> {:ok, ExWebRTC.RTP.Depayloader.Opus} "audio/pcma" -> {:ok, ExWebRTC.RTP.Depayloader.G711} "audio/pcmu" -> {:ok, ExWebRTC.RTP.Depayloader.G711} diff --git a/lib/ex_webrtc/rtp/h264/depayloader.ex b/lib/ex_webrtc/rtp/h264/depayloader.ex index d23b0015..38118081 100644 --- a/lib/ex_webrtc/rtp/h264/depayloader.ex +++ b/lib/ex_webrtc/rtp/h264/depayloader.ex @@ -12,27 +12,20 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do alias ExWebRTC.RTP.H264.{FU, NAL, StapA} - @frame_prefix <<1::32>> - @annexb_prefix <<1::4>> - - defmodule State do - @moduledoc false - defstruct parser_acc: nil - end + @annexb_prefix <<1::32>> @type t() :: %__MODULE__{ - current_nal: nil, - current_timestamp: nil + current_timestamp: nil, + fu_parser_acc: nil } - defstruct [:current_nal, :current_timestamp] + defstruct [:current_timestamp, :fu_parser_acc] @impl true def new() do %__MODULE__{} end - # TODO: handle timestamps @impl true def depayload(depayloader, packet) @@ -41,8 +34,9 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do def depayload(depayloader, packet) do with {:ok, {header, _payload} = nal} <- NAL.Header.parse_unit_header(packet.payload), unit_type = NAL.Header.decode_type(header), - {:ok, {nalu, depayloader}} <- handle_unit_type(unit_type, depayloader, packet, nal) do - {nalu, depayloader} + {:ok, {nal, depayloader}} <- + handle_unit_type(unit_type, depayloader, packet, nal) do + {nal, depayloader} else {:error, reason} -> Logger.warning(""" @@ -54,18 +48,18 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do end end - defp handle_unit_type(:single_nalu, _depayloader, _packet, nal) do - {header, payload} = nal - {:ok, {prefix_annexb(payload), depayloader}} + defp handle_unit_type(:single_nalu, depayloader, packet, {_header, payload}) do + {:ok, + {prefix_annexb(payload), %__MODULE__{depayloader | current_timestamp: packet.timestamp}}} end defp handle_unit_type( :fu_a, - {current_nal, current_timestamp} = depayloader, + %{current_timestamp: current_timestamp, fu_parser_acc: fu_parser_acc}, packet, - {header, payload} = nal + {header, payload} ) do - if current_nal != nil and current_timestamp != packet.timestamp do + if fu_parser_acc != nil and current_timestamp != packet.timestamp do {:error, "fu-a colliding rtp timestamps"} Logger.debug(""" @@ -74,25 +68,37 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do """) end - case FU.parse(payload, current_nal) do - {:ok, {payload, type}} -> - {:ok, result} + case FU.parse(payload, fu_parser_acc || %FU{}) do + {:ok, {data, type}} -> + data = NAL.Header.add_header(data, 0, header.nal_ref_idc, type) + + {:ok, + {prefix_annexb(data), + %__MODULE__{current_timestamp: packet.timestamp, fu_parser_acc: nil}}} - {:incomplete, tmp} -> - {:ok, {nil, %{depayloader | current_nal: curent_nal <> tmp}}} + {:incomplete, fu} -> + {:ok, {nil, %__MODULE__{fu_parser_acc: fu}}} {:error, _reason} = error -> error end end - defp handle_unit_type(:stap_a, depayloader, {_header, data}, buffer, state) do - with {:ok, result} <- StapA.parse(data) do + defp handle_unit_type(:stap_a, depayloader, packet, {_header, payload}) do + with {:ok, result} <- StapA.parse(payload) do nals = Enum.reduce(result, <<>>, fn nal, acc -> acc <> prefix_annexb(nal) end) - {:ok, {nals, depayloader}} + {:ok, {nals, %__MODULE__{depayloader | current_timestamp: packet.timestamp}}} end end + defp handle_unit_type(unsupported_type, _depayloader, _packet, _nal) do + {:error, "unsupported nal type #{unsupported_type}"} + + Logger.debug(""" + Received packet with unsupported NAL type. Supported types are: NAL unit types, STAP-A, FU-A. Dropping packet. + """) + end + defp prefix_annexb(nal) do @annexb_prefix <> nal end diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex index 4c437288..85de4a47 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex @@ -4,7 +4,7 @@ defmodule ExWebRTC.RTP.H264.FU do """ use Bunch alias __MODULE__ - alias Membrane.RTP.H264.NAL + alias ExWebRTC.RTP.H264.NAL defstruct data: [] diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex index 6b256e2a..1e3c0a63 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex @@ -12,7 +12,7 @@ defmodule ExWebRTC.RTP.H264.FU.Header do ``` """ - alias Membrane.RTP.H264.NAL + alias ExWebRTC.RTP.H264.NAL @typedoc """ MUST be set to true only in the first packet in a sequence. diff --git a/lib/ex_webrtc/rtp/h264/nal_header.ex b/lib/ex_webrtc/rtp/h264/nal_header.ex index 7c1c22ea..9c1fbe86 100644 --- a/lib/ex_webrtc/rtp/h264/nal_header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_header.ex @@ -1,4 +1,4 @@ -defmodule ExWebRTC.RTP.H264.NALHeader do +defmodule ExWebRTC.RTP.H264.NAL.Header do @moduledoc """ Defines a structure representing Network Abstraction Layer Unit Header From 3976b903feb194c1048e36d55bc539f12b542453 Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Fri, 4 Jul 2025 15:42:18 +0200 Subject: [PATCH 03/12] cleanup --- lib/ex_webrtc/rtp/h264/depayloader.ex | 14 ++- lib/ex_webrtc/rtp/h264/payload.ex | 150 -------------------------- 2 files changed, 6 insertions(+), 158 deletions(-) delete mode 100644 lib/ex_webrtc/rtp/h264/payload.ex diff --git a/lib/ex_webrtc/rtp/h264/depayloader.ex b/lib/ex_webrtc/rtp/h264/depayloader.ex index 38118081..12b7e769 100644 --- a/lib/ex_webrtc/rtp/h264/depayloader.ex +++ b/lib/ex_webrtc/rtp/h264/depayloader.ex @@ -1,6 +1,6 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do @moduledoc """ - Depayloads H264 RTP payloads into H264 NAL Units. + Extracts H264 NAL Units from RTP packets. Based on [RFC 6184](https://tools.ietf.org/html/rfc6184). @@ -27,8 +27,6 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do end @impl true - def depayload(depayloader, packet) - def depayload(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {nil, depayloader} def depayload(depayloader, packet) do @@ -60,11 +58,11 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do {header, payload} ) do if fu_parser_acc != nil and current_timestamp != packet.timestamp do - {:error, "fu-a colliding rtp timestamps"} + {:error, "Invalid timestamp inside FU-A"} Logger.debug(""" - Received packet with timestamp from a new frame that is not a beginning of this frame \ - and without finishing the previous frame. Dropping both.\ + Received packet with FU-A type payload that is not a start of Fragmentation Unit with timestamp \ + different than last start and without finishing the previous FU. Dropping FU.\ """) end @@ -92,10 +90,10 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do end defp handle_unit_type(unsupported_type, _depayloader, _packet, _nal) do - {:error, "unsupported nal type #{unsupported_type}"} + {:error, "Unsupported nal type #{unsupported_type}"} Logger.debug(""" - Received packet with unsupported NAL type. Supported types are: NAL unit types, STAP-A, FU-A. Dropping packet. + Received packet with unsupported NAL type. Supported types are: Single NALU, STAP-A, FU-A. Dropping packet. """) end diff --git a/lib/ex_webrtc/rtp/h264/payload.ex b/lib/ex_webrtc/rtp/h264/payload.ex deleted file mode 100644 index 60a6b557..00000000 --- a/lib/ex_webrtc/rtp/h264/payload.ex +++ /dev/null @@ -1,150 +0,0 @@ -defmodule ExWebRTC.RTP.H264.Payload do - @moduledoc false - # Defines VP8 payload structure stored in RTP packet payload. - # - # Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). - - @type t() :: %__MODULE__{ - n: 0 | 1, - s: 0 | 1, - pid: non_neg_integer(), - picture_id: non_neg_integer() | nil, - tl0picidx: non_neg_integer() | nil, - tid: non_neg_integer() | nil, - y: 0 | 1 | nil, - keyidx: non_neg_integer() | nil, - payload: binary() - } - - @enforce_keys [:n, :s, :pid, :payload] - defstruct @enforce_keys ++ [:picture_id, :tl0picidx, :tid, :y, :keyidx] - - @doc """ - Parses RTP payload as H264 payload. - """ - @spec parse(binary()) :: {:ok, t()} | {:error, :invalid_packet} - def parse(rtp_payload) - - def parse(<<>>), do: {:error, :invalid_packet} - - def parse(<<0::1, 0::1, n::1, s::1, 0::1, pid::3, payload::binary>>) do - if payload == <<>> do - {:error, :invalid_packet} - else - {:ok, - %__MODULE__{ - n: n, - s: s, - pid: pid, - payload: payload - }} - end - end - - def parse(<<1::1, 0::1, n::1, s::1, 0::1, pid::3, i::1, l::1, t::1, k::1, 0::4, rest::binary>>) do - with {:ok, picture_id, rest} <- parse_picture_id(i, rest), - {:ok, tl0picidx, rest} <- parse_tl0picidx(l, rest), - {:ok, tid, y, keyidx, rest} <- parse_tidykeyidx(t, k, rest) do - if rest == <<>> do - {:error, :invalid_packet} - else - {:ok, - %__MODULE__{ - n: n, - s: s, - pid: pid, - picture_id: picture_id, - tl0picidx: tl0picidx, - tid: tid, - y: y, - keyidx: keyidx, - payload: rest - }} - end - end - end - - def parse(_), do: {:error, :invalid_packet} - - defp parse_picture_id(0, rest), - do: {:ok, nil, rest} - - defp parse_picture_id(1, <<0::1, picture_id::7, rest::binary>>), do: {:ok, picture_id, rest} - defp parse_picture_id(1, <<1::1, picture_id::15, rest::binary>>), do: {:ok, picture_id, rest} - defp parse_picture_id(_, _), do: {:error, :invalid_packet} - - defp parse_tl0picidx(0, rest), do: {:ok, nil, rest} - defp parse_tl0picidx(1, <>), do: {:ok, tl0picidx, rest} - defp parse_tl0picidx(_, _), do: {:error, :invalid_packet} - - defp parse_tidykeyidx(0, 0, rest), do: {:ok, nil, nil, nil, rest} - - defp parse_tidykeyidx(1, 0, <>), - do: {:ok, tid, y, nil, rest} - - # note that both pion and web browser always set y bit to 0 in this case - # but RFC 7741, sec. 4.2 (definition for Y bit) explicitly states that Y bit - # can be set when T is 0 and K is 1 - defp parse_tidykeyidx(0, 1, <<_tid::2, y::1, keyidx::5, rest::binary>>), - do: {:ok, nil, y, keyidx, rest} - - defp parse_tidykeyidx(1, 1, <>), - do: {:ok, tid, y, keyidx, rest} - - defp parse_tidykeyidx(_, _, _), do: {:error, :invalid_packet} - - @spec serialize(t()) :: binary() - def serialize( - %__MODULE__{ - picture_id: nil, - tl0picidx: nil, - tid: nil, - y: nil, - keyidx: nil - } = vp8_payload - ) do - p = vp8_payload - <<0::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, p.payload::binary>> - end - - def serialize(vp8_payload) do - p = vp8_payload - i = if p.picture_id, do: 1, else: 0 - l = if p.tl0picidx, do: 1, else: 0 - t = if p.tid, do: 1, else: 0 - k = if p.keyidx, do: 1, else: 0 - - payload = - <<1::1, 0::1, p.n::1, p.s::1, 0::1, p.pid::3, i::1, l::1, t::1, k::1, 0::4>> - |> add_picture_id(p.picture_id) - |> add_tl0picidx(p.tl0picidx) - |> add_tidykeyidx(p.tid, p.y, p.keyidx) - - <> - end - - defp add_picture_id(payload, nil), do: payload - - defp add_picture_id(payload, picture_id) when picture_id in 0..127 do - <> - end - - defp add_picture_id(payload, picture_id) when picture_id in 128..32_767 do - <> - end - - defp add_tl0picidx(payload, nil), do: payload - - defp add_tl0picidx(payload, tl0picidx) do - <> - end - - defp add_tidykeyidx(payload, nil, nil, nil), do: payload - - defp add_tidykeyidx(_payload, tid, nil, _keyidx) when tid != nil, - do: raise("VP8 Y bit has to be set when TID is set") - - defp add_tidykeyidx(payload, tid, y, keyidx) do - <> - end -end From b6b74b088e3c088d557dd1fc466d2b5a0b40792d Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Wed, 9 Jul 2025 11:59:17 +0200 Subject: [PATCH 04/12] add tests, fix depayloader --- lib/ex_webrtc/rtp/h264/depayloader.ex | 38 ++++----- test/ex_webrtc/rtp/h264/depayloader_test.exs | 83 ++++++++++++++++++++ 2 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 test/ex_webrtc/rtp/h264/depayloader_test.exs diff --git a/lib/ex_webrtc/rtp/h264/depayloader.ex b/lib/ex_webrtc/rtp/h264/depayloader.ex index 12b7e769..7372592c 100644 --- a/lib/ex_webrtc/rtp/h264/depayloader.ex +++ b/lib/ex_webrtc/rtp/h264/depayloader.ex @@ -42,7 +42,7 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do Resetting depayloader state. Payload: #{inspect(packet.payload)}.\ """) - {:ok, %{depayloader | current_nal: nil, current_timestamp: nil}} + {nil, %{depayloader | current_timestamp: nil, fu_parser_acc: nil}} end end @@ -58,27 +58,27 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do {header, payload} ) do if fu_parser_acc != nil and current_timestamp != packet.timestamp do - {:error, "Invalid timestamp inside FU-A"} - - Logger.debug(""" + Logger.warning(""" Received packet with FU-A type payload that is not a start of Fragmentation Unit with timestamp \ different than last start and without finishing the previous FU. Dropping FU.\ """) - end - case FU.parse(payload, fu_parser_acc || %FU{}) do - {:ok, {data, type}} -> - data = NAL.Header.add_header(data, 0, header.nal_ref_idc, type) + {:error, "Invalid timestamp inside FU-A"} + else + case FU.parse(payload, fu_parser_acc || %FU{}) do + {:ok, {data, type}} -> + data = NAL.Header.add_header(data, 0, header.nal_ref_idc, type) - {:ok, - {prefix_annexb(data), - %__MODULE__{current_timestamp: packet.timestamp, fu_parser_acc: nil}}} + {:ok, + {prefix_annexb(data), + %__MODULE__{current_timestamp: packet.timestamp, fu_parser_acc: nil}}} - {:incomplete, fu} -> - {:ok, {nil, %__MODULE__{fu_parser_acc: fu}}} + {:incomplete, fu} -> + {:ok, {nil, %__MODULE__{fu_parser_acc: fu, current_timestamp: packet.timestamp}}} - {:error, _reason} = error -> - error + {:error, _reason} = error -> + error + end end end @@ -90,11 +90,11 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do end defp handle_unit_type(unsupported_type, _depayloader, _packet, _nal) do - {:error, "Unsupported nal type #{unsupported_type}"} - - Logger.debug(""" - Received packet with unsupported NAL type. Supported types are: Single NALU, STAP-A, FU-A. Dropping packet. + Logger.warning(""" + Received packet with unsupported NAL type: #{unsupported_type}. Supported types are: Single NALU, STAP-A, FU-A. Dropping packet. """) + + {:error, "Unsupported nal type #{unsupported_type}"} end defp prefix_annexb(nal) do diff --git a/test/ex_webrtc/rtp/h264/depayloader_test.exs b/test/ex_webrtc/rtp/h264/depayloader_test.exs new file mode 100644 index 00000000..f8c16548 --- /dev/null +++ b/test/ex_webrtc/rtp/h264/depayloader_test.exs @@ -0,0 +1,83 @@ +defmodule ExWebRTC.RTP.H264.DepayloaderTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTP.Depayloader + + test "depayload/2" do + payload_stapa = <<56, 0, 1, 128, 0, 1, 129>> + payload_stapa_out = <<0, 0, 0, 1, 128, 0, 0, 0, 1, 129>> + + payload_fuas = <<60, 133, 128>> + payload_fua = <<60, 5, 129>> + payload_fuae = <<60, 69, 130>> + payload_fua_out = <<0, 0, 0, 1, 37, 128, 129, 130>> + + payload_single = <<53, 131>> + payload_single_out = <<0, 0, 0, 1, 131>> + + # reserved NALu types (22, 23, 30, 31) + payloads_nalu_reserved = [<<55, 131>>, <<56, 131>>, <<62, 131>>, <<63, 131>>] + # malformed STAP-A payload. First NAL should be 1-byte long, but is 2-bytes long + payload_invalid = <<56, 0, 1, 128, 12, 0, 1, 129>> + + depayloader = Depayloader.H264.new() + + # Single NAL Unit + packet = ExRTP.Packet.new(payload_single, timestamp: 123) + + assert {^payload_single_out, %{current_timestamp: 123, fu_parser_acc: nil}} = + Depayloader.H264.depayload(depayloader, packet) + + # STAP-A NAL + packet = ExRTP.Packet.new(payload_stapa, timestamp: 123) + + assert {^payload_stapa_out, %{current_timestamp: 123, fu_parser_acc: nil}} = + Depayloader.H264.depayload(depayloader, packet) + + # FU-A NAL + packet1 = ExRTP.Packet.new(payload_fuas, timestamp: 10) + packet2 = ExRTP.Packet.new(payload_fua, timestamp: 10) + packet3 = ExRTP.Packet.new(payload_fuae, timestamp: 10) + + {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) + + assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<128>>]}}} = + {bin, depayloader} + + {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet2) + + assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<129>>, <<128>>]}}} = + {bin, depayloader} + + assert {^payload_fua_out, %{current_timestamp: 10, fu_parser_acc: nil}} = + Depayloader.H264.depayload(depayloader, packet3) + + # Colliding timestamps in one FU-A + packet1 = ExRTP.Packet.new(payload_fuas, timestamp: 10) + packet2 = ExRTP.Packet.new(payload_fua, timestamp: 11) + + {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) + + assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<128>>]}}} = + {bin, depayloader} + + {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet2) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + {bin, depayloader} + + # Check all reserved NAL types + Enum.map(payloads_nalu_reserved, fn payload -> + packet = ExRTP.Packet.new(payload, timestamp: 123) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + Depayloader.H264.depayload(depayloader, packet) + end) + + # Check malformed NAL + packet = ExRTP.Packet.new(payload_invalid, timestamp: 123) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + Depayloader.H264.depayload(depayloader, packet) + end +end From de0ac2a62a81f49f679ad6f639508f41f0a0e37c Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Wed, 9 Jul 2025 16:50:36 +0200 Subject: [PATCH 05/12] fix tests --- lib/ex_webrtc/rtp/h264/nal_header.ex | 1 + test/ex_webrtc/rtp/depayloader_test.exs | 7 ++- test/ex_webrtc/rtp/h264/depayloader_test.exs | 57 ++++++++++++-------- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/lib/ex_webrtc/rtp/h264/nal_header.ex b/lib/ex_webrtc/rtp/h264/nal_header.ex index 9c1fbe86..d7372331 100644 --- a/lib/ex_webrtc/rtp/h264/nal_header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_header.ex @@ -93,6 +93,7 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do defp do_decode_type(28), do: :fu_a defp do_decode_type(29), do: :fu_b defp do_decode_type(number) when number in [30, 31], do: :reserved + defp do_decode_type(_), do: :invalid @doc """ Encodes given NAL type diff --git a/test/ex_webrtc/rtp/depayloader_test.exs b/test/ex_webrtc/rtp/depayloader_test.exs index f25df040..149e835a 100644 --- a/test/ex_webrtc/rtp/depayloader_test.exs +++ b/test/ex_webrtc/rtp/depayloader_test.exs @@ -75,9 +75,12 @@ defmodule ExWebRTC.RTP.DepayloaderTest do Depayloader.DTMF.depayload(depayloader, @packet) end - test "returns error if no depayloader exists for given codec" do - assert {:error, :no_depayloader_for_codec} = + test "creates a H264 depayloader and dispatches calls to its module" do + assert {:ok, depayloader} = %RTPCodecParameters{payload_type: 97, mime_type: "video/H264", clock_rate: 90_000} |> Depayloader.new() + + assert Depayloader.depayload(depayloader, @packet) == + Depayloader.H264.depayload(depayloader, @packet) end end diff --git a/test/ex_webrtc/rtp/h264/depayloader_test.exs b/test/ex_webrtc/rtp/h264/depayloader_test.exs index f8c16548..6db6dd25 100644 --- a/test/ex_webrtc/rtp/h264/depayloader_test.exs +++ b/test/ex_webrtc/rtp/h264/depayloader_test.exs @@ -3,38 +3,36 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do alias ExWebRTC.RTP.Depayloader - test "depayload/2" do - payload_stapa = <<56, 0, 1, 128, 0, 1, 129>> - payload_stapa_out = <<0, 0, 0, 1, 128, 0, 0, 0, 1, 129>> - - payload_fuas = <<60, 133, 128>> - payload_fua = <<60, 5, 129>> - payload_fuae = <<60, 69, 130>> - payload_fua_out = <<0, 0, 0, 1, 37, 128, 129, 130>> - + test "Check valid Single NAL Unit" do payload_single = <<53, 131>> payload_single_out = <<0, 0, 0, 1, 131>> - # reserved NALu types (22, 23, 30, 31) - payloads_nalu_reserved = [<<55, 131>>, <<56, 131>>, <<62, 131>>, <<63, 131>>] - # malformed STAP-A payload. First NAL should be 1-byte long, but is 2-bytes long - payload_invalid = <<56, 0, 1, 128, 12, 0, 1, 129>> - depayloader = Depayloader.H264.new() - - # Single NAL Unit packet = ExRTP.Packet.new(payload_single, timestamp: 123) assert {^payload_single_out, %{current_timestamp: 123, fu_parser_acc: nil}} = Depayloader.H264.depayload(depayloader, packet) + end - # STAP-A NAL + test "Check valid STAP-A NAL" do + payload_stapa = <<56, 0, 1, 128, 0, 1, 129>> + payload_stapa_out = <<0, 0, 0, 1, 128, 0, 0, 0, 1, 129>> + + depayloader = Depayloader.H264.new() packet = ExRTP.Packet.new(payload_stapa, timestamp: 123) assert {^payload_stapa_out, %{current_timestamp: 123, fu_parser_acc: nil}} = Depayloader.H264.depayload(depayloader, packet) + end + + test "Check valid FU-A NAL" do + payload_fuas = <<60, 133, 128>> + payload_fua = <<60, 5, 129>> + payload_fuae = <<60, 69, 130>> + payload_fua_out = <<0, 0, 0, 1, 37, 128, 129, 130>> + + depayloader = Depayloader.H264.new() - # FU-A NAL packet1 = ExRTP.Packet.new(payload_fuas, timestamp: 10) packet2 = ExRTP.Packet.new(payload_fua, timestamp: 10) packet3 = ExRTP.Packet.new(payload_fuae, timestamp: 10) @@ -51,8 +49,14 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do assert {^payload_fua_out, %{current_timestamp: 10, fu_parser_acc: nil}} = Depayloader.H264.depayload(depayloader, packet3) + end + + test "Check colliding timestamps in one FU-A" do + payload_fuas = <<60, 133, 128>> + payload_fua = <<60, 5, 129>> + + depayloader = Depayloader.H264.new() - # Colliding timestamps in one FU-A packet1 = ExRTP.Packet.new(payload_fuas, timestamp: 10) packet2 = ExRTP.Packet.new(payload_fua, timestamp: 11) @@ -65,16 +69,27 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = {bin, depayloader} + end + + test "Check all reserved NAL types" do + # reserved NALu types (22, 23, 30, 31) + payloads_nalu_reserved = [<<55, 131>>, <<56, 131>>, <<62, 131>>, <<63, 131>>] + + depayloader = Depayloader.H264.new() - # Check all reserved NAL types Enum.map(payloads_nalu_reserved, fn payload -> packet = ExRTP.Packet.new(payload, timestamp: 123) assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = Depayloader.H264.depayload(depayloader, packet) end) + end - # Check malformed NAL + test "Check malformed NAL" do + # malformed STAP-A payload. First NAL should be 1-byte long, but is 2-bytes long + payload_invalid = <<56, 0, 1, 128, 12, 0, 1, 129>> + + depayloader = Depayloader.H264.new() packet = ExRTP.Packet.new(payload_invalid, timestamp: 123) assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = From 73531651adbd2d5658533fd8ed14162f4a63caed Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Wed, 9 Jul 2025 17:26:52 +0200 Subject: [PATCH 06/12] add FU-A start constraint --- lib/ex_webrtc/rtp/h264/nal_formats/fu.ex | 15 +++++++++------ test/ex_webrtc/rtp/h264/depayloader_test.exs | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex index 85de4a47..6c448484 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex @@ -70,21 +70,24 @@ defmodule ExWebRTC.RTP.H264.FU do defp do_parse(header, packet, acc) - defp do_parse(%FU.Header{start_bit: true}, packet, acc), - do: {:incomplete, %__MODULE__{acc | data: [packet]}} + defp do_parse(%FU.Header{start_bit: true}, data, %{data: []} = acc), + do: {:incomplete, %__MODULE__{acc | data: [data]}} + + defp do_parse(%FU.Header{start_bit: true}, _data, _acc), + do: {:error, :last_fu_not_finished} defp do_parse(%FU.Header{start_bit: false}, _data, %__MODULE__{data: []}), do: {:error, :invalid_first_packet} - defp do_parse(%FU.Header{end_bit: true, type: type}, packet, %__MODULE__{data: acc_data}) do + defp do_parse(%FU.Header{end_bit: true, type: type}, data, %__MODULE__{data: acc_data}) do result = - [packet | acc_data] + [data | acc_data] |> Enum.reverse() |> Enum.join() {:ok, {result, type}} end - defp do_parse(_header, packet, %__MODULE__{data: acc_data} = fu), - do: {:incomplete, %__MODULE__{fu | data: [packet | acc_data]}} + defp do_parse(_header, data, %__MODULE__{data: acc_data} = fu), + do: {:incomplete, %__MODULE__{fu | data: [data | acc_data]}} end diff --git a/test/ex_webrtc/rtp/h264/depayloader_test.exs b/test/ex_webrtc/rtp/h264/depayloader_test.exs index 6db6dd25..d8a62576 100644 --- a/test/ex_webrtc/rtp/h264/depayloader_test.exs +++ b/test/ex_webrtc/rtp/h264/depayloader_test.exs @@ -71,6 +71,26 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do {bin, depayloader} end + test "Check starting new without ending previous FU-A" do + payload_fuas = <<60, 133, 128>> + payload_fua = <<60, 133, 129>> + + depayloader = Depayloader.H264.new() + + packet1 = ExRTP.Packet.new(payload_fuas, timestamp: 10) + packet2 = ExRTP.Packet.new(payload_fua, timestamp: 10) + + {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) + + assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<128>>]}}} = + {bin, depayloader} + + {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet2) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + {bin, depayloader} + end + test "Check all reserved NAL types" do # reserved NALu types (22, 23, 30, 31) payloads_nalu_reserved = [<<55, 131>>, <<56, 131>>, <<62, 131>>, <<63, 131>>] From 43407a35d0305292bb9b9466353e5193270e92cd Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Fri, 11 Jul 2025 11:03:22 +0200 Subject: [PATCH 07/12] review fixes, unused code removal --- lib/ex_webrtc/rtp/h264/depayloader.ex | 76 +++++++++++--------- lib/ex_webrtc/rtp/h264/nal_formats/fu.ex | 61 +++------------- lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex | 14 ---- lib/ex_webrtc/rtp/h264/nal_header.ex | 19 +---- test/ex_webrtc/rtp/depayloader_test.exs | 6 ++ test/ex_webrtc/rtp/h264/depayloader_test.exs | 51 +++++++------ 6 files changed, 88 insertions(+), 139 deletions(-) diff --git a/lib/ex_webrtc/rtp/h264/depayloader.ex b/lib/ex_webrtc/rtp/h264/depayloader.ex index 7372592c..0c2c6fa7 100644 --- a/lib/ex_webrtc/rtp/h264/depayloader.ex +++ b/lib/ex_webrtc/rtp/h264/depayloader.ex @@ -1,11 +1,11 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do - @moduledoc """ - Extracts H264 NAL Units from RTP packets. + @moduledoc false + # Extracts H264 NAL Units from RTP packets. + # + # Based on [RFC 6184](https://tools.ietf.org/html/rfc6184). + # + # Supported types: Single NALU, FU-A, STAP-A. - Based on [RFC 6184](https://tools.ietf.org/html/rfc6184). - - Supported types: Single NALU, FU-A, STAP-A. - """ @behaviour ExWebRTC.RTP.Depayloader.Behaviour require Logger @@ -15,11 +15,11 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do @annexb_prefix <<1::32>> @type t() :: %__MODULE__{ - current_timestamp: nil, - fu_parser_acc: nil + current_timestamp: non_neg_integer() | nil, + fu_parser_acc: [binary()] } - defstruct [:current_timestamp, :fu_parser_acc] + defstruct current_timestamp: nil, fu_parser_acc: [] @impl true def new() do @@ -33,7 +33,7 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do with {:ok, {header, _payload} = nal} <- NAL.Header.parse_unit_header(packet.payload), unit_type = NAL.Header.decode_type(header), {:ok, {nal, depayloader}} <- - handle_unit_type(unit_type, depayloader, packet, nal) do + do_depayload(unit_type, depayloader, packet, nal) do {nal, depayloader} else {:error, reason} -> @@ -42,54 +42,60 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do Resetting depayloader state. Payload: #{inspect(packet.payload)}.\ """) - {nil, %{depayloader | current_timestamp: nil, fu_parser_acc: nil}} + {nil, %{depayloader | current_timestamp: nil, fu_parser_acc: []}} end end - defp handle_unit_type(:single_nalu, depayloader, packet, {_header, payload}) do + defp do_depayload(:single_nalu, depayloader, packet, {_header, payload}) do {:ok, {prefix_annexb(payload), %__MODULE__{depayloader | current_timestamp: packet.timestamp}}} end - defp handle_unit_type( + defp do_depayload( :fu_a, %{current_timestamp: current_timestamp, fu_parser_acc: fu_parser_acc}, packet, + {_header, _payload} + ) + when fu_parser_acc != [] and current_timestamp != packet.timestamp do + Logger.warning(""" + received packet with fu-a type payload that is not a start of fragmentation unit with timestamp \ + different than last start and without finishing the previous fu. dropping fu.\ + """) + + {:error, "invalid timestamp inside fu-a"} + end + + defp do_depayload( + :fu_a, + %{fu_parser_acc: fu_parser_acc}, + packet, {header, payload} ) do - if fu_parser_acc != nil and current_timestamp != packet.timestamp do - Logger.warning(""" - Received packet with FU-A type payload that is not a start of Fragmentation Unit with timestamp \ - different than last start and without finishing the previous FU. Dropping FU.\ - """) - - {:error, "Invalid timestamp inside FU-A"} - else - case FU.parse(payload, fu_parser_acc || %FU{}) do - {:ok, {data, type}} -> - data = NAL.Header.add_header(data, 0, header.nal_ref_idc, type) + case FU.parse(payload, fu_parser_acc || []) do + {:ok, {data, type}} -> + data = NAL.Header.add_header(data, 0, header.nal_ref_idc, type) - {:ok, - {prefix_annexb(data), - %__MODULE__{current_timestamp: packet.timestamp, fu_parser_acc: nil}}} + {:ok, + {prefix_annexb(data), + %__MODULE__{current_timestamp: packet.timestamp, fu_parser_acc: []}}} - {:incomplete, fu} -> - {:ok, {nil, %__MODULE__{fu_parser_acc: fu, current_timestamp: packet.timestamp}}} + {:incomplete, fu} -> + {:ok, {nil, %__MODULE__{fu_parser_acc: fu, current_timestamp: packet.timestamp}}} - {:error, _reason} = error -> - error - end + {:error, _reason} = error -> + error end end - defp handle_unit_type(:stap_a, depayloader, packet, {_header, payload}) do + defp do_depayload(:stap_a, depayloader, packet, {_header, payload}) do with {:ok, result} <- StapA.parse(payload) do - nals = Enum.reduce(result, <<>>, fn nal, acc -> acc <> prefix_annexb(nal) end) + nals = result |> Stream.map(&prefix_annexb/1) |> Enum.join() {:ok, {nals, %__MODULE__{depayloader | current_timestamp: packet.timestamp}}} end end - defp handle_unit_type(unsupported_type, _depayloader, _packet, _nal) do + defp do_depayload(unsupported_type, _depayloader, _packet, _nal) do Logger.warning(""" Received packet with unsupported NAL type: #{unsupported_type}. Supported types are: Single NALU, STAP-A, FU-A. Dropping packet. """) diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex index 6c448484..1a334848 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex @@ -2,14 +2,9 @@ defmodule ExWebRTC.RTP.H264.FU do @moduledoc """ Module responsible for parsing H264 Fragmentation Unit. """ - use Bunch alias __MODULE__ alias ExWebRTC.RTP.H264.NAL - defstruct data: [] - - @type t :: %__MODULE__{data: [binary()]} - @doc """ Parses H264 Fragmentation Unit @@ -18,68 +13,28 @@ defmodule ExWebRTC.RTP.H264.FU do In case of last packet `{:ok, {type, data}}` tuple will be returned, where data is `NAL Unit` created by concatenating subsequent Fragmentation Units. """ - @spec parse(binary(), t) :: + @spec parse(binary(), [binary()]) :: {:ok, {binary(), NAL.Header.type()}} | {:error, :packet_malformed | :invalid_first_packet} - | {:incomplete, t()} + | {:incomplete, [binary()]} def parse(packet, acc) do with {:ok, {header, value}} <- FU.Header.parse(packet) do do_parse(header, value, acc) end end - @doc """ - Serialize H264 unit into list of FU-A payloads - """ - @spec serialize(binary(), pos_integer()) :: list(binary()) | {:error, :unit_too_small} - def serialize(data, preferred_size) do - case data do - <> -> - <> = header - - payload = - head - |> FU.Header.add_header(1, 0, type) - |> NAL.Header.add_header(r, nri, NAL.Header.encode_type(:fu_a)) - - [payload | do_serialize(rest, r, nri, type, preferred_size)] - - _data -> - {:error, :unit_too_small} - end - end - - defp do_serialize(data, r, nri, type, preferred_size) do - case data do - <> when byte_size(rest) > 0 -> - payload = - head - |> FU.Header.add_header(0, 0, type) - |> NAL.Header.add_header(r, nri, NAL.Header.encode_type(:fu_a)) - - [payload] ++ do_serialize(rest, r, nri, type, preferred_size) - - rest -> - [ - rest - |> FU.Header.add_header(0, 1, type) - |> NAL.Header.add_header(r, nri, NAL.Header.encode_type(:fu_a)) - ] - end - end - defp do_parse(header, packet, acc) - defp do_parse(%FU.Header{start_bit: true}, data, %{data: []} = acc), - do: {:incomplete, %__MODULE__{acc | data: [data]}} + defp do_parse(%FU.Header{start_bit: true}, data, []), + do: {:incomplete, [data]} defp do_parse(%FU.Header{start_bit: true}, _data, _acc), do: {:error, :last_fu_not_finished} - defp do_parse(%FU.Header{start_bit: false}, _data, %__MODULE__{data: []}), + defp do_parse(%FU.Header{start_bit: false}, _data, []), do: {:error, :invalid_first_packet} - defp do_parse(%FU.Header{end_bit: true, type: type}, data, %__MODULE__{data: acc_data}) do + defp do_parse(%FU.Header{end_bit: true, type: type}, data, acc_data) do result = [data | acc_data] |> Enum.reverse() @@ -88,6 +43,6 @@ defmodule ExWebRTC.RTP.H264.FU do {:ok, {result, type}} end - defp do_parse(_header, data, %__MODULE__{data: acc_data} = fu), - do: {:incomplete, %__MODULE__{fu | data: [data | acc_data]}} + defp do_parse(_header, data, acc_data), + do: {:incomplete, [data | acc_data]} end diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex b/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex index 8970b94e..e1d04a6a 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex @@ -24,8 +24,6 @@ defmodule ExWebRTC.RTP.H264.StapA do +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` """ - use Bunch - alias ExWebRTC.RTP.H264.NAL @spec parse(binary()) :: {:ok, [binary()]} | {:error, :packet_malformed} @@ -39,16 +37,4 @@ defmodule ExWebRTC.RTP.H264.StapA do do: do_parse(rest, [nalu | acc]) defp do_parse(_data, _acc), do: {:error, :packet_malformed} - - @spec aggregation_unit_size(binary()) :: pos_integer() - def aggregation_unit_size(nalu), do: byte_size(nalu) + 2 - - @spec serialize([binary], 0..1, 0..3) :: binary - def serialize(payloads, f, nri) do - payloads - |> Enum.reverse() - |> Enum.map(&<>) - |> IO.iodata_to_binary() - |> NAL.Header.add_header(f, nri, NAL.Header.encode_type(:stap_a)) - end end diff --git a/lib/ex_webrtc/rtp/h264/nal_header.ex b/lib/ex_webrtc/rtp/h264/nal_header.ex index d7372331..02a1cfe3 100644 --- a/lib/ex_webrtc/rtp/h264/nal_header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_header.ex @@ -44,7 +44,7 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do | Reserved | 30-31 | """ - @type type :: 1..31 + @type rbsp_type :: 1..31 @type supported_types :: :stap_a | :fu_a | :single_nalu @type unsupported_types :: :stap_b | :mtap_16 | :mtap_24 | :fu_b @type types :: supported_types | unsupported_types | :reserved @@ -53,7 +53,7 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do @type t :: %__MODULE__{ nal_ref_idc: nri(), - type: type() + type: rbsp_type() } @spec parse_unit_header(binary()) :: {:error, :malformed_data} | {:ok, {t(), binary()}} @@ -74,7 +74,7 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do @doc """ Adds NAL header to payload """ - @spec add_header(binary(), 0 | 1, nri(), type()) :: binary() + @spec add_header(binary(), 0 | 1, nri(), rbsp_type()) :: binary() def add_header(payload, f, nri, type), do: <> <> payload @@ -94,17 +94,4 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do defp do_decode_type(29), do: :fu_b defp do_decode_type(number) when number in [30, 31], do: :reserved defp do_decode_type(_), do: :invalid - - @doc """ - Encodes given NAL type - """ - @spec encode_type(types()) :: type() - def encode_type(:single_nalu), do: 1 - def encode_type(:stap_a), do: 24 - def encode_type(:stap_b), do: 25 - def encode_type(:mtap_16), do: 26 - def encode_type(:mtap_24), do: 27 - def encode_type(:fu_a), do: 28 - def encode_type(:fu_b), do: 29 - def encode_type(:reserved), do: 30 end diff --git a/test/ex_webrtc/rtp/depayloader_test.exs b/test/ex_webrtc/rtp/depayloader_test.exs index 149e835a..448dc043 100644 --- a/test/ex_webrtc/rtp/depayloader_test.exs +++ b/test/ex_webrtc/rtp/depayloader_test.exs @@ -83,4 +83,10 @@ defmodule ExWebRTC.RTP.DepayloaderTest do assert Depayloader.depayload(depayloader, @packet) == Depayloader.H264.depayload(depayloader, @packet) end + + test "returns error if no depayloader exists for given codec" do + assert {:error, :no_depayloader_for_codec} = + %RTPCodecParameters{payload_type: 45, mime_type: "video/AV1", clock_rate: 90_000} + |> Depayloader.new() + end end diff --git a/test/ex_webrtc/rtp/h264/depayloader_test.exs b/test/ex_webrtc/rtp/h264/depayloader_test.exs index d8a62576..c222c2eb 100644 --- a/test/ex_webrtc/rtp/h264/depayloader_test.exs +++ b/test/ex_webrtc/rtp/h264/depayloader_test.exs @@ -3,29 +3,29 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do alias ExWebRTC.RTP.Depayloader - test "Check valid Single NAL Unit" do + test "valid Single NAL Unit" do payload_single = <<53, 131>> payload_single_out = <<0, 0, 0, 1, 131>> depayloader = Depayloader.H264.new() packet = ExRTP.Packet.new(payload_single, timestamp: 123) - assert {^payload_single_out, %{current_timestamp: 123, fu_parser_acc: nil}} = + assert {^payload_single_out, %{current_timestamp: 123}} = Depayloader.H264.depayload(depayloader, packet) end - test "Check valid STAP-A NAL" do + test "valid STAP-A NAL" do payload_stapa = <<56, 0, 1, 128, 0, 1, 129>> payload_stapa_out = <<0, 0, 0, 1, 128, 0, 0, 0, 1, 129>> depayloader = Depayloader.H264.new() packet = ExRTP.Packet.new(payload_stapa, timestamp: 123) - assert {^payload_stapa_out, %{current_timestamp: 123, fu_parser_acc: nil}} = + assert {^payload_stapa_out, %{current_timestamp: 123}} = Depayloader.H264.depayload(depayloader, packet) end - test "Check valid FU-A NAL" do + test "valid FU-A NAL" do payload_fuas = <<60, 133, 128>> payload_fua = <<60, 5, 129>> payload_fuae = <<60, 69, 130>> @@ -39,19 +39,19 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) - assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<128>>]}}} = + assert {nil, %{current_timestamp: 10, fu_parser_acc: [<<128>>]}} = {bin, depayloader} {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet2) - assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<129>>, <<128>>]}}} = + assert {nil, %{current_timestamp: 10, fu_parser_acc: [<<129>>, <<128>>]}} = {bin, depayloader} - assert {^payload_fua_out, %{current_timestamp: 10, fu_parser_acc: nil}} = + assert {^payload_fua_out, %{current_timestamp: 10, fu_parser_acc: []}} = Depayloader.H264.depayload(depayloader, packet3) end - test "Check colliding timestamps in one FU-A" do + test "colliding timestamps in one FU-A" do payload_fuas = <<60, 133, 128>> payload_fua = <<60, 5, 129>> @@ -62,16 +62,16 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) - assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<128>>]}}} = + assert {nil, %{current_timestamp: 10, fu_parser_acc: [<<128>>]}} = {bin, depayloader} {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet2) - assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = {bin, depayloader} end - test "Check starting new without ending previous FU-A" do + test "starting new without ending previous FU-A" do payload_fuas = <<60, 133, 128>> payload_fua = <<60, 133, 129>> @@ -82,37 +82,46 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) - assert {nil, %{current_timestamp: 10, fu_parser_acc: %{data: [<<128>>]}}} = + assert {nil, %{current_timestamp: 10, fu_parser_acc: [<<128>>]}} = {bin, depayloader} {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet2) - assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = {bin, depayloader} end - test "Check all reserved NAL types" do - # reserved NALu types (22, 23, 30, 31) - payloads_nalu_reserved = [<<55, 131>>, <<56, 131>>, <<62, 131>>, <<63, 131>>] + test "all unsupported NAL types" do + # reserved (22, 23, 30, 31) and unsupported NALu types (STAP-B: 25, MTAP-16: 26, MTAP-24: 27, FU-B: 29) + payloads_nalu_unsupported = [ + <<54, 131>>, + <<55, 131>>, + <<62, 131>>, + <<63, 131>>, + <<57, 131>>, + <<58, 131>>, + <<59, 131>>, + <<61, 131>> + ] depayloader = Depayloader.H264.new() - Enum.map(payloads_nalu_reserved, fn payload -> + Enum.map(payloads_nalu_unsupported, fn payload -> packet = ExRTP.Packet.new(payload, timestamp: 123) - assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = Depayloader.H264.depayload(depayloader, packet) end) end - test "Check malformed NAL" do + test "malformed NAL" do # malformed STAP-A payload. First NAL should be 1-byte long, but is 2-bytes long payload_invalid = <<56, 0, 1, 128, 12, 0, 1, 129>> depayloader = Depayloader.H264.new() packet = ExRTP.Packet.new(payload_invalid, timestamp: 123) - assert {nil, %{current_timestamp: nil, fu_parser_acc: nil}} = + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = Depayloader.H264.depayload(depayloader, packet) end end From ecc81f3eba8d173df5d6214e289798824830cf77 Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Fri, 11 Jul 2025 11:14:28 +0200 Subject: [PATCH 08/12] fix warning --- lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex b/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex index e1d04a6a..b2c30227 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex @@ -24,7 +24,6 @@ defmodule ExWebRTC.RTP.H264.StapA do +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` """ - alias ExWebRTC.RTP.H264.NAL @spec parse(binary()) :: {:ok, [binary()]} | {:error, :packet_malformed} def parse(data) do From f068bb674b4de83313686e5c1f5038da6ddef5f3 Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Fri, 11 Jul 2025 11:19:46 +0200 Subject: [PATCH 09/12] dialyzer fixes --- lib/ex_webrtc/rtp/h264/nal_formats/fu.ex | 2 +- lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex index 1a334848..18bdc68c 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex @@ -14,7 +14,7 @@ defmodule ExWebRTC.RTP.H264.FU do is `NAL Unit` created by concatenating subsequent Fragmentation Units. """ @spec parse(binary(), [binary()]) :: - {:ok, {binary(), NAL.Header.type()}} + {:ok, {binary(), NAL.Header.rbsp_type()}} | {:error, :packet_malformed | :invalid_first_packet} | {:incomplete, [binary()]} def parse(packet, acc) do diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex index 1e3c0a63..6c132b11 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex @@ -30,7 +30,7 @@ defmodule ExWebRTC.RTP.H264.FU.Header do @type t :: %__MODULE__{ start_bit: start_flag(), end_bit: end_flag(), - type: NAL.Header.type() + type: NAL.Header.rbsp_type() } defguardp valid_frame_boundary(start, finish) when start != 1 or finish != 1 @@ -59,7 +59,7 @@ defmodule ExWebRTC.RTP.H264.FU.Header do @doc """ Adds FU header """ - @spec add_header(binary(), 0 | 1, 0 | 1, NAL.Header.type()) :: binary() + @spec add_header(binary(), 0 | 1, 0 | 1, NAL.Header.rbsp_type()) :: binary() def add_header(payload, start_bit, end_bit, type), do: <> <> payload end From 5e64f056adb299243a4b8f35a383ff305b7fcbd8 Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Fri, 11 Jul 2025 14:44:24 +0200 Subject: [PATCH 10/12] test coverage --- .../rtp/h264/nal_formats/fu/header.ex | 7 --- test/ex_webrtc/rtp/h264/depayloader_test.exs | 51 +++++++++++++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex index 6c132b11..037eb978 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex @@ -55,11 +55,4 @@ defmodule ExWebRTC.RTP.H264.FU.Header do end def parse(_binary), do: {:error, :packet_malformed} - - @doc """ - Adds FU header - """ - @spec add_header(binary(), 0 | 1, 0 | 1, NAL.Header.rbsp_type()) :: binary() - def add_header(payload, start_bit, end_bit, type), - do: <> <> payload end diff --git a/test/ex_webrtc/rtp/h264/depayloader_test.exs b/test/ex_webrtc/rtp/h264/depayloader_test.exs index c222c2eb..cf7850af 100644 --- a/test/ex_webrtc/rtp/h264/depayloader_test.exs +++ b/test/ex_webrtc/rtp/h264/depayloader_test.exs @@ -72,13 +72,13 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do end test "starting new without ending previous FU-A" do - payload_fuas = <<60, 133, 128>> - payload_fua = <<60, 133, 129>> + payload_fuas1 = <<60, 133, 128>> + payload_fuas2 = <<60, 133, 129>> depayloader = Depayloader.H264.new() - packet1 = ExRTP.Packet.new(payload_fuas, timestamp: 10) - packet2 = ExRTP.Packet.new(payload_fua, timestamp: 10) + packet1 = ExRTP.Packet.new(payload_fuas1, timestamp: 10) + packet2 = ExRTP.Packet.new(payload_fuas2, timestamp: 10) {bin, depayloader} = Depayloader.H264.depayload(depayloader, packet1) @@ -91,6 +91,28 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do {bin, depayloader} end + test "non-start packet without starting FU-A beforehand" do + payload_fua = <<60, 5, 128>> + + depayloader = Depayloader.H264.new() + + packet = ExRTP.Packet.new(payload_fua, timestamp: 10) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = + Depayloader.H264.depayload(depayloader, packet) + end + + test "non-fragmented FU-A (start and end bits set to 1)" do + payload_fua = <<60, 197, 129>> + + depayloader = Depayloader.H264.new() + + packet = ExRTP.Packet.new(payload_fua, timestamp: 10) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = + Depayloader.H264.depayload(depayloader, packet) + end + test "all unsupported NAL types" do # reserved (22, 23, 30, 31) and unsupported NALu types (STAP-B: 25, MTAP-16: 26, MTAP-24: 27, FU-B: 29) payloads_nalu_unsupported = [ @@ -114,7 +136,28 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do end) end + test "empty RTP payload" do + payload_empty = <<>> + + depayloader = Depayloader.H264.new() + packet = ExRTP.Packet.new(payload_empty, timestamp: 123) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = + Depayloader.H264.depayload(depayloader, packet) + end + test "malformed NAL" do + # forbidden zero bit set to 1 + payload_invalid = <<181, 0>> + + depayloader = Depayloader.H264.new() + packet = ExRTP.Packet.new(payload_invalid, timestamp: 123) + + assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = + Depayloader.H264.depayload(depayloader, packet) + end + + test "malformed STAP-A" do # malformed STAP-A payload. First NAL should be 1-byte long, but is 2-bytes long payload_invalid = <<56, 0, 1, 128, 12, 0, 1, 129>> From 5ce91571d36f5e4f208a4556731b599971b0b26d Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Fri, 11 Jul 2025 14:52:55 +0200 Subject: [PATCH 11/12] drop rtp padding test --- test/ex_webrtc/rtp/h264/depayloader_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ex_webrtc/rtp/h264/depayloader_test.exs b/test/ex_webrtc/rtp/h264/depayloader_test.exs index cf7850af..adccc361 100644 --- a/test/ex_webrtc/rtp/h264/depayloader_test.exs +++ b/test/ex_webrtc/rtp/h264/depayloader_test.exs @@ -136,17 +136,17 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do end) end - test "empty RTP payload" do + test "drop RTP padding packets" do payload_empty = <<>> depayloader = Depayloader.H264.new() - packet = ExRTP.Packet.new(payload_empty, timestamp: 123) + packet = ExRTP.Packet.new(payload_empty, padding: true, timestamp: 123) assert {nil, %{current_timestamp: nil, fu_parser_acc: []}} = Depayloader.H264.depayload(depayloader, packet) end - test "malformed NAL" do + test "drop malformed NAL" do # forbidden zero bit set to 1 payload_invalid = <<181, 0>> @@ -157,7 +157,7 @@ defmodule ExWebRTC.RTP.H264.DepayloaderTest do Depayloader.H264.depayload(depayloader, packet) end - test "malformed STAP-A" do + test "drop malformed STAP-A" do # malformed STAP-A payload. First NAL should be 1-byte long, but is 2-bytes long payload_invalid = <<56, 0, 1, 128, 12, 0, 1, 129>> From be1ad656d68932534b3aa68cb9fdf94b347fbe0b Mon Sep 17 00:00:00 2001 From: bartosz rzepa Date: Mon, 21 Jul 2025 09:28:01 +0200 Subject: [PATCH 12/12] more cleanup --- lib/ex_webrtc/rtp/h264/depayloader.ex | 12 ++++++------ lib/ex_webrtc/rtp/h264/nal_formats/fu.ex | 6 +++--- lib/ex_webrtc/rtp/h264/nal_header.ex | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/ex_webrtc/rtp/h264/depayloader.ex b/lib/ex_webrtc/rtp/h264/depayloader.ex index 0c2c6fa7..f207bdac 100644 --- a/lib/ex_webrtc/rtp/h264/depayloader.ex +++ b/lib/ex_webrtc/rtp/h264/depayloader.ex @@ -30,14 +30,14 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do def depayload(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {nil, depayloader} def depayload(depayloader, packet) do - with {:ok, {header, _payload} = nal} <- NAL.Header.parse_unit_header(packet.payload), + with {:ok, {header, _payload} = nal} <- NAL.Header.parse(packet.payload), unit_type = NAL.Header.decode_type(header), {:ok, {nal, depayloader}} <- do_depayload(unit_type, depayloader, packet, nal) do {nal, depayloader} else {:error, reason} -> - Logger.warning(""" + Logger.debug(""" Couldn't parse payload, reason: #{reason}. \ Resetting depayloader state. Payload: #{inspect(packet.payload)}.\ """) @@ -63,7 +63,7 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do different than last start and without finishing the previous fu. dropping fu.\ """) - {:error, "invalid timestamp inside fu-a"} + {:error, :invalid_timestamp} end defp do_depayload( @@ -74,7 +74,7 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do ) do case FU.parse(payload, fu_parser_acc || []) do {:ok, {data, type}} -> - data = NAL.Header.add_header(data, 0, header.nal_ref_idc, type) + data = NAL.Header.add(data, 0, header.nal_ref_idc, type) {:ok, {prefix_annexb(data), @@ -90,7 +90,7 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do defp do_depayload(:stap_a, depayloader, packet, {_header, payload}) do with {:ok, result} <- StapA.parse(payload) do - nals = result |> Stream.map(&prefix_annexb/1) |> Enum.join() + nals = result |> Enum.map_join(&prefix_annexb/1) {:ok, {nals, %__MODULE__{depayloader | current_timestamp: packet.timestamp}}} end end @@ -100,7 +100,7 @@ defmodule ExWebRTC.RTP.Depayloader.H264 do Received packet with unsupported NAL type: #{unsupported_type}. Supported types are: Single NALU, STAP-A, FU-A. Dropping packet. """) - {:error, "Unsupported nal type #{unsupported_type}"} + {:error, :unsupported_nal_type} end defp prefix_annexb(nal) do diff --git a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex index 18bdc68c..7810a215 100644 --- a/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex +++ b/lib/ex_webrtc/rtp/h264/nal_formats/fu.ex @@ -1,7 +1,7 @@ defmodule ExWebRTC.RTP.H264.FU do - @moduledoc """ - Module responsible for parsing H264 Fragmentation Unit. - """ + @moduledoc false + # Module responsible for parsing H264 Fragmentation Unit. + alias __MODULE__ alias ExWebRTC.RTP.H264.NAL diff --git a/lib/ex_webrtc/rtp/h264/nal_header.ex b/lib/ex_webrtc/rtp/h264/nal_header.ex index 02a1cfe3..c2796566 100644 --- a/lib/ex_webrtc/rtp/h264/nal_header.ex +++ b/lib/ex_webrtc/rtp/h264/nal_header.ex @@ -56,10 +56,10 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do type: rbsp_type() } - @spec parse_unit_header(binary()) :: {:error, :malformed_data} | {:ok, {t(), binary()}} - def parse_unit_header(raw_nal) + @spec parse(binary()) :: {:error, :malformed_data} | {:ok, {t(), binary()}} + def parse(raw_nal) - def parse_unit_header(<<0::1, nri::2, type::5, rest::binary>>) do + def parse(<<0::1, nri::2, type::5, rest::binary>>) do nal = %__MODULE__{ nal_ref_idc: nri, type: type @@ -69,13 +69,13 @@ defmodule ExWebRTC.RTP.H264.NAL.Header do end # If first bit is not set to 0 packet is flagged as malformed - def parse_unit_header(_binary), do: {:error, :malformed_data} + def parse(_binary), do: {:error, :malformed_data} @doc """ Adds NAL header to payload """ - @spec add_header(binary(), 0 | 1, nri(), rbsp_type()) :: binary() - def add_header(payload, f, nri, type), + @spec add(binary(), 0 | 1, nri(), rbsp_type()) :: binary() + def add(payload, f, nri, type), do: <> <> payload @doc """