Skip to content

Commit d165ccb

Browse files
brzepbrzep
authored
Add H264 RTP depayloader (#224)
1 parent 2dc5768 commit d165ccb

File tree

8 files changed

+532
-1
lines changed

8 files changed

+532
-1
lines changed

‎lib/ex_webrtc/rtp/depayloader.ex‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ defmodule ExWebRTC.RTP.Depayloader do
3636
defp to_depayloader_module(mime_type) do
3737
case String.downcase(mime_type) do
3838
"video/vp8" -> {:ok, ExWebRTC.RTP.Depayloader.VP8}
39+
"video/h264" -> {:ok, ExWebRTC.RTP.Depayloader.H264}
3940
"audio/opus" -> {:ok, ExWebRTC.RTP.Depayloader.Opus}
4041
"audio/pcma" -> {:ok, ExWebRTC.RTP.Depayloader.G711}
4142
"audio/pcmu" -> {:ok, ExWebRTC.RTP.Depayloader.G711}

‎lib/ex_webrtc/rtp/h264/depayloader.ex‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
defmodule ExWebRTC.RTP.Depayloader.H264 do
2+
@moduledoc false
3+
# Extracts H264 NAL Units from RTP packets.
4+
#
5+
# Based on [RFC 6184](https://tools.ietf.org/html/rfc6184).
6+
#
7+
# Supported types: Single NALU, FU-A, STAP-A.
8+
9+
@behaviour ExWebRTC.RTP.Depayloader.Behaviour
10+
11+
require Logger
12+
13+
alias ExWebRTC.RTP.H264.{FU, NAL, StapA}
14+
15+
@annexb_prefix <<1::32>>
16+
17+
@type t() :: %__MODULE__{
18+
current_timestamp: non_neg_integer() | nil,
19+
fu_parser_acc: [binary()]
20+
}
21+
22+
defstruct current_timestamp: nil, fu_parser_acc: []
23+
24+
@impl true
25+
def new() do
26+
%__MODULE__{}
27+
end
28+
29+
@impl true
30+
def depayload(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {nil, depayloader}
31+
32+
def depayload(depayloader, packet) do
33+
with {:ok, {header, _payload} = nal} <- NAL.Header.parse(packet.payload),
34+
unit_type = NAL.Header.decode_type(header),
35+
{:ok, {nal, depayloader}} <-
36+
do_depayload(unit_type, depayloader, packet, nal) do
37+
{nal, depayloader}
38+
else
39+
{:error, reason} ->
40+
Logger.debug("""
41+
Couldn't parse payload, reason: #{reason}. \
42+
Resetting depayloader state. Payload: #{inspect(packet.payload)}.\
43+
""")
44+
45+
{nil, %{depayloader | current_timestamp: nil, fu_parser_acc: []}}
46+
end
47+
end
48+
49+
defp do_depayload(:single_nalu, depayloader, packet, {_header, payload}) do
50+
{:ok,
51+
{prefix_annexb(payload), %__MODULE__{depayloader | current_timestamp: packet.timestamp}}}
52+
end
53+
54+
defp do_depayload(
55+
:fu_a,
56+
%{current_timestamp: current_timestamp, fu_parser_acc: fu_parser_acc},
57+
packet,
58+
{_header, _payload}
59+
)
60+
when fu_parser_acc != [] and current_timestamp != packet.timestamp do
61+
Logger.warning("""
62+
received packet with fu-a type payload that is not a start of fragmentation unit with timestamp \
63+
different than last start and without finishing the previous fu. dropping fu.\
64+
""")
65+
66+
{:error, :invalid_timestamp}
67+
end
68+
69+
defp do_depayload(
70+
:fu_a,
71+
%{fu_parser_acc: fu_parser_acc},
72+
packet,
73+
{header, payload}
74+
) do
75+
case FU.parse(payload, fu_parser_acc || []) do
76+
{:ok, {data, type}} ->
77+
data = NAL.Header.add(data, 0, header.nal_ref_idc, type)
78+
79+
{:ok,
80+
{prefix_annexb(data),
81+
%__MODULE__{current_timestamp: packet.timestamp, fu_parser_acc: []}}}
82+
83+
{:incomplete, fu} ->
84+
{:ok, {nil, %__MODULE__{fu_parser_acc: fu, current_timestamp: packet.timestamp}}}
85+
86+
{:error, _reason} = error ->
87+
error
88+
end
89+
end
90+
91+
defp do_depayload(:stap_a, depayloader, packet, {_header, payload}) do
92+
with {:ok, result} <- StapA.parse(payload) do
93+
nals = result |> Enum.map_join(&prefix_annexb/1)
94+
{:ok, {nals, %__MODULE__{depayloader | current_timestamp: packet.timestamp}}}
95+
end
96+
end
97+
98+
defp do_depayload(unsupported_type, _depayloader, _packet, _nal) do
99+
Logger.warning("""
100+
Received packet with unsupported NAL type: #{unsupported_type}. Supported types are: Single NALU, STAP-A, FU-A. Dropping packet.
101+
""")
102+
103+
{:error, :unsupported_nal_type}
104+
end
105+
106+
defp prefix_annexb(nal) do
107+
@annexb_prefix <> nal
108+
end
109+
end

‎lib/ex_webrtc/rtp/h264/nal_formats/fu.ex‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
defmodule ExWebRTC.RTP.H264.FU do
2+
@moduledoc false
3+
# Module responsible for parsing H264 Fragmentation Unit.
4+
5+
alias __MODULE__
6+
alias ExWebRTC.RTP.H264.NAL
7+
8+
@doc """
9+
Parses H264 Fragmentation Unit
10+
11+
If a packet that is being parsed is not considered last then a `{:incomplete, t()}`
12+
tuple will be returned.
13+
In case of last packet `{:ok, {type, data}}` tuple will be returned, where data
14+
is `NAL Unit` created by concatenating subsequent Fragmentation Units.
15+
"""
16+
@spec parse(binary(), [binary()]) ::
17+
{:ok, {binary(), NAL.Header.rbsp_type()}}
18+
| {:error, :packet_malformed | :invalid_first_packet}
19+
| {:incomplete, [binary()]}
20+
def parse(packet, acc) do
21+
with {:ok, {header, value}} <- FU.Header.parse(packet) do
22+
do_parse(header, value, acc)
23+
end
24+
end
25+
26+
defp do_parse(header, packet, acc)
27+
28+
defp do_parse(%FU.Header{start_bit: true}, data, []),
29+
do: {:incomplete, [data]}
30+
31+
defp do_parse(%FU.Header{start_bit: true}, _data, _acc),
32+
do: {:error, :last_fu_not_finished}
33+
34+
defp do_parse(%FU.Header{start_bit: false}, _data, []),
35+
do: {:error, :invalid_first_packet}
36+
37+
defp do_parse(%FU.Header{end_bit: true, type: type}, data, acc_data) do
38+
result =
39+
[data | acc_data]
40+
|> Enum.reverse()
41+
|> Enum.join()
42+
43+
{:ok, {result, type}}
44+
end
45+
46+
defp do_parse(_header, data, acc_data),
47+
do: {:incomplete, [data | acc_data]}
48+
end

‎lib/ex_webrtc/rtp/h264/nal_formats/fu/header.ex‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
defmodule ExWebRTC.RTP.H264.FU.Header do
2+
@moduledoc """
3+
Defines a structure representing Fragmentation Unit (FU) header
4+
which is defined in [RFC6184](https://tools.ietf.org/html/rfc6184#page-31)
5+
6+
```
7+
+---------------+
8+
|0|1|2|3|4|5|6|7|
9+
+-+-+-+-+-+-+-+-+
10+
|S|E|R| Type |
11+
+---------------+
12+
```
13+
"""
14+
15+
alias ExWebRTC.RTP.H264.NAL
16+
17+
@typedoc """
18+
MUST be set to true only in the first packet in a sequence.
19+
"""
20+
@type start_flag :: boolean()
21+
22+
@typedoc """
23+
MUST be set to true only in the last packet in a sequence.
24+
"""
25+
@type end_flag :: boolean()
26+
27+
@enforce_keys [:type]
28+
defstruct start_bit: false, end_bit: false, type: 0
29+
30+
@type t :: %__MODULE__{
31+
start_bit: start_flag(),
32+
end_bit: end_flag(),
33+
type: NAL.Header.rbsp_type()
34+
}
35+
36+
defguardp valid_frame_boundary(start, finish) when start != 1 or finish != 1
37+
38+
@doc """
39+
Parses Fragmentation Unit Header
40+
41+
It will fail if the Start bit and End bit are both set to one in the
42+
same Fragmentation Unit Header, because a fragmented NAL unit
43+
MUST NOT be transmitted in one FU.
44+
"""
45+
@spec parse(data :: binary()) :: {:error, :packet_malformed} | {:ok, {t(), nal :: binary()}}
46+
def parse(<<start::1, finish::1, 0::1, nal_type::5, rest::binary>>)
47+
when nal_type in 1..23 and valid_frame_boundary(start, finish) do
48+
header = %__MODULE__{
49+
start_bit: start == 1,
50+
end_bit: finish == 1,
51+
type: nal_type
52+
}
53+
54+
{:ok, {header, rest}}
55+
end
56+
57+
def parse(_binary), do: {:error, :packet_malformed}
58+
end

‎lib/ex_webrtc/rtp/h264/nal_formats/stap_a.ex‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
defmodule ExWebRTC.RTP.H264.StapA do
2+
@moduledoc """
3+
Module responsible for parsing Single Time Agregation Packets type A.
4+
5+
Documented in [RFC6184](https://tools.ietf.org/html/rfc6184#page-22)
6+
7+
```
8+
0 1 2 3
9+
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
10+
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
11+
| RTP Header |
12+
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
13+
|STAP-A NAL HDR | NALU 1 Size | NALU 1 HDR |
14+
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
15+
| NALU 1 Data |
16+
: :
17+
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
18+
| | NALU 2 Size | NALU 2 HDR |
19+
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
20+
| NALU 2 Data |
21+
: :
22+
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
23+
| :...OPTIONAL RTP padding |
24+
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
25+
```
26+
"""
27+
28+
@spec parse(binary()) :: {:ok, [binary()]} | {:error, :packet_malformed}
29+
def parse(data) do
30+
do_parse(data, [])
31+
end
32+
33+
defp do_parse(<<>>, acc), do: {:ok, Enum.reverse(acc)}
34+
35+
defp do_parse(<<size::16, nalu::binary-size(size), rest::binary>>, acc),
36+
do: do_parse(rest, [nalu | acc])
37+
38+
defp do_parse(_data, _acc), do: {:error, :packet_malformed}
39+
end

‎lib/ex_webrtc/rtp/h264/nal_header.ex‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
defmodule ExWebRTC.RTP.H264.NAL.Header do
2+
@moduledoc """
3+
Defines a structure representing Network Abstraction Layer Unit Header
4+
5+
Defined in [RFC 6184](https://tools.ietf.org/html/rfc6184#section-5.3)
6+
7+
```
8+
+---------------+
9+
|0|1|2|3|4|5|6|7|
10+
+-+-+-+-+-+-+-+-+
11+
|F|NRI| Type |
12+
+---------------+
13+
```
14+
"""
15+
16+
@typedoc """
17+
NRI stands for nal_ref_idc. This value represents importance of
18+
frame that is being parsed.
19+
20+
The higher the value the more important frame is (for example key
21+
frames have nri value of 3) and a value of 00 indicates that the
22+
content of the NAL unit is not used to reconstruct reference pictures
23+
for inter picture prediction. NAL units with NRI equal 0 can be discarded
24+
without risking the integrity of the reference pictures, although these
25+
payloads might contain metadata.
26+
"""
27+
@type nri :: 0..3
28+
29+
@typedoc """
30+
Specifies the type of RBSP (Raw Byte Sequence Payload) data structure contained in the NAL unit.
31+
32+
Types are defined as follows.
33+
34+
| ID | RBSP Type |
35+
|----------|----------------|
36+
| 0 | Unspecified |
37+
| 1-23 | NAL unit types |
38+
| 24 | STAP-A |
39+
| 25 | STAP-B |
40+
| 26 | MTAP-16 |
41+
| 27 | MTAP-24 |
42+
| 28 | FU-A |
43+
| 29 | FU-B |
44+
| Reserved | 30-31 |
45+
46+
"""
47+
@type rbsp_type :: 1..31
48+
@type supported_types :: :stap_a | :fu_a | :single_nalu
49+
@type unsupported_types :: :stap_b | :mtap_16 | :mtap_24 | :fu_b
50+
@type types :: supported_types | unsupported_types | :reserved
51+
52+
defstruct [:nal_ref_idc, :type]
53+
54+
@type t :: %__MODULE__{
55+
nal_ref_idc: nri(),
56+
type: rbsp_type()
57+
}
58+
59+
@spec parse(binary()) :: {:error, :malformed_data} | {:ok, {t(), binary()}}
60+
def parse(raw_nal)
61+
62+
def parse(<<0::1, nri::2, type::5, rest::binary>>) do
63+
nal = %__MODULE__{
64+
nal_ref_idc: nri,
65+
type: type
66+
}
67+
68+
{:ok, {nal, rest}}
69+
end
70+
71+
# If first bit is not set to 0 packet is flagged as malformed
72+
def parse(_binary), do: {:error, :malformed_data}
73+
74+
@doc """
75+
Adds NAL header to payload
76+
"""
77+
@spec add(binary(), 0 | 1, nri(), rbsp_type()) :: binary()
78+
def add(payload, f, nri, type),
79+
do: <<f::1, nri::2, type::5>> <> payload
80+
81+
@doc """
82+
Parses type stored in NAL Header
83+
"""
84+
@spec decode_type(t) :: types()
85+
def decode_type(%__MODULE__{type: type}), do: do_decode_type(type)
86+
87+
defp do_decode_type(number) when number in 1..21, do: :single_nalu
88+
defp do_decode_type(number) when number in [22, 23], do: :reserved
89+
defp do_decode_type(24), do: :stap_a
90+
defp do_decode_type(25), do: :stap_b
91+
defp do_decode_type(26), do: :mtap_16
92+
defp do_decode_type(27), do: :mtap_24
93+
defp do_decode_type(28), do: :fu_a
94+
defp do_decode_type(29), do: :fu_b
95+
defp do_decode_type(number) when number in [30, 31], do: :reserved
96+
defp do_decode_type(_), do: :invalid
97+
end

‎test/ex_webrtc/rtp/depayloader_test.exs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,18 @@ defmodule ExWebRTC.RTP.DepayloaderTest do
7575
Depayloader.DTMF.depayload(depayloader, @packet)
7676
end
7777

78+
test "creates a H264 depayloader and dispatches calls to its module" do
79+
assert {:ok, depayloader} =
80+
%RTPCodecParameters{payload_type: 97, mime_type: "video/H264", clock_rate: 90_000}
81+
|> Depayloader.new()
82+
83+
assert Depayloader.depayload(depayloader, @packet) ==
84+
Depayloader.H264.depayload(depayloader, @packet)
85+
end
86+
7887
test "returns error if no depayloader exists for given codec" do
7988
assert {:error, :no_depayloader_for_codec} =
80-
%RTPCodecParameters{payload_type: 97, mime_type: "video/H264", clock_rate: 90_000}
89+
%RTPCodecParameters{payload_type: 45, mime_type: "video/AV1", clock_rate: 90_000}
8190
|> Depayloader.new()
8291
end
8392
end

0 commit comments

Comments
 (0)