Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at https://mozilla.org/MPL/2.0/.
4#
5# This file incorporates work covered by the following copyright and
6# permission notice:
7#
8# Copyright (c) Jeremy Lainé.
9# All rights reserved.
10#
11# Redistribution and use in source and binary forms, with or without
12# modification, are permitted provided that the following conditions are met:
13#
14# * Redistributions of source code must retain the above copyright notice,
15# this list of conditions and the following disclaimer.
16# * Redistributions in binary form must reproduce the above copyright notice,
17# this list of conditions and the following disclaimer in the documentation
18# and/or other materials provided with the distribution.
19# * Neither the name of aiortc nor the names of its contributors may
20# be used to endorse or promote products derived from this software without
21# specific prior written permission.
22#
23# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
24# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
27# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
30# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34from typing import Optional, Union
36from ..rtcrtpparameters import (
37 ParametersDict,
38 RTCRtcpFeedback,
39 RTCRtpCapabilities,
40 RTCRtpCodecCapability,
41 RTCRtpCodecParameters,
42 RTCRtpHeaderExtensionCapability,
43 RTCRtpHeaderExtensionParameters,
44)
45from ..rtp import ABS_CAPTURE_TIME_URI, DEPENDENCY_DESCRIPTOR_URI
46from .base import Decoder, Encoder
47from .g711 import PcmaDecoder, PcmaEncoder, PcmuDecoder, PcmuEncoder
48from .g722 import G722Decoder, G722Encoder
49from collections.abc import Callable
51from .av1 import Av1Decoder, Av1Encoder, av1_assemble
52from .h264 import H264Decoder, H264Encoder, h264_depayload
53from .h265 import H265Decoder, H265Encoder, h265_depayload
54from .opus import OpusDecoder, OpusEncoder
55from .red import RedOpusEncoder, red_block_payload_type
56from .vp9 import Vp9Decoder, Vp9Encoder, vp9_depayload
57from .vpx import Vp8Decoder, Vp8Encoder, vp8_depayload
59# The clockrate for G.722 is 8kHz even though the sampling rate is 16kHz.
60# See https://datatracker.ietf.org/doc/html/rfc3551
61G722_CODEC = RTCRtpCodecParameters(
62 mimeType="audio/G722", clockRate=8000, channels=1, payloadType=9
63)
64PCMU_CODEC = RTCRtpCodecParameters(
65 mimeType="audio/PCMU", clockRate=8000, channels=1, payloadType=0
66)
67PCMA_CODEC = RTCRtpCodecParameters(
68 mimeType="audio/PCMA", clockRate=8000, channels=1, payloadType=8
69)
70# Opus wrapped in RFC 2198 RED; offered first so peers negotiate redundancy,
71# with plain opus (PT96) as the fallback for peers that decline red.
72RED_CODEC = RTCRtpCodecParameters(
73 mimeType="audio/red",
74 clockRate=48000,
75 channels=2,
76 payloadType=63,
77 parameters={"96/96": None},
78)
80CODECS: dict[str, list[RTCRtpCodecParameters]] = {
81 "audio": [
82 RED_CODEC,
83 RTCRtpCodecParameters(
84 mimeType="audio/opus",
85 clockRate=48000,
86 channels=2,
87 payloadType=96,
88 rtcpFeedback=[RTCRtcpFeedback(type="transport-cc")],
89 ),
90 G722_CODEC,
91 PCMU_CODEC,
92 PCMA_CODEC,
93 ],
94 "video": [],
95}
97# Chromium's multistream-Opus surround layouts (matching libwebrtc).
98MULTIOPUS_LAYOUTS: dict[int, dict[str, str]] = {
99 6: {"channel_mapping": "0,4,1,2,3,5", "num_streams": "4", "coupled_streams": "2"},
100 8: {
101 "channel_mapping": "0,6,1,2,3,4,5,7",
102 "num_streams": "5",
103 "coupled_streams": "3",
104 },
105}
108def configure_multiopus(channels: int) -> None:
109 """Offer surround audio as Chromium's non-standard `multiopus` codec, ahead of the
110 stereo RED and opus a peer without it answers with instead (browsers do not RED
111 multiopus). No engine lists multiopus in an answer of its own, so the client puts
112 it there where its engine decodes it. The payload type is the first dynamic one no
113 other codec of either kind holds, since bundled sections share the space. No-op for
114 unknown layouts."""
115 layout = MULTIOPUS_LAYOUTS.get(channels)
116 if layout is None:
117 return
118 stereo = [c for c in CODECS["audio"] if c.mimeType.lower() != "audio/multiopus"]
119 taken = {c.payloadType for c in stereo + CODECS["video"]}
120 CODECS["audio"] = [
121 RTCRtpCodecParameters(
122 mimeType="audio/multiopus",
123 clockRate=48000,
124 channels=channels,
125 payloadType=next(pt for pt in range(96, 128) if pt not in taken),
126 rtcpFeedback=[RTCRtcpFeedback(type="transport-cc")],
127 parameters=dict(layout),
128 ),
129 ] + stereo
130# Note, the id space for these extensions is shared across media types when BUNDLE
131# is negotiated. If you add a audio- or video-specific extension, make sure it has
132# a unique id.
133HEADER_EXTENSIONS: dict[str, list[RTCRtpHeaderExtensionParameters]] = {
134 "audio": [
135 RTCRtpHeaderExtensionParameters(
136 id=1, uri="urn:ietf:params:rtp-hdrext:sdes:mid"
137 ),
138 RTCRtpHeaderExtensionParameters(
139 id=2, uri="urn:ietf:params:rtp-hdrext:ssrc-audio-level"
140 ),
141 RTCRtpHeaderExtensionParameters(
142 id=5,
143 uri="http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
144 ),
145 ],
146 "video": [
147 RTCRtpHeaderExtensionParameters(
148 id=1, uri="urn:ietf:params:rtp-hdrext:sdes:mid"
149 ),
150 RTCRtpHeaderExtensionParameters(
151 id=3, uri="http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"
152 ),
153 RTCRtpHeaderExtensionParameters(
154 id=4, uri="http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"
155 ),
156 RTCRtpHeaderExtensionParameters(
157 id=5,
158 uri="http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
159 ),
160 RTCRtpHeaderExtensionParameters(
161 id=6, uri="http://www.webrtc.org/experiments/rtp-hdrext/video-timing"
162 ),
163 RTCRtpHeaderExtensionParameters(
164 id=7, uri="http://www.webrtc.org/experiments/rtp-hdrext/color-space"
165 ),
166 RTCRtpHeaderExtensionParameters(id=8, uri=DEPENDENCY_DESCRIPTOR_URI),
167 RTCRtpHeaderExtensionParameters(id=9, uri=ABS_CAPTURE_TIME_URI),
168 ],
169}
172def init_codecs() -> None:
173 dynamic_pt = 97
175 def add_video_codec(
176 mimeType: str, parameters: Optional[ParametersDict] = None
177 ) -> None:
178 nonlocal dynamic_pt
180 clockRate = 90000
181 CODECS["video"] += [
182 RTCRtpCodecParameters(
183 mimeType=mimeType,
184 clockRate=clockRate,
185 payloadType=dynamic_pt,
186 rtcpFeedback=[
187 RTCRtcpFeedback(type="nack"),
188 RTCRtcpFeedback(type="nack", parameter="pli"),
189 RTCRtcpFeedback(type="ccm", parameter="fir"),
190 RTCRtcpFeedback(type="goog-remb"),
191 RTCRtcpFeedback(type="transport-cc"),
192 # Offered so a libwebrtc receiver sends reference times (RTCP XR)
193 # for the round trip a receive-only stream needs to place its
194 # frames' capture time; its answer never echoes it, so a sender
195 # answers whatever reference time arrives.
196 RTCRtcpFeedback(type="rrtr"),
197 ],
198 parameters=parameters or {},
199 ),
200 RTCRtpCodecParameters(
201 mimeType="video/rtx",
202 clockRate=clockRate,
203 payloadType=dynamic_pt + 1,
204 parameters={"apt": dynamic_pt},
205 ),
206 ]
207 dynamic_pt += 2
209 # f4001f = High 4:4:4 Predictive: offered so a full-color (4:4:4) session
210 # negotiates natively instead of relying on SDP text alone (the sender
211 # payloads pixelflux's bitstream as-is, whatever the profile says).
212 for profile_level_id in ("42001f", "42e01f", "f4001f"):
213 add_video_codec(
214 "video/H264",
215 {
216 "level-asymmetry-allowed": "1",
217 "packetization-mode": "1",
218 "profile-level-id": profile_level_id,
219 },
220 )
221 # The parameters name the profile and level a 4K stream stays within; a
222 # browser matches these codecs by MIME type alone.
223 add_video_codec("video/VP8")
224 add_video_codec("video/VP9", {"profile-id": "0"})
225 add_video_codec("video/AV1", {"level-idx": "5", "profile": "0", "tier": "0"})
226 add_video_codec(
227 "video/H265",
228 {"level-id": "153", "profile-id": "1", "tier-flag": "0", "tx-mode": "SRST"},
229 )
231 # FlexFEC (draft-03): one m-line-level repair stream (its SSRC is announced via
232 # the FEC-FR ssrc-group); Chrome negotiates it by default, other browsers drop it.
233 CODECS["video"].append(
234 RTCRtpCodecParameters(
235 mimeType="video/flexfec-03",
236 clockRate=90000,
237 payloadType=dynamic_pt,
238 parameters={"repair-window": "10000000"},
239 )
240 )
241 dynamic_pt += 1
244def depayload(codec: RTCRtpCodecParameters, payload: bytes) -> bytes:
245 name = codec.name.upper()
246 if name == "VP8":
247 return vp8_depayload(payload)
248 elif name == "VP9":
249 return vp9_depayload(payload)
250 elif name == "H264":
251 return h264_depayload(payload)
252 elif name == "H265":
253 return h265_depayload(payload)
254 elif name == "AV1":
255 # One OBU's fragments span packets: the frame assembler joins them.
256 return payload
257 else:
258 return payload
261def frame_assembler(codec: RTCRtpCodecParameters) -> Optional[Callable[[list[bytes]], bytes]]:
262 """The function that turns a frame's depayloaded packets into its bitstream,
263 for a codec whose packets cannot simply be joined; None for the rest."""
264 if codec.name.upper() == "AV1":
265 return av1_assemble
266 return None
269def get_capabilities(kind: str) -> RTCRtpCapabilities:
270 if kind not in CODECS:
271 raise ValueError(f"cannot get capabilities for unknown media {kind}")
273 codecs = []
274 rtx_added = False
275 for params in CODECS[kind]:
276 if not is_rtx(params):
277 codecs.append(
278 RTCRtpCodecCapability(
279 mimeType=params.mimeType,
280 clockRate=params.clockRate,
281 channels=params.channels,
282 parameters=params.parameters,
283 )
284 )
285 elif not rtx_added:
286 # There will only be a single entry in codecs[] for retransmission
287 # via RTX, with sdpFmtpLine not present.
288 codecs.append(
289 RTCRtpCodecCapability(
290 mimeType=params.mimeType, clockRate=params.clockRate
291 )
292 )
293 rtx_added = True
295 headerExtensions = []
296 for extension in HEADER_EXTENSIONS[kind]:
297 headerExtensions.append(RTCRtpHeaderExtensionCapability(uri=extension.uri))
298 return RTCRtpCapabilities(codecs=codecs, headerExtensions=headerExtensions)
301def get_decoder(codec: RTCRtpCodecParameters) -> Decoder:
302 mimeType = codec.mimeType.lower()
304 if mimeType == "audio/g722":
305 return G722Decoder()
306 elif mimeType in ("audio/opus", "audio/multiopus"):
307 return OpusDecoder()
308 elif mimeType == "audio/pcma":
309 return PcmaDecoder()
310 elif mimeType == "audio/pcmu":
311 return PcmuDecoder()
312 elif mimeType == "video/h264":
313 return H264Decoder()
314 elif mimeType == "video/h265":
315 return H265Decoder()
316 elif mimeType == "video/vp8":
317 return Vp8Decoder()
318 elif mimeType == "video/vp9":
319 return Vp9Decoder()
320 elif mimeType == "video/av1":
321 return Av1Decoder()
322 else:
323 raise ValueError(f"No decoder found for MIME type `{mimeType}`")
326def get_encoder(codec: RTCRtpCodecParameters) -> Encoder:
327 mimeType = codec.mimeType.lower()
329 if mimeType == "audio/g722":
330 return G722Encoder()
331 elif mimeType in ("audio/opus", "audio/multiopus"):
332 # Multiopus reaches this encoder only as pre-encoded packets (pack()), which
333 # is channel-count-agnostic; the PCM encode() path stays stereo-only.
334 return OpusEncoder()
335 elif mimeType == "audio/red":
336 return RedOpusEncoder(block_pt=red_block_payload_type(codec.parameters))
337 elif mimeType == "audio/pcma":
338 return PcmaEncoder()
339 elif mimeType == "audio/pcmu":
340 return PcmuEncoder()
341 elif mimeType == "video/h264":
342 return H264Encoder()
343 elif mimeType == "video/h265":
344 return H265Encoder()
345 elif mimeType == "video/vp8":
346 return Vp8Encoder()
347 elif mimeType == "video/vp9":
348 return Vp9Encoder()
349 elif mimeType == "video/av1":
350 return Av1Encoder()
351 else:
352 raise ValueError(f"No encoder found for MIME type `{mimeType}`")
355def is_rtx(codec: Union[RTCRtpCodecCapability, RTCRtpCodecParameters]) -> bool:
356 return codec.name.lower() == "rtx"
359def is_red(codec: Union[RTCRtpCodecCapability, RTCRtpCodecParameters]) -> bool:
360 return codec.name.lower() == "red"
363init_codecs()