/src/FreeRDP/libfreerdp/codec/h264.c
Line | Count | Source |
1 | | /** |
2 | | * FreeRDP: A Remote Desktop Protocol Implementation |
3 | | * H.264 Bitmap Compression |
4 | | * |
5 | | * Copyright 2014 Mike McDonald <Mike.McDonald@software.dell.com> |
6 | | * Copyright 2017 David Fort <contact@hardening-consulting.com> |
7 | | * |
8 | | * Licensed under the Apache License, Version 2.0 (the "License"); |
9 | | * you may not use this file except in compliance with the License. |
10 | | * You may obtain a copy of the License at |
11 | | * |
12 | | * http://www.apache.org/licenses/LICENSE-2.0 |
13 | | * |
14 | | * Unless required by applicable law or agreed to in writing, software |
15 | | * distributed under the License is distributed on an "AS IS" BASIS, |
16 | | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
17 | | * See the License for the specific language governing permissions and |
18 | | * limitations under the License. |
19 | | */ |
20 | | |
21 | | #include <freerdp/config.h> |
22 | | |
23 | | #include <winpr/crt.h> |
24 | | #include <winpr/print.h> |
25 | | #include <winpr/library.h> |
26 | | #include <winpr/bitstream.h> |
27 | | #include <winpr/synch.h> |
28 | | |
29 | | #include <freerdp/primitives.h> |
30 | | #include <freerdp/codec/h264.h> |
31 | | #include <freerdp/codec/yuv.h> |
32 | | #include <freerdp/log.h> |
33 | | |
34 | | #include "h264.h" |
35 | | |
36 | 0 | #define TAG FREERDP_TAG("codec") |
37 | | |
38 | | static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight); |
39 | | |
40 | | static BOOL yuv_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height) |
41 | 0 | { |
42 | 0 | BOOL isNull = FALSE; |
43 | 0 | UINT32 pheight = height; |
44 | |
|
45 | 0 | if (!h264) |
46 | 0 | return FALSE; |
47 | | |
48 | 0 | if (stride == 0) |
49 | 0 | stride = width; |
50 | |
|
51 | 0 | if (stride % 16 != 0) |
52 | 0 | stride += 16 - stride % 16; |
53 | |
|
54 | 0 | if (pheight % 16 != 0) |
55 | 0 | pheight += 16 - pheight % 16; |
56 | |
|
57 | 0 | const size_t nPlanes = h264->hwAccel ? 2 : 3; |
58 | |
|
59 | 0 | for (size_t x = 0; x < nPlanes; x++) |
60 | 0 | { |
61 | 0 | if (!h264->pYUVData[x] || !h264->pOldYUVData[x]) |
62 | 0 | isNull = TRUE; |
63 | 0 | } |
64 | |
|
65 | 0 | if (pheight == 0) |
66 | 0 | return FALSE; |
67 | 0 | if (stride == 0) |
68 | 0 | return FALSE; |
69 | | |
70 | 0 | if (isNull || (width != h264->width) || (height != h264->height) || |
71 | 0 | (stride != h264->iStride[0])) |
72 | 0 | { |
73 | 0 | if (h264->hwAccel) /* NV12 */ |
74 | 0 | { |
75 | 0 | h264->iStride[0] = stride; |
76 | 0 | h264->iStride[1] = stride; |
77 | 0 | h264->iStride[2] = 0; |
78 | 0 | } |
79 | 0 | else /* I420 */ |
80 | 0 | { |
81 | 0 | h264->iStride[0] = stride; |
82 | 0 | h264->iStride[1] = (stride + 1) / 2; |
83 | 0 | h264->iStride[2] = (stride + 1) / 2; |
84 | 0 | } |
85 | |
|
86 | 0 | for (size_t x = 0; x < nPlanes; x++) |
87 | 0 | { |
88 | 0 | BYTE* tmp1 = winpr_aligned_recalloc(h264->pYUVData[x], h264->iStride[x], pheight, 16); |
89 | 0 | BYTE* tmp2 = |
90 | 0 | winpr_aligned_recalloc(h264->pOldYUVData[x], h264->iStride[x], pheight, 16); |
91 | 0 | if (tmp1) |
92 | 0 | h264->pYUVData[x] = tmp1; |
93 | 0 | if (tmp2) |
94 | 0 | h264->pOldYUVData[x] = tmp2; |
95 | 0 | if (!tmp1 || !tmp2) |
96 | 0 | return FALSE; |
97 | 0 | } |
98 | 0 | h264->width = width; |
99 | 0 | h264->height = height; |
100 | 0 | } |
101 | | |
102 | 0 | return TRUE; |
103 | 0 | } |
104 | | |
105 | | BOOL avc420_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height) |
106 | 0 | { |
107 | 0 | return yuv_ensure_buffer(h264, stride, width, height); |
108 | 0 | } |
109 | | |
110 | | static BOOL isRectValid(UINT32 width, UINT32 height, const RECTANGLE_16* rect) |
111 | 0 | { |
112 | 0 | WINPR_ASSERT(rect); |
113 | 0 | if (rect->left > width) |
114 | 0 | return FALSE; |
115 | 0 | if (rect->right > width) |
116 | 0 | return FALSE; |
117 | 0 | if (rect->left >= rect->right) |
118 | 0 | return FALSE; |
119 | 0 | if (rect->top > height) |
120 | 0 | return FALSE; |
121 | 0 | if (rect->bottom > height) |
122 | 0 | return FALSE; |
123 | 0 | if (rect->top >= rect->bottom) |
124 | 0 | return FALSE; |
125 | 0 | return TRUE; |
126 | 0 | } |
127 | | |
128 | | static BOOL areRectsValid(UINT32 width, UINT32 height, const RECTANGLE_16* rects, UINT32 count) |
129 | 0 | { |
130 | 0 | WINPR_ASSERT(rects || (count == 0)); |
131 | 0 | for (size_t x = 0; x < count; x++) |
132 | 0 | { |
133 | 0 | const RECTANGLE_16* rect = &rects[x]; |
134 | 0 | if (!isRectValid(width, height, rect)) |
135 | 0 | return FALSE; |
136 | 0 | } |
137 | 0 | return TRUE; |
138 | 0 | } |
139 | | |
140 | | INT32 avc420_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize, BYTE* pDstData, |
141 | | DWORD DstFormat, UINT32 nDstStep, WINPR_ATTR_UNUSED UINT32 nDstWidth, |
142 | | WINPR_ATTR_UNUSED UINT32 nDstHeight, const RECTANGLE_16* regionRects, |
143 | | UINT32 numRegionRects) |
144 | 0 | { |
145 | 0 | int status = 0; |
146 | 0 | const BYTE* pYUVData[3]; |
147 | |
|
148 | 0 | if (!h264 || h264->Compressor) |
149 | 0 | return -1001; |
150 | | |
151 | 0 | if (!areRectsValid(nDstWidth, nDstHeight, regionRects, numRegionRects)) |
152 | 0 | return -1013; |
153 | | |
154 | 0 | status = h264->subsystem->Decompress(h264, pSrcData, SrcSize); |
155 | |
|
156 | 0 | if (status == 0) |
157 | 0 | return 1; |
158 | | |
159 | 0 | if (status < 0) |
160 | 0 | return status; |
161 | | |
162 | 0 | pYUVData[0] = h264->pYUVData[0]; |
163 | 0 | pYUVData[1] = h264->pYUVData[1]; |
164 | 0 | pYUVData[2] = h264->pYUVData[2]; |
165 | 0 | if (!yuv420_context_decode(h264->yuv, pYUVData, h264->iStride, h264->height, DstFormat, |
166 | 0 | pDstData, nDstStep, regionRects, numRegionRects)) |
167 | 0 | return -1002; |
168 | | |
169 | 0 | return 1; |
170 | 0 | } |
171 | | |
172 | | static BOOL allocate_h264_metablock(UINT32 QP, RECTANGLE_16* rectangles, |
173 | | RDPGFX_H264_METABLOCK* meta, size_t count) |
174 | 0 | { |
175 | | /* [MS-RDPEGFX] 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY */ |
176 | 0 | if (!meta || (QP > UINT8_MAX)) |
177 | 0 | { |
178 | 0 | free(rectangles); |
179 | 0 | return FALSE; |
180 | 0 | } |
181 | | |
182 | 0 | meta->regionRects = rectangles; |
183 | 0 | if (count == 0) |
184 | 0 | return TRUE; |
185 | | |
186 | 0 | if (count > UINT32_MAX) |
187 | 0 | return FALSE; |
188 | | |
189 | 0 | meta->quantQualityVals = calloc(count, sizeof(RDPGFX_H264_QUANT_QUALITY)); |
190 | |
|
191 | 0 | if (!meta->quantQualityVals || !meta->regionRects) |
192 | 0 | return FALSE; |
193 | 0 | meta->numRegionRects = (UINT32)count; |
194 | 0 | for (size_t x = 0; x < count; x++) |
195 | 0 | { |
196 | 0 | RDPGFX_H264_QUANT_QUALITY* cur = &meta->quantQualityVals[x]; |
197 | 0 | cur->qp = (UINT8)QP; |
198 | | |
199 | | /* qpVal bit 6 and 7 are flags, so mask them out here. |
200 | | * qualityVal is [0-100] so 100 - qpVal [0-64] is always in range */ |
201 | 0 | cur->qualityVal = 100 - (QP & 0x3F); |
202 | 0 | } |
203 | 0 | return TRUE; |
204 | 0 | } |
205 | | |
206 | | static inline BOOL diff_tile(const RECTANGLE_16* regionRect, BYTE* pYUVData[3], |
207 | | BYTE* pOldYUVData[3], UINT32 const iStride[3]) |
208 | 0 | { |
209 | 0 | size_t size = 0; |
210 | |
|
211 | 0 | if (!regionRect || !pYUVData || !pOldYUVData || !iStride) |
212 | 0 | return FALSE; |
213 | 0 | size = regionRect->right - regionRect->left; |
214 | 0 | if (regionRect->right > iStride[0]) |
215 | 0 | return FALSE; |
216 | 0 | if (regionRect->right / 2u > iStride[1]) |
217 | 0 | return FALSE; |
218 | 0 | if (regionRect->right / 2u > iStride[2]) |
219 | 0 | return FALSE; |
220 | | |
221 | 0 | for (size_t y = regionRect->top; y < regionRect->bottom; y++) |
222 | 0 | { |
223 | 0 | const BYTE* cur0 = &pYUVData[0][y * iStride[0]]; |
224 | 0 | const BYTE* cur1 = &pYUVData[1][y * iStride[1]]; |
225 | 0 | const BYTE* cur2 = &pYUVData[2][y * iStride[2]]; |
226 | 0 | const BYTE* old0 = &pOldYUVData[0][y * iStride[0]]; |
227 | 0 | const BYTE* old1 = &pOldYUVData[1][y * iStride[1]]; |
228 | 0 | const BYTE* old2 = &pOldYUVData[2][y * iStride[2]]; |
229 | |
|
230 | 0 | if (memcmp(&cur0[regionRect->left], &old0[regionRect->left], size) != 0) |
231 | 0 | return TRUE; |
232 | 0 | if (memcmp(&cur1[regionRect->left / 2], &old1[regionRect->left / 2], size / 2) != 0) |
233 | 0 | return TRUE; |
234 | 0 | if (memcmp(&cur2[regionRect->left / 2], &old2[regionRect->left / 2], size / 2) != 0) |
235 | 0 | return TRUE; |
236 | 0 | } |
237 | 0 | return FALSE; |
238 | 0 | } |
239 | | |
240 | | static BOOL detect_changes(BOOL firstFrameDone, const UINT32 QP, const RECTANGLE_16* regionRect, |
241 | | BYTE* pYUVData[3], BYTE* pOldYUVData[3], UINT32 const iStride[3], |
242 | | RDPGFX_H264_METABLOCK* meta) |
243 | 0 | { |
244 | 0 | size_t count = 0; |
245 | 0 | size_t wc = 0; |
246 | 0 | size_t hc = 0; |
247 | 0 | RECTANGLE_16* rectangles = nullptr; |
248 | |
|
249 | 0 | if (!regionRect || !pYUVData || !pOldYUVData || !iStride || !meta) |
250 | 0 | return FALSE; |
251 | | |
252 | 0 | wc = (regionRect->right - regionRect->left) / 64 + 1; |
253 | 0 | hc = (regionRect->bottom - regionRect->top) / 64 + 1; |
254 | 0 | rectangles = calloc(wc * hc, sizeof(RECTANGLE_16)); |
255 | 0 | if (!rectangles) |
256 | 0 | return FALSE; |
257 | 0 | if (!firstFrameDone) |
258 | 0 | { |
259 | 0 | rectangles[0] = *regionRect; |
260 | 0 | count = 1; |
261 | 0 | } |
262 | 0 | else |
263 | 0 | { |
264 | 0 | for (size_t y = regionRect->top; y < regionRect->bottom; y += 64) |
265 | 0 | { |
266 | 0 | for (size_t x = regionRect->left; x < regionRect->right; x += 64) |
267 | 0 | { |
268 | 0 | RECTANGLE_16 rect; |
269 | 0 | rect.left = (UINT16)MIN(UINT16_MAX, regionRect->left + x); |
270 | 0 | rect.top = (UINT16)MIN(UINT16_MAX, regionRect->top + y); |
271 | 0 | rect.right = |
272 | 0 | (UINT16)MIN(UINT16_MAX, MIN(regionRect->left + x + 64, regionRect->right)); |
273 | 0 | rect.bottom = |
274 | 0 | (UINT16)MIN(UINT16_MAX, MIN(regionRect->top + y + 64, regionRect->bottom)); |
275 | 0 | if (diff_tile(&rect, pYUVData, pOldYUVData, iStride)) |
276 | 0 | rectangles[count++] = rect; |
277 | 0 | } |
278 | 0 | } |
279 | 0 | } |
280 | 0 | if (!allocate_h264_metablock(QP, rectangles, meta, count)) |
281 | 0 | return FALSE; |
282 | 0 | return TRUE; |
283 | 0 | } |
284 | | |
285 | | INT32 h264_get_yuv_buffer(H264_CONTEXT* h264, UINT32 nSrcStride, UINT32 nSrcWidth, |
286 | | UINT32 nSrcHeight, BYTE* YUVData[3], UINT32 stride[3]) |
287 | 0 | { |
288 | 0 | if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress) |
289 | 0 | return -1; |
290 | | |
291 | 0 | if (!yuv_ensure_buffer(h264, nSrcStride, nSrcWidth, nSrcHeight)) |
292 | 0 | return -1; |
293 | | |
294 | 0 | for (size_t x = 0; x < 3; x++) |
295 | 0 | { |
296 | 0 | YUVData[x] = h264->pYUVData[x]; |
297 | 0 | stride[x] = h264->iStride[x]; |
298 | 0 | } |
299 | |
|
300 | 0 | return 0; |
301 | 0 | } |
302 | | |
303 | | INT32 h264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize) |
304 | 0 | { |
305 | 0 | if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress) |
306 | 0 | return -1; |
307 | | |
308 | 0 | const BYTE* pcYUVData[3] = { h264->pYUVData[0], h264->pYUVData[1], h264->pYUVData[2] }; |
309 | |
|
310 | 0 | return h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize); |
311 | 0 | } |
312 | | |
313 | | INT32 avc420_compress(H264_CONTEXT* h264, const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep, |
314 | | UINT32 nSrcWidth, UINT32 nSrcHeight, const RECTANGLE_16* regionRect, |
315 | | BYTE** ppDstData, UINT32* pDstSize, RDPGFX_H264_METABLOCK* meta) |
316 | 0 | { |
317 | 0 | INT32 rc = -1; |
318 | 0 | BYTE* pYUVData[3] = WINPR_C_ARRAY_INIT; |
319 | 0 | const BYTE* pcYUVData[3] = WINPR_C_ARRAY_INIT; |
320 | 0 | BYTE* pOldYUVData[3] = WINPR_C_ARRAY_INIT; |
321 | |
|
322 | 0 | if (!h264 || !regionRect || !meta || !h264->Compressor) |
323 | 0 | return -1; |
324 | | |
325 | 0 | if (!h264->subsystem->Compress) |
326 | 0 | return -1; |
327 | | |
328 | 0 | if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight)) |
329 | 0 | return -1; |
330 | | |
331 | 0 | if (h264->encodingBuffer) |
332 | 0 | { |
333 | 0 | for (size_t x = 0; x < 3; x++) |
334 | 0 | { |
335 | 0 | pYUVData[x] = h264->pYUVData[x]; |
336 | 0 | pOldYUVData[x] = h264->pOldYUVData[x]; |
337 | 0 | } |
338 | 0 | } |
339 | 0 | else |
340 | 0 | { |
341 | 0 | for (size_t x = 0; x < 3; x++) |
342 | 0 | { |
343 | 0 | pYUVData[x] = h264->pOldYUVData[x]; |
344 | 0 | pOldYUVData[x] = h264->pYUVData[x]; |
345 | 0 | } |
346 | 0 | } |
347 | 0 | h264->encodingBuffer = !h264->encodingBuffer; |
348 | |
|
349 | 0 | if (!yuv420_context_encode(h264->yuv, pSrcData, nSrcStep, SrcFormat, h264->iStride, pYUVData, |
350 | 0 | regionRect, 1)) |
351 | 0 | goto fail; |
352 | | |
353 | 0 | if (!detect_changes(h264->firstLumaFrameDone, h264->QP, regionRect, pYUVData, pOldYUVData, |
354 | 0 | h264->iStride, meta)) |
355 | 0 | goto fail; |
356 | | |
357 | 0 | if (meta->numRegionRects == 0) |
358 | 0 | { |
359 | 0 | rc = 0; |
360 | 0 | goto fail; |
361 | 0 | } |
362 | | |
363 | 0 | for (size_t x = 0; x < 3; x++) |
364 | 0 | pcYUVData[x] = pYUVData[x]; |
365 | |
|
366 | 0 | rc = h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize); |
367 | 0 | if (rc >= 0) |
368 | 0 | h264->firstLumaFrameDone = TRUE; |
369 | |
|
370 | 0 | fail: |
371 | 0 | if (rc < 0) |
372 | 0 | free_h264_metablock(meta); |
373 | 0 | return rc; |
374 | 0 | } |
375 | | |
376 | | INT32 avc444_compress(H264_CONTEXT* h264, const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep, |
377 | | UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE version, const RECTANGLE_16* region, |
378 | | BYTE* op, BYTE** ppDstData, UINT32* pDstSize, BYTE** ppAuxDstData, |
379 | | UINT32* pAuxDstSize, RDPGFX_H264_METABLOCK* meta, |
380 | | RDPGFX_H264_METABLOCK* auxMeta) |
381 | 0 | { |
382 | 0 | int rc = -1; |
383 | 0 | BYTE* coded = nullptr; |
384 | 0 | UINT32 codedSize = 0; |
385 | 0 | BYTE** pYUV444Data = nullptr; |
386 | 0 | BYTE** pOldYUV444Data = nullptr; |
387 | 0 | BYTE** pYUVData = nullptr; |
388 | 0 | BYTE** pOldYUVData = nullptr; |
389 | |
|
390 | 0 | if (!h264 || !h264->Compressor) |
391 | 0 | return -1; |
392 | | |
393 | 0 | if (!h264->subsystem->Compress) |
394 | 0 | return -1; |
395 | | |
396 | 0 | if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight)) |
397 | 0 | return -1; |
398 | | |
399 | 0 | if (!avc444_ensure_buffer(h264, nSrcHeight)) |
400 | 0 | return -1; |
401 | | |
402 | 0 | if (h264->encodingBuffer) |
403 | 0 | { |
404 | 0 | pYUV444Data = h264->pOldYUV444Data; |
405 | 0 | pOldYUV444Data = h264->pYUV444Data; |
406 | 0 | pYUVData = h264->pOldYUVData; |
407 | 0 | pOldYUVData = h264->pYUVData; |
408 | 0 | } |
409 | 0 | else |
410 | 0 | { |
411 | 0 | pYUV444Data = h264->pYUV444Data; |
412 | 0 | pOldYUV444Data = h264->pOldYUV444Data; |
413 | 0 | pYUVData = h264->pYUVData; |
414 | 0 | pOldYUVData = h264->pOldYUVData; |
415 | 0 | } |
416 | 0 | h264->encodingBuffer = !h264->encodingBuffer; |
417 | |
|
418 | 0 | if (!yuv444_context_encode(h264->yuv, version, pSrcData, nSrcStep, SrcFormat, h264->iStride, |
419 | 0 | pYUV444Data, pYUVData, region, 1)) |
420 | 0 | goto fail; |
421 | | |
422 | 0 | if (!detect_changes(h264->firstLumaFrameDone, h264->QP, region, pYUV444Data, pOldYUV444Data, |
423 | 0 | h264->iStride, meta)) |
424 | 0 | goto fail; |
425 | 0 | if (!detect_changes(h264->firstChromaFrameDone, h264->QP, region, pYUVData, pOldYUVData, |
426 | 0 | h264->iStride, auxMeta)) |
427 | 0 | goto fail; |
428 | | |
429 | | /* [MS-RDPEGFX] 2.2.4.5 RFX_AVC444_BITMAP_STREAM |
430 | | * LC: |
431 | | * 0 ... Luma & Chroma |
432 | | * 1 ... Luma |
433 | | * 2 ... Chroma |
434 | | */ |
435 | 0 | if ((meta->numRegionRects > 0) && (auxMeta->numRegionRects > 0)) |
436 | 0 | *op = 0; |
437 | 0 | else if (meta->numRegionRects > 0) |
438 | 0 | *op = 1; |
439 | 0 | else if (auxMeta->numRegionRects > 0) |
440 | 0 | *op = 2; |
441 | 0 | else |
442 | 0 | { |
443 | 0 | WLog_Print(h264->log, WLOG_TRACE, "no changes detected for luma or chroma frame"); |
444 | 0 | rc = 0; |
445 | 0 | goto fail; |
446 | 0 | } |
447 | | |
448 | 0 | if ((*op == 0) || (*op == 1)) |
449 | 0 | { |
450 | 0 | const BYTE* pcYUV444Data[3] = { pYUV444Data[0], pYUV444Data[1], pYUV444Data[2] }; |
451 | |
|
452 | 0 | if (h264->subsystem->Compress(h264, pcYUV444Data, h264->iStride, &coded, &codedSize) < 0) |
453 | 0 | goto fail; |
454 | 0 | h264->firstLumaFrameDone = TRUE; |
455 | 0 | memcpy(h264->lumaData, coded, codedSize); |
456 | 0 | *ppDstData = h264->lumaData; |
457 | 0 | *pDstSize = codedSize; |
458 | 0 | } |
459 | | |
460 | 0 | if ((*op == 0) || (*op == 2)) |
461 | 0 | { |
462 | 0 | const BYTE* pcYUVData[3] = { pYUVData[0], pYUVData[1], pYUVData[2] }; |
463 | |
|
464 | 0 | if (h264->subsystem->Compress(h264, pcYUVData, h264->iStride, &coded, &codedSize) < 0) |
465 | 0 | goto fail; |
466 | 0 | h264->firstChromaFrameDone = TRUE; |
467 | 0 | *ppAuxDstData = coded; |
468 | 0 | *pAuxDstSize = codedSize; |
469 | 0 | } |
470 | | |
471 | 0 | rc = 1; |
472 | 0 | fail: |
473 | 0 | if (rc < 0) |
474 | 0 | { |
475 | 0 | free_h264_metablock(meta); |
476 | 0 | free_h264_metablock(auxMeta); |
477 | 0 | } |
478 | 0 | return rc; |
479 | 0 | } |
480 | | |
481 | | static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight) |
482 | 0 | { |
483 | 0 | WINPR_ASSERT(h264); |
484 | |
|
485 | 0 | const UINT32* piMainStride = h264->iStride; |
486 | 0 | UINT32* piDstSize = h264->iYUV444Size; |
487 | 0 | UINT32* piDstStride = h264->iYUV444Stride; |
488 | 0 | BYTE** ppYUVDstData = h264->pYUV444Data; |
489 | 0 | BYTE** ppOldYUVDstData = h264->pOldYUV444Data; |
490 | |
|
491 | 0 | nDstHeight = MAX(h264->height, nDstHeight); |
492 | 0 | const UINT32 pad = nDstHeight % 16; |
493 | 0 | UINT32 padDstHeight = nDstHeight; /* Need alignment to 16x16 blocks */ |
494 | |
|
495 | 0 | if (pad != 0) |
496 | 0 | padDstHeight += 16 - pad; |
497 | |
|
498 | 0 | if ((piMainStride[0] != piDstStride[0]) || |
499 | 0 | (piDstSize[0] != 1ull * piMainStride[0] * padDstHeight)) |
500 | 0 | { |
501 | 0 | for (UINT32 x = 0; x < 3; x++) |
502 | 0 | { |
503 | 0 | piDstStride[x] = piMainStride[0]; |
504 | 0 | piDstSize[x] = piDstStride[x] * padDstHeight; |
505 | |
|
506 | 0 | if (piDstSize[x] == 0) |
507 | 0 | return FALSE; |
508 | | |
509 | 0 | BYTE* tmp1 = winpr_aligned_recalloc(ppYUVDstData[x], piDstSize[x], 1, 16); |
510 | 0 | if (!tmp1) |
511 | 0 | return FALSE; |
512 | 0 | ppYUVDstData[x] = tmp1; |
513 | 0 | BYTE* tmp2 = winpr_aligned_recalloc(ppOldYUVDstData[x], piDstSize[x], 1, 16); |
514 | 0 | if (!tmp2) |
515 | 0 | return FALSE; |
516 | 0 | ppOldYUVDstData[x] = tmp2; |
517 | 0 | } |
518 | | |
519 | 0 | { |
520 | 0 | BYTE* tmp = winpr_aligned_recalloc(h264->lumaData, piDstSize[0], 4, 16); |
521 | 0 | if (!tmp) |
522 | 0 | goto fail; |
523 | 0 | h264->lumaData = tmp; |
524 | 0 | } |
525 | 0 | } |
526 | | |
527 | 0 | for (UINT32 x = 0; x < 3; x++) |
528 | 0 | { |
529 | 0 | if (!ppOldYUVDstData[x] || !ppYUVDstData[x] || (piDstSize[x] == 0) || (piDstStride[x] == 0)) |
530 | 0 | { |
531 | 0 | WLog_Print(h264->log, WLOG_ERROR, |
532 | 0 | "YUV buffer not initialized! check your decoder settings"); |
533 | 0 | goto fail; |
534 | 0 | } |
535 | 0 | } |
536 | | |
537 | 0 | if (!h264->lumaData) |
538 | 0 | goto fail; |
539 | | |
540 | 0 | return TRUE; |
541 | 0 | fail: |
542 | 0 | return FALSE; |
543 | 0 | } |
544 | | |
545 | | static BOOL avc444_process_rects(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize, |
546 | | BYTE* pDstData, UINT32 DstFormat, UINT32 nDstStep, |
547 | | WINPR_ATTR_UNUSED UINT32 nDstWidth, UINT32 nDstHeight, |
548 | | const RECTANGLE_16* rects, UINT32 nrRects, avc444_frame_type type) |
549 | 0 | { |
550 | 0 | const BYTE* pYUVData[3]; |
551 | 0 | BYTE* pYUVDstData[3]; |
552 | 0 | UINT32* piDstStride = h264->iYUV444Stride; |
553 | 0 | BYTE** ppYUVDstData = h264->pYUV444Data; |
554 | 0 | const UINT32* piStride = h264->iStride; |
555 | |
|
556 | 0 | if (h264->subsystem->Decompress(h264, pSrcData, SrcSize) < 0) |
557 | 0 | return FALSE; |
558 | | |
559 | 0 | pYUVData[0] = h264->pYUVData[0]; |
560 | 0 | pYUVData[1] = h264->pYUVData[1]; |
561 | 0 | pYUVData[2] = h264->pYUVData[2]; |
562 | 0 | if (!avc444_ensure_buffer(h264, nDstHeight)) |
563 | 0 | return FALSE; |
564 | | |
565 | 0 | pYUVDstData[0] = ppYUVDstData[0]; |
566 | 0 | pYUVDstData[1] = ppYUVDstData[1]; |
567 | 0 | pYUVDstData[2] = ppYUVDstData[2]; |
568 | 0 | return (yuv444_context_decode(h264->yuv, (BYTE)type, pYUVData, piStride, h264->height, |
569 | 0 | pYUVDstData, piDstStride, DstFormat, pDstData, nDstStep, rects, |
570 | 0 | nrRects)); |
571 | 0 | } |
572 | | |
573 | | #if defined(AVC444_FRAME_STAT) |
574 | | static UINT64 op1 = 0; |
575 | | static double op1sum = 0; |
576 | | static UINT64 op2 = 0; |
577 | | static double op2sum = 0; |
578 | | static UINT64 op3 = 0; |
579 | | static double op3sum = 0; |
580 | | static double avg(UINT64* count, double old, double size) |
581 | | { |
582 | | double tmp = size + *count * old; |
583 | | (*count)++; |
584 | | tmp = tmp / *count; |
585 | | return tmp; |
586 | | } |
587 | | #endif |
588 | | |
589 | | INT32 avc444_decompress(H264_CONTEXT* h264, BYTE op, const RECTANGLE_16* regionRects, |
590 | | UINT32 numRegionRects, const BYTE* pSrcData, UINT32 SrcSize, |
591 | | const RECTANGLE_16* auxRegionRects, UINT32 numAuxRegionRect, |
592 | | const BYTE* pAuxSrcData, UINT32 AuxSrcSize, BYTE* pDstData, DWORD DstFormat, |
593 | | UINT32 nDstStep, UINT32 nDstWidth, UINT32 nDstHeight, UINT32 codecId) |
594 | 0 | { |
595 | 0 | INT32 status = -1; |
596 | 0 | avc444_frame_type chroma = |
597 | 0 | (codecId == RDPGFX_CODECID_AVC444) ? AVC444_CHROMAv1 : AVC444_CHROMAv2; |
598 | |
|
599 | 0 | if (!h264 || !regionRects || !pSrcData || !pDstData || h264->Compressor) |
600 | 0 | return -1001; |
601 | | |
602 | 0 | if (!areRectsValid(nDstWidth, nDstHeight, regionRects, numRegionRects)) |
603 | 0 | return -1013; |
604 | 0 | if (!areRectsValid(nDstWidth, nDstHeight, auxRegionRects, numAuxRegionRect)) |
605 | 0 | return -1014; |
606 | | |
607 | 0 | switch (op) |
608 | 0 | { |
609 | 0 | case 0: /* YUV420 in stream 1 |
610 | | * Chroma420 in stream 2 */ |
611 | 0 | if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep, |
612 | 0 | nDstWidth, nDstHeight, regionRects, numRegionRects, |
613 | 0 | AVC444_LUMA)) |
614 | 0 | status = -1; |
615 | 0 | else if (!avc444_process_rects(h264, pAuxSrcData, AuxSrcSize, pDstData, DstFormat, |
616 | 0 | nDstStep, nDstWidth, nDstHeight, auxRegionRects, |
617 | 0 | numAuxRegionRect, chroma)) |
618 | 0 | status = -1; |
619 | 0 | else |
620 | 0 | status = 0; |
621 | |
|
622 | 0 | break; |
623 | | |
624 | 0 | case 2: /* Chroma420 in stream 1 */ |
625 | 0 | if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep, |
626 | 0 | nDstWidth, nDstHeight, regionRects, numRegionRects, chroma)) |
627 | 0 | status = -1; |
628 | 0 | else |
629 | 0 | status = 0; |
630 | |
|
631 | 0 | break; |
632 | | |
633 | 0 | case 1: /* YUV420 in stream 1 */ |
634 | 0 | if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep, |
635 | 0 | nDstWidth, nDstHeight, regionRects, numRegionRects, |
636 | 0 | AVC444_LUMA)) |
637 | 0 | status = -1; |
638 | 0 | else |
639 | 0 | status = 0; |
640 | |
|
641 | 0 | break; |
642 | | |
643 | 0 | default: /* WTF? */ |
644 | 0 | break; |
645 | 0 | } |
646 | | |
647 | | #if defined(AVC444_FRAME_STAT) |
648 | | |
649 | | switch (op) |
650 | | { |
651 | | case 0: |
652 | | op1sum = avg(&op1, op1sum, SrcSize + AuxSrcSize); |
653 | | break; |
654 | | |
655 | | case 1: |
656 | | op2sum = avg(&op2, op2sum, SrcSize); |
657 | | break; |
658 | | |
659 | | case 2: |
660 | | op3sum = avg(&op3, op3sum, SrcSize); |
661 | | break; |
662 | | |
663 | | default: |
664 | | break; |
665 | | } |
666 | | |
667 | | WLog_Print(h264->log, WLOG_INFO, |
668 | | "luma=%" PRIu64 " [avg=%lf] chroma=%" PRIu64 " [avg=%lf] combined=%" PRIu64 |
669 | | " [avg=%lf]", |
670 | | op1, op1sum, op2, op2sum, op3, op3sum); |
671 | | #endif |
672 | 0 | return status; |
673 | 0 | } |
674 | | |
675 | 0 | #define MAX_SUBSYSTEMS 10 |
676 | | static INIT_ONCE subsystems_once = INIT_ONCE_STATIC_INIT; |
677 | | static const H264_CONTEXT_SUBSYSTEM* subSystems[MAX_SUBSYSTEMS] = WINPR_C_ARRAY_INIT; |
678 | | |
679 | | static BOOL CALLBACK h264_register_subsystems(WINPR_ATTR_UNUSED PINIT_ONCE once, |
680 | | WINPR_ATTR_UNUSED PVOID param, |
681 | | WINPR_ATTR_UNUSED PVOID* context) |
682 | 0 | { |
683 | 0 | int i = 0; |
684 | |
|
685 | | #ifdef WITH_MEDIACODEC |
686 | | { |
687 | | subSystems[i] = &g_Subsystem_mediacodec; |
688 | | i++; |
689 | | } |
690 | | #endif |
691 | | #if defined(_WIN32) && defined(WITH_MEDIA_FOUNDATION) |
692 | | { |
693 | | subSystems[i] = &g_Subsystem_MF; |
694 | | i++; |
695 | | } |
696 | | #endif |
697 | | #ifdef WITH_OPENH264 |
698 | | { |
699 | | subSystems[i] = &g_Subsystem_OpenH264; |
700 | | i++; |
701 | | } |
702 | | #endif |
703 | | #ifdef WITH_VIDEO_FFMPEG |
704 | | { |
705 | | subSystems[i] = &g_Subsystem_libavcodec; |
706 | | i++; |
707 | | } |
708 | | #endif |
709 | 0 | return i > 0; |
710 | 0 | } |
711 | | |
712 | | static BOOL h264_context_init(H264_CONTEXT* h264) |
713 | 0 | { |
714 | 0 | if (!h264) |
715 | 0 | return FALSE; |
716 | | |
717 | 0 | h264->subsystem = nullptr; |
718 | 0 | if (!InitOnceExecuteOnce(&subsystems_once, h264_register_subsystems, nullptr, nullptr)) |
719 | 0 | return FALSE; |
720 | | |
721 | 0 | for (size_t i = 0; i < MAX_SUBSYSTEMS; i++) |
722 | 0 | { |
723 | 0 | const H264_CONTEXT_SUBSYSTEM* subsystem = subSystems[i]; |
724 | |
|
725 | 0 | if (!subsystem || !subsystem->Init) |
726 | 0 | break; |
727 | | |
728 | 0 | if (subsystem->Init(h264)) |
729 | 0 | { |
730 | 0 | h264->subsystem = subsystem; |
731 | 0 | return TRUE; |
732 | 0 | } |
733 | 0 | } |
734 | | |
735 | 0 | return FALSE; |
736 | 0 | } |
737 | | |
738 | | BOOL h264_context_reset(H264_CONTEXT* h264, UINT32 width, UINT32 height) |
739 | 0 | { |
740 | 0 | if (!h264) |
741 | 0 | return FALSE; |
742 | | |
743 | 0 | h264->width = width; |
744 | 0 | h264->height = height; |
745 | |
|
746 | 0 | if (h264->subsystem && h264->subsystem->Uninit) |
747 | 0 | h264->subsystem->Uninit(h264); |
748 | 0 | if (!h264_context_init(h264)) |
749 | 0 | return FALSE; |
750 | | |
751 | 0 | return yuv_context_reset(h264->yuv, width, height); |
752 | 0 | } |
753 | | |
754 | | H264_CONTEXT* h264_context_new(BOOL Compressor) |
755 | 0 | { |
756 | 0 | H264_CONTEXT* h264 = (H264_CONTEXT*)calloc(1, sizeof(H264_CONTEXT)); |
757 | 0 | if (!h264) |
758 | 0 | return nullptr; |
759 | | |
760 | 0 | h264->log = WLog_Get(TAG); |
761 | |
|
762 | 0 | if (!h264->log) |
763 | 0 | goto fail; |
764 | | |
765 | 0 | h264->Compressor = Compressor; |
766 | 0 | if (Compressor) |
767 | 0 | { |
768 | | /* Default compressor settings, may be changed by caller */ |
769 | 0 | h264->BitRate = 1000000; |
770 | 0 | h264->FrameRate = 30; |
771 | 0 | } |
772 | |
|
773 | 0 | if (!h264_context_init(h264)) |
774 | 0 | goto fail; |
775 | | |
776 | 0 | h264->yuv = yuv_context_new(Compressor, 0); |
777 | 0 | if (!h264->yuv) |
778 | 0 | goto fail; |
779 | | |
780 | 0 | return h264; |
781 | | |
782 | 0 | fail: |
783 | 0 | WINPR_PRAGMA_DIAG_PUSH |
784 | 0 | WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC |
785 | 0 | h264_context_free(h264); |
786 | 0 | WINPR_PRAGMA_DIAG_POP |
787 | 0 | return nullptr; |
788 | 0 | } |
789 | | |
790 | | void h264_context_free(H264_CONTEXT* h264) |
791 | 0 | { |
792 | 0 | if (h264) |
793 | 0 | { |
794 | 0 | if (h264->subsystem) |
795 | 0 | { |
796 | 0 | WINPR_ASSERT(h264->subsystem->Uninit); |
797 | 0 | h264->subsystem->Uninit(h264); |
798 | 0 | } |
799 | |
|
800 | 0 | for (size_t x = 0; x < 3; x++) |
801 | 0 | { |
802 | 0 | if (h264->Compressor) |
803 | 0 | { |
804 | 0 | winpr_aligned_free(h264->pYUVData[x]); |
805 | 0 | winpr_aligned_free(h264->pOldYUVData[x]); |
806 | 0 | } |
807 | 0 | winpr_aligned_free(h264->pYUV444Data[x]); |
808 | 0 | winpr_aligned_free(h264->pOldYUV444Data[x]); |
809 | 0 | } |
810 | 0 | winpr_aligned_free(h264->lumaData); |
811 | |
|
812 | 0 | yuv_context_free(h264->yuv); |
813 | 0 | free(h264); |
814 | 0 | } |
815 | 0 | } |
816 | | |
817 | | void free_h264_metablock(RDPGFX_H264_METABLOCK* meta) |
818 | 0 | { |
819 | 0 | RDPGFX_H264_METABLOCK m = WINPR_C_ARRAY_INIT; |
820 | 0 | if (!meta) |
821 | 0 | return; |
822 | 0 | free(meta->quantQualityVals); |
823 | 0 | free(meta->regionRects); |
824 | 0 | *meta = m; |
825 | 0 | } |
826 | | |
827 | | BOOL h264_context_set_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option, UINT32 value) |
828 | 0 | { |
829 | 0 | WINPR_ASSERT(h264); |
830 | 0 | switch (option) |
831 | 0 | { |
832 | 0 | case H264_CONTEXT_OPTION_BITRATE: |
833 | 0 | h264->BitRate = value; |
834 | 0 | return TRUE; |
835 | 0 | case H264_CONTEXT_OPTION_FRAMERATE: |
836 | 0 | h264->FrameRate = value; |
837 | 0 | return TRUE; |
838 | 0 | case H264_CONTEXT_OPTION_RATECONTROL: |
839 | 0 | { |
840 | 0 | switch (value) |
841 | 0 | { |
842 | 0 | case H264_RATECONTROL_VBR: |
843 | 0 | h264->RateControlMode = H264_RATECONTROL_VBR; |
844 | 0 | return TRUE; |
845 | 0 | case H264_RATECONTROL_CQP: |
846 | 0 | h264->RateControlMode = H264_RATECONTROL_CQP; |
847 | 0 | return TRUE; |
848 | 0 | default: |
849 | 0 | WLog_Print(h264->log, WLOG_WARN, |
850 | 0 | "Unknown H264_CONTEXT_OPTION_RATECONTROL value [0x%08" PRIx32 "]", |
851 | 0 | value); |
852 | 0 | return FALSE; |
853 | 0 | } |
854 | 0 | return TRUE; |
855 | 0 | } |
856 | 0 | case H264_CONTEXT_OPTION_QP: |
857 | 0 | h264->QP = value; |
858 | 0 | return TRUE; |
859 | 0 | case H264_CONTEXT_OPTION_USAGETYPE: |
860 | 0 | h264->UsageType = value; |
861 | 0 | return TRUE; |
862 | 0 | case H264_CONTEXT_OPTION_HW_ACCEL: |
863 | 0 | h264->hwAccel = (value); |
864 | 0 | return TRUE; |
865 | 0 | default: |
866 | 0 | WLog_Print(h264->log, WLOG_WARN, "Unknown H264_CONTEXT_OPTION[0x%08" PRIx32 "]", |
867 | 0 | option); |
868 | 0 | return FALSE; |
869 | 0 | } |
870 | 0 | } |
871 | | |
872 | | UINT32 h264_context_get_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option) |
873 | 0 | { |
874 | 0 | WINPR_ASSERT(h264); |
875 | 0 | switch (option) |
876 | 0 | { |
877 | 0 | case H264_CONTEXT_OPTION_BITRATE: |
878 | 0 | return h264->BitRate; |
879 | 0 | case H264_CONTEXT_OPTION_FRAMERATE: |
880 | 0 | return h264->FrameRate; |
881 | 0 | case H264_CONTEXT_OPTION_RATECONTROL: |
882 | 0 | return h264->RateControlMode; |
883 | 0 | case H264_CONTEXT_OPTION_QP: |
884 | 0 | return h264->QP; |
885 | 0 | case H264_CONTEXT_OPTION_USAGETYPE: |
886 | 0 | return h264->UsageType; |
887 | 0 | case H264_CONTEXT_OPTION_HW_ACCEL: |
888 | 0 | return h264->hwAccel; |
889 | 0 | default: |
890 | 0 | WLog_Print(h264->log, WLOG_WARN, "Unknown H264_CONTEXT_OPTION[0x%08" PRIx32 "]", |
891 | 0 | option); |
892 | 0 | return 0; |
893 | 0 | } |
894 | 0 | } |