Coverage Report

Created: 2026-06-15 06:57

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/FreeRDP/libfreerdp/codec/h264.c
Line
Count
Source
1
/**
2
 * FreeRDP: A Remote Desktop Protocol Implementation
3
 * H.264 Bitmap Compression
4
 *
5
 * Copyright 2014 Mike McDonald <Mike.McDonald@software.dell.com>
6
 * Copyright 2017 David Fort <contact@hardening-consulting.com>
7
 *
8
 * Licensed under the Apache License, Version 2.0 (the "License");
9
 * you may not use this file except in compliance with the License.
10
 * You may obtain a copy of the License at
11
 *
12
 *     http://www.apache.org/licenses/LICENSE-2.0
13
 *
14
 * Unless required by applicable law or agreed to in writing, software
15
 * distributed under the License is distributed on an "AS IS" BASIS,
16
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
 * See the License for the specific language governing permissions and
18
 * limitations under the License.
19
 */
20
21
#include <freerdp/config.h>
22
23
#include <winpr/crt.h>
24
#include <winpr/print.h>
25
#include <winpr/library.h>
26
#include <winpr/bitstream.h>
27
#include <winpr/synch.h>
28
29
#include <freerdp/primitives.h>
30
#include <freerdp/codec/h264.h>
31
#include <freerdp/codec/yuv.h>
32
#include <freerdp/log.h>
33
34
#include "h264.h"
35
36
0
#define TAG FREERDP_TAG("codec")
37
38
static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight);
39
40
static BOOL yuv_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height)
41
0
{
42
0
  BOOL isNull = FALSE;
43
0
  UINT32 pheight = height;
44
45
0
  if (!h264)
46
0
    return FALSE;
47
48
0
  if (stride == 0)
49
0
    stride = width;
50
51
0
  if (stride % 16 != 0)
52
0
    stride += 16 - stride % 16;
53
54
0
  if (pheight % 16 != 0)
55
0
    pheight += 16 - pheight % 16;
56
57
0
  const size_t nPlanes = h264->hwAccel ? 2 : 3;
58
59
0
  for (size_t x = 0; x < nPlanes; x++)
60
0
  {
61
0
    if (!h264->pYUVData[x] || !h264->pOldYUVData[x])
62
0
      isNull = TRUE;
63
0
  }
64
65
0
  if (pheight == 0)
66
0
    return FALSE;
67
0
  if (stride == 0)
68
0
    return FALSE;
69
70
0
  if (isNull || (width != h264->width) || (height != h264->height) ||
71
0
      (stride != h264->iStride[0]))
72
0
  {
73
0
    if (h264->hwAccel) /* NV12 */
74
0
    {
75
0
      h264->iStride[0] = stride;
76
0
      h264->iStride[1] = stride;
77
0
      h264->iStride[2] = 0;
78
0
    }
79
0
    else /* I420 */
80
0
    {
81
0
      h264->iStride[0] = stride;
82
0
      h264->iStride[1] = (stride + 1) / 2;
83
0
      h264->iStride[2] = (stride + 1) / 2;
84
0
    }
85
86
0
    for (size_t x = 0; x < nPlanes; x++)
87
0
    {
88
0
      BYTE* tmp1 = winpr_aligned_recalloc(h264->pYUVData[x], h264->iStride[x], pheight, 16);
89
0
      BYTE* tmp2 =
90
0
          winpr_aligned_recalloc(h264->pOldYUVData[x], h264->iStride[x], pheight, 16);
91
0
      if (tmp1)
92
0
        h264->pYUVData[x] = tmp1;
93
0
      if (tmp2)
94
0
        h264->pOldYUVData[x] = tmp2;
95
0
      if (!tmp1 || !tmp2)
96
0
        return FALSE;
97
0
    }
98
0
    h264->width = width;
99
0
    h264->height = height;
100
0
  }
101
102
0
  return TRUE;
103
0
}
104
105
BOOL avc420_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height)
106
0
{
107
0
  return yuv_ensure_buffer(h264, stride, width, height);
108
0
}
109
110
static BOOL isRectValid(UINT32 width, UINT32 height, const RECTANGLE_16* rect)
111
0
{
112
0
  WINPR_ASSERT(rect);
113
0
  if (rect->left > width)
114
0
    return FALSE;
115
0
  if (rect->right > width)
116
0
    return FALSE;
117
0
  if (rect->left >= rect->right)
118
0
    return FALSE;
119
0
  if (rect->top > height)
120
0
    return FALSE;
121
0
  if (rect->bottom > height)
122
0
    return FALSE;
123
0
  if (rect->top >= rect->bottom)
124
0
    return FALSE;
125
0
  return TRUE;
126
0
}
127
128
static BOOL areRectsValid(UINT32 width, UINT32 height, const RECTANGLE_16* rects, UINT32 count)
129
0
{
130
0
  WINPR_ASSERT(rects || (count == 0));
131
0
  for (size_t x = 0; x < count; x++)
132
0
  {
133
0
    const RECTANGLE_16* rect = &rects[x];
134
0
    if (!isRectValid(width, height, rect))
135
0
      return FALSE;
136
0
  }
137
0
  return TRUE;
138
0
}
139
140
INT32 avc420_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize, BYTE* pDstData,
141
                        DWORD DstFormat, UINT32 nDstStep, WINPR_ATTR_UNUSED UINT32 nDstWidth,
142
                        WINPR_ATTR_UNUSED UINT32 nDstHeight, const RECTANGLE_16* regionRects,
143
                        UINT32 numRegionRects)
144
0
{
145
0
  int status = 0;
146
0
  const BYTE* pYUVData[3];
147
148
0
  if (!h264 || h264->Compressor)
149
0
    return -1001;
150
151
0
  if (!areRectsValid(nDstWidth, nDstHeight, regionRects, numRegionRects))
152
0
    return -1013;
153
154
0
  status = h264->subsystem->Decompress(h264, pSrcData, SrcSize);
155
156
0
  if (status == 0)
157
0
    return 1;
158
159
0
  if (status < 0)
160
0
    return status;
161
162
0
  pYUVData[0] = h264->pYUVData[0];
163
0
  pYUVData[1] = h264->pYUVData[1];
164
0
  pYUVData[2] = h264->pYUVData[2];
165
0
  if (!yuv420_context_decode(h264->yuv, pYUVData, h264->iStride, h264->height, DstFormat,
166
0
                             pDstData, nDstStep, regionRects, numRegionRects))
167
0
    return -1002;
168
169
0
  return 1;
170
0
}
171
172
static BOOL allocate_h264_metablock(UINT32 QP, RECTANGLE_16* rectangles,
173
                                    RDPGFX_H264_METABLOCK* meta, size_t count)
174
0
{
175
  /* [MS-RDPEGFX] 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY */
176
0
  if (!meta || (QP > UINT8_MAX))
177
0
  {
178
0
    free(rectangles);
179
0
    return FALSE;
180
0
  }
181
182
0
  meta->regionRects = rectangles;
183
0
  if (count == 0)
184
0
    return TRUE;
185
186
0
  if (count > UINT32_MAX)
187
0
    return FALSE;
188
189
0
  meta->quantQualityVals = calloc(count, sizeof(RDPGFX_H264_QUANT_QUALITY));
190
191
0
  if (!meta->quantQualityVals || !meta->regionRects)
192
0
    return FALSE;
193
0
  meta->numRegionRects = (UINT32)count;
194
0
  for (size_t x = 0; x < count; x++)
195
0
  {
196
0
    RDPGFX_H264_QUANT_QUALITY* cur = &meta->quantQualityVals[x];
197
0
    cur->qp = (UINT8)QP;
198
199
    /* qpVal bit 6 and 7 are flags, so mask them out here.
200
     * qualityVal is [0-100] so 100 - qpVal [0-64] is always in range */
201
0
    cur->qualityVal = 100 - (QP & 0x3F);
202
0
  }
203
0
  return TRUE;
204
0
}
205
206
static inline BOOL diff_tile(const RECTANGLE_16* regionRect, BYTE* pYUVData[3],
207
                             BYTE* pOldYUVData[3], UINT32 const iStride[3])
208
0
{
209
0
  size_t size = 0;
210
211
0
  if (!regionRect || !pYUVData || !pOldYUVData || !iStride)
212
0
    return FALSE;
213
0
  size = regionRect->right - regionRect->left;
214
0
  if (regionRect->right > iStride[0])
215
0
    return FALSE;
216
0
  if (regionRect->right / 2u > iStride[1])
217
0
    return FALSE;
218
0
  if (regionRect->right / 2u > iStride[2])
219
0
    return FALSE;
220
221
0
  for (size_t y = regionRect->top; y < regionRect->bottom; y++)
222
0
  {
223
0
    const BYTE* cur0 = &pYUVData[0][y * iStride[0]];
224
0
    const BYTE* cur1 = &pYUVData[1][y * iStride[1]];
225
0
    const BYTE* cur2 = &pYUVData[2][y * iStride[2]];
226
0
    const BYTE* old0 = &pOldYUVData[0][y * iStride[0]];
227
0
    const BYTE* old1 = &pOldYUVData[1][y * iStride[1]];
228
0
    const BYTE* old2 = &pOldYUVData[2][y * iStride[2]];
229
230
0
    if (memcmp(&cur0[regionRect->left], &old0[regionRect->left], size) != 0)
231
0
      return TRUE;
232
0
    if (memcmp(&cur1[regionRect->left / 2], &old1[regionRect->left / 2], size / 2) != 0)
233
0
      return TRUE;
234
0
    if (memcmp(&cur2[regionRect->left / 2], &old2[regionRect->left / 2], size / 2) != 0)
235
0
      return TRUE;
236
0
  }
237
0
  return FALSE;
238
0
}
239
240
static BOOL detect_changes(BOOL firstFrameDone, const UINT32 QP, const RECTANGLE_16* regionRect,
241
                           BYTE* pYUVData[3], BYTE* pOldYUVData[3], UINT32 const iStride[3],
242
                           RDPGFX_H264_METABLOCK* meta)
243
0
{
244
0
  size_t count = 0;
245
0
  size_t wc = 0;
246
0
  size_t hc = 0;
247
0
  RECTANGLE_16* rectangles = nullptr;
248
249
0
  if (!regionRect || !pYUVData || !pOldYUVData || !iStride || !meta)
250
0
    return FALSE;
251
252
0
  wc = (regionRect->right - regionRect->left) / 64 + 1;
253
0
  hc = (regionRect->bottom - regionRect->top) / 64 + 1;
254
0
  rectangles = calloc(wc * hc, sizeof(RECTANGLE_16));
255
0
  if (!rectangles)
256
0
    return FALSE;
257
0
  if (!firstFrameDone)
258
0
  {
259
0
    rectangles[0] = *regionRect;
260
0
    count = 1;
261
0
  }
262
0
  else
263
0
  {
264
0
    for (size_t y = regionRect->top; y < regionRect->bottom; y += 64)
265
0
    {
266
0
      for (size_t x = regionRect->left; x < regionRect->right; x += 64)
267
0
      {
268
0
        RECTANGLE_16 rect;
269
0
        rect.left = (UINT16)MIN(UINT16_MAX, regionRect->left + x);
270
0
        rect.top = (UINT16)MIN(UINT16_MAX, regionRect->top + y);
271
0
        rect.right =
272
0
            (UINT16)MIN(UINT16_MAX, MIN(regionRect->left + x + 64, regionRect->right));
273
0
        rect.bottom =
274
0
            (UINT16)MIN(UINT16_MAX, MIN(regionRect->top + y + 64, regionRect->bottom));
275
0
        if (diff_tile(&rect, pYUVData, pOldYUVData, iStride))
276
0
          rectangles[count++] = rect;
277
0
      }
278
0
    }
279
0
  }
280
0
  if (!allocate_h264_metablock(QP, rectangles, meta, count))
281
0
    return FALSE;
282
0
  return TRUE;
283
0
}
284
285
INT32 h264_get_yuv_buffer(H264_CONTEXT* h264, UINT32 nSrcStride, UINT32 nSrcWidth,
286
                          UINT32 nSrcHeight, BYTE* YUVData[3], UINT32 stride[3])
287
0
{
288
0
  if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress)
289
0
    return -1;
290
291
0
  if (!yuv_ensure_buffer(h264, nSrcStride, nSrcWidth, nSrcHeight))
292
0
    return -1;
293
294
0
  for (size_t x = 0; x < 3; x++)
295
0
  {
296
0
    YUVData[x] = h264->pYUVData[x];
297
0
    stride[x] = h264->iStride[x];
298
0
  }
299
300
0
  return 0;
301
0
}
302
303
INT32 h264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize)
304
0
{
305
0
  if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress)
306
0
    return -1;
307
308
0
  const BYTE* pcYUVData[3] = { h264->pYUVData[0], h264->pYUVData[1], h264->pYUVData[2] };
309
310
0
  return h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize);
311
0
}
312
313
INT32 avc420_compress(H264_CONTEXT* h264, const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep,
314
                      UINT32 nSrcWidth, UINT32 nSrcHeight, const RECTANGLE_16* regionRect,
315
                      BYTE** ppDstData, UINT32* pDstSize, RDPGFX_H264_METABLOCK* meta)
316
0
{
317
0
  INT32 rc = -1;
318
0
  BYTE* pYUVData[3] = WINPR_C_ARRAY_INIT;
319
0
  const BYTE* pcYUVData[3] = WINPR_C_ARRAY_INIT;
320
0
  BYTE* pOldYUVData[3] = WINPR_C_ARRAY_INIT;
321
322
0
  if (!h264 || !regionRect || !meta || !h264->Compressor)
323
0
    return -1;
324
325
0
  if (!h264->subsystem->Compress)
326
0
    return -1;
327
328
0
  if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight))
329
0
    return -1;
330
331
0
  if (h264->encodingBuffer)
332
0
  {
333
0
    for (size_t x = 0; x < 3; x++)
334
0
    {
335
0
      pYUVData[x] = h264->pYUVData[x];
336
0
      pOldYUVData[x] = h264->pOldYUVData[x];
337
0
    }
338
0
  }
339
0
  else
340
0
  {
341
0
    for (size_t x = 0; x < 3; x++)
342
0
    {
343
0
      pYUVData[x] = h264->pOldYUVData[x];
344
0
      pOldYUVData[x] = h264->pYUVData[x];
345
0
    }
346
0
  }
347
0
  h264->encodingBuffer = !h264->encodingBuffer;
348
349
0
  if (!yuv420_context_encode(h264->yuv, pSrcData, nSrcStep, SrcFormat, h264->iStride, pYUVData,
350
0
                             regionRect, 1))
351
0
    goto fail;
352
353
0
  if (!detect_changes(h264->firstLumaFrameDone, h264->QP, regionRect, pYUVData, pOldYUVData,
354
0
                      h264->iStride, meta))
355
0
    goto fail;
356
357
0
  if (meta->numRegionRects == 0)
358
0
  {
359
0
    rc = 0;
360
0
    goto fail;
361
0
  }
362
363
0
  for (size_t x = 0; x < 3; x++)
364
0
    pcYUVData[x] = pYUVData[x];
365
366
0
  rc = h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize);
367
0
  if (rc >= 0)
368
0
    h264->firstLumaFrameDone = TRUE;
369
370
0
fail:
371
0
  if (rc < 0)
372
0
    free_h264_metablock(meta);
373
0
  return rc;
374
0
}
375
376
INT32 avc444_compress(H264_CONTEXT* h264, const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep,
377
                      UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE version, const RECTANGLE_16* region,
378
                      BYTE* op, BYTE** ppDstData, UINT32* pDstSize, BYTE** ppAuxDstData,
379
                      UINT32* pAuxDstSize, RDPGFX_H264_METABLOCK* meta,
380
                      RDPGFX_H264_METABLOCK* auxMeta)
381
0
{
382
0
  int rc = -1;
383
0
  BYTE* coded = nullptr;
384
0
  UINT32 codedSize = 0;
385
0
  BYTE** pYUV444Data = nullptr;
386
0
  BYTE** pOldYUV444Data = nullptr;
387
0
  BYTE** pYUVData = nullptr;
388
0
  BYTE** pOldYUVData = nullptr;
389
390
0
  if (!h264 || !h264->Compressor)
391
0
    return -1;
392
393
0
  if (!h264->subsystem->Compress)
394
0
    return -1;
395
396
0
  if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight))
397
0
    return -1;
398
399
0
  if (!avc444_ensure_buffer(h264, nSrcHeight))
400
0
    return -1;
401
402
0
  if (h264->encodingBuffer)
403
0
  {
404
0
    pYUV444Data = h264->pOldYUV444Data;
405
0
    pOldYUV444Data = h264->pYUV444Data;
406
0
    pYUVData = h264->pOldYUVData;
407
0
    pOldYUVData = h264->pYUVData;
408
0
  }
409
0
  else
410
0
  {
411
0
    pYUV444Data = h264->pYUV444Data;
412
0
    pOldYUV444Data = h264->pOldYUV444Data;
413
0
    pYUVData = h264->pYUVData;
414
0
    pOldYUVData = h264->pOldYUVData;
415
0
  }
416
0
  h264->encodingBuffer = !h264->encodingBuffer;
417
418
0
  if (!yuv444_context_encode(h264->yuv, version, pSrcData, nSrcStep, SrcFormat, h264->iStride,
419
0
                             pYUV444Data, pYUVData, region, 1))
420
0
    goto fail;
421
422
0
  if (!detect_changes(h264->firstLumaFrameDone, h264->QP, region, pYUV444Data, pOldYUV444Data,
423
0
                      h264->iStride, meta))
424
0
    goto fail;
425
0
  if (!detect_changes(h264->firstChromaFrameDone, h264->QP, region, pYUVData, pOldYUVData,
426
0
                      h264->iStride, auxMeta))
427
0
    goto fail;
428
429
  /* [MS-RDPEGFX] 2.2.4.5 RFX_AVC444_BITMAP_STREAM
430
   * LC:
431
   * 0 ... Luma & Chroma
432
   * 1 ... Luma
433
   * 2 ... Chroma
434
   */
435
0
  if ((meta->numRegionRects > 0) && (auxMeta->numRegionRects > 0))
436
0
    *op = 0;
437
0
  else if (meta->numRegionRects > 0)
438
0
    *op = 1;
439
0
  else if (auxMeta->numRegionRects > 0)
440
0
    *op = 2;
441
0
  else
442
0
  {
443
0
    WLog_Print(h264->log, WLOG_TRACE, "no changes detected for luma or chroma frame");
444
0
    rc = 0;
445
0
    goto fail;
446
0
  }
447
448
0
  if ((*op == 0) || (*op == 1))
449
0
  {
450
0
    const BYTE* pcYUV444Data[3] = { pYUV444Data[0], pYUV444Data[1], pYUV444Data[2] };
451
452
0
    if (h264->subsystem->Compress(h264, pcYUV444Data, h264->iStride, &coded, &codedSize) < 0)
453
0
      goto fail;
454
0
    h264->firstLumaFrameDone = TRUE;
455
0
    memcpy(h264->lumaData, coded, codedSize);
456
0
    *ppDstData = h264->lumaData;
457
0
    *pDstSize = codedSize;
458
0
  }
459
460
0
  if ((*op == 0) || (*op == 2))
461
0
  {
462
0
    const BYTE* pcYUVData[3] = { pYUVData[0], pYUVData[1], pYUVData[2] };
463
464
0
    if (h264->subsystem->Compress(h264, pcYUVData, h264->iStride, &coded, &codedSize) < 0)
465
0
      goto fail;
466
0
    h264->firstChromaFrameDone = TRUE;
467
0
    *ppAuxDstData = coded;
468
0
    *pAuxDstSize = codedSize;
469
0
  }
470
471
0
  rc = 1;
472
0
fail:
473
0
  if (rc < 0)
474
0
  {
475
0
    free_h264_metablock(meta);
476
0
    free_h264_metablock(auxMeta);
477
0
  }
478
0
  return rc;
479
0
}
480
481
static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight)
482
0
{
483
0
  WINPR_ASSERT(h264);
484
485
0
  const UINT32* piMainStride = h264->iStride;
486
0
  UINT32* piDstSize = h264->iYUV444Size;
487
0
  UINT32* piDstStride = h264->iYUV444Stride;
488
0
  BYTE** ppYUVDstData = h264->pYUV444Data;
489
0
  BYTE** ppOldYUVDstData = h264->pOldYUV444Data;
490
491
0
  nDstHeight = MAX(h264->height, nDstHeight);
492
0
  const UINT32 pad = nDstHeight % 16;
493
0
  UINT32 padDstHeight = nDstHeight; /* Need alignment to 16x16 blocks */
494
495
0
  if (pad != 0)
496
0
    padDstHeight += 16 - pad;
497
498
0
  if ((piMainStride[0] == 0) || (padDstHeight == 0))
499
0
    return FALSE;
500
501
0
  const uint64_t dstsize = 1ull * piMainStride[0] * padDstHeight;
502
0
  if (dstsize > UINT32_MAX)
503
0
    return FALSE;
504
505
0
  if ((piMainStride[0] != piDstStride[0]) || (piDstSize[0] != dstsize))
506
0
  {
507
0
    for (UINT32 x = 0; x < 3; x++)
508
0
    {
509
0
      piDstStride[x] = piMainStride[0];
510
511
0
      const uint64_t dstride = 1ull * piDstStride[x] * padDstHeight;
512
0
      if (dstride > UINT32_MAX)
513
0
        return FALSE;
514
515
0
      piDstSize[x] = WINPR_ASSERTING_INT_CAST(UINT32, dstride);
516
0
      if (piDstSize[x] == 0)
517
0
        return FALSE;
518
519
0
      BYTE* tmp1 = winpr_aligned_recalloc(ppYUVDstData[x], piDstSize[x], 1, 16);
520
0
      if (!tmp1)
521
0
        return FALSE;
522
0
      ppYUVDstData[x] = tmp1;
523
0
      BYTE* tmp2 = winpr_aligned_recalloc(ppOldYUVDstData[x], piDstSize[x], 1, 16);
524
0
      if (!tmp2)
525
0
        return FALSE;
526
0
      ppOldYUVDstData[x] = tmp2;
527
0
    }
528
529
0
    {
530
0
      BYTE* tmp = winpr_aligned_recalloc(h264->lumaData, piDstSize[0], 4, 16);
531
0
      if (!tmp)
532
0
        goto fail;
533
0
      h264->lumaData = tmp;
534
0
    }
535
0
  }
536
537
0
  for (UINT32 x = 0; x < 3; x++)
538
0
  {
539
0
    if (!ppOldYUVDstData[x] || !ppYUVDstData[x] || (piDstSize[x] == 0) || (piDstStride[x] == 0))
540
0
    {
541
0
      WLog_Print(h264->log, WLOG_ERROR,
542
0
                 "YUV buffer not initialized! check your decoder settings");
543
0
      goto fail;
544
0
    }
545
0
  }
546
547
0
  if (!h264->lumaData)
548
0
    goto fail;
549
550
0
  return TRUE;
551
0
fail:
552
0
  return FALSE;
553
0
}
554
555
static BOOL avc444_process_rects(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize,
556
                                 BYTE* pDstData, UINT32 DstFormat, UINT32 nDstStep,
557
                                 WINPR_ATTR_UNUSED UINT32 nDstWidth, UINT32 nDstHeight,
558
                                 const RECTANGLE_16* rects, UINT32 nrRects, avc444_frame_type type)
559
0
{
560
0
  const BYTE* pYUVData[3];
561
0
  BYTE* pYUVDstData[3];
562
0
  UINT32* piDstStride = h264->iYUV444Stride;
563
0
  BYTE** ppYUVDstData = h264->pYUV444Data;
564
0
  const UINT32* piStride = h264->iStride;
565
566
0
  if (h264->subsystem->Decompress(h264, pSrcData, SrcSize) < 0)
567
0
    return FALSE;
568
569
0
  pYUVData[0] = h264->pYUVData[0];
570
0
  pYUVData[1] = h264->pYUVData[1];
571
0
  pYUVData[2] = h264->pYUVData[2];
572
0
  if (!avc444_ensure_buffer(h264, nDstHeight))
573
0
    return FALSE;
574
575
0
  pYUVDstData[0] = ppYUVDstData[0];
576
0
  pYUVDstData[1] = ppYUVDstData[1];
577
0
  pYUVDstData[2] = ppYUVDstData[2];
578
0
  return (yuv444_context_decode(h264->yuv, (BYTE)type, pYUVData, piStride, h264->height,
579
0
                                pYUVDstData, piDstStride, DstFormat, pDstData, nDstStep, rects,
580
0
                                nrRects));
581
0
}
582
583
#if defined(AVC444_FRAME_STAT)
584
static UINT64 op1 = 0;
585
static double op1sum = 0;
586
static UINT64 op2 = 0;
587
static double op2sum = 0;
588
static UINT64 op3 = 0;
589
static double op3sum = 0;
590
static double avg(UINT64* count, double old, double size)
591
{
592
  double tmp = size + *count * old;
593
  (*count)++;
594
  tmp = tmp / *count;
595
  return tmp;
596
}
597
#endif
598
599
INT32 avc444_decompress(H264_CONTEXT* h264, BYTE op, const RECTANGLE_16* regionRects,
600
                        UINT32 numRegionRects, const BYTE* pSrcData, UINT32 SrcSize,
601
                        const RECTANGLE_16* auxRegionRects, UINT32 numAuxRegionRect,
602
                        const BYTE* pAuxSrcData, UINT32 AuxSrcSize, BYTE* pDstData, DWORD DstFormat,
603
                        UINT32 nDstStep, UINT32 nDstWidth, UINT32 nDstHeight, UINT32 codecId)
604
0
{
605
0
  INT32 status = -1;
606
0
  avc444_frame_type chroma =
607
0
      (codecId == RDPGFX_CODECID_AVC444) ? AVC444_CHROMAv1 : AVC444_CHROMAv2;
608
609
0
  if (!h264 || !regionRects || !pSrcData || !pDstData || h264->Compressor)
610
0
    return -1001;
611
612
0
  if (!areRectsValid(nDstWidth, nDstHeight, regionRects, numRegionRects))
613
0
    return -1013;
614
0
  if (!areRectsValid(nDstWidth, nDstHeight, auxRegionRects, numAuxRegionRect))
615
0
    return -1014;
616
617
0
  switch (op)
618
0
  {
619
0
    case 0: /* YUV420 in stream 1
620
             * Chroma420 in stream 2 */
621
0
      if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
622
0
                                nDstWidth, nDstHeight, regionRects, numRegionRects,
623
0
                                AVC444_LUMA))
624
0
        status = -1;
625
0
      else if (!avc444_process_rects(h264, pAuxSrcData, AuxSrcSize, pDstData, DstFormat,
626
0
                                     nDstStep, nDstWidth, nDstHeight, auxRegionRects,
627
0
                                     numAuxRegionRect, chroma))
628
0
        status = -1;
629
0
      else
630
0
        status = 0;
631
632
0
      break;
633
634
0
    case 2: /* Chroma420 in stream 1 */
635
0
      if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
636
0
                                nDstWidth, nDstHeight, regionRects, numRegionRects, chroma))
637
0
        status = -1;
638
0
      else
639
0
        status = 0;
640
641
0
      break;
642
643
0
    case 1: /* YUV420 in stream 1 */
644
0
      if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
645
0
                                nDstWidth, nDstHeight, regionRects, numRegionRects,
646
0
                                AVC444_LUMA))
647
0
        status = -1;
648
0
      else
649
0
        status = 0;
650
651
0
      break;
652
653
0
    default: /* WTF? */
654
0
      break;
655
0
  }
656
657
#if defined(AVC444_FRAME_STAT)
658
659
  switch (op)
660
  {
661
    case 0:
662
      op1sum = avg(&op1, op1sum, SrcSize + AuxSrcSize);
663
      break;
664
665
    case 1:
666
      op2sum = avg(&op2, op2sum, SrcSize);
667
      break;
668
669
    case 2:
670
      op3sum = avg(&op3, op3sum, SrcSize);
671
      break;
672
673
    default:
674
      break;
675
  }
676
677
  WLog_Print(h264->log, WLOG_INFO,
678
             "luma=%" PRIu64 " [avg=%lf] chroma=%" PRIu64 " [avg=%lf] combined=%" PRIu64
679
             " [avg=%lf]",
680
             op1, op1sum, op2, op2sum, op3, op3sum);
681
#endif
682
0
  return status;
683
0
}
684
685
0
#define MAX_SUBSYSTEMS 10
686
static INIT_ONCE subsystems_once = INIT_ONCE_STATIC_INIT;
687
static const H264_CONTEXT_SUBSYSTEM* subSystems[MAX_SUBSYSTEMS] = WINPR_C_ARRAY_INIT;
688
689
static BOOL CALLBACK h264_register_subsystems(WINPR_ATTR_UNUSED PINIT_ONCE once,
690
                                              WINPR_ATTR_UNUSED PVOID param,
691
                                              WINPR_ATTR_UNUSED PVOID* context)
692
0
{
693
0
  int i = 0;
694
695
#ifdef WITH_MEDIACODEC
696
  {
697
    subSystems[i] = &g_Subsystem_mediacodec;
698
    i++;
699
  }
700
#endif
701
#if defined(_WIN32) && defined(WITH_MEDIA_FOUNDATION)
702
  {
703
    subSystems[i] = &g_Subsystem_MF;
704
    i++;
705
  }
706
#endif
707
#ifdef WITH_OPENH264
708
  {
709
    subSystems[i] = &g_Subsystem_OpenH264;
710
    i++;
711
  }
712
#endif
713
#ifdef WITH_VIDEO_FFMPEG
714
  {
715
    subSystems[i] = &g_Subsystem_libavcodec;
716
    i++;
717
  }
718
#endif
719
0
  return i > 0;
720
0
}
721
722
static BOOL h264_context_init(H264_CONTEXT* h264)
723
0
{
724
0
  if (!h264)
725
0
    return FALSE;
726
727
0
  h264->subsystem = nullptr;
728
0
  if (!InitOnceExecuteOnce(&subsystems_once, h264_register_subsystems, nullptr, nullptr))
729
0
    return FALSE;
730
731
0
  for (size_t i = 0; i < MAX_SUBSYSTEMS; i++)
732
0
  {
733
0
    const H264_CONTEXT_SUBSYSTEM* subsystem = subSystems[i];
734
735
0
    if (!subsystem || !subsystem->Init)
736
0
      break;
737
738
0
    if (subsystem->Init(h264))
739
0
    {
740
0
      h264->subsystem = subsystem;
741
0
      return TRUE;
742
0
    }
743
0
  }
744
745
0
  return FALSE;
746
0
}
747
748
BOOL h264_context_reset(H264_CONTEXT* h264, UINT32 width, UINT32 height)
749
0
{
750
0
  if (!h264)
751
0
    return FALSE;
752
753
0
  h264->width = width;
754
0
  h264->height = height;
755
756
0
  if (h264->subsystem && h264->subsystem->Uninit)
757
0
    h264->subsystem->Uninit(h264);
758
0
  if (!h264_context_init(h264))
759
0
    return FALSE;
760
761
0
  return yuv_context_reset(h264->yuv, width, height);
762
0
}
763
764
H264_CONTEXT* h264_context_new(BOOL Compressor)
765
0
{
766
0
  H264_CONTEXT* h264 = (H264_CONTEXT*)calloc(1, sizeof(H264_CONTEXT));
767
0
  if (!h264)
768
0
    return nullptr;
769
770
0
  h264->log = WLog_Get(TAG);
771
772
0
  if (!h264->log)
773
0
    goto fail;
774
775
0
  h264->Compressor = Compressor;
776
0
  if (Compressor)
777
0
  {
778
    /* Default compressor settings, may be changed by caller */
779
0
    h264->BitRate = 1000000;
780
0
    h264->FrameRate = 30;
781
0
  }
782
783
0
  if (!h264_context_init(h264))
784
0
    goto fail;
785
786
0
  h264->yuv = yuv_context_new(Compressor, 0);
787
0
  if (!h264->yuv)
788
0
    goto fail;
789
790
0
  return h264;
791
792
0
fail:
793
0
  WINPR_PRAGMA_DIAG_PUSH
794
0
  WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
795
0
  h264_context_free(h264);
796
0
  WINPR_PRAGMA_DIAG_POP
797
0
  return nullptr;
798
0
}
799
800
void h264_context_free(H264_CONTEXT* h264)
801
0
{
802
0
  if (h264)
803
0
  {
804
0
    if (h264->subsystem)
805
0
    {
806
0
      WINPR_ASSERT(h264->subsystem->Uninit);
807
0
      h264->subsystem->Uninit(h264);
808
0
    }
809
810
0
    for (size_t x = 0; x < 3; x++)
811
0
    {
812
0
      if (h264->Compressor)
813
0
      {
814
0
        winpr_aligned_free(h264->pYUVData[x]);
815
0
        winpr_aligned_free(h264->pOldYUVData[x]);
816
0
      }
817
0
      winpr_aligned_free(h264->pYUV444Data[x]);
818
0
      winpr_aligned_free(h264->pOldYUV444Data[x]);
819
0
    }
820
0
    winpr_aligned_free(h264->lumaData);
821
822
0
    yuv_context_free(h264->yuv);
823
0
    free(h264);
824
0
  }
825
0
}
826
827
void free_h264_metablock(RDPGFX_H264_METABLOCK* meta)
828
0
{
829
0
  RDPGFX_H264_METABLOCK m = WINPR_C_ARRAY_INIT;
830
0
  if (!meta)
831
0
    return;
832
0
  free(meta->quantQualityVals);
833
0
  free(meta->regionRects);
834
0
  *meta = m;
835
0
}
836
837
BOOL h264_context_set_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option, UINT32 value)
838
0
{
839
0
  WINPR_ASSERT(h264);
840
0
  switch (option)
841
0
  {
842
0
    case H264_CONTEXT_OPTION_BITRATE:
843
0
      h264->BitRate = value;
844
0
      return TRUE;
845
0
    case H264_CONTEXT_OPTION_FRAMERATE:
846
0
      h264->FrameRate = value;
847
0
      return TRUE;
848
0
    case H264_CONTEXT_OPTION_RATECONTROL:
849
0
    {
850
0
      switch (value)
851
0
      {
852
0
        case H264_RATECONTROL_VBR:
853
0
          h264->RateControlMode = H264_RATECONTROL_VBR;
854
0
          return TRUE;
855
0
        case H264_RATECONTROL_CQP:
856
0
          h264->RateControlMode = H264_RATECONTROL_CQP;
857
0
          return TRUE;
858
0
        default:
859
0
          WLog_Print(h264->log, WLOG_WARN,
860
0
                     "Unknown H264_CONTEXT_OPTION_RATECONTROL value [0x%08" PRIx32 "]",
861
0
                     value);
862
0
          return FALSE;
863
0
      }
864
0
      return TRUE;
865
0
    }
866
0
    case H264_CONTEXT_OPTION_QP:
867
0
      h264->QP = value;
868
0
      return TRUE;
869
0
    case H264_CONTEXT_OPTION_USAGETYPE:
870
0
      h264->UsageType = value;
871
0
      return TRUE;
872
0
    case H264_CONTEXT_OPTION_HW_ACCEL:
873
0
      h264->hwAccel = (value);
874
0
      return TRUE;
875
0
    default:
876
0
      WLog_Print(h264->log, WLOG_WARN, "Unknown H264_CONTEXT_OPTION[0x%08" PRIx32 "]",
877
0
                 option);
878
0
      return FALSE;
879
0
  }
880
0
}
881
882
UINT32 h264_context_get_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option)
883
0
{
884
0
  WINPR_ASSERT(h264);
885
0
  switch (option)
886
0
  {
887
0
    case H264_CONTEXT_OPTION_BITRATE:
888
0
      return h264->BitRate;
889
0
    case H264_CONTEXT_OPTION_FRAMERATE:
890
0
      return h264->FrameRate;
891
0
    case H264_CONTEXT_OPTION_RATECONTROL:
892
0
      return h264->RateControlMode;
893
0
    case H264_CONTEXT_OPTION_QP:
894
0
      return h264->QP;
895
0
    case H264_CONTEXT_OPTION_USAGETYPE:
896
0
      return h264->UsageType;
897
0
    case H264_CONTEXT_OPTION_HW_ACCEL:
898
0
      return h264->hwAccel;
899
0
    default:
900
0
      WLog_Print(h264->log, WLOG_WARN, "Unknown H264_CONTEXT_OPTION[0x%08" PRIx32 "]",
901
0
                 option);
902
0
      return 0;
903
0
  }
904
0
}