Coverage Report

Created: 2024-09-08 06:20

/src/FreeRDP/libfreerdp/codec/nsc.c
Line
Count
Source (jump to first uncovered line)
1
/**
2
 * FreeRDP: A Remote Desktop Protocol Implementation
3
 * NSCodec Codec
4
 *
5
 * Copyright 2011 Samsung, Author Jiten Pathy
6
 * Copyright 2012 Vic Lee
7
 * Copyright 2016 Armin Novak <armin.novak@thincast.com>
8
 * Copyright 2016 Thincast Technologies GmbH
9
 *
10
 * Licensed under the Apache License, Version 2.0 (the "License");
11
 * you may not use this file except in compliance with the License.
12
 * You may obtain a copy of the License at
13
 *
14
 *   http://www.apache.org/licenses/LICENSE-2.0
15
 *
16
 * Unless required by applicable law or agreed to in writing, software
17
 * distributed under the License is distributed on an "AS IS" BASIS,
18
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
 * See the License for the specific language governing permissions and
20
 * limitations under the License.
21
 */
22
23
#include <freerdp/config.h>
24
25
#include <stdio.h>
26
#include <stdlib.h>
27
#include <string.h>
28
29
#include <winpr/crt.h>
30
31
#include <freerdp/codec/nsc.h>
32
#include <freerdp/codec/color.h>
33
34
#include "nsc_types.h"
35
#include "nsc_encode.h"
36
37
#include "sse/nsc_sse2.h"
38
#include "neon/nsc_neon.h"
39
40
#include <freerdp/log.h>
41
#define TAG FREERDP_TAG("codec.nsc")
42
43
#ifndef NSC_INIT_SIMD
44
#define NSC_INIT_SIMD(_nsc_context) \
45
  do                              \
46
  {                               \
47
  } while (0)
48
#endif
49
50
static BOOL nsc_decode(NSC_CONTEXT* WINPR_RESTRICT context)
51
275
{
52
275
  UINT16 rw = 0;
53
275
  BYTE shift = 0;
54
275
  BYTE* bmpdata = NULL;
55
275
  size_t pos = 0;
56
57
275
  if (!context)
58
0
    return FALSE;
59
60
275
  rw = ROUND_UP_TO(context->width, 8);
61
275
  shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
62
275
  bmpdata = context->BitmapData;
63
64
275
  if (!bmpdata)
65
0
    return FALSE;
66
67
1.58k
  for (size_t y = 0; y < context->height; y++)
68
1.30k
  {
69
1.30k
    const BYTE* yplane = NULL;
70
1.30k
    const BYTE* coplane = NULL;
71
1.30k
    const BYTE* cgplane = NULL;
72
1.30k
    const BYTE* aplane = context->priv->PlaneBuffers[3] + y * context->width; /* A */
73
74
1.30k
    if (context->ChromaSubsamplingLevel)
75
935
    {
76
935
      yplane = context->priv->PlaneBuffers[0] + y * rw;                /* Y */
77
935
      coplane = context->priv->PlaneBuffers[1] + (y >> 1) * (rw >> 1); /* Co, supersampled */
78
935
      cgplane = context->priv->PlaneBuffers[2] + (y >> 1) * (rw >> 1); /* Cg, supersampled */
79
935
    }
80
372
    else
81
372
    {
82
372
      yplane = context->priv->PlaneBuffers[0] + y * context->width;  /* Y */
83
372
      coplane = context->priv->PlaneBuffers[1] + y * context->width; /* Co */
84
372
      cgplane = context->priv->PlaneBuffers[2] + y * context->width; /* Cg */
85
372
    }
86
87
29.5k
    for (UINT32 x = 0; x < context->width; x++)
88
28.2k
    {
89
28.2k
      INT16 y_val = (INT16)*yplane;
90
28.2k
      INT16 co_val = (INT16)(INT8)(((INT16)*coplane) << shift);
91
28.2k
      INT16 cg_val = (INT16)(INT8)(((INT16)*cgplane) << shift);
92
28.2k
      INT16 r_val = y_val + co_val - cg_val;
93
28.2k
      INT16 g_val = y_val + cg_val;
94
28.2k
      INT16 b_val = y_val - co_val - cg_val;
95
96
28.2k
      if (pos + 4 > context->BitmapDataLength)
97
0
        return FALSE;
98
99
28.2k
      pos += 4;
100
28.2k
      *bmpdata++ = MINMAX(b_val, 0, 0xFF);
101
28.2k
      *bmpdata++ = MINMAX(g_val, 0, 0xFF);
102
28.2k
      *bmpdata++ = MINMAX(r_val, 0, 0xFF);
103
28.2k
      *bmpdata++ = *aplane;
104
28.2k
      yplane++;
105
28.2k
      coplane += (context->ChromaSubsamplingLevel ? x % 2 : 1);
106
28.2k
      cgplane += (context->ChromaSubsamplingLevel ? x % 2 : 1);
107
28.2k
      aplane++;
108
28.2k
    }
109
1.30k
  }
110
111
275
  return TRUE;
112
275
}
113
114
static BOOL nsc_rle_decode(const BYTE* WINPR_RESTRICT in, size_t inSize, BYTE* WINPR_RESTRICT out,
115
                           UINT32 outSize, UINT32 originalSize)
116
332
{
117
332
  UINT32 left = originalSize;
118
119
3.81k
  while (left > 4)
120
3.62k
  {
121
3.62k
    if (inSize < 1)
122
12
      return FALSE;
123
3.61k
    inSize--;
124
125
3.61k
    const BYTE value = *in++;
126
3.61k
    UINT32 len = 0;
127
128
3.61k
    if (left == 5)
129
126
    {
130
126
      if (outSize < 1)
131
0
        return FALSE;
132
133
126
      outSize--;
134
126
      *out++ = value;
135
126
      left--;
136
126
    }
137
3.48k
    else if (inSize < 1)
138
34
      return FALSE;
139
3.45k
    else if (value == *in)
140
1.34k
    {
141
1.34k
      inSize--;
142
1.34k
      in++;
143
144
1.34k
      if (inSize < 1)
145
13
        return FALSE;
146
1.33k
      else if (*in < 0xFF)
147
1.16k
      {
148
1.16k
        inSize--;
149
1.16k
        len = (UINT32)*in++;
150
1.16k
        len += 2;
151
1.16k
      }
152
170
      else
153
170
      {
154
170
        if (inSize < 5)
155
7
          return FALSE;
156
163
        inSize -= 5;
157
163
        in++;
158
163
        len = ((UINT32)(*in++));
159
163
        len |= ((UINT32)(*in++)) << 8U;
160
163
        len |= ((UINT32)(*in++)) << 16U;
161
163
        len |= ((UINT32)(*in++)) << 24U;
162
163
      }
163
164
1.32k
      if ((outSize < len) || (left < len))
165
79
        return FALSE;
166
167
1.24k
      outSize -= len;
168
1.24k
      FillMemory(out, len, value);
169
1.24k
      out += len;
170
1.24k
      left -= len;
171
1.24k
    }
172
2.10k
    else
173
2.10k
    {
174
2.10k
      if (outSize < 1)
175
0
        return FALSE;
176
177
2.10k
      outSize--;
178
2.10k
      *out++ = value;
179
2.10k
      left--;
180
2.10k
    }
181
3.61k
  }
182
183
187
  if ((outSize < 4) || (left < 4))
184
15
    return FALSE;
185
186
172
  if (inSize < 4)
187
23
    return FALSE;
188
149
  memcpy(out, in, 4);
189
149
  return TRUE;
190
172
}
191
192
static BOOL nsc_rle_decompress_data(NSC_CONTEXT* WINPR_RESTRICT context)
193
458
{
194
458
  if (!context)
195
0
    return FALSE;
196
197
458
  const BYTE* rle = context->Planes;
198
458
  size_t rleSize = context->PlanesSize;
199
458
  WINPR_ASSERT(rle);
200
201
1.68k
  for (size_t i = 0; i < 4; i++)
202
1.41k
  {
203
1.41k
    const UINT32 originalSize = context->OrgByteCount[i];
204
1.41k
    const UINT32 planeSize = context->PlaneByteCount[i];
205
206
1.41k
    if (rleSize < planeSize)
207
0
      return FALSE;
208
209
1.41k
    if (planeSize == 0)
210
885
    {
211
885
      if (context->priv->PlaneBuffersLength < originalSize)
212
0
        return FALSE;
213
214
885
      FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
215
885
    }
216
526
    else if (planeSize < originalSize)
217
332
    {
218
332
      if (!nsc_rle_decode(rle, rleSize, context->priv->PlaneBuffers[i],
219
332
                          context->priv->PlaneBuffersLength, originalSize))
220
183
        return FALSE;
221
332
    }
222
194
    else
223
194
    {
224
194
      if (context->priv->PlaneBuffersLength < originalSize)
225
0
        return FALSE;
226
227
194
      if (rleSize < originalSize)
228
0
        return FALSE;
229
230
194
      CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
231
194
    }
232
233
1.22k
    rle += planeSize;
234
1.22k
    rleSize -= planeSize;
235
1.22k
  }
236
237
275
  return TRUE;
238
458
}
239
240
static BOOL nsc_stream_initialize(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RESTRICT s)
241
524
{
242
524
  WINPR_ASSERT(context);
243
524
  WINPR_ASSERT(context->priv);
244
524
  if (!Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, 20))
245
45
    return FALSE;
246
247
479
  size_t total = 0;
248
2.39k
  for (size_t i = 0; i < 4; i++)
249
1.91k
  {
250
1.91k
    Stream_Read_UINT32(s, context->PlaneByteCount[i]);
251
1.91k
    total += context->PlaneByteCount[i];
252
1.91k
  }
253
254
479
  Stream_Read_UINT8(s, context->ColorLossLevel);         /* ColorLossLevel (1 byte) */
255
479
  if ((context->ColorLossLevel < 1) || (context->ColorLossLevel > 7))
256
18
  {
257
18
    WLog_Print(context->priv->log, WLOG_ERROR,
258
18
               "ColorLossLevel=%" PRIu8 " out of range, must be [1,7] inclusive",
259
18
               context->ColorLossLevel);
260
18
    return FALSE;
261
18
  }
262
461
  Stream_Read_UINT8(s, context->ChromaSubsamplingLevel); /* ChromaSubsamplingLevel (1 byte) */
263
461
  Stream_Seek(s, 2);                                     /* Reserved (2 bytes) */
264
461
  context->Planes = Stream_Pointer(s);
265
461
  context->PlanesSize = total;
266
461
  return Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, total);
267
479
}
268
269
static BOOL nsc_context_initialize(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RESTRICT s)
270
524
{
271
524
  if (!nsc_stream_initialize(context, s))
272
66
    return FALSE;
273
274
458
  const size_t blength = 4ull * context->width * context->height;
275
276
458
  if (!context->BitmapData || (blength > context->BitmapDataLength))
277
442
  {
278
442
    void* tmp = winpr_aligned_recalloc(context->BitmapData, blength + 16, sizeof(BYTE), 32);
279
280
442
    if (!tmp)
281
0
      return FALSE;
282
283
442
    context->BitmapData = tmp;
284
442
    context->BitmapDataLength = blength;
285
442
  }
286
287
458
  const UINT32 tempWidth = ROUND_UP_TO(context->width, 8);
288
458
  const UINT32 tempHeight = ROUND_UP_TO(context->height, 2);
289
  /* The maximum length a decoded plane can reach in all cases */
290
458
  const size_t plength = 1ull * tempWidth * tempHeight;
291
292
458
  if (plength > context->priv->PlaneBuffersLength)
293
393
  {
294
1.96k
    for (size_t i = 0; i < 4; i++)
295
1.57k
    {
296
1.57k
      void* tmp = (BYTE*)winpr_aligned_recalloc(context->priv->PlaneBuffers[i], plength,
297
1.57k
                                                sizeof(BYTE), 32);
298
299
1.57k
      if (!tmp)
300
0
        return FALSE;
301
302
1.57k
      context->priv->PlaneBuffers[i] = tmp;
303
1.57k
    }
304
305
393
    context->priv->PlaneBuffersLength = plength;
306
393
  }
307
308
2.29k
  for (size_t i = 0; i < 4; i++)
309
1.83k
    context->OrgByteCount[i] = context->width * context->height;
310
311
458
  if (context->ChromaSubsamplingLevel)
312
303
  {
313
303
    context->OrgByteCount[0] = tempWidth * context->height;
314
303
    context->OrgByteCount[1] = (tempWidth >> 1) * (tempHeight >> 1);
315
303
    context->OrgByteCount[2] = context->OrgByteCount[1];
316
303
  }
317
318
458
  return TRUE;
319
458
}
320
321
static void nsc_profiler_print(NSC_CONTEXT_PRIV* WINPR_RESTRICT priv)
322
16.2k
{
323
16.2k
  WINPR_UNUSED(priv);
324
325
16.2k
  PROFILER_PRINT_HEADER
326
16.2k
  PROFILER_PRINT(priv->prof_nsc_rle_decompress_data)
327
16.2k
  PROFILER_PRINT(priv->prof_nsc_decode)
328
16.2k
  PROFILER_PRINT(priv->prof_nsc_rle_compress_data)
329
16.2k
  PROFILER_PRINT(priv->prof_nsc_encode)
330
16.2k
  PROFILER_PRINT_FOOTER
331
16.2k
}
332
333
BOOL nsc_context_reset(NSC_CONTEXT* WINPR_RESTRICT context, UINT32 width, UINT32 height)
334
0
{
335
0
  if (!context)
336
0
    return FALSE;
337
338
0
  if ((width > UINT16_MAX) || (height > UINT16_MAX))
339
0
    return FALSE;
340
341
0
  context->width = (UINT16)width;
342
0
  context->height = (UINT16)height;
343
0
  return TRUE;
344
0
}
345
346
NSC_CONTEXT* nsc_context_new(void)
347
16.2k
{
348
16.2k
  NSC_CONTEXT* context = (NSC_CONTEXT*)winpr_aligned_calloc(1, sizeof(NSC_CONTEXT), 32);
349
350
16.2k
  if (!context)
351
0
    return NULL;
352
353
16.2k
  context->priv = (NSC_CONTEXT_PRIV*)winpr_aligned_calloc(1, sizeof(NSC_CONTEXT_PRIV), 32);
354
355
16.2k
  if (!context->priv)
356
0
    goto error;
357
358
16.2k
  context->priv->log = WLog_Get("com.freerdp.codec.nsc");
359
16.2k
  WLog_OpenAppender(context->priv->log);
360
16.2k
  context->BitmapData = NULL;
361
16.2k
  context->decode = nsc_decode;
362
16.2k
  context->encode = nsc_encode;
363
364
16.2k
  PROFILER_CREATE(context->priv->prof_nsc_rle_decompress_data, "nsc_rle_decompress_data")
365
16.2k
  PROFILER_CREATE(context->priv->prof_nsc_decode, "nsc_decode")
366
16.2k
  PROFILER_CREATE(context->priv->prof_nsc_rle_compress_data, "nsc_rle_compress_data")
367
16.2k
  PROFILER_CREATE(context->priv->prof_nsc_encode, "nsc_encode")
368
  /* Default encoding parameters */
369
16.2k
  context->ColorLossLevel = 3;
370
16.2k
  context->ChromaSubsamplingLevel = 1;
371
  /* init optimized methods */
372
16.2k
  nsc_init_sse2(context);
373
16.2k
  nsc_init_neon(context);
374
16.2k
  return context;
375
0
error:
376
0
  WINPR_PRAGMA_DIAG_PUSH
377
0
  WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
378
0
  nsc_context_free(context);
379
0
  WINPR_PRAGMA_DIAG_POP
380
0
  return NULL;
381
16.2k
}
382
383
void nsc_context_free(NSC_CONTEXT* context)
384
16.2k
{
385
16.2k
  if (!context)
386
0
    return;
387
388
16.2k
  if (context->priv)
389
16.2k
  {
390
97.3k
    for (size_t i = 0; i < 5; i++)
391
81.0k
      winpr_aligned_free(context->priv->PlaneBuffers[i]);
392
393
16.2k
    nsc_profiler_print(context->priv);
394
16.2k
    PROFILER_FREE(context->priv->prof_nsc_rle_decompress_data)
395
16.2k
    PROFILER_FREE(context->priv->prof_nsc_decode)
396
16.2k
    PROFILER_FREE(context->priv->prof_nsc_rle_compress_data)
397
16.2k
    PROFILER_FREE(context->priv->prof_nsc_encode)
398
16.2k
    winpr_aligned_free(context->priv);
399
16.2k
  }
400
401
16.2k
  winpr_aligned_free(context->BitmapData);
402
16.2k
  winpr_aligned_free(context);
403
16.2k
}
404
405
#if defined(WITH_FREERDP_DEPRECATED)
406
BOOL nsc_context_set_pixel_format(NSC_CONTEXT* context, UINT32 pixel_format)
407
{
408
  return nsc_context_set_parameters(context, NSC_COLOR_FORMAT, pixel_format);
409
}
410
#endif
411
412
BOOL nsc_context_set_parameters(NSC_CONTEXT* context, NSC_PARAMETER what, UINT32 value)
413
32.4k
{
414
32.4k
  if (!context)
415
0
    return FALSE;
416
417
32.4k
  switch (what)
418
32.4k
  {
419
0
    case NSC_COLOR_LOSS_LEVEL:
420
0
      context->ColorLossLevel = value;
421
0
      break;
422
0
    case NSC_ALLOW_SUBSAMPLING:
423
0
      context->ChromaSubsamplingLevel = value;
424
0
      break;
425
0
    case NSC_DYNAMIC_COLOR_FIDELITY:
426
0
      context->DynamicColorFidelity = value != 0;
427
0
      break;
428
32.4k
    case NSC_COLOR_FORMAT:
429
32.4k
      context->format = value;
430
32.4k
      break;
431
0
    default:
432
0
      return FALSE;
433
32.4k
  }
434
32.4k
  return TRUE;
435
32.4k
}
436
437
BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp, UINT32 width, UINT32 height,
438
                         const BYTE* WINPR_RESTRICT data, UINT32 length,
439
                         BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat, UINT32 nDstStride,
440
                         UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 flip)
441
524
{
442
524
  wStream* s = NULL;
443
524
  wStream sbuffer = { 0 };
444
524
  BOOL ret = 0;
445
524
  if (!context || !data || !pDstData)
446
0
    return FALSE;
447
448
524
  s = Stream_StaticConstInit(&sbuffer, data, length);
449
450
524
  if (!s)
451
0
    return FALSE;
452
453
524
  if (nDstStride == 0)
454
524
    nDstStride = nWidth * FreeRDPGetBytesPerPixel(DstFormat);
455
456
524
  switch (bpp)
457
524
  {
458
524
    case 32:
459
524
      context->format = PIXEL_FORMAT_BGRA32;
460
524
      break;
461
462
0
    case 24:
463
0
      context->format = PIXEL_FORMAT_BGR24;
464
0
      break;
465
466
0
    case 16:
467
0
      context->format = PIXEL_FORMAT_BGR16;
468
0
      break;
469
470
0
    case 8:
471
0
      context->format = PIXEL_FORMAT_RGB8;
472
0
      break;
473
474
0
    case 4:
475
0
      context->format = PIXEL_FORMAT_A4;
476
0
      break;
477
478
0
    default:
479
0
      return FALSE;
480
524
  }
481
482
524
  context->width = width;
483
524
  context->height = height;
484
524
  ret = nsc_context_initialize(context, s);
485
486
524
  if (!ret)
487
66
    return FALSE;
488
489
  /* RLE decode */
490
458
  {
491
458
    BOOL rc = 0;
492
458
    PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
493
458
    rc = nsc_rle_decompress_data(context);
494
458
    PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)
495
496
458
    if (!rc)
497
183
      return FALSE;
498
458
  }
499
  /* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
500
275
  {
501
275
    BOOL rc = 0;
502
275
    PROFILER_ENTER(context->priv->prof_nsc_decode)
503
275
    rc = context->decode(context);
504
275
    PROFILER_EXIT(context->priv->prof_nsc_decode)
505
506
275
    if (!rc)
507
0
      return FALSE;
508
275
  }
509
510
275
  if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStride, nXDst, nYDst, width, height,
511
275
                                     context->BitmapData, PIXEL_FORMAT_BGRA32, 0, 0, 0, NULL,
512
275
                                     flip))
513
0
    return FALSE;
514
515
275
  return TRUE;
516
275
}