Coverage Report

Created: 2024-05-20 06:11

/src/FreeRDP/libfreerdp/codec/nsc.c
Line
Count
Source (jump to first uncovered line)
1
/**
2
 * FreeRDP: A Remote Desktop Protocol Implementation
3
 * NSCodec Codec
4
 *
5
 * Copyright 2011 Samsung, Author Jiten Pathy
6
 * Copyright 2012 Vic Lee
7
 * Copyright 2016 Armin Novak <armin.novak@thincast.com>
8
 * Copyright 2016 Thincast Technologies GmbH
9
 *
10
 * Licensed under the Apache License, Version 2.0 (the "License");
11
 * you may not use this file except in compliance with the License.
12
 * You may obtain a copy of the License at
13
 *
14
 *   http://www.apache.org/licenses/LICENSE-2.0
15
 *
16
 * Unless required by applicable law or agreed to in writing, software
17
 * distributed under the License is distributed on an "AS IS" BASIS,
18
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
 * See the License for the specific language governing permissions and
20
 * limitations under the License.
21
 */
22
23
#include <freerdp/config.h>
24
25
#include <stdio.h>
26
#include <stdlib.h>
27
#include <string.h>
28
29
#include <winpr/crt.h>
30
31
#include <freerdp/codec/nsc.h>
32
#include <freerdp/codec/color.h>
33
34
#include "nsc_types.h"
35
#include "nsc_encode.h"
36
37
#include "nsc_sse2.h"
38
39
#include <freerdp/log.h>
40
#define TAG FREERDP_TAG("codec.nsc")
41
42
#ifndef NSC_INIT_SIMD
43
#define NSC_INIT_SIMD(_nsc_context) \
44
17.1k
  do                              \
45
17.1k
  {                               \
46
17.1k
  } while (0)
47
#endif
48
49
static BOOL nsc_decode(NSC_CONTEXT* context)
50
261
{
51
261
  UINT16 rw = 0;
52
261
  BYTE shift = 0;
53
261
  BYTE* bmpdata = NULL;
54
261
  size_t pos = 0;
55
56
261
  if (!context)
57
0
    return FALSE;
58
59
261
  rw = ROUND_UP_TO(context->width, 8);
60
261
  shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
61
261
  bmpdata = context->BitmapData;
62
63
261
  if (!bmpdata)
64
0
    return FALSE;
65
66
1.37k
  for (UINT32 y = 0; y < context->height; y++)
67
1.11k
  {
68
1.11k
    const BYTE* yplane = NULL;
69
1.11k
    const BYTE* coplane = NULL;
70
1.11k
    const BYTE* cgplane = NULL;
71
1.11k
    const BYTE* aplane = context->priv->PlaneBuffers[3] + y * context->width; /* A */
72
73
1.11k
    if (context->ChromaSubsamplingLevel)
74
770
    {
75
770
      yplane = context->priv->PlaneBuffers[0] + y * rw;                /* Y */
76
770
      coplane = context->priv->PlaneBuffers[1] + (y >> 1) * (rw >> 1); /* Co, supersampled */
77
770
      cgplane = context->priv->PlaneBuffers[2] + (y >> 1) * (rw >> 1); /* Cg, supersampled */
78
770
    }
79
341
    else
80
341
    {
81
341
      yplane = context->priv->PlaneBuffers[0] + y * context->width;  /* Y */
82
341
      coplane = context->priv->PlaneBuffers[1] + y * context->width; /* Co */
83
341
      cgplane = context->priv->PlaneBuffers[2] + y * context->width; /* Cg */
84
341
    }
85
86
15.7k
    for (UINT32 x = 0; x < context->width; x++)
87
14.6k
    {
88
14.6k
      INT16 y_val = (INT16)*yplane;
89
14.6k
      INT16 co_val = (INT16)(INT8)(((INT16)*coplane) << shift);
90
14.6k
      INT16 cg_val = (INT16)(INT8)(((INT16)*cgplane) << shift);
91
14.6k
      INT16 r_val = y_val + co_val - cg_val;
92
14.6k
      INT16 g_val = y_val + cg_val;
93
14.6k
      INT16 b_val = y_val - co_val - cg_val;
94
95
14.6k
      if (pos + 4 > context->BitmapDataLength)
96
0
        return FALSE;
97
98
14.6k
      pos += 4;
99
14.6k
      *bmpdata++ = MINMAX(b_val, 0, 0xFF);
100
14.6k
      *bmpdata++ = MINMAX(g_val, 0, 0xFF);
101
14.6k
      *bmpdata++ = MINMAX(r_val, 0, 0xFF);
102
14.6k
      *bmpdata++ = *aplane;
103
14.6k
      yplane++;
104
14.6k
      coplane += (context->ChromaSubsamplingLevel ? x % 2 : 1);
105
14.6k
      cgplane += (context->ChromaSubsamplingLevel ? x % 2 : 1);
106
14.6k
      aplane++;
107
14.6k
    }
108
1.11k
  }
109
110
261
  return TRUE;
111
261
}
112
113
static BOOL nsc_rle_decode(const BYTE* WINPR_RESTRICT in, size_t inSize, BYTE* WINPR_RESTRICT out,
114
                           UINT32 outSize, UINT32 originalSize)
115
253
{
116
253
  UINT32 left = originalSize;
117
118
2.84k
  while (left > 4)
119
2.73k
  {
120
2.73k
    if (inSize < 1)
121
15
      return FALSE;
122
2.71k
    inSize--;
123
124
2.71k
    const BYTE value = *in++;
125
2.71k
    UINT32 len = 0;
126
127
2.71k
    if (left == 5)
128
77
    {
129
77
      if (outSize < 1)
130
0
        return FALSE;
131
132
77
      outSize--;
133
77
      *out++ = value;
134
77
      left--;
135
77
    }
136
2.63k
    else if (inSize < 1)
137
18
      return FALSE;
138
2.62k
    else if (value == *in)
139
1.10k
    {
140
1.10k
      inSize--;
141
1.10k
      in++;
142
143
1.10k
      if (inSize < 1)
144
8
        return FALSE;
145
1.10k
      else if (*in < 0xFF)
146
964
      {
147
964
        inSize--;
148
964
        len = (UINT32)*in++;
149
964
        len += 2;
150
964
      }
151
137
      else
152
137
      {
153
137
        if (inSize < 5)
154
7
          return FALSE;
155
130
        inSize -= 5;
156
130
        in++;
157
130
        len = ((UINT32)(*in++));
158
130
        len |= ((UINT32)(*in++)) << 8U;
159
130
        len |= ((UINT32)(*in++)) << 16U;
160
130
        len |= ((UINT32)(*in++)) << 24U;
161
130
      }
162
163
1.09k
      if ((outSize < len) || (left < len))
164
95
        return FALSE;
165
166
999
      outSize -= len;
167
999
      FillMemory(out, len, value);
168
999
      out += len;
169
999
      left -= len;
170
999
    }
171
1.51k
    else
172
1.51k
    {
173
1.51k
      if (outSize < 1)
174
0
        return FALSE;
175
176
1.51k
      outSize--;
177
1.51k
      *out++ = value;
178
1.51k
      left--;
179
1.51k
    }
180
2.71k
  }
181
182
110
  if ((outSize < 4) || (left < 4))
183
13
    return FALSE;
184
185
97
  if (inSize < 4)
186
8
    return FALSE;
187
89
  memcpy(out, in, 4);
188
89
  return TRUE;
189
97
}
190
191
static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
192
425
{
193
425
  if (!context)
194
0
    return FALSE;
195
196
425
  const BYTE* rle = context->Planes;
197
425
  size_t rleSize = context->PlanesSize;
198
425
  WINPR_ASSERT(rle);
199
200
1.51k
  for (size_t i = 0; i < 4; i++)
201
1.25k
  {
202
1.25k
    const UINT32 originalSize = context->OrgByteCount[i];
203
1.25k
    const UINT32 planeSize = context->PlaneByteCount[i];
204
205
1.25k
    if (rleSize < planeSize)
206
0
      return FALSE;
207
208
1.25k
    if (planeSize == 0)
209
791
    {
210
791
      if (context->priv->PlaneBuffersLength < originalSize)
211
0
        return FALSE;
212
213
791
      FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
214
791
    }
215
465
    else if (planeSize < originalSize)
216
253
    {
217
253
      if (!nsc_rle_decode(rle, rleSize, context->priv->PlaneBuffers[i],
218
253
                          context->priv->PlaneBuffersLength, originalSize))
219
164
        return FALSE;
220
253
    }
221
212
    else
222
212
    {
223
212
      if (context->priv->PlaneBuffersLength < originalSize)
224
0
        return FALSE;
225
226
212
      if (rleSize < originalSize)
227
0
        return FALSE;
228
229
212
      CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
230
212
    }
231
232
1.09k
    rle += planeSize;
233
1.09k
    rleSize -= planeSize;
234
1.09k
  }
235
236
261
  return TRUE;
237
425
}
238
239
static BOOL nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
240
605
{
241
605
  WINPR_ASSERT(context);
242
605
  WINPR_ASSERT(context->priv);
243
605
  if (!Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, 20))
244
114
    return FALSE;
245
246
491
  size_t total = 0;
247
2.45k
  for (size_t i = 0; i < 4; i++)
248
1.96k
  {
249
1.96k
    Stream_Read_UINT32(s, context->PlaneByteCount[i]);
250
1.96k
    total += context->PlaneByteCount[i];
251
1.96k
  }
252
253
491
  Stream_Read_UINT8(s, context->ColorLossLevel);         /* ColorLossLevel (1 byte) */
254
491
  if ((context->ColorLossLevel < 1) || (context->ColorLossLevel > 7))
255
56
  {
256
56
    WLog_Print(context->priv->log, WLOG_ERROR,
257
56
               "ColorLossLevel=%" PRIu8 " out of range, must be [1,7] inclusive",
258
56
               context->ColorLossLevel);
259
56
    return FALSE;
260
56
  }
261
435
  Stream_Read_UINT8(s, context->ChromaSubsamplingLevel); /* ChromaSubsamplingLevel (1 byte) */
262
435
  Stream_Seek(s, 2);                                     /* Reserved (2 bytes) */
263
435
  context->Planes = Stream_Pointer(s);
264
435
  context->PlanesSize = total;
265
435
  return Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, total);
266
491
}
267
268
static BOOL nsc_context_initialize(NSC_CONTEXT* context, wStream* s)
269
605
{
270
605
  if (!nsc_stream_initialize(context, s))
271
180
    return FALSE;
272
273
425
  const size_t blength = 4ull * context->width * context->height;
274
275
425
  if (!context->BitmapData || (blength > context->BitmapDataLength))
276
417
  {
277
417
    void* tmp = winpr_aligned_recalloc(context->BitmapData, blength + 16, sizeof(BYTE), 32);
278
279
417
    if (!tmp)
280
0
      return FALSE;
281
282
417
    context->BitmapData = tmp;
283
417
    context->BitmapDataLength = blength;
284
417
  }
285
286
425
  const UINT32 tempWidth = ROUND_UP_TO(context->width, 8);
287
425
  const UINT32 tempHeight = ROUND_UP_TO(context->height, 2);
288
  /* The maximum length a decoded plane can reach in all cases */
289
425
  const size_t plength = 1ull * tempWidth * tempHeight;
290
291
425
  if (plength > context->priv->PlaneBuffersLength)
292
357
  {
293
1.78k
    for (size_t i = 0; i < 4; i++)
294
1.42k
    {
295
1.42k
      void* tmp = (BYTE*)winpr_aligned_recalloc(context->priv->PlaneBuffers[i], plength,
296
1.42k
                                                sizeof(BYTE), 32);
297
298
1.42k
      if (!tmp)
299
0
        return FALSE;
300
301
1.42k
      context->priv->PlaneBuffers[i] = tmp;
302
1.42k
    }
303
304
357
    context->priv->PlaneBuffersLength = plength;
305
357
  }
306
307
2.12k
  for (size_t i = 0; i < 4; i++)
308
1.70k
    context->OrgByteCount[i] = context->width * context->height;
309
310
425
  if (context->ChromaSubsamplingLevel)
311
299
  {
312
299
    context->OrgByteCount[0] = tempWidth * context->height;
313
299
    context->OrgByteCount[1] = (tempWidth >> 1) * (tempHeight >> 1);
314
299
    context->OrgByteCount[2] = context->OrgByteCount[1];
315
299
  }
316
317
425
  return TRUE;
318
425
}
319
320
static void nsc_profiler_print(NSC_CONTEXT_PRIV* priv)
321
17.1k
{
322
17.1k
  WINPR_UNUSED(priv);
323
324
17.1k
  PROFILER_PRINT_HEADER
325
17.1k
  PROFILER_PRINT(priv->prof_nsc_rle_decompress_data)
326
17.1k
  PROFILER_PRINT(priv->prof_nsc_decode)
327
17.1k
  PROFILER_PRINT(priv->prof_nsc_rle_compress_data)
328
17.1k
  PROFILER_PRINT(priv->prof_nsc_encode)
329
17.1k
  PROFILER_PRINT_FOOTER
330
17.1k
}
331
332
BOOL nsc_context_reset(NSC_CONTEXT* context, UINT32 width, UINT32 height)
333
0
{
334
0
  if (!context)
335
0
    return FALSE;
336
337
0
  if ((width > UINT16_MAX) || (height > UINT16_MAX))
338
0
    return FALSE;
339
340
0
  context->width = (UINT16)width;
341
0
  context->height = (UINT16)height;
342
0
  return TRUE;
343
0
}
344
345
NSC_CONTEXT* nsc_context_new(void)
346
17.1k
{
347
17.1k
  NSC_CONTEXT* context = (NSC_CONTEXT*)winpr_aligned_calloc(1, sizeof(NSC_CONTEXT), 32);
348
349
17.1k
  if (!context)
350
0
    return NULL;
351
352
17.1k
  context->priv = (NSC_CONTEXT_PRIV*)winpr_aligned_calloc(1, sizeof(NSC_CONTEXT_PRIV), 32);
353
354
17.1k
  if (!context->priv)
355
0
    goto error;
356
357
17.1k
  context->priv->log = WLog_Get("com.freerdp.codec.nsc");
358
17.1k
  WLog_OpenAppender(context->priv->log);
359
17.1k
  context->BitmapData = NULL;
360
17.1k
  context->decode = nsc_decode;
361
17.1k
  context->encode = nsc_encode;
362
363
17.1k
  PROFILER_CREATE(context->priv->prof_nsc_rle_decompress_data, "nsc_rle_decompress_data")
364
17.1k
  PROFILER_CREATE(context->priv->prof_nsc_decode, "nsc_decode")
365
17.1k
  PROFILER_CREATE(context->priv->prof_nsc_rle_compress_data, "nsc_rle_compress_data")
366
17.1k
  PROFILER_CREATE(context->priv->prof_nsc_encode, "nsc_encode")
367
  /* Default encoding parameters */
368
17.1k
  context->ColorLossLevel = 3;
369
17.1k
  context->ChromaSubsamplingLevel = 1;
370
  /* init optimized methods */
371
17.1k
  NSC_INIT_SIMD(context);
372
17.1k
  return context;
373
0
error:
374
0
  WINPR_PRAGMA_DIAG_PUSH
375
0
  WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
376
0
  nsc_context_free(context);
377
0
  WINPR_PRAGMA_DIAG_POP
378
0
  return NULL;
379
17.1k
}
380
381
void nsc_context_free(NSC_CONTEXT* context)
382
17.1k
{
383
17.1k
  if (!context)
384
0
    return;
385
386
17.1k
  if (context->priv)
387
17.1k
  {
388
102k
    for (size_t i = 0; i < 5; i++)
389
85.7k
      winpr_aligned_free(context->priv->PlaneBuffers[i]);
390
391
17.1k
    nsc_profiler_print(context->priv);
392
17.1k
    PROFILER_FREE(context->priv->prof_nsc_rle_decompress_data)
393
17.1k
    PROFILER_FREE(context->priv->prof_nsc_decode)
394
17.1k
    PROFILER_FREE(context->priv->prof_nsc_rle_compress_data)
395
17.1k
    PROFILER_FREE(context->priv->prof_nsc_encode)
396
17.1k
    winpr_aligned_free(context->priv);
397
17.1k
  }
398
399
17.1k
  winpr_aligned_free(context->BitmapData);
400
17.1k
  winpr_aligned_free(context);
401
17.1k
}
402
403
#if defined(WITH_FREERDP_DEPRECATED)
404
BOOL nsc_context_set_pixel_format(NSC_CONTEXT* context, UINT32 pixel_format)
405
{
406
  return nsc_context_set_parameters(context, NSC_COLOR_FORMAT, pixel_format);
407
}
408
#endif
409
410
BOOL nsc_context_set_parameters(NSC_CONTEXT* context, NSC_PARAMETER what, UINT32 value)
411
34.2k
{
412
34.2k
  if (!context)
413
0
    return FALSE;
414
415
34.2k
  switch (what)
416
34.2k
  {
417
0
    case NSC_COLOR_LOSS_LEVEL:
418
0
      context->ColorLossLevel = value;
419
0
      break;
420
0
    case NSC_ALLOW_SUBSAMPLING:
421
0
      context->ChromaSubsamplingLevel = value;
422
0
      break;
423
0
    case NSC_DYNAMIC_COLOR_FIDELITY:
424
0
      context->DynamicColorFidelity = value != 0;
425
0
      break;
426
34.2k
    case NSC_COLOR_FORMAT:
427
34.2k
      context->format = value;
428
34.2k
      break;
429
0
    default:
430
0
      return FALSE;
431
34.2k
  }
432
34.2k
  return TRUE;
433
34.2k
}
434
435
BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp, UINT32 width, UINT32 height,
436
                         const BYTE* WINPR_RESTRICT data, UINT32 length,
437
                         BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat, UINT32 nDstStride,
438
                         UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 flip)
439
605
{
440
605
  wStream* s = NULL;
441
605
  wStream sbuffer = { 0 };
442
605
  BOOL ret = 0;
443
605
  if (!context || !data || !pDstData)
444
0
    return FALSE;
445
446
605
  s = Stream_StaticConstInit(&sbuffer, data, length);
447
448
605
  if (!s)
449
0
    return FALSE;
450
451
605
  if (nDstStride == 0)
452
605
    nDstStride = nWidth * FreeRDPGetBytesPerPixel(DstFormat);
453
454
605
  switch (bpp)
455
605
  {
456
605
    case 32:
457
605
      context->format = PIXEL_FORMAT_BGRA32;
458
605
      break;
459
460
0
    case 24:
461
0
      context->format = PIXEL_FORMAT_BGR24;
462
0
      break;
463
464
0
    case 16:
465
0
      context->format = PIXEL_FORMAT_BGR16;
466
0
      break;
467
468
0
    case 8:
469
0
      context->format = PIXEL_FORMAT_RGB8;
470
0
      break;
471
472
0
    case 4:
473
0
      context->format = PIXEL_FORMAT_A4;
474
0
      break;
475
476
0
    default:
477
0
      return FALSE;
478
605
  }
479
480
605
  context->width = width;
481
605
  context->height = height;
482
605
  ret = nsc_context_initialize(context, s);
483
484
605
  if (!ret)
485
180
    return FALSE;
486
487
  /* RLE decode */
488
425
  {
489
425
    BOOL rc = 0;
490
425
    PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
491
425
    rc = nsc_rle_decompress_data(context);
492
425
    PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)
493
494
425
    if (!rc)
495
164
      return FALSE;
496
425
  }
497
  /* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
498
261
  {
499
261
    BOOL rc = 0;
500
261
    PROFILER_ENTER(context->priv->prof_nsc_decode)
501
261
    rc = context->decode(context);
502
261
    PROFILER_EXIT(context->priv->prof_nsc_decode)
503
504
261
    if (!rc)
505
0
      return FALSE;
506
261
  }
507
508
261
  if (!freerdp_image_copy(pDstData, DstFormat, nDstStride, nXDst, nYDst, width, height,
509
261
                          context->BitmapData, PIXEL_FORMAT_BGRA32, 0, 0, 0, NULL, flip))
510
0
    return FALSE;
511
512
261
  return TRUE;
513
261
}