Coverage Report

Created: 2023-09-25 06:56

/src/FreeRDP/libfreerdp/codec/nsc_encode.c
Line
Count
Source (jump to first uncovered line)
1
/**
2
 * FreeRDP: A Remote Desktop Protocol Implementation
3
 * NSCodec Encoder
4
 *
5
 * Copyright 2012 Vic Lee
6
 * Copyright 2016 Armin Novak <armin.novak@thincast.com>
7
 * Copyright 2016 Thincast Technologies GmbH
8
 *
9
 * Licensed under the Apache License, Version 2.0 (the "License");
10
 * you may not use this file except in compliance with the License.
11
 * You may obtain a copy of the License at
12
 *
13
 *     http://www.apache.org/licenses/LICENSE-2.0
14
 *
15
 * Unless required by applicable law or agreed to in writing, software
16
 * distributed under the License is distributed on an "AS IS" BASIS,
17
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
 * See the License for the specific language governing permissions and
19
 * limitations under the License.
20
 */
21
22
#include <freerdp/config.h>
23
24
#include <stdio.h>
25
#include <stdlib.h>
26
#include <string.h>
27
28
#include <winpr/crt.h>
29
30
#include <freerdp/codec/nsc.h>
31
#include <freerdp/codec/color.h>
32
33
#include "nsc_types.h"
34
#include "nsc_encode.h"
35
36
typedef struct
37
{
38
  UINT32 x;
39
  UINT32 y;
40
  UINT32 width;
41
  UINT32 height;
42
  const BYTE* data;
43
  UINT32 scanline;
44
  BYTE* PlaneBuffer;
45
  UINT32 MaxPlaneSize;
46
  BYTE* PlaneBuffers[5];
47
  UINT32 OrgByteCount[4];
48
49
  UINT32 LumaPlaneByteCount;
50
  UINT32 OrangeChromaPlaneByteCount;
51
  UINT32 GreenChromaPlaneByteCount;
52
  UINT32 AlphaPlaneByteCount;
53
  UINT8 ColorLossLevel;
54
  UINT8 ChromaSubsamplingLevel;
55
} NSC_MESSAGE;
56
57
static BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, const NSC_MESSAGE* message);
58
59
static BOOL nsc_context_initialize_encode(NSC_CONTEXT* context)
60
0
{
61
0
  int i;
62
0
  UINT32 length;
63
0
  UINT32 tempWidth;
64
0
  UINT32 tempHeight;
65
0
  tempWidth = ROUND_UP_TO(context->width, 8);
66
0
  tempHeight = ROUND_UP_TO(context->height, 2);
67
  /* The maximum length a decoded plane can reach in all cases */
68
0
  length = tempWidth * tempHeight + 16;
69
70
0
  if (length > context->priv->PlaneBuffersLength)
71
0
  {
72
0
    for (i = 0; i < 5; i++)
73
0
    {
74
0
      BYTE* tmp = (BYTE*)winpr_aligned_recalloc(context->priv->PlaneBuffers[i], length,
75
0
                                                sizeof(BYTE), 32);
76
77
0
      if (!tmp)
78
0
        goto fail;
79
80
0
      context->priv->PlaneBuffers[i] = tmp;
81
0
    }
82
83
0
    context->priv->PlaneBuffersLength = length;
84
0
  }
85
86
0
  if (context->ChromaSubsamplingLevel)
87
0
  {
88
0
    context->OrgByteCount[0] = tempWidth * context->height;
89
0
    context->OrgByteCount[1] = tempWidth * tempHeight / 4;
90
0
    context->OrgByteCount[2] = tempWidth * tempHeight / 4;
91
0
    context->OrgByteCount[3] = context->width * context->height;
92
0
  }
93
0
  else
94
0
  {
95
0
    context->OrgByteCount[0] = context->width * context->height;
96
0
    context->OrgByteCount[1] = context->width * context->height;
97
0
    context->OrgByteCount[2] = context->width * context->height;
98
0
    context->OrgByteCount[3] = context->width * context->height;
99
0
  }
100
101
0
  return TRUE;
102
0
fail:
103
104
0
  if (length > context->priv->PlaneBuffersLength)
105
0
  {
106
0
    for (i = 0; i < 5; i++)
107
0
      winpr_aligned_free(context->priv->PlaneBuffers[i]);
108
0
  }
109
110
0
  return FALSE;
111
0
}
112
113
static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UINT32 scanline)
114
0
{
115
0
  UINT16 x;
116
0
  UINT16 y;
117
0
  UINT16 rw;
118
0
  BYTE ccl;
119
0
  const BYTE* src;
120
0
  const UINT32* src_32;
121
0
  const UINT16* src_16;
122
0
  BYTE* yplane = NULL;
123
0
  BYTE* coplane = NULL;
124
0
  BYTE* cgplane = NULL;
125
0
  BYTE* aplane = NULL;
126
0
  INT16 r_val;
127
0
  INT16 g_val;
128
0
  INT16 b_val;
129
0
  BYTE a_val;
130
0
  UINT32 tempWidth;
131
132
0
  tempWidth = ROUND_UP_TO(context->width, 8);
133
0
  rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
134
0
  ccl = context->ColorLossLevel;
135
136
0
  for (y = 0; y < context->height; y++)
137
0
  {
138
0
    src = data + (context->height - 1 - y) * scanline;
139
0
    yplane = context->priv->PlaneBuffers[0] + y * rw;
140
0
    coplane = context->priv->PlaneBuffers[1] + y * rw;
141
0
    cgplane = context->priv->PlaneBuffers[2] + y * rw;
142
0
    aplane = context->priv->PlaneBuffers[3] + y * context->width;
143
0
    src_32 = (UINT32*)src;
144
0
    src_16 = (UINT16*)src;
145
146
0
    for (x = 0; x < context->width; x++)
147
0
    {
148
0
      switch (context->format)
149
0
      {
150
0
        case PIXEL_FORMAT_BGRX32:
151
0
          b_val = (INT16)(*src_32 & 0xFF);
152
0
          g_val = (INT16)((*src_32 >> 8) & 0xFF);
153
0
          r_val = (INT16)((*src_32 >> 16) & 0xFF);
154
0
          a_val = 0xFF;
155
0
          src_32++;
156
0
          break;
157
158
0
        case PIXEL_FORMAT_BGRA32:
159
0
          b_val = (INT16)(*src_32 & 0xFF);
160
0
          g_val = (INT16)((*src_32 >> 8) & 0xFF);
161
0
          r_val = (INT16)((*src_32 >> 16) & 0xFF);
162
0
          a_val = (INT16)((*src_32 >> 24) & 0xFF);
163
0
          src_32++;
164
0
          break;
165
166
0
        case PIXEL_FORMAT_RGBX32:
167
0
          r_val = (INT16)(*src_32 & 0xFF);
168
0
          g_val = (INT16)((*src_32 >> 8) & 0xFF);
169
0
          b_val = (INT16)((*src_32 >> 16) & 0xFF);
170
0
          a_val = 0xFF;
171
0
          src_32++;
172
0
          break;
173
174
0
        case PIXEL_FORMAT_RGBA32:
175
0
          r_val = (INT16)(*src_32 & 0xFF);
176
0
          g_val = (INT16)((*src_32 >> 8) & 0xFF);
177
0
          b_val = (INT16)((*src_32 >> 16) & 0xFF);
178
0
          a_val = (INT16)((*src_32 >> 24) & 0xFF);
179
0
          src_32++;
180
0
          break;
181
182
0
        case PIXEL_FORMAT_BGR24:
183
0
#ifdef __LITTLE_ENDIAN__
184
0
          b_val = *src++;
185
0
          g_val = *src++;
186
0
          r_val = *src++;
187
#else
188
          r_val = *src++;
189
          g_val = *src++;
190
          b_val = *src++;
191
#endif
192
0
          a_val = 0xFF;
193
0
          break;
194
195
0
        case PIXEL_FORMAT_RGB24:
196
0
#ifdef __LITTLE_ENDIAN__
197
0
          r_val = *src++;
198
0
          g_val = *src++;
199
0
          b_val = *src++;
200
#else
201
          b_val = *src++;
202
          g_val = *src++;
203
          r_val = *src++;
204
#endif
205
0
          a_val = 0xFF;
206
0
          break;
207
208
0
        case PIXEL_FORMAT_BGR16:
209
0
          b_val = (INT16)((*src_16) & 0x1F);
210
0
          g_val = (INT16)((*src_16 >> 5) & 0x3F);
211
0
          r_val = (INT16)((*src_16 >> 11) & 0x1F);
212
0
          a_val = 0xFF;
213
0
          src_16++;
214
0
          break;
215
216
0
        case PIXEL_FORMAT_RGB16:
217
0
          r_val = (INT16)((*src_16) & 0x1F);
218
0
          g_val = (INT16)((*src_16 >> 5) & 0x3F);
219
0
          b_val = (INT16)((*src_16 >> 11) & 0x1F);
220
0
          a_val = 0xFF;
221
0
          src_16++;
222
0
          break;
223
224
0
        case PIXEL_FORMAT_A4:
225
0
        {
226
0
          int shift;
227
0
          BYTE idx;
228
0
          shift = (7 - (x % 8));
229
0
          idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1);
230
0
          idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1);
231
0
          idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2);
232
0
          idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3);
233
0
          idx *= 3;
234
0
          r_val = (INT16)context->palette[idx];
235
0
          g_val = (INT16)context->palette[idx + 1];
236
0
          b_val = (INT16)context->palette[idx + 2];
237
238
0
          if (shift == 0)
239
0
            src_32++;
240
0
        }
241
242
0
          a_val = 0xFF;
243
0
          break;
244
245
0
        case PIXEL_FORMAT_RGB8:
246
0
        {
247
0
          int idx = (*src) * 3;
248
0
          r_val = (INT16)context->palette[idx];
249
0
          g_val = (INT16)context->palette[idx + 1];
250
0
          b_val = (INT16)context->palette[idx + 2];
251
0
          src++;
252
0
        }
253
254
0
          a_val = 0xFF;
255
0
          break;
256
257
0
        default:
258
0
          r_val = g_val = b_val = a_val = 0;
259
0
          break;
260
0
      }
261
262
0
      *yplane++ = (BYTE)((r_val >> 2) + (g_val >> 1) + (b_val >> 2));
263
      /* Perform color loss reduction here */
264
0
      *coplane++ = (BYTE)((r_val - b_val) >> ccl);
265
0
      *cgplane++ = (BYTE)((-(r_val >> 1) + g_val - (b_val >> 1)) >> ccl);
266
0
      *aplane++ = a_val;
267
0
    }
268
269
0
    if (context->ChromaSubsamplingLevel && (x % 2) == 1)
270
0
    {
271
0
      *yplane = *(yplane - 1);
272
0
      *coplane = *(coplane - 1);
273
0
      *cgplane = *(cgplane - 1);
274
0
    }
275
0
  }
276
277
0
  if (context->ChromaSubsamplingLevel && (y % 2) == 1)
278
0
  {
279
0
    yplane = context->priv->PlaneBuffers[0] + y * rw;
280
0
    coplane = context->priv->PlaneBuffers[1] + y * rw;
281
0
    cgplane = context->priv->PlaneBuffers[2] + y * rw;
282
0
    CopyMemory(yplane, yplane - rw, rw);
283
0
    CopyMemory(coplane, coplane - rw, rw);
284
0
    CopyMemory(cgplane, cgplane - rw, rw);
285
0
  }
286
287
0
  return TRUE;
288
0
}
289
290
static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
291
0
{
292
0
  UINT32 y;
293
0
  UINT32 tempWidth;
294
0
  UINT32 tempHeight;
295
296
0
  if (!context)
297
0
    return FALSE;
298
299
0
  tempWidth = ROUND_UP_TO(context->width, 8);
300
0
  tempHeight = ROUND_UP_TO(context->height, 2);
301
302
0
  if (tempHeight == 0)
303
0
    return FALSE;
304
305
0
  if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
306
0
    return FALSE;
307
308
0
  for (y = 0; y < tempHeight >> 1; y++)
309
0
  {
310
0
    UINT32 x;
311
0
    BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
312
0
    BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
313
0
    const INT8* co_src0 = (INT8*)context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
314
0
    const INT8* co_src1 = co_src0 + tempWidth;
315
0
    const INT8* cg_src0 = (INT8*)context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
316
0
    const INT8* cg_src1 = cg_src0 + tempWidth;
317
318
0
    for (x = 0; x < tempWidth >> 1; x++)
319
0
    {
320
0
      *co_dst++ = (BYTE)(((INT16)*co_src0 + (INT16) * (co_src0 + 1) + (INT16)*co_src1 +
321
0
                          (INT16) * (co_src1 + 1)) >>
322
0
                         2);
323
0
      *cg_dst++ = (BYTE)(((INT16)*cg_src0 + (INT16) * (cg_src0 + 1) + (INT16)*cg_src1 +
324
0
                          (INT16) * (cg_src1 + 1)) >>
325
0
                         2);
326
0
      co_src0 += 2;
327
0
      co_src1 += 2;
328
0
      cg_src0 += 2;
329
0
      cg_src1 += 2;
330
0
    }
331
0
  }
332
333
0
  return TRUE;
334
0
}
335
336
BOOL nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata, UINT32 rowstride)
337
0
{
338
0
  if (!context || !bmpdata || (rowstride == 0))
339
0
    return FALSE;
340
341
0
  if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
342
0
    return FALSE;
343
344
0
  if (context->ChromaSubsamplingLevel)
345
0
  {
346
0
    if (!nsc_encode_subsampling(context))
347
0
      return FALSE;
348
0
  }
349
350
0
  return TRUE;
351
0
}
352
353
static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 originalSize)
354
0
{
355
0
  UINT32 left;
356
0
  UINT32 runlength = 1;
357
0
  UINT32 planeSize = 0;
358
0
  left = originalSize;
359
360
  /**
361
   * We quit the loop if the running compressed size is larger than the original.
362
   * In such cases data will be sent uncompressed.
363
   */
364
0
  while (left > 4 && planeSize < originalSize - 4)
365
0
  {
366
0
    if (left > 5 && *in == *(in + 1))
367
0
    {
368
0
      runlength++;
369
0
    }
370
0
    else if (runlength == 1)
371
0
    {
372
0
      *out++ = *in;
373
0
      planeSize++;
374
0
    }
375
0
    else if (runlength < 256)
376
0
    {
377
0
      *out++ = *in;
378
0
      *out++ = *in;
379
0
      *out++ = runlength - 2;
380
0
      runlength = 1;
381
0
      planeSize += 3;
382
0
    }
383
0
    else
384
0
    {
385
0
      *out++ = *in;
386
0
      *out++ = *in;
387
0
      *out++ = 0xFF;
388
0
      *out++ = (runlength & 0x000000FF);
389
0
      *out++ = (runlength & 0x0000FF00) >> 8;
390
0
      *out++ = (runlength & 0x00FF0000) >> 16;
391
0
      *out++ = (runlength & 0xFF000000) >> 24;
392
0
      runlength = 1;
393
0
      planeSize += 7;
394
0
    }
395
396
0
    in++;
397
0
    left--;
398
0
  }
399
400
0
  if (planeSize < originalSize - 4)
401
0
    CopyMemory(out, in, 4);
402
403
0
  planeSize += 4;
404
0
  return planeSize;
405
0
}
406
407
static void nsc_rle_compress_data(NSC_CONTEXT* context)
408
0
{
409
0
  UINT16 i;
410
0
  UINT32 planeSize;
411
0
  UINT32 originalSize;
412
413
0
  for (i = 0; i < 4; i++)
414
0
  {
415
0
    originalSize = context->OrgByteCount[i];
416
417
0
    if (originalSize == 0)
418
0
    {
419
0
      planeSize = 0;
420
0
    }
421
0
    else
422
0
    {
423
0
      planeSize = nsc_rle_encode(context->priv->PlaneBuffers[i],
424
0
                                 context->priv->PlaneBuffers[4], originalSize);
425
426
0
      if (planeSize < originalSize)
427
0
        CopyMemory(context->priv->PlaneBuffers[i], context->priv->PlaneBuffers[4],
428
0
                   planeSize);
429
0
      else
430
0
        planeSize = originalSize;
431
0
    }
432
433
0
    context->PlaneByteCount[i] = planeSize;
434
0
  }
435
0
}
436
437
static UINT32 nsc_compute_byte_count(NSC_CONTEXT* context, UINT32* ByteCount, UINT32 width,
438
                                     UINT32 height)
439
0
{
440
0
  UINT32 tempWidth;
441
0
  UINT32 tempHeight;
442
0
  UINT32 maxPlaneSize;
443
0
  tempWidth = ROUND_UP_TO(width, 8);
444
0
  tempHeight = ROUND_UP_TO(height, 2);
445
0
  maxPlaneSize = tempWidth * tempHeight + 16;
446
0
447
0
  if (context->ChromaSubsamplingLevel)
448
0
  {
449
0
    ByteCount[0] = tempWidth * height;
450
0
    ByteCount[1] = tempWidth * tempHeight / 4;
451
0
    ByteCount[2] = tempWidth * tempHeight / 4;
452
0
    ByteCount[3] = width * height;
453
0
  }
454
0
  else
455
0
  {
456
0
    ByteCount[0] = width * height;
457
0
    ByteCount[1] = width * height;
458
0
    ByteCount[2] = width * height;
459
0
    ByteCount[3] = width * height;
460
0
  }
461
0
462
0
  return maxPlaneSize;
463
0
}
464
465
BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, const NSC_MESSAGE* message)
466
0
{
467
0
  UINT32 totalPlaneByteCount;
468
0
  totalPlaneByteCount = message->LumaPlaneByteCount + message->OrangeChromaPlaneByteCount +
469
0
                        message->GreenChromaPlaneByteCount + message->AlphaPlaneByteCount;
470
471
0
  if (!Stream_EnsureRemainingCapacity(s, 20 + totalPlaneByteCount))
472
0
    return FALSE;
473
474
0
  Stream_Write_UINT32(s, message->LumaPlaneByteCount); /* LumaPlaneByteCount (4 bytes) */
475
0
  Stream_Write_UINT32(
476
0
      s, message->OrangeChromaPlaneByteCount); /* OrangeChromaPlaneByteCount (4 bytes) */
477
0
  Stream_Write_UINT32(
478
0
      s, message->GreenChromaPlaneByteCount);           /* GreenChromaPlaneByteCount (4 bytes) */
479
0
  Stream_Write_UINT32(s, message->AlphaPlaneByteCount); /* AlphaPlaneByteCount (4 bytes) */
480
0
  Stream_Write_UINT8(s, message->ColorLossLevel);       /* ColorLossLevel (1 byte) */
481
0
  Stream_Write_UINT8(s, message->ChromaSubsamplingLevel); /* ChromaSubsamplingLevel (1 byte) */
482
0
  Stream_Write_UINT16(s, 0);                              /* Reserved (2 bytes) */
483
484
0
  if (message->LumaPlaneByteCount)
485
0
    Stream_Write(s, message->PlaneBuffers[0], message->LumaPlaneByteCount); /* LumaPlane */
486
487
0
  if (message->OrangeChromaPlaneByteCount)
488
0
    Stream_Write(s, message->PlaneBuffers[1],
489
0
                 message->OrangeChromaPlaneByteCount); /* OrangeChromaPlane */
490
491
0
  if (message->GreenChromaPlaneByteCount)
492
0
    Stream_Write(s, message->PlaneBuffers[2],
493
0
                 message->GreenChromaPlaneByteCount); /* GreenChromaPlane */
494
495
0
  if (message->AlphaPlaneByteCount)
496
0
    Stream_Write(s, message->PlaneBuffers[3], message->AlphaPlaneByteCount); /* AlphaPlane */
497
498
0
  return TRUE;
499
0
}
500
501
BOOL nsc_compose_message(NSC_CONTEXT* context, wStream* s, const BYTE* data, UINT32 width,
502
                         UINT32 height, UINT32 scanline)
503
0
{
504
0
  BOOL rc;
505
0
  NSC_MESSAGE message = { 0 };
506
507
0
  if (!context || !s || !data)
508
0
    return FALSE;
509
510
0
  context->width = width;
511
0
  context->height = height;
512
513
0
  if (!nsc_context_initialize_encode(context))
514
0
    return FALSE;
515
516
  /* ARGB to AYCoCg conversion, chroma subsampling and colorloss reduction */
517
0
  PROFILER_ENTER(context->priv->prof_nsc_encode)
518
0
  rc = context->encode(context, data, scanline);
519
0
  PROFILER_EXIT(context->priv->prof_nsc_encode)
520
0
  if (!rc)
521
0
    return FALSE;
522
523
  /* RLE encode */
524
0
  PROFILER_ENTER(context->priv->prof_nsc_rle_compress_data)
525
0
  nsc_rle_compress_data(context);
526
0
  PROFILER_EXIT(context->priv->prof_nsc_rle_compress_data)
527
0
  message.PlaneBuffers[0] = context->priv->PlaneBuffers[0];
528
0
  message.PlaneBuffers[1] = context->priv->PlaneBuffers[1];
529
0
  message.PlaneBuffers[2] = context->priv->PlaneBuffers[2];
530
0
  message.PlaneBuffers[3] = context->priv->PlaneBuffers[3];
531
0
  message.LumaPlaneByteCount = context->PlaneByteCount[0];
532
0
  message.OrangeChromaPlaneByteCount = context->PlaneByteCount[1];
533
0
  message.GreenChromaPlaneByteCount = context->PlaneByteCount[2];
534
0
  message.AlphaPlaneByteCount = context->PlaneByteCount[3];
535
0
  message.ColorLossLevel = context->ColorLossLevel;
536
0
  message.ChromaSubsamplingLevel = context->ChromaSubsamplingLevel;
537
0
  return nsc_write_message(context, s, &message);
538
0
}
539
540
BOOL nsc_decompose_message(NSC_CONTEXT* context, wStream* s, BYTE* bmpdata, UINT32 x, UINT32 y,
541
                           UINT32 width, UINT32 height, UINT32 rowstride, UINT32 format,
542
                           UINT32 flip)
543
0
{
544
0
  size_t size = Stream_GetRemainingLength(s);
545
546
0
  if (size > UINT32_MAX)
547
0
    return FALSE;
548
549
0
  if (!nsc_process_message(context, (UINT16)FreeRDPGetBitsPerPixel(context->format), width,
550
0
                           height, Stream_Pointer(s), (UINT32)size, bmpdata, format, rowstride, x,
551
0
                           y, width, height, flip))
552
0
    return FALSE;
553
0
  Stream_Seek(s, size);
554
0
  return TRUE;
555
0
}