Coverage Report

Created: 2025-07-01 06:46

/src/FreeRDP/libfreerdp/codec/nsc_encode.c
Line
Count
Source (jump to first uncovered line)
1
/**
2
 * FreeRDP: A Remote Desktop Protocol Implementation
3
 * NSCodec Encoder
4
 *
5
 * Copyright 2012 Vic Lee
6
 * Copyright 2016 Armin Novak <armin.novak@thincast.com>
7
 * Copyright 2016 Thincast Technologies GmbH
8
 *
9
 * Licensed under the Apache License, Version 2.0 (the "License");
10
 * you may not use this file except in compliance with the License.
11
 * You may obtain a copy of the License at
12
 *
13
 *     http://www.apache.org/licenses/LICENSE-2.0
14
 *
15
 * Unless required by applicable law or agreed to in writing, software
16
 * distributed under the License is distributed on an "AS IS" BASIS,
17
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
 * See the License for the specific language governing permissions and
19
 * limitations under the License.
20
 */
21
22
#include <freerdp/config.h>
23
24
#include <stdio.h>
25
#include <stdlib.h>
26
#include <string.h>
27
28
#include <winpr/assert.h>
29
#include <winpr/cast.h>
30
#include <winpr/crt.h>
31
32
#include <freerdp/codec/nsc.h>
33
#include <freerdp/codec/color.h>
34
35
#include "nsc_types.h"
36
#include "nsc_encode.h"
37
38
typedef struct
39
{
40
  UINT32 x;
41
  UINT32 y;
42
  UINT32 width;
43
  UINT32 height;
44
  const BYTE* data;
45
  UINT32 scanline;
46
  BYTE* PlaneBuffer;
47
  UINT32 MaxPlaneSize;
48
  BYTE* PlaneBuffers[5];
49
  UINT32 OrgByteCount[4];
50
51
  UINT32 LumaPlaneByteCount;
52
  UINT32 OrangeChromaPlaneByteCount;
53
  UINT32 GreenChromaPlaneByteCount;
54
  UINT32 AlphaPlaneByteCount;
55
  UINT8 ColorLossLevel;
56
  UINT8 ChromaSubsamplingLevel;
57
} NSC_MESSAGE;
58
59
static BOOL nsc_write_message(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RESTRICT s,
60
                              const NSC_MESSAGE* WINPR_RESTRICT message);
61
62
static BOOL nsc_context_initialize_encode(NSC_CONTEXT* WINPR_RESTRICT context)
63
0
{
64
0
  UINT32 length = 0;
65
0
  UINT32 tempWidth = 0;
66
0
  UINT32 tempHeight = 0;
67
0
  tempWidth = ROUND_UP_TO(context->width, 8);
68
0
  tempHeight = ROUND_UP_TO(context->height, 2);
69
  /* The maximum length a decoded plane can reach in all cases */
70
0
  length = tempWidth * tempHeight + 16;
71
72
0
  if (length > context->priv->PlaneBuffersLength)
73
0
  {
74
0
    for (int i = 0; i < 5; i++)
75
0
    {
76
0
      BYTE* tmp = (BYTE*)winpr_aligned_recalloc(context->priv->PlaneBuffers[i], length,
77
0
                                                sizeof(BYTE), 32);
78
79
0
      if (!tmp)
80
0
        goto fail;
81
82
0
      context->priv->PlaneBuffers[i] = tmp;
83
0
    }
84
85
0
    context->priv->PlaneBuffersLength = length;
86
0
  }
87
88
0
  if (context->ChromaSubsamplingLevel)
89
0
  {
90
0
    context->OrgByteCount[0] = tempWidth * context->height;
91
0
    context->OrgByteCount[1] = tempWidth * tempHeight / 4;
92
0
    context->OrgByteCount[2] = tempWidth * tempHeight / 4;
93
0
    context->OrgByteCount[3] = context->width * context->height;
94
0
  }
95
0
  else
96
0
  {
97
0
    context->OrgByteCount[0] = context->width * context->height;
98
0
    context->OrgByteCount[1] = context->width * context->height;
99
0
    context->OrgByteCount[2] = context->width * context->height;
100
0
    context->OrgByteCount[3] = context->width * context->height;
101
0
  }
102
103
0
  return TRUE;
104
0
fail:
105
106
0
  if (length > context->priv->PlaneBuffersLength)
107
0
  {
108
0
    for (int i = 0; i < 5; i++)
109
0
      winpr_aligned_free(context->priv->PlaneBuffers[i]);
110
0
  }
111
112
0
  return FALSE;
113
0
}
114
115
static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* WINPR_RESTRICT context,
116
                                      const BYTE* WINPR_RESTRICT data, UINT32 scanline)
117
0
{
118
0
  size_t y = 0;
119
0
  const BYTE* src = NULL;
120
0
  BYTE* yplane = NULL;
121
0
  BYTE* coplane = NULL;
122
0
  BYTE* cgplane = NULL;
123
0
  BYTE* aplane = NULL;
124
0
  INT16 r_val = 0;
125
0
  INT16 g_val = 0;
126
0
  INT16 b_val = 0;
127
0
  BYTE a_val = 0;
128
129
0
  UINT16 tempWidth = ROUND_UP_TO(context->width, 8);
130
0
  const UINT16 rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
131
0
  const BYTE ccl = WINPR_ASSERTING_INT_CAST(BYTE, context->ColorLossLevel);
132
133
0
  for (; y < context->height; y++)
134
0
  {
135
0
    src = data + (context->height - 1 - y) * scanline;
136
0
    yplane = context->priv->PlaneBuffers[0] + y * rw;
137
0
    coplane = context->priv->PlaneBuffers[1] + y * rw;
138
0
    cgplane = context->priv->PlaneBuffers[2] + y * rw;
139
0
    aplane = context->priv->PlaneBuffers[3] + y * context->width;
140
141
0
    UINT16 x = 0;
142
0
    for (; x < context->width; x++)
143
0
    {
144
0
      switch (context->format)
145
0
      {
146
0
        case PIXEL_FORMAT_BGRX32:
147
0
          b_val = *src++;
148
0
          g_val = *src++;
149
0
          r_val = *src++;
150
0
          src++;
151
0
          a_val = 0xFF;
152
0
          break;
153
154
0
        case PIXEL_FORMAT_BGRA32:
155
0
          b_val = *src++;
156
0
          g_val = *src++;
157
0
          r_val = *src++;
158
0
          a_val = *src++;
159
0
          break;
160
161
0
        case PIXEL_FORMAT_RGBX32:
162
0
          r_val = *src++;
163
0
          g_val = *src++;
164
0
          b_val = *src++;
165
0
          src++;
166
0
          a_val = 0xFF;
167
0
          break;
168
169
0
        case PIXEL_FORMAT_RGBA32:
170
0
          r_val = *src++;
171
0
          g_val = *src++;
172
0
          b_val = *src++;
173
0
          a_val = *src++;
174
0
          break;
175
176
0
        case PIXEL_FORMAT_BGR24:
177
0
          b_val = *src++;
178
0
          g_val = *src++;
179
0
          r_val = *src++;
180
0
          a_val = 0xFF;
181
0
          break;
182
183
0
        case PIXEL_FORMAT_RGB24:
184
0
          r_val = *src++;
185
0
          g_val = *src++;
186
0
          b_val = *src++;
187
0
          a_val = 0xFF;
188
0
          break;
189
190
0
        case PIXEL_FORMAT_BGR16:
191
0
          b_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
192
0
          g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
193
0
          r_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
194
0
          a_val = 0xFF;
195
0
          src += 2;
196
0
          break;
197
198
0
        case PIXEL_FORMAT_RGB16:
199
0
          r_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
200
0
          g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
201
0
          b_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
202
0
          a_val = 0xFF;
203
0
          src += 2;
204
0
          break;
205
206
0
        case PIXEL_FORMAT_A4:
207
0
        {
208
0
          int shift = 0;
209
0
          BYTE idx = 0;
210
0
          shift = (7 - (x % 8));
211
0
          idx = ((*src) >> shift) & 1;
212
0
          idx |= (((*(src + 1)) >> shift) & 1) << 1;
213
0
          idx |= (((*(src + 2)) >> shift) & 1) << 2;
214
0
          idx |= (((*(src + 3)) >> shift) & 1) << 3;
215
0
          idx *= 3;
216
0
          r_val = (INT16)context->palette[idx];
217
0
          g_val = (INT16)context->palette[idx + 1];
218
0
          b_val = (INT16)context->palette[idx + 2];
219
220
0
          if (shift == 0)
221
0
            src += 4;
222
0
        }
223
224
0
          a_val = 0xFF;
225
0
          break;
226
227
0
        case PIXEL_FORMAT_RGB8:
228
0
        {
229
0
          int idx = (*src) * 3;
230
0
          r_val = (INT16)context->palette[idx];
231
0
          g_val = (INT16)context->palette[idx + 1];
232
0
          b_val = (INT16)context->palette[idx + 2];
233
0
          src++;
234
0
        }
235
236
0
          a_val = 0xFF;
237
0
          break;
238
239
0
        default:
240
0
          r_val = g_val = b_val = a_val = 0;
241
0
          break;
242
0
      }
243
244
0
      *yplane++ = (BYTE)((r_val >> 2) + (g_val >> 1) + (b_val >> 2));
245
      /* Perform color loss reduction here */
246
0
      *coplane++ = (BYTE)((r_val - b_val) >> ccl);
247
0
      *cgplane++ = (BYTE)((-(r_val >> 1) + g_val - (b_val >> 1)) >> ccl);
248
0
      *aplane++ = a_val;
249
0
    }
250
251
0
    if (context->ChromaSubsamplingLevel && (x % 2) == 1)
252
0
    {
253
0
      *yplane = *(yplane - 1);
254
0
      *coplane = *(coplane - 1);
255
0
      *cgplane = *(cgplane - 1);
256
0
    }
257
0
  }
258
259
0
  if (context->ChromaSubsamplingLevel && (y % 2) == 1)
260
0
  {
261
0
    yplane = context->priv->PlaneBuffers[0] + y * rw;
262
0
    coplane = context->priv->PlaneBuffers[1] + y * rw;
263
0
    cgplane = context->priv->PlaneBuffers[2] + y * rw;
264
0
    CopyMemory(yplane, yplane - rw, rw);
265
0
    CopyMemory(coplane, coplane - rw, rw);
266
0
    CopyMemory(cgplane, cgplane - rw, rw);
267
0
  }
268
269
0
  return TRUE;
270
0
}
271
272
static BOOL nsc_encode_subsampling(NSC_CONTEXT* WINPR_RESTRICT context)
273
0
{
274
0
  UINT32 tempWidth = 0;
275
0
  UINT32 tempHeight = 0;
276
277
0
  if (!context)
278
0
    return FALSE;
279
280
0
  tempWidth = ROUND_UP_TO(context->width, 8);
281
0
  tempHeight = ROUND_UP_TO(context->height, 2);
282
283
0
  if (tempHeight == 0)
284
0
    return FALSE;
285
286
0
  if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
287
0
    return FALSE;
288
289
0
  for (size_t y = 0; y < tempHeight >> 1; y++)
290
0
  {
291
0
    BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
292
0
    BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
293
0
    const INT8* co_src0 = (INT8*)context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
294
0
    const INT8* co_src1 = co_src0 + tempWidth;
295
0
    const INT8* cg_src0 = (INT8*)context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
296
0
    const INT8* cg_src1 = cg_src0 + tempWidth;
297
298
0
    for (UINT32 x = 0; x < tempWidth >> 1; x++)
299
0
    {
300
0
      *co_dst++ = (BYTE)(((INT16)*co_src0 + (INT16) * (co_src0 + 1) + (INT16)*co_src1 +
301
0
                          (INT16) * (co_src1 + 1)) >>
302
0
                         2);
303
0
      *cg_dst++ = (BYTE)(((INT16)*cg_src0 + (INT16) * (cg_src0 + 1) + (INT16)*cg_src1 +
304
0
                          (INT16) * (cg_src1 + 1)) >>
305
0
                         2);
306
0
      co_src0 += 2;
307
0
      co_src1 += 2;
308
0
      cg_src0 += 2;
309
0
      cg_src1 += 2;
310
0
    }
311
0
  }
312
313
0
  return TRUE;
314
0
}
315
316
BOOL nsc_encode(NSC_CONTEXT* WINPR_RESTRICT context, const BYTE* WINPR_RESTRICT bmpdata,
317
                UINT32 rowstride)
318
0
{
319
0
  if (!context || !bmpdata || (rowstride == 0))
320
0
    return FALSE;
321
322
0
  if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
323
0
    return FALSE;
324
325
0
  if (context->ChromaSubsamplingLevel)
326
0
  {
327
0
    if (!nsc_encode_subsampling(context))
328
0
      return FALSE;
329
0
  }
330
331
0
  return TRUE;
332
0
}
333
334
static UINT32 nsc_rle_encode(const BYTE* WINPR_RESTRICT in, BYTE* WINPR_RESTRICT out,
335
                             UINT32 originalSize)
336
0
{
337
0
  UINT32 left = 0;
338
0
  UINT32 runlength = 1;
339
0
  UINT32 planeSize = 0;
340
0
  left = originalSize;
341
342
  /**
343
   * We quit the loop if the running compressed size is larger than the original.
344
   * In such cases data will be sent uncompressed.
345
   */
346
0
  while (left > 4 && planeSize < originalSize - 4)
347
0
  {
348
0
    if (left > 5 && *in == *(in + 1))
349
0
    {
350
0
      runlength++;
351
0
    }
352
0
    else if (runlength == 1)
353
0
    {
354
0
      *out++ = *in;
355
0
      planeSize++;
356
0
    }
357
0
    else if (runlength < 256)
358
0
    {
359
0
      *out++ = *in;
360
0
      *out++ = *in;
361
0
      WINPR_ASSERT(runlength >= 2);
362
0
      *out++ = WINPR_ASSERTING_INT_CAST(BYTE, runlength - 2);
363
0
      runlength = 1;
364
0
      planeSize += 3;
365
0
    }
366
0
    else
367
0
    {
368
0
      *out++ = *in;
369
0
      *out++ = *in;
370
0
      *out++ = 0xFF;
371
0
      *out++ = (runlength & 0x000000FF);
372
0
      *out++ = (runlength & 0x0000FF00) >> 8;
373
0
      *out++ = (runlength & 0x00FF0000) >> 16;
374
0
      *out++ = (runlength & 0xFF000000) >> 24;
375
0
      runlength = 1;
376
0
      planeSize += 7;
377
0
    }
378
379
0
    in++;
380
0
    left--;
381
0
  }
382
383
0
  if (planeSize < originalSize - 4)
384
0
    CopyMemory(out, in, 4);
385
386
0
  planeSize += 4;
387
0
  return planeSize;
388
0
}
389
390
static void nsc_rle_compress_data(NSC_CONTEXT* WINPR_RESTRICT context)
391
0
{
392
0
  UINT32 planeSize = 0;
393
0
  UINT32 originalSize = 0;
394
395
0
  for (UINT16 i = 0; i < 4; i++)
396
0
  {
397
0
    originalSize = context->OrgByteCount[i];
398
399
0
    if (originalSize == 0)
400
0
    {
401
0
      planeSize = 0;
402
0
    }
403
0
    else
404
0
    {
405
0
      planeSize = nsc_rle_encode(context->priv->PlaneBuffers[i],
406
0
                                 context->priv->PlaneBuffers[4], originalSize);
407
408
0
      if (planeSize < originalSize)
409
0
        CopyMemory(context->priv->PlaneBuffers[i], context->priv->PlaneBuffers[4],
410
0
                   planeSize);
411
0
      else
412
0
        planeSize = originalSize;
413
0
    }
414
415
0
    context->PlaneByteCount[i] = planeSize;
416
0
  }
417
0
}
418
419
BOOL nsc_write_message(WINPR_ATTR_UNUSED NSC_CONTEXT* WINPR_RESTRICT context,
420
                       wStream* WINPR_RESTRICT s, const NSC_MESSAGE* WINPR_RESTRICT message)
421
0
{
422
0
  UINT32 totalPlaneByteCount = 0;
423
0
  totalPlaneByteCount = message->LumaPlaneByteCount + message->OrangeChromaPlaneByteCount +
424
0
                        message->GreenChromaPlaneByteCount + message->AlphaPlaneByteCount;
425
426
0
  if (!Stream_EnsureRemainingCapacity(s, 20 + totalPlaneByteCount))
427
0
    return FALSE;
428
429
0
  Stream_Write_UINT32(s, message->LumaPlaneByteCount); /* LumaPlaneByteCount (4 bytes) */
430
0
  Stream_Write_UINT32(
431
0
      s, message->OrangeChromaPlaneByteCount); /* OrangeChromaPlaneByteCount (4 bytes) */
432
0
  Stream_Write_UINT32(
433
0
      s, message->GreenChromaPlaneByteCount);           /* GreenChromaPlaneByteCount (4 bytes) */
434
0
  Stream_Write_UINT32(s, message->AlphaPlaneByteCount); /* AlphaPlaneByteCount (4 bytes) */
435
0
  Stream_Write_UINT8(s, message->ColorLossLevel);       /* ColorLossLevel (1 byte) */
436
0
  Stream_Write_UINT8(s, message->ChromaSubsamplingLevel); /* ChromaSubsamplingLevel (1 byte) */
437
0
  Stream_Write_UINT16(s, 0);                              /* Reserved (2 bytes) */
438
439
0
  if (message->LumaPlaneByteCount)
440
0
    Stream_Write(s, message->PlaneBuffers[0], message->LumaPlaneByteCount); /* LumaPlane */
441
442
0
  if (message->OrangeChromaPlaneByteCount)
443
0
    Stream_Write(s, message->PlaneBuffers[1],
444
0
                 message->OrangeChromaPlaneByteCount); /* OrangeChromaPlane */
445
446
0
  if (message->GreenChromaPlaneByteCount)
447
0
    Stream_Write(s, message->PlaneBuffers[2],
448
0
                 message->GreenChromaPlaneByteCount); /* GreenChromaPlane */
449
450
0
  if (message->AlphaPlaneByteCount)
451
0
    Stream_Write(s, message->PlaneBuffers[3], message->AlphaPlaneByteCount); /* AlphaPlane */
452
453
0
  return TRUE;
454
0
}
455
456
BOOL nsc_compose_message(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RESTRICT s,
457
                         const BYTE* WINPR_RESTRICT data, UINT32 width, UINT32 height,
458
                         UINT32 scanline)
459
0
{
460
0
  BOOL rc = 0;
461
0
  NSC_MESSAGE message = { 0 };
462
463
0
  if (!context || !s || !data)
464
0
    return FALSE;
465
466
0
  context->width = WINPR_ASSERTING_INT_CAST(UINT16, width);
467
0
  context->height = WINPR_ASSERTING_INT_CAST(UINT16, height);
468
469
0
  if (!nsc_context_initialize_encode(context))
470
0
    return FALSE;
471
472
  /* ARGB to AYCoCg conversion, chroma subsampling and colorloss reduction */
473
0
  PROFILER_ENTER(context->priv->prof_nsc_encode)
474
0
  rc = context->encode(context, data, scanline);
475
0
  PROFILER_EXIT(context->priv->prof_nsc_encode)
476
0
  if (!rc)
477
0
    return FALSE;
478
479
  /* RLE encode */
480
0
  PROFILER_ENTER(context->priv->prof_nsc_rle_compress_data)
481
0
  nsc_rle_compress_data(context);
482
0
  PROFILER_EXIT(context->priv->prof_nsc_rle_compress_data)
483
0
  message.PlaneBuffers[0] = context->priv->PlaneBuffers[0];
484
0
  message.PlaneBuffers[1] = context->priv->PlaneBuffers[1];
485
0
  message.PlaneBuffers[2] = context->priv->PlaneBuffers[2];
486
0
  message.PlaneBuffers[3] = context->priv->PlaneBuffers[3];
487
0
  message.LumaPlaneByteCount = context->PlaneByteCount[0];
488
0
  message.OrangeChromaPlaneByteCount = context->PlaneByteCount[1];
489
0
  message.GreenChromaPlaneByteCount = context->PlaneByteCount[2];
490
0
  message.AlphaPlaneByteCount = context->PlaneByteCount[3];
491
492
0
  message.ColorLossLevel = WINPR_ASSERTING_INT_CAST(BYTE, context->ColorLossLevel);
493
0
  message.ChromaSubsamplingLevel =
494
0
      WINPR_ASSERTING_INT_CAST(BYTE, context->ChromaSubsamplingLevel);
495
0
  return nsc_write_message(context, s, &message);
496
0
}
497
498
BOOL nsc_decompose_message(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RESTRICT s,
499
                           BYTE* WINPR_RESTRICT bmpdata, UINT32 x, UINT32 y, UINT32 width,
500
                           UINT32 height, UINT32 rowstride, UINT32 format, UINT32 flip)
501
0
{
502
0
  size_t size = Stream_GetRemainingLength(s);
503
504
0
  if (size > UINT32_MAX)
505
0
    return FALSE;
506
507
0
  if (!nsc_process_message(context, (UINT16)FreeRDPGetBitsPerPixel(context->format), width,
508
0
                           height, Stream_Pointer(s), (UINT32)size, bmpdata, format, rowstride, x,
509
0
                           y, width, height, flip))
510
0
    return FALSE;
511
0
  Stream_Seek(s, size);
512
0
  return TRUE;
513
0
}