Coverage Report

Created: 2025-12-31 06:44

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libvncserver/src/libvncserver/translate.c
Line
Count
Source
1
/*
2
 * translate.c - translate between different pixel formats
3
 */
4
5
/*
6
 *  OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
7
 *  Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.  
8
 *  All Rights Reserved.
9
 *
10
 *  This is free software; you can redistribute it and/or modify
11
 *  it under the terms of the GNU General Public License as published by
12
 *  the Free Software Foundation; either version 2 of the License, or
13
 *  (at your option) any later version.
14
 *
15
 *  This software is distributed in the hope that it will be useful,
16
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
17
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18
 *  GNU General Public License for more details.
19
 *
20
 *  You should have received a copy of the GNU General Public License
21
 *  along with this software; if not, write to the Free Software
22
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
23
 *  USA.
24
 */
25
26
#include <rfb/rfb.h>
27
#include <rfb/rfbregion.h>
28
29
static void PrintPixelFormat(rfbPixelFormat *pf);
30
static rfbBool rfbSetClientColourMapBGR233(rfbClientPtr cl);
31
32
rfbBool rfbEconomicTranslate = FALSE;
33
34
/*
35
 * Some standard pixel formats.
36
 */
37
38
static const rfbPixelFormat BGR233Format = {
39
    8, 8, 0, 1, 7, 7, 3, 0, 3, 6, 0, 0
40
};
41
42
43
/*
44
 * Macro to compare pixel formats.
45
 */
46
47
#define PF_EQ(x,y)                                                      \
48
10.6k
        ((x.bitsPerPixel == y.bitsPerPixel) &&                          \
49
10.6k
         (x.depth == y.depth) &&                                        \
50
10.6k
         ((x.bigEndian == y.bigEndian) || (x.bitsPerPixel == 8)) &&     \
51
10.6k
         (!x.trueColour == !y.trueColour) &&                            \
52
10.6k
         (!x.trueColour || ((x.redMax == y.redMax) &&                   \
53
3.39k
                            (x.greenMax == y.greenMax) &&               \
54
3.39k
                            (x.blueMax == y.blueMax) &&                 \
55
3.39k
                            (x.redShift == y.redShift) &&               \
56
3.39k
                            (x.greenShift == y.greenShift) &&           \
57
3.39k
                            (x.blueShift == y.blueShift))))
58
59
2.21M
#define CONCAT2(a,b) a##b
60
2.23M
#define CONCAT2E(a,b) CONCAT2(a,b)
61
25.4k
#define CONCAT3(a,b,c) a##b##c
62
25.4k
#define CONCAT3E(a,b,c) CONCAT3(a,b,c)
63
#define CONCAT4(a,b,c,d) a##b##c##d
64
#define CONCAT4E(a,b,c,d) CONCAT4(a,b,c,d)
65
66
#undef OUT
67
#undef IN
68
69
#define OUT 8
70
#include "tableinitcmtemplate.c"
71
#include "tableinittctemplate.c"
72
#define IN 8
73
#include "tabletranstemplate.c"
74
#undef IN
75
#define IN 16
76
#include "tabletranstemplate.c"
77
#undef IN
78
#define IN 32
79
#include "tabletranstemplate.c"
80
#undef IN
81
#undef OUT
82
83
#define OUT 16
84
#include "tableinitcmtemplate.c"
85
#include "tableinittctemplate.c"
86
#define IN 8
87
#include "tabletranstemplate.c"
88
#undef IN
89
#define IN 16
90
#include "tabletranstemplate.c"
91
#undef IN
92
#define IN 32
93
#include "tabletranstemplate.c"
94
#undef IN
95
#undef OUT
96
97
#define OUT 32
98
#include "tableinitcmtemplate.c"
99
#include "tableinittctemplate.c"
100
#define IN 8
101
#include "tabletranstemplate.c"
102
#undef IN
103
#define IN 16
104
#include "tabletranstemplate.c"
105
#undef IN
106
#define IN 32
107
#include "tabletranstemplate.c"
108
#undef IN
109
#undef OUT
110
111
#ifdef LIBVNCSERVER_ALLOW24BPP
112
#define COUNT_OFFSETS 4
113
30.9k
#define BPP2OFFSET(bpp) ((bpp)/8-1)
114
#include "tableinit24.c"
115
#define BPP 8
116
#include "tabletrans24template.c"
117
#undef BPP
118
#define BPP 16
119
#include "tabletrans24template.c"
120
#undef BPP
121
#define BPP 24
122
#include "tabletrans24template.c"
123
#undef BPP
124
#define BPP 32
125
#include "tabletrans24template.c"
126
#undef BPP
127
#else
128
#define COUNT_OFFSETS 3
129
#define BPP2OFFSET(bpp) ((int)(bpp)/16)
130
#endif
131
132
typedef void (*rfbInitCMTableFnType)(char **table, rfbPixelFormat *in,
133
                                   rfbPixelFormat *out,rfbColourMap* cm);
134
typedef void (*rfbInitTableFnType)(char **table, rfbPixelFormat *in,
135
                                   rfbPixelFormat *out);
136
137
static rfbInitCMTableFnType rfbInitColourMapSingleTableFns[COUNT_OFFSETS] = {
138
    rfbInitColourMapSingleTable8,
139
    rfbInitColourMapSingleTable16,
140
#ifdef LIBVNCSERVER_ALLOW24BPP
141
    rfbInitColourMapSingleTable24,
142
#endif
143
    rfbInitColourMapSingleTable32
144
};
145
146
static rfbInitTableFnType rfbInitTrueColourSingleTableFns[COUNT_OFFSETS] = {
147
    rfbInitTrueColourSingleTable8,
148
    rfbInitTrueColourSingleTable16,
149
#ifdef LIBVNCSERVER_ALLOW24BPP
150
    rfbInitTrueColourSingleTable24,
151
#endif
152
    rfbInitTrueColourSingleTable32
153
};
154
155
static rfbInitTableFnType rfbInitTrueColourRGBTablesFns[COUNT_OFFSETS] = {
156
    rfbInitTrueColourRGBTables8,
157
    rfbInitTrueColourRGBTables16,
158
#ifdef LIBVNCSERVER_ALLOW24BPP
159
    rfbInitTrueColourRGBTables24,
160
#endif
161
    rfbInitTrueColourRGBTables32
162
};
163
164
static rfbTranslateFnType rfbTranslateWithSingleTableFns[COUNT_OFFSETS][COUNT_OFFSETS] = {
165
    { rfbTranslateWithSingleTable8to8,
166
      rfbTranslateWithSingleTable8to16,
167
#ifdef LIBVNCSERVER_ALLOW24BPP
168
      rfbTranslateWithSingleTable8to24,
169
#endif
170
      rfbTranslateWithSingleTable8to32 },
171
    { rfbTranslateWithSingleTable16to8,
172
      rfbTranslateWithSingleTable16to16,
173
#ifdef LIBVNCSERVER_ALLOW24BPP
174
      rfbTranslateWithSingleTable16to24,
175
#endif
176
      rfbTranslateWithSingleTable16to32 },
177
#ifdef LIBVNCSERVER_ALLOW24BPP
178
    { rfbTranslateWithSingleTable24to8,
179
      rfbTranslateWithSingleTable24to16,
180
      rfbTranslateWithSingleTable24to24,
181
      rfbTranslateWithSingleTable24to32 },
182
#endif
183
    { rfbTranslateWithSingleTable32to8,
184
      rfbTranslateWithSingleTable32to16,
185
#ifdef LIBVNCSERVER_ALLOW24BPP
186
      rfbTranslateWithSingleTable32to24,
187
#endif
188
      rfbTranslateWithSingleTable32to32 }
189
};
190
191
static rfbTranslateFnType rfbTranslateWithRGBTablesFns[COUNT_OFFSETS][COUNT_OFFSETS] = {
192
    { rfbTranslateWithRGBTables8to8,
193
      rfbTranslateWithRGBTables8to16,
194
#ifdef LIBVNCSERVER_ALLOW24BPP
195
      rfbTranslateWithRGBTables8to24,
196
#endif
197
      rfbTranslateWithRGBTables8to32 },
198
    { rfbTranslateWithRGBTables16to8,
199
      rfbTranslateWithRGBTables16to16,
200
#ifdef LIBVNCSERVER_ALLOW24BPP
201
      rfbTranslateWithRGBTables16to24,
202
#endif
203
      rfbTranslateWithRGBTables16to32 },
204
#ifdef LIBVNCSERVER_ALLOW24BPP
205
    { rfbTranslateWithRGBTables24to8,
206
      rfbTranslateWithRGBTables24to16,
207
      rfbTranslateWithRGBTables24to24,
208
      rfbTranslateWithRGBTables24to32 },
209
#endif
210
    { rfbTranslateWithRGBTables32to8,
211
      rfbTranslateWithRGBTables32to16,
212
#ifdef LIBVNCSERVER_ALLOW24BPP
213
      rfbTranslateWithRGBTables32to24,
214
#endif
215
      rfbTranslateWithRGBTables32to32 }
216
};
217
218
219
220
/*
221
 * rfbTranslateNone is used when no translation is required.
222
 */
223
224
void
225
rfbTranslateNone(char *table, rfbPixelFormat *in, rfbPixelFormat *out,
226
                 char *iptr, char *optr, int bytesBetweenInputLines,
227
                 int width, int height)
228
0
{
229
0
    int bytesPerOutputLine = width * (out->bitsPerPixel / 8);
230
231
0
    while (height > 0) {
232
0
        memcpy(optr, iptr, bytesPerOutputLine);
233
0
        iptr += bytesBetweenInputLines;
234
0
        optr += bytesPerOutputLine;
235
0
        height--;
236
0
    }
237
0
}
238
239
240
/*
241
 * rfbSetTranslateFunction sets the translation function.
242
 */
243
244
rfbBool
245
rfbSetTranslateFunction(rfbClientPtr cl)
246
10.6k
{
247
10.6k
    rfbLog("Pixel format for client %s:\n",cl->host);
248
10.6k
    PrintPixelFormat(&cl->format);
249
250
    /*
251
     * Check that bits per pixel values are valid
252
     */
253
254
10.6k
    if ((cl->screen->serverFormat.bitsPerPixel != 8) &&
255
10.6k
        (cl->screen->serverFormat.bitsPerPixel != 16) &&
256
10.6k
#ifdef LIBVNCSERVER_ALLOW24BPP
257
10.6k
  (cl->screen->serverFormat.bitsPerPixel != 24) &&
258
10.6k
#endif
259
10.6k
        (cl->screen->serverFormat.bitsPerPixel != 32))
260
0
    {
261
0
        rfbErr("%s: server bits per pixel not 8, 16 or 32 (is %d)\n",
262
0
         "rfbSetTranslateFunction", 
263
0
         cl->screen->serverFormat.bitsPerPixel);
264
0
        rfbCloseClient(cl);
265
0
        return FALSE;
266
0
    }
267
268
10.6k
    if ((cl->format.bitsPerPixel != 8) &&
269
8.30k
        (cl->format.bitsPerPixel != 16) &&
270
6.46k
#ifdef LIBVNCSERVER_ALLOW24BPP
271
6.46k
  (cl->format.bitsPerPixel != 24) &&
272
4.62k
#endif
273
4.62k
        (cl->format.bitsPerPixel != 32))
274
23
    {
275
23
        rfbErr("%s: client bits per pixel not 8, 16 or 32\n",
276
23
                "rfbSetTranslateFunction");
277
23
        rfbCloseClient(cl);
278
23
        return FALSE;
279
23
    }
280
281
10.6k
    if (!cl->format.trueColour && (cl->format.bitsPerPixel != 8)) {
282
1
        rfbErr("rfbSetTranslateFunction: client has colour map "
283
1
                "but %d-bit - can only cope with 8-bit colour maps\n",
284
1
                cl->format.bitsPerPixel);
285
1
        rfbCloseClient(cl);
286
1
        return FALSE;
287
1
    }
288
289
    /*
290
     * bpp is valid, now work out how to translate
291
     */
292
293
10.6k
    if (!cl->format.trueColour) {
294
        /*
295
         * truecolour -> colour map
296
         *
297
         * Set client's colour map to BGR233, then effectively it's
298
         * truecolour as well
299
         */
300
301
2.22k
        if (!rfbSetClientColourMapBGR233(cl))
302
0
            return FALSE;
303
304
2.22k
        cl->format = BGR233Format;
305
2.22k
    }
306
307
    /* truecolour -> truecolour */
308
309
10.6k
    if (PF_EQ(cl->format,cl->screen->serverFormat)) {
310
311
        /* client & server the same */
312
313
315
        rfbLog("no translation needed\n");
314
315
        cl->translateFn = rfbTranslateNone;
315
315
        return TRUE;
316
315
    }
317
318
10.3k
    if ((cl->screen->serverFormat.bitsPerPixel < 16) ||
319
10.3k
        ((!cl->screen->serverFormat.trueColour || !rfbEconomicTranslate) &&
320
10.3k
     (cl->screen->serverFormat.bitsPerPixel == 16))) {
321
322
        /* we can use a single lookup table for <= 16 bpp */
323
324
0
        cl->translateFn = rfbTranslateWithSingleTableFns
325
0
                              [BPP2OFFSET(cl->screen->serverFormat.bitsPerPixel)]
326
0
                                  [BPP2OFFSET(cl->format.bitsPerPixel)];
327
328
0
  if(cl->screen->serverFormat.trueColour)
329
0
    (*rfbInitTrueColourSingleTableFns
330
0
     [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
331
0
               &(cl->screen->serverFormat), &cl->format);
332
0
  else
333
0
    (*rfbInitColourMapSingleTableFns
334
0
     [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
335
0
               &(cl->screen->serverFormat), &cl->format,&cl->screen->colourMap);
336
337
10.3k
    } else {
338
339
        /* otherwise we use three separate tables for red, green and blue */
340
341
10.3k
        cl->translateFn = rfbTranslateWithRGBTablesFns
342
10.3k
                              [BPP2OFFSET(cl->screen->serverFormat.bitsPerPixel)]
343
10.3k
                                  [BPP2OFFSET(cl->format.bitsPerPixel)];
344
345
10.3k
        (*rfbInitTrueColourRGBTablesFns
346
10.3k
            [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
347
10.3k
                                             &(cl->screen->serverFormat), &cl->format);
348
10.3k
    }
349
350
10.3k
    return TRUE;
351
10.6k
}
352
353
354
355
/*
356
 * rfbSetClientColourMapBGR233 sets the client's colour map so that it's
357
 * just like an 8-bit BGR233 true colour client.
358
 */
359
360
static rfbBool
361
rfbSetClientColourMapBGR233(rfbClientPtr cl)
362
2.22k
{
363
2.22k
    union {
364
2.22k
        char bytes[sz_rfbSetColourMapEntriesMsg + 256 * 3 * 2];
365
2.22k
        rfbSetColourMapEntriesMsg msg;
366
2.22k
    } buf;
367
2.22k
    rfbSetColourMapEntriesMsg *scme = &buf.msg;
368
2.22k
    uint16_t *rgb = (uint16_t *)(&buf.bytes[sz_rfbSetColourMapEntriesMsg]);
369
2.22k
    int i, len;
370
2.22k
    int r, g, b;
371
372
2.22k
    if (cl->format.bitsPerPixel != 8 ) {
373
0
        rfbErr("%s: client not 8 bits per pixel\n",
374
0
                "rfbSetClientColourMapBGR233");
375
0
        rfbCloseClient(cl);
376
0
        return FALSE;
377
0
    }
378
    
379
2.22k
    scme->type = rfbSetColourMapEntries;
380
381
2.22k
    scme->firstColour = Swap16IfLE(0);
382
2.22k
    scme->nColours = Swap16IfLE(256);
383
384
2.22k
    len = sz_rfbSetColourMapEntriesMsg;
385
386
2.22k
    i = 0;
387
388
11.1k
    for (b = 0; b < 4; b++) {
389
80.0k
        for (g = 0; g < 8; g++) {
390
640k
            for (r = 0; r < 8; r++) {
391
569k
                rgb[i++] = Swap16IfLE(r * 65535 / 7);
392
569k
                rgb[i++] = Swap16IfLE(g * 65535 / 7);
393
569k
                rgb[i++] = Swap16IfLE(b * 65535 / 3);
394
569k
            }
395
71.1k
        }
396
8.89k
    }
397
398
2.22k
    len += 256 * 3 * 2;
399
400
2.22k
    if (rfbWriteExact(cl, buf.bytes, len) < 0) {
401
0
        rfbLogPerror("rfbSetClientColourMapBGR233: write");
402
0
        rfbCloseClient(cl);
403
0
        return FALSE;
404
0
    }
405
2.22k
    return TRUE;
406
2.22k
}
407
408
/* this function is not called very often, so it needn't be
409
   efficient. */
410
411
/*
412
 * rfbSetClientColourMap is called to set the client's colour map.  If the
413
 * client is a true colour client, we simply update our own translation table
414
 * and mark the whole screen as having been modified.
415
 */
416
417
rfbBool
418
rfbSetClientColourMap(rfbClientPtr cl, int firstColour, int nColours)
419
0
{
420
0
    if (cl->screen->serverFormat.trueColour || !cl->readyForSetColourMapEntries) {
421
0
  return TRUE;
422
0
    }
423
424
0
    if (nColours == 0) {
425
0
  nColours = cl->screen->colourMap.count;
426
0
    }
427
428
0
    if (cl->format.trueColour) {
429
0
  LOCK(cl->updateMutex);
430
0
  (*rfbInitColourMapSingleTableFns
431
0
      [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
432
0
               &cl->screen->serverFormat, &cl->format,&cl->screen->colourMap);
433
434
0
  sraRgnDestroy(cl->modifiedRegion);
435
0
  cl->modifiedRegion =
436
0
    sraRgnCreateRect(0,0,cl->screen->width,cl->screen->height);
437
0
  UNLOCK(cl->updateMutex);
438
439
0
  return TRUE;
440
0
    }
441
442
0
    return rfbSendSetColourMapEntries(cl, firstColour, nColours);
443
0
}
444
445
446
/*
447
 * rfbSetClientColourMaps sets the colour map for each RFB client.
448
 */
449
450
void
451
rfbSetClientColourMaps(rfbScreenInfoPtr rfbScreen, int firstColour, int nColours)
452
0
{
453
0
    rfbClientIteratorPtr i;
454
0
    rfbClientPtr cl;
455
456
0
    i = rfbGetClientIterator(rfbScreen);
457
0
    while((cl = rfbClientIteratorNext(i)))
458
0
      rfbSetClientColourMap(cl, firstColour, nColours);
459
0
    rfbReleaseClientIterator(i);
460
0
}
461
462
static void
463
PrintPixelFormat(rfbPixelFormat *pf)
464
10.6k
{
465
10.6k
    if (pf->bitsPerPixel == 1) {
466
2
        rfbLog("  1 bpp, %s sig bit in each byte is leftmost on the screen.\n",
467
2
               (pf->bigEndian ? "most" : "least"));
468
10.6k
    } else {
469
10.6k
        rfbLog("  %d bpp, depth %d%s\n",pf->bitsPerPixel,pf->depth,
470
10.6k
               ((pf->bitsPerPixel == 8) ? ""
471
10.6k
                : (pf->bigEndian ? ", big endian" : ", little endian")));
472
10.6k
        if (pf->trueColour) {
473
8.42k
            rfbLog("  true colour: max r %d g %d b %d, shift r %d g %d b %d\n",
474
8.42k
                   pf->redMax, pf->greenMax, pf->blueMax,
475
8.42k
                   pf->redShift, pf->greenShift, pf->blueShift);
476
8.42k
        } else {
477
2.23k
            rfbLog("  uses a colour map (not true colour).\n");
478
2.23k
        }
479
10.6k
    }
480
10.6k
}