Coverage Report

Created: 2025-07-18 07:06

/src/libvncserver/src/libvncserver/zlib.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * zlib.c
3
 *
4
 * Routines to implement zlib based encoding (deflate).
5
 */
6
7
/*
8
 *  Copyright (C) 2000 Tridia Corporation.  All Rights Reserved.
9
 *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
10
 *
11
 *  This is free software; you can redistribute it and/or modify
12
 *  it under the terms of the GNU General Public License as published by
13
 *  the Free Software Foundation; either version 2 of the License, or
14
 *  (at your option) any later version.
15
 *
16
 *  This software is distributed in the hope that it will be useful,
17
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
18
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19
 *  GNU General Public License for more details.
20
 *
21
 *  You should have received a copy of the GNU General Public License
22
 *  along with this software; if not, write to the Free Software
23
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
24
 *  USA.
25
 *
26
 * For the latest source code, please check:
27
 *
28
 * http://www.developVNC.org/
29
 *
30
 * or send email to feedback@developvnc.org.
31
 */
32
33
#include <rfb/rfb.h>
34
35
/*
36
 * cl->beforeEncBuf contains pixel data in the client's format.
37
 * cl->afterEncBuf contains the zlib (deflated) encoding version.
38
 * If the zlib compressed/encoded version is
39
 * larger than the raw data or if it exceeds cl->afterEncBufSize then
40
 * raw encoding is used instead.
41
 */
42
43
/*
44
 * rfbSendOneRectEncodingZlib - send a given rectangle using one Zlib
45
 *                              rectangle encoding.
46
 */
47
48
static rfbBool
49
rfbSendOneRectEncodingZlib(rfbClientPtr cl,
50
                           int x,
51
                           int y,
52
                           int w,
53
                           int h)
54
0
{
55
0
    rfbFramebufferUpdateRectHeader rect;
56
0
    rfbZlibHeader hdr;
57
0
    int deflateResult;
58
0
    int previousOut;
59
0
    int i;
60
0
    char *fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y)
61
0
         + (x * (cl->scaledScreen->bitsPerPixel / 8)));
62
63
0
    int maxRawSize;
64
0
    int maxCompSize;
65
66
0
    maxRawSize = (cl->scaledScreen->width * cl->scaledScreen->height
67
0
                  * (cl->format.bitsPerPixel / 8));
68
69
0
    if (!cl->beforeEncBuf || cl->beforeEncBufSize < maxRawSize) {
70
0
        if (cl->beforeEncBuf == NULL)
71
0
            cl->beforeEncBuf = (char *)malloc(maxRawSize);
72
0
        else {
73
0
            char *reallocedBeforeEncBuf = (char *)realloc(cl->beforeEncBuf, maxRawSize);
74
0
            if (!reallocedBeforeEncBuf) return FALSE;
75
0
            cl->beforeEncBuf = reallocedBeforeEncBuf;
76
0
        }
77
0
        if(cl->beforeEncBuf)
78
0
            cl->beforeEncBufSize = maxRawSize;
79
0
    }
80
81
    /* zlib compression is not useful for very small data sets.
82
     * So, we just send these raw without any compression.
83
     */
84
0
    if (( w * h * (cl->scaledScreen->bitsPerPixel / 8)) <
85
0
          VNC_ENCODE_ZLIB_MIN_COMP_SIZE ) {
86
87
0
        int result;
88
89
        /* The translation function (used also by the in raw encoding)
90
         * requires 4/2/1 byte alignment in the output buffer (which is
91
         * updateBuf for the raw encoding) based on the bitsPerPixel of
92
         * the viewer/client.  This prevents SIGBUS errors on some
93
         * architectures like SPARC, PARISC...
94
         */
95
0
        if (( cl->format.bitsPerPixel > 8 ) &&
96
0
            ( cl->ublen % ( cl->format.bitsPerPixel / 8 )) != 0 ) {
97
0
            if (!rfbSendUpdateBuf(cl))
98
0
                return FALSE;
99
0
        }
100
101
0
        result = rfbSendRectEncodingRaw(cl, x, y, w, h);
102
103
0
        return result;
104
105
0
    }
106
107
    /*
108
     * zlib requires output buffer to be slightly larger than the input
109
     * buffer, in the worst case.
110
     */
111
0
    maxCompSize = maxRawSize + (( maxRawSize + 99 ) / 100 ) + 12;
112
113
0
    if (!cl->afterEncBuf || cl->afterEncBufSize < maxCompSize) {
114
0
        if (cl->afterEncBuf == NULL)
115
0
            cl->afterEncBuf = (char *)malloc(maxCompSize);
116
0
        else {
117
0
            char *reallocedAfterEncBuf = (char *)realloc(cl->afterEncBuf, maxCompSize);
118
0
            if (!reallocedAfterEncBuf) return FALSE;
119
0
            cl->afterEncBuf = reallocedAfterEncBuf;
120
0
        }
121
0
        if(cl->afterEncBuf)
122
0
            cl->afterEncBufSize = maxCompSize;
123
0
    }
124
125
0
    if (!cl->beforeEncBuf || !cl->afterEncBuf)
126
0
    {
127
0
        rfbLog("rfbSendOneRectEncodingZlib: failed to allocate memory\n");
128
0
        return FALSE;
129
0
    }
130
131
    /* 
132
     * Convert pixel data to client format.
133
     */
134
0
    (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
135
0
               &cl->format, fbptr, cl->beforeEncBuf,
136
0
           cl->scaledScreen->paddedWidthInBytes, w, h);
137
138
0
    cl->compStream.next_in = ( Bytef * )cl->beforeEncBuf;
139
0
    cl->compStream.avail_in = w * h * (cl->format.bitsPerPixel / 8);
140
0
    cl->compStream.next_out = ( Bytef * )cl->afterEncBuf;
141
0
    cl->compStream.avail_out = maxCompSize;
142
0
    cl->compStream.data_type = Z_BINARY;
143
144
    /* Initialize the deflation state. */
145
0
    if ( cl->compStreamInited == FALSE ) {
146
147
0
        cl->compStream.total_in = 0;
148
0
        cl->compStream.total_out = 0;
149
0
        cl->compStream.zalloc = Z_NULL;
150
0
        cl->compStream.zfree = Z_NULL;
151
0
        cl->compStream.opaque = Z_NULL;
152
153
0
        deflateInit2( &(cl->compStream),
154
0
                        cl->zlibCompressLevel,
155
0
                        Z_DEFLATED,
156
0
                        MAX_WBITS,
157
0
                        MAX_MEM_LEVEL,
158
0
                        Z_DEFAULT_STRATEGY );
159
        /* deflateInit( &(cl->compStream), Z_BEST_COMPRESSION ); */
160
        /* deflateInit( &(cl->compStream), Z_BEST_SPEED ); */
161
0
        cl->compStreamInited = TRUE;
162
163
0
    }
164
165
0
    previousOut = cl->compStream.total_out;
166
167
    /* Perform the compression here. */
168
0
    deflateResult = deflate( &(cl->compStream), Z_SYNC_FLUSH );
169
170
    /* Find the total size of the resulting compressed data. */
171
0
    cl->afterEncBufLen = cl->compStream.total_out - previousOut;
172
173
0
    if ( deflateResult != Z_OK ) {
174
0
        rfbErr("zlib deflation error: %s\n", cl->compStream.msg);
175
0
        return FALSE;
176
0
    }
177
178
    /* Note that it is not possible to switch zlib parameters based on
179
     * the results of the compression pass.  The reason is
180
     * that we rely on the compressor and decompressor states being
181
     * in sync.  Compressing and then discarding the results would
182
     * cause lose of synchronization.
183
     */
184
185
    /* Update statics */
186
0
    rfbStatRecordEncodingSent(cl, rfbEncodingZlib, sz_rfbFramebufferUpdateRectHeader + sz_rfbZlibHeader + cl->afterEncBufLen,
187
0
        + w * (cl->format.bitsPerPixel / 8) * h);
188
189
0
    if (cl->ublen + sz_rfbFramebufferUpdateRectHeader + sz_rfbZlibHeader
190
0
  > UPDATE_BUF_SIZE)
191
0
    {
192
0
  if (!rfbSendUpdateBuf(cl))
193
0
      return FALSE;
194
0
    }
195
196
0
    rect.r.x = Swap16IfLE(x);
197
0
    rect.r.y = Swap16IfLE(y);
198
0
    rect.r.w = Swap16IfLE(w);
199
0
    rect.r.h = Swap16IfLE(h);
200
0
    rect.encoding = Swap32IfLE(rfbEncodingZlib);
201
202
0
    memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
203
0
     sz_rfbFramebufferUpdateRectHeader);
204
0
    cl->ublen += sz_rfbFramebufferUpdateRectHeader;
205
206
0
    hdr.nBytes = Swap32IfLE(cl->afterEncBufLen);
207
208
0
    memcpy(&cl->updateBuf[cl->ublen], (char *)&hdr, sz_rfbZlibHeader);
209
0
    cl->ublen += sz_rfbZlibHeader;
210
211
0
    for (i = 0; i < cl->afterEncBufLen;) {
212
213
0
  int bytesToCopy = UPDATE_BUF_SIZE - cl->ublen;
214
215
0
    if (i + bytesToCopy > cl->afterEncBufLen) {
216
0
        bytesToCopy = cl->afterEncBufLen - i;
217
0
  }
218
219
0
    memcpy(&cl->updateBuf[cl->ublen], &cl->afterEncBuf[i], bytesToCopy);
220
221
0
  cl->ublen += bytesToCopy;
222
0
  i += bytesToCopy;
223
224
0
  if (cl->ublen == UPDATE_BUF_SIZE) {
225
0
      if (!rfbSendUpdateBuf(cl))
226
0
    return FALSE;
227
0
  }
228
0
    }
229
230
0
    return TRUE;
231
232
0
}
233
234
235
/*
236
 * rfbSendRectEncodingZlib - send a given rectangle using one or more
237
 *                           Zlib encoding rectangles.
238
 */
239
240
rfbBool
241
rfbSendRectEncodingZlib(rfbClientPtr cl,
242
                        int x,
243
                        int y,
244
                        int w,
245
                        int h)
246
0
{
247
0
    int  maxLines;
248
0
    int  linesRemaining;
249
0
    rfbRectangle partialRect;
250
251
0
    partialRect.x = x;
252
0
    partialRect.y = y;
253
0
    partialRect.w = w;
254
0
    partialRect.h = h;
255
256
    /* Determine maximum pixel/scan lines allowed per rectangle. */
257
0
    maxLines = ( ZLIB_MAX_SIZE(w) / w );
258
259
    /* Initialize number of scan lines left to do. */
260
0
    linesRemaining = h;
261
262
    /* Loop until all work is done. */
263
0
    while ( linesRemaining > 0 ) {
264
265
0
        int linesToComp;
266
267
0
        if ( maxLines < linesRemaining )
268
0
            linesToComp = maxLines;
269
0
        else
270
0
            linesToComp = linesRemaining;
271
272
0
        partialRect.h = linesToComp;
273
274
        /* Encode (compress) and send the next rectangle. */
275
0
        if ( ! rfbSendOneRectEncodingZlib( cl,
276
0
                                           partialRect.x,
277
0
                                           partialRect.y,
278
0
                                           partialRect.w,
279
0
                                           partialRect.h )) {
280
281
0
            return FALSE;
282
0
        }
283
284
        /* Technically, flushing the buffer here is not extremely
285
         * efficient.  However, this improves the overall throughput
286
         * of the system over very slow networks.  By flushing
287
         * the buffer with every maximum size zlib rectangle, we
288
         * improve the pipelining usage of the server CPU, network,
289
         * and viewer CPU components.  Insuring that these components
290
         * are working in parallel actually improves the performance
291
         * seen by the user.
292
         * Since, zlib is most useful for slow networks, this flush
293
         * is appropriate for the desired behavior of the zlib encoding.
294
         */
295
0
        if (( cl->ublen > 0 ) &&
296
0
            ( linesToComp == maxLines )) {
297
0
            if (!rfbSendUpdateBuf(cl)) {
298
299
0
                return FALSE;
300
0
            }
301
0
        }
302
303
        /* Update remaining and incremental rectangle location. */
304
0
        linesRemaining -= linesToComp;
305
0
        partialRect.y += linesToComp;
306
307
0
    }
308
309
0
    return TRUE;
310
311
0
}
312