/src/FreeRDP/libfreerdp/codec/planar.c

Source
/**
 * FreeRDP: A Remote Desktop Protocol Implementation
 * RDP6 Planar Codec
 *
 * Copyright 2013 Marc-Andre Moreau <marcandre.moreau@gmail.com>
 * Copyright 2016 Armin Novak <armin.novak@thincast.com>
 * Copyright 2016 Thincast Technologies GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <freerdp/config.h>

#include <winpr/crt.h>
#include <winpr/wtypes.h>
#include <winpr/assert.h>
#include <winpr/cast.h>
#include <winpr/print.h>

#include <freerdp/primitives.h>
#include <freerdp/log.h>
#include <freerdp/codec/bitmap.h>
#include <freerdp/codec/planar.h>

#define TAG FREERDP_TAG("codec")

#define PLANAR_ALIGN(val, align) \
  ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))

typedef struct
{
  /**
   * controlByte:
   * [0-3]: nRunLength
   * [4-7]: cRawBytes
   */
  BYTE controlByte;
  BYTE* rawValues;
} RDP6_RLE_SEGMENT;

typedef struct
{
  UINT32 cSegments;
  RDP6_RLE_SEGMENT* segments;
} RDP6_RLE_SEGMENTS;

typedef struct
{
  /**
   * formatHeader:
   * [0-2]: Color Loss Level (CLL)
   *  [3] : Chroma Subsampling (CS)
   *  [4] : Run Length Encoding (RLE)
   *  [5] : No Alpha (NA)
   * [6-7]: Reserved
   */
  BYTE formatHeader;
} RDP6_BITMAP_STREAM;

struct S_BITMAP_PLANAR_CONTEXT
{
  UINT32 maxWidth;
  UINT32 maxHeight;
  UINT32 maxPlaneSize;

  BOOL AllowSkipAlpha;
  BOOL AllowRunLengthEncoding;
  BOOL AllowColorSubsampling;
  BOOL AllowDynamicColorFidelity;

  UINT32 ColorLossLevel;

  BYTE* planes[4];
  BYTE* planesBuffer;

  BYTE* deltaPlanes[4];
  BYTE* deltaPlanesBuffer;

  BYTE* rlePlanes[4];
  BYTE* rlePlanesBuffer;

  BYTE* pTempData;
  UINT32 nTempStep;

  BOOL bgr;
  BOOL topdown;
};

static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes)
{
  return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4)));
}

static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte)
{
  return (controlByte & 0x0F);
}
static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
{
  return ((controlByte >> 4) & 0x0F);
}

static inline UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
                                          UINT32 DstFormat)
{
  WINPR_ASSERT(planar);
  if (planar->bgr && alpha)
  {
    switch (DstFormat)
    {
      case PIXEL_FORMAT_ARGB32:
        DstFormat = PIXEL_FORMAT_ABGR32;
        break;
      case PIXEL_FORMAT_XRGB32:
        DstFormat = PIXEL_FORMAT_XBGR32;
        break;
      case PIXEL_FORMAT_ABGR32:
        DstFormat = PIXEL_FORMAT_ARGB32;
        break;
      case PIXEL_FORMAT_XBGR32:
        DstFormat = PIXEL_FORMAT_XRGB32;
        break;
      case PIXEL_FORMAT_BGRA32:
        DstFormat = PIXEL_FORMAT_RGBA32;
        break;
      case PIXEL_FORMAT_BGRX32:
        DstFormat = PIXEL_FORMAT_RGBX32;
        break;
      case PIXEL_FORMAT_RGBA32:
        DstFormat = PIXEL_FORMAT_BGRA32;
        break;
      case PIXEL_FORMAT_RGBX32:
        DstFormat = PIXEL_FORMAT_BGRX32;
        break;
      case PIXEL_FORMAT_RGB24:
        DstFormat = PIXEL_FORMAT_BGR24;
        break;
      case PIXEL_FORMAT_BGR24:
        DstFormat = PIXEL_FORMAT_RGB24;
        break;
      case PIXEL_FORMAT_RGB16:
        DstFormat = PIXEL_FORMAT_BGR16;
        break;
      case PIXEL_FORMAT_BGR16:
        DstFormat = PIXEL_FORMAT_RGB16;
        break;
      case PIXEL_FORMAT_ARGB15:
        DstFormat = PIXEL_FORMAT_ABGR15;
        break;
      case PIXEL_FORMAT_RGB15:
        DstFormat = PIXEL_FORMAT_BGR15;
        break;
      case PIXEL_FORMAT_ABGR15:
        DstFormat = PIXEL_FORMAT_ARGB15;
        break;
      case PIXEL_FORMAT_BGR15:
        DstFormat = PIXEL_FORMAT_RGB15;
        break;
      default:
        break;
    }
  }
  return DstFormat;
}

static inline BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
                                                            UINT32 width, UINT32 height,
                                                            BYTE* WINPR_RESTRICT outPlane,
                                                            UINT32* WINPR_RESTRICT dstSize);
static inline BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
                                                             UINT32 width, UINT32 height,
                                                             BYTE* WINPR_RESTRICT outPlane);

static inline INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
                                          UINT32 nWidth, UINT32 nHeight)
{
  UINT32 used = 0;

  WINPR_ASSERT(pSrcData);

  for (UINT32 y = 0; y < nHeight; y++)
  {
    for (UINT32 x = 0; x < nWidth;)
    {
      if (used >= SrcSize)
      {
        WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
                 SrcSize);
        return -1;
      }

      const uint8_t controlByte = pSrcData[used++];
      uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
      uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);

      if (nRunLength == 1)
      {
        nRunLength = cRawBytes + 16;
        cRawBytes = 0;
      }
      else if (nRunLength == 2)
      {
        nRunLength = cRawBytes + 32;
        cRawBytes = 0;
      }

      used += cRawBytes;
      x += cRawBytes;
      x += nRunLength;

      if (x > nWidth)
      {
        WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
        return -1;
      }

      if (used > SrcSize)
      {
        WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRId32, used,
                 INT32_MAX);
        return -1;
      }
    }
  }

  if (used > INT32_MAX)
  {
    WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
    return -1;
  }
  return (INT32)used;
}

static inline UINT8 clamp(INT16 val)
{
  return (UINT8)val;
}

static inline bool check_source_available(const BYTE* buffer, size_t length, const BYTE* cur,
                                          size_t required)
{
  WINPR_ASSERT(cur >= buffer);
  const size_t len = WINPR_ASSERTING_INT_CAST(size_t, (cur - buffer));

  if ((SIZE_MAX - len < required) || (len + required > length))
  {
    WLog_ERR(TAG, "error reading input buffer");
    return false;
  }
  return true;
}

static inline INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
                                                     UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
                                                     UINT32 nWidth, UINT32 nHeight)
{
  BYTE* previousScanline = nullptr;
  const BYTE* srcp = pSrcData;

  WINPR_ASSERT(nHeight <= INT32_MAX);
  WINPR_ASSERT(nWidth <= INT32_MAX);

  for (UINT32 y = 0; y < nHeight; y++)
  {
    BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
    INT16 pixel = 0;
    BYTE* currentScanline = dstp;

    for (UINT32 x = 0; x < nWidth;)
    {
      if (!check_source_available(pSrcData, SrcSize, srcp, 1))
        return -1;

      const BYTE controlByte = *srcp++;

      UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
      UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);

      if (nRunLength == 1)
      {
        nRunLength = cRawBytes + 16;
        cRawBytes = 0;
      }
      else if (nRunLength == 2)
      {
        nRunLength = cRawBytes + 32;
        cRawBytes = 0;
      }

      if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
      {
        WLog_ERR(TAG, "too many pixels in scanline");
        return -1;
      }

      if (!previousScanline)
      {
        if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
          return -1;
        /* first scanline, absolute values */
        while (cRawBytes > 0)
        {
          pixel = *srcp++;
          *dstp = clamp(pixel);
          dstp++;
          x++;
          cRawBytes--;
        }

        while (nRunLength > 0)
        {
          *dstp = clamp(pixel);
          dstp++;
          x++;
          nRunLength--;
        }
      }
      else
      {
        if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
          return -1;

        /* delta values relative to previous scanline */
        while (cRawBytes > 0)
        {
          UINT8 deltaValue = *srcp++;

          if (deltaValue & 1)
          {
            deltaValue = deltaValue >> 1;
            deltaValue = deltaValue + 1;
            pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
          }
          else
          {
            deltaValue = deltaValue >> 1;
            pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
          }

          const INT16 delta =
              WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
          *dstp = clamp(delta);
          dstp++;
          x++;
          cRawBytes--;
        }

        while (nRunLength > 0)
        {
          const INT16 deltaValue =
              WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
          *dstp = clamp(deltaValue);
          dstp++;
          x++;
          nRunLength--;
        }
      }
    }

    previousScanline = currentScanline;
  }

  return (INT32)(srcp - pSrcData);
}

static inline INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
                                                BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
                                                UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
                                                UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
{
  INT32 beg = 0;
  INT32 end = 0;
  INT32 inc = 0;
  BYTE* previousScanline = nullptr;
  const BYTE* srcp = pSrcData;

  WINPR_ASSERT(nHeight <= INT32_MAX);
  WINPR_ASSERT(nWidth <= INT32_MAX);
  WINPR_ASSERT(nDstStep <= INT32_MAX);

  if (vFlip)
  {
    beg = (INT32)nHeight - 1;
    end = -1;
    inc = -1;
  }
  else
  {
    beg = 0;
    end = (INT32)nHeight;
    inc = 1;
  }

  for (INT32 y = beg; y != end; y += inc)
  {
    const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
    BYTE* dstp = &pDstData[off];
    BYTE* currentScanline = dstp;
    INT16 pixel = 0;

    for (INT32 x = 0; x < (INT32)nWidth;)
    {
      if (!check_source_available(pSrcData, SrcSize, srcp, 1))
        return -1;

      const BYTE controlByte = *srcp++;

      UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
      UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);

      if (nRunLength == 1)
      {
        nRunLength = cRawBytes + 16;
        cRawBytes = 0;
      }
      else if (nRunLength == 2)
      {
        nRunLength = cRawBytes + 32;
        cRawBytes = 0;
      }

      if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
      {
        WLog_ERR(TAG, "too many pixels in scanline");
        return -1;
      }

      if (!previousScanline)
      {
        if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
          return -1;

        /* first scanline, absolute values */
        while (cRawBytes > 0)
        {
          pixel = *srcp++;

          *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
          dstp += 4;
          x++;
          cRawBytes--;
        }

        while (nRunLength > 0)
        {
          *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
          dstp += 4;
          x++;
          nRunLength--;
        }
      }
      else
      {
        if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
          return -1;

        /* delta values relative to previous scanline */
        while (cRawBytes > 0)
        {
          BYTE deltaValue = *srcp++;

          if (deltaValue & 1)
          {
            deltaValue = deltaValue >> 1;
            deltaValue = deltaValue + 1;
            pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
          }
          else
          {
            deltaValue = deltaValue >> 1;
            pixel = deltaValue;
          }

          const INT16 delta =
              WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
          *dstp = clamp(delta);
          dstp += 4;
          x++;
          cRawBytes--;
        }

        while (nRunLength > 0)
        {
          const INT16 deltaValue =
              WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
          *dstp = clamp(deltaValue);
          dstp += 4;
          x++;
          nRunLength--;
        }
      }
    }

    previousScanline = currentScanline;
  }

  return (INT32)(srcp - pSrcData);
}

static inline INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
                                     UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
                                     BOOL vFlip)
{
  INT32 beg = 0;
  INT32 end = (INT32)nHeight;
  INT32 inc = 1;

  WINPR_ASSERT(nHeight <= INT32_MAX);
  WINPR_ASSERT(nWidth <= INT32_MAX);
  WINPR_ASSERT(nDstStep <= INT32_MAX);

  if (vFlip)
  {
    beg = (INT32)nHeight - 1;
    end = -1;
    inc = -1;
  }

  for (INT32 y = beg; y != end; y += inc)
  {
    const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
    BYTE* dstp = &pDstData[off];

    for (INT32 x = 0; x < (INT32)nWidth; ++x)
    {
      *dstp = bValue;
      dstp += 4;
    }
  }

  return 0;
}

static inline BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
                             const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
                             const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
{
  WINPR_ASSERT(ppRgba);
  WINPR_ASSERT(ppR);
  WINPR_ASSERT(ppG);
  WINPR_ASSERT(ppB);

  switch (DstFormat)
  {
    case PIXEL_FORMAT_BGRA32:
      for (UINT32 x = 0; x < width; x++)
      {
        *(*ppRgba)++ = *(*ppB)++;
        *(*ppRgba)++ = *(*ppG)++;
        *(*ppRgba)++ = *(*ppR)++;
        *(*ppRgba)++ = *(*ppA)++;
      }

      return TRUE;

    case PIXEL_FORMAT_BGRX32:
      for (UINT32 x = 0; x < width; x++)
      {
        *(*ppRgba)++ = *(*ppB)++;
        *(*ppRgba)++ = *(*ppG)++;
        *(*ppRgba)++ = *(*ppR)++;
        *(*ppRgba)++ = 0xFF;
      }

      return TRUE;

    default:
      if (ppA)
      {
        for (UINT32 x = 0; x < width; x++)
        {
          BYTE alpha = *(*ppA)++;
          UINT32 color =
              FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
          FreeRDPWriteColor(*ppRgba, DstFormat, color);
          *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
        }
      }
      else
      {
        const BYTE alpha = 0xFF;

        for (UINT32 x = 0; x < width; x++)
        {
          UINT32 color =
              FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
          FreeRDPWriteColor(*ppRgba, DstFormat, color);
          *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
        }
      }

      return TRUE;
  }
}

static inline BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
                                                BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
                                                UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
                                                UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
                                                UINT32 totalHeight)
{
  INT32 beg = 0;
  INT32 end = 0;
  INT32 inc = 0;
  const BYTE* pR = pSrcData[0];
  const BYTE* pG = pSrcData[1];
  const BYTE* pB = pSrcData[2];
  const BYTE* pA = pSrcData[3];
  const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);

  if (vFlip)
  {
    beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
    end = -1;
    inc = -1;
  }
  else
  {
    beg = 0;
    end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
    inc = 1;
  }

  if (nYDst + nHeight > totalHeight)
  {
    WLog_ERR(TAG,
             "planar plane destination Y %" PRIu32 " + height %" PRIu32
             " exceeds totalHeight %" PRIu32,
             nYDst, nHeight, totalHeight);
    return FALSE;
  }

  if ((nXDst + nWidth) * bpp > nDstStep)
  {
    WLog_ERR(TAG,
             "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
             " exceeds stride %" PRIu32,
             nXDst, nWidth, bpp, nDstStep);
    return FALSE;
  }

  for (INT32 y = beg; y != end; y += inc)
  {
    BYTE* pRGB = nullptr;

    if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
    {
      WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
               nHeight);
      return FALSE;
    }

    const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
    pRGB = &pDstData[off];

    if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
      return FALSE;
  }

  return TRUE;
}

static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
                                    UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
                                    UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
{
  size_t pos = 0;
  WINPR_UNUSED(planeLength);

  WINPR_ASSERT(plane);
  WINPR_ASSERT(deltaPlane);

  if (nWidth > nPlaneWidth * 2)
  {
    WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
             nPlaneWidth);
    return FALSE;
  }

  if (nHeight > nPlaneHeight * 2)
  {
    WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
             nPlaneHeight);
    return FALSE;
  }

  for (size_t y = 0; y < nHeight; y++)
  {
    const BYTE* src = plane + y / 2 * nPlaneWidth;

    for (UINT32 x = 0; x < nWidth; x++)
    {
      deltaPlane[pos++] = src[x / 2];
    }
  }

  return TRUE;
}

#if !defined(WITHOUT_FREERDP_3x_DEPRECATED)
BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
                       const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
                       UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
                       UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
                       UINT32 nDstHeight, BOOL vFlip)
{
  return freerdp_bitmap_decompress_planar(planar, pSrcData, SrcSize, nSrcWidth, nSrcHeight,
                                          pDstData, DstFormat, nDstStep, nXDst, nYDst, nDstWidth,
                                          nDstHeight, vFlip);
}
#endif

BOOL freerdp_bitmap_decompress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
                                      const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
                                      UINT32 nSrcWidth, UINT32 nSrcHeight,
                                      BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
                                      UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
                                      UINT32 nDstHeight, BOOL vFlip)
{
  BOOL useAlpha = FALSE;
  INT32 status = 0;
  INT32 rleSizes[4] = { 0, 0, 0, 0 };
  UINT32 rawSizes[4] = WINPR_C_ARRAY_INIT;
  UINT32 rawWidths[4] = WINPR_C_ARRAY_INIT;
  UINT32 rawHeights[4] = WINPR_C_ARRAY_INIT;
  const BYTE* planes[4] = WINPR_C_ARRAY_INIT;
  const UINT32 w = MIN(nSrcWidth, nDstWidth);
  const UINT32 h = MIN(nSrcHeight, nDstHeight);
  const primitives_t* prims = primitives_get();

  WINPR_ASSERT(planar);
  WINPR_ASSERT(prims);

  if (planar->maxWidth < nSrcWidth)
  {
    WLog_ERR(TAG, "planar->maxWidth %" PRIu32 " < nSrcWidth %" PRIu32, planar->maxWidth,
             nSrcWidth);
    return FALSE;
  }
  if (planar->maxHeight < nSrcHeight)
  {
    WLog_ERR(TAG, "planar->maxHeight %" PRIu32 " < nSrcHeight %" PRIu32, planar->maxHeight,
             nSrcHeight);
    return FALSE;
  }

  const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
  if (nDstStep <= 0)
    nDstStep = nDstWidth * bpp;

  const BYTE* srcp = pSrcData;

  if (!pSrcData || (SrcSize < 1))
  {
    WLog_ERR(TAG, "Invalid argument pSrcData=%p [size=%" PRIu32 "]",
             WINPR_CXX_COMPAT_CAST(const void*, pSrcData), SrcSize);
    return FALSE;
  }

  if (!pDstData)
  {
    WLog_ERR(TAG, "Invalid argument pDstData=nullptr");
    return FALSE;
  }

  const BYTE FormatHeader = *srcp++;
  const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
  const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) != 0;
  const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) != 0;
  const BYTE alpha = !(FormatHeader & PLANAR_FORMAT_HEADER_NA);

  DstFormat = planar_invert_format(planar, alpha, DstFormat);

  if (alpha)
    useAlpha = FreeRDPColorHasAlpha(DstFormat);

  // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
  // alpha);

  if (!cll && cs)
  {
    WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
    return FALSE; /* Chroma subsampling requires YCoCg */
  }

  const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
  const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
  const UINT32 planeSize = nSrcWidth * nSrcHeight;
  const UINT32 subSize = subWidth * subHeight;

  if (!cs)
  {
    rawSizes[0] = planeSize; /* LumaOrRedPlane */
    rawWidths[0] = nSrcWidth;
    rawHeights[0] = nSrcHeight;
    rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
    rawWidths[1] = nSrcWidth;
    rawHeights[1] = nSrcHeight;
    rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
    rawWidths[2] = nSrcWidth;
    rawHeights[2] = nSrcHeight;
    rawSizes[3] = planeSize; /* AlphaPlane */
    rawWidths[3] = nSrcWidth;
    rawHeights[3] = nSrcHeight;
  }
  else /* Chroma Subsampling */
  {
    rawSizes[0] = planeSize; /* LumaOrRedPlane */
    rawWidths[0] = nSrcWidth;
    rawHeights[0] = nSrcHeight;
    rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
    rawWidths[1] = subWidth;
    rawHeights[1] = subHeight;
    rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
    rawWidths[2] = subWidth;
    rawHeights[2] = subHeight;
    rawSizes[3] = planeSize; /* AlphaPlane */
    rawWidths[3] = nSrcWidth;
    rawHeights[3] = nSrcHeight;
  }

  const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
  if (SrcSize < diff)
  {
    WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
    return FALSE;
  }

  if (!rle) /* RAW */
  {

    UINT32 base = planeSize * 3;
    if (cs)
      base = planeSize + planeSize / 2;

    if (alpha)
    {
      if ((SrcSize - diff) < (planeSize + base))
      {
        WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
                 (planeSize + base));
        return FALSE;
      }

      planes[3] = srcp;                    /* AlphaPlane */
      planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
      planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
      planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */

      if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
      {
        WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
                 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
                 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
        return FALSE;
      }
    }
    else
    {
      if ((SrcSize - diff) < base)
      {
        WLog_ERR(TAG, "plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, base);
        return FALSE;
      }

      planes[0] = srcp;                    /* LumaOrRedPlane */
      planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
      planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */

      if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
      {
        WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
                 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
                 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
        return FALSE;
      }
    }
  }
  else /* RLE */
  {
    if (alpha)
    {
      planes[3] = srcp;
      rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
                                          rawHeights[3]); /* AlphaPlane */

      if (rleSizes[3] < 0)
        return FALSE;

      planes[0] = planes[3] + rleSizes[3];
    }
    else
      planes[0] = srcp;

    const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
    if (SrcSize < diff0)
    {
      WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
      return FALSE;
    }
    rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
                                        rawHeights[0]); /* RedPlane */

    if (rleSizes[0] < 0)
      return FALSE;

    planes[1] = planes[0] + rleSizes[0];

    const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
    if (SrcSize < diff1)
    {
      WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
      return FALSE;
    }
    rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
                                        rawHeights[1]); /* GreenPlane */

    if (rleSizes[1] < 1)
      return FALSE;

    planes[2] = planes[1] + rleSizes[1];
    const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
    if (SrcSize < diff2)
    {
      WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
      return FALSE;
    }
    rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
                                        rawHeights[2]); /* BluePlane */

    if (rleSizes[2] < 1)
      return FALSE;
  }

  if (!cll) /* RGB */
  {
    UINT32 TempFormat = 0;
    BYTE* pTempData = pDstData;
    UINT32 nTempStep = nDstStep;
    UINT32 nTotalHeight = nYDst + nDstHeight;

    if (useAlpha)
      TempFormat = PIXEL_FORMAT_BGRA32;
    else
      TempFormat = PIXEL_FORMAT_BGRX32;

    TempFormat = planar_invert_format(planar, alpha, TempFormat);

    if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
    {
      pTempData = planar->pTempData;
      nTempStep = planar->nTempStep;
      nTotalHeight = planar->maxHeight;
    }

    if (!rle) /* RAW */
    {
      if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
                                        nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
        return FALSE;

      if (alpha)
        srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
      else /* NoAlpha */
        srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];

      if ((SrcSize - (srcp - pSrcData)) == 1)
        srcp++; /* pad */
    }
    else /* RLE */
    {
      if (nYDst + nSrcHeight > nTotalHeight)
      {
        WLog_ERR(TAG,
                 "planar plane destination Y %" PRIu32 " + height %" PRIu32
                 " exceeds totalHeight %" PRIu32,
                 nYDst, nSrcHeight, nTotalHeight);
        return FALSE;
      }

      if ((nXDst + nSrcWidth) * bpp > nTempStep)
      {
        WLog_ERR(TAG,
                 "planar plane destination (X %" PRIu32 " + width %" PRIu32
                 ") * bpp %" PRIu32 " exceeds stride %" PRIu32,
                 nXDst, nSrcWidth, bpp, nTempStep);
        return FALSE;
      }

      status = planar_decompress_plane_rle(
          planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
          nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */

      if (status < 0)
        return FALSE;

      status = planar_decompress_plane_rle(
          planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
          nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */

      if (status < 0)
        return FALSE;

      status = planar_decompress_plane_rle(
          planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
          nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */

      if (status < 0)
        return FALSE;

      srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];

      if (useAlpha)
      {
        status = planar_decompress_plane_rle(
            planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
            nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
      }
      else
        status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
                                  nSrcHeight, 3, vFlip);

      if (status < 0)
        return FALSE;

      if (alpha)
        srcp += rleSizes[3];
    }

    if (pTempData != pDstData)
    {
      if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
                                         pTempData, TempFormat, nTempStep, nXDst, nYDst,
                                         nullptr, FREERDP_FLIP_NONE))
      {
        WLog_ERR(TAG, "planar image copy failed");
        return FALSE;
      }
    }
  }
  else /* YCoCg */
  {
    UINT32 TempFormat = 0;
    BYTE* pTempData = planar->pTempData;
    UINT32 nTempStep = planar->nTempStep;
    UINT32 nTotalHeight = planar->maxHeight;
    BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];

    if (useAlpha)
      TempFormat = PIXEL_FORMAT_BGRA32;
    else
      TempFormat = PIXEL_FORMAT_BGRX32;

    if (!pTempData)
    {
      WLog_ERR(TAG, "pTempData == NULL");
      return FALSE;
    }

    if (rle) /* RLE encoded data. Decode and handle it like raw data. */
    {
      BYTE* rleBuffer[4] = WINPR_C_ARRAY_INIT;

      if (!planar->rlePlanesBuffer)
      {
        WLog_ERR(TAG, "planar->rlePlanesBuffer == NULL");
        return FALSE;
      }

      rleBuffer[3] = planar->rlePlanesBuffer;  /* AlphaPlane */
      rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
      rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
      rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
      if (useAlpha)
      {
        status = planar_decompress_plane_rle_only(
            planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
            rawWidths[3], rawHeights[3]); /* AlphaPlane */

        if (status < 0)
          return FALSE;
      }

      if (alpha)
        srcp += rleSizes[3];

      status = planar_decompress_plane_rle_only(
          planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
          rawWidths[0], rawHeights[0]); /* LumaPlane */

      if (status < 0)
        return FALSE;

      status = planar_decompress_plane_rle_only(
          planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
          rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */

      if (status < 0)
        return FALSE;

      status = planar_decompress_plane_rle_only(
          planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
          rawWidths[2], rawHeights[2]); /* GreenChromaPlane */

      if (status < 0)
        return FALSE;

      planes[0] = rleBuffer[0];
      planes[1] = rleBuffer[1];
      planes[2] = rleBuffer[2];
      planes[3] = rleBuffer[3];
    }

    /* RAW */
    {
      if (cs)
      { /* Chroma subsampling for Co and Cg:
         * Each pixel contains the value that should be expanded to
         * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
        if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
                                     rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
          return FALSE;

        planes[1] = planar->deltaPlanes[0];
        rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
        rawWidths[1] = nSrcWidth;
        rawHeights[1] = nSrcHeight;

        if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
                                     rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
          return FALSE;

        planes[2] = planar->deltaPlanes[1];
        rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
        rawWidths[2] = nSrcWidth;
        rawHeights[2] = nSrcHeight;
      }

      if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
                                        nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
        return FALSE;

      if (alpha)
        srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
      else /* NoAlpha */
        srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];

      if ((SrcSize - (srcp - pSrcData)) == 1)
        srcp++; /* pad */
    }

    WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
    int rc = prims->YCoCgToRGB_8u_AC4R(
        pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
        WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
    if (rc != PRIMITIVES_SUCCESS)
    {
      WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
      return FALSE;
    }
  }

  WINPR_UNUSED(srcp);
  return TRUE;
}

static inline BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
                                              const BYTE* WINPR_RESTRICT data, UINT32 format,
                                              UINT32 width, UINT32 height, UINT32 scanline,
                                              BYTE* WINPR_RESTRICT planes[4])
{
  WINPR_ASSERT(planar);

  if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
    return FALSE;

  if (scanline == 0)
    scanline = width * FreeRDPGetBytesPerPixel(format);

  if (planar->topdown)
  {
    UINT32 k = 0;
    for (UINT32 i = 0; i < height; i++)
    {
      const BYTE* pixel = &data[1ULL * scanline * i];

      for (UINT32 j = 0; j < width; j++)
      {
        const UINT32 color = FreeRDPReadColor(pixel, format);
        pixel += FreeRDPGetBytesPerPixel(format);
        FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
                          &planes[0][k], nullptr);
        k++;
      }
    }
  }
  else
  {
    UINT32 k = 0;

    for (INT64 i = (INT64)height - 1; i >= 0; i--)
    {
      const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];

      for (UINT32 j = 0; j < width; j++)
      {
        const UINT32 color = FreeRDPReadColor(pixel, format);
        pixel += FreeRDPGetBytesPerPixel(format);
        FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
                          &planes[0][k], nullptr);
        k++;
      }
    }
  }
  return TRUE;
}

static inline UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
                                                           UINT32 cRawBytes, UINT32 nRunLength,
                                                           BYTE* WINPR_RESTRICT pOutBuffer,
                                                           UINT32 outBufferSize)
{
  const BYTE* pInput = pInBuffer;
  BYTE* pOutput = pOutBuffer;
  BYTE controlByte = 0;
  UINT32 nBytesToWrite = 0;

  if (!cRawBytes && !nRunLength)
    return 0;

  if (nRunLength < 3)
  {
    cRawBytes += nRunLength;
    nRunLength = 0;
  }

  while (cRawBytes)
  {
    if (cRawBytes < 16)
    {
      if (nRunLength > 15)
      {
        if (nRunLength < 18)
        {
          controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
          nRunLength -= 13;
          cRawBytes = 0;
        }
        else
        {
          controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
          nRunLength -= 15;
          cRawBytes = 0;
        }
      }
      else
      {
        controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
        nRunLength = 0;
        cRawBytes = 0;
      }
    }
    else
    {
      controlByte = PLANAR_CONTROL_BYTE(0, 15);
      cRawBytes -= 15;
    }

    if (outBufferSize < 1)
      return 0;

    outBufferSize--;
    *pOutput = controlByte;
    pOutput++;
    nBytesToWrite = (controlByte >> 4);

    if (nBytesToWrite)
    {
      if (outBufferSize < nBytesToWrite)
        return 0;

      outBufferSize -= nBytesToWrite;
      CopyMemory(pOutput, pInput, nBytesToWrite);
      pOutput += nBytesToWrite;
      pInput += nBytesToWrite;
    }
  }

  while (nRunLength)
  {
    if (nRunLength > 47)
    {
      if (nRunLength < 50)
      {
        controlByte = PLANAR_CONTROL_BYTE(2, 13);
        nRunLength -= 45;
      }
      else
      {
        controlByte = PLANAR_CONTROL_BYTE(2, 15);
        nRunLength -= 47;
      }
    }
    else if (nRunLength > 31)
    {
      controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
      nRunLength = 0;
    }
    else if (nRunLength > 15)
    {
      controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
      nRunLength = 0;
    }
    else
    {
      controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
      nRunLength = 0;
    }

    if (outBufferSize < 1)
      return 0;

    --outBufferSize;
    *pOutput = controlByte;
    pOutput++;
  }

  const intptr_t diff = (pOutput - pOutBuffer);
  if ((diff < 0) || (diff > UINT32_MAX))
    return 0;
  return (UINT32)diff;
}

static inline UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
                                                            UINT32 inBufferSize,
                                                            BYTE* WINPR_RESTRICT pOutBuffer,
                                                            UINT32 outBufferSize)
{
  BYTE symbol = 0;
  const BYTE* pInput = pInBuffer;
  BYTE* pOutput = pOutBuffer;
  const BYTE* pBytes = nullptr;
  UINT32 cRawBytes = 0;
  UINT32 nRunLength = 0;
  UINT32 nBytesWritten = 0;
  UINT32 nTotalBytesWritten = 0;

  if (!outBufferSize)
    return 0;

  do
  {
    if (!inBufferSize)
      break;

    const UINT32 bSymbolMatch = (symbol == *pInput) != 0;
    symbol = *pInput;
    pInput++;
    inBufferSize--;

    if (nRunLength && !bSymbolMatch)
    {
      if (nRunLength < 3)
      {
        cRawBytes += nRunLength;
        nRunLength = 0;
      }
      else
      {
        pBytes = pInput - (cRawBytes + nRunLength + 1);
        nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
                                                              pOutput, outBufferSize);
        nRunLength = 0;

        if (!nBytesWritten || (nBytesWritten > outBufferSize))
          return nRunLength;

        nTotalBytesWritten += nBytesWritten;
        outBufferSize -= nBytesWritten;
        pOutput += nBytesWritten;
        cRawBytes = 0;
      }
    }

    nRunLength += bSymbolMatch;
    cRawBytes += (!bSymbolMatch) != 0;
  } while (outBufferSize);

  if (cRawBytes || nRunLength)
  {
    pBytes = pInput - (cRawBytes + nRunLength);
    nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
                                                          pOutput, outBufferSize);

    if (!nBytesWritten)
      return 0;

    nTotalBytesWritten += nBytesWritten;
  }

  if (inBufferSize)
    return 0;

  return nTotalBytesWritten;
}

BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
                                              UINT32 height, BYTE* WINPR_RESTRICT outPlane,
                                              UINT32* WINPR_RESTRICT dstSize)
{
  if (!outPlane)
    return FALSE;

  UINT32 index = 0;
  const BYTE* pInput = inPlane;
  BYTE* pOutput = outPlane;
  UINT32 outBufferSize = *dstSize;
  UINT32 nTotalBytesWritten = 0;

  while (outBufferSize > 0)
  {
    const UINT32 nBytesWritten =
        freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);

    if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
      return FALSE;

    outBufferSize -= nBytesWritten;
    nTotalBytesWritten += nBytesWritten;
    pOutput += nBytesWritten;
    pInput += width;
    index++;

    if (index >= height)
      break;
  }

  *dstSize = nTotalBytesWritten;
  return TRUE;
}

static inline BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
                                                             UINT32 width, UINT32 height,
                                                             BYTE* WINPR_RESTRICT outPlanes,
                                                             UINT32* WINPR_RESTRICT dstSizes,
                                                             BOOL skipAlpha)
{
  UINT32 outPlanesSize = width * height * 4;

  /* AlphaPlane */
  if (skipAlpha)
  {
    dstSizes[0] = 0;
  }
  else
  {
    dstSizes[0] = outPlanesSize;

    if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
                                                  &dstSizes[0]))
      return FALSE;

    outPlanes += dstSizes[0];
    outPlanesSize -= dstSizes[0];
  }

  /* LumaOrRedPlane */
  dstSizes[1] = outPlanesSize;

  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
                                                &dstSizes[1]))
    return FALSE;

  outPlanes += dstSizes[1];
  outPlanesSize -= dstSizes[1];
  /* OrangeChromaOrGreenPlane */
  dstSizes[2] = outPlanesSize;

  if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
                                                &dstSizes[2]))
    return FALSE;

  outPlanes += dstSizes[2];
  outPlanesSize -= dstSizes[2];
  /* GreenChromeOrBluePlane */
  dstSizes[3] = outPlanesSize;

  return (freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
                                                   &dstSizes[3]));
}

BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
                                               UINT32 height, BYTE* WINPR_RESTRICT outPlane)
{
  if (!outPlane)
  {
    if (width * height == 0)
      return nullptr;

    outPlane = (BYTE*)calloc(height, width);
    if (!outPlane)
      return nullptr;
  }

  // first line is copied as is
  CopyMemory(outPlane, inPlane, width);

  for (UINT32 y = 1; y < height; y++)
  {
    const size_t off = 1ull * width * y;
    BYTE* outPtr = &outPlane[off];
    const BYTE* srcPtr = &inPlane[off];
    const BYTE* prevLinePtr = &inPlane[off - width];
    for (UINT32 x = 0; x < width; x++)
    {
      const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
      const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
      const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
      const uint32_t s2c =
          (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
      outPtr[x] = (BYTE)s2c;
    }
  }

  return outPlane;
}

static inline BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
                                                             UINT32 width, UINT32 height,
                                                             BYTE* WINPR_RESTRICT outPlanes[4])
{
  for (UINT32 i = 0; i < 4; i++)
  {
    outPlanes[i] =
        freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);

    if (!outPlanes[i])
      return FALSE;
  }

  return TRUE;
}

BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
                                     const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
                                     UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
                                     UINT32* WINPR_RESTRICT pDstSize)
{
  UINT32 size = 0;
  BYTE* dstp = nullptr;
  UINT32 dstSizes[4] = WINPR_C_ARRAY_INIT;
  BYTE FormatHeader = 0;

  if (!context || !context->rlePlanesBuffer)
    return nullptr;

  if (context->AllowSkipAlpha)
    FormatHeader |= PLANAR_FORMAT_HEADER_NA;

  const UINT32 planeSize = width * height;

  if (!context->AllowSkipAlpha)
    format = planar_invert_format(context, TRUE, format);

  if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
                                  context->planes))
    return nullptr;

  if (context->AllowRunLengthEncoding)
  {
    if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
                                                   context->deltaPlanes))
      return nullptr;

    if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
                                                   context->rlePlanesBuffer, dstSizes,
                                                   context->AllowSkipAlpha))
      return nullptr;

    {
      uint32_t offset = 0;
      FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
      context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
      offset += dstSizes[0];
      context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
      offset += dstSizes[1];
      context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
      offset += dstSizes[2];
      context->rlePlanes[3] = &context->rlePlanesBuffer[offset];

#if defined(WITH_DEBUG_CODECS)
      WLog_DBG(TAG,
               "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
               " / %" PRIu32 "] ",
               dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
#endif
    }
  }

  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
  {
    if (!context->AllowRunLengthEncoding)
      return nullptr;

    if (context->rlePlanes[0] == nullptr)
      return nullptr;

    if (context->rlePlanes[1] == nullptr)
      return nullptr;

    if (context->rlePlanes[2] == nullptr)
      return nullptr;

    if (context->rlePlanes[3] == nullptr)
      return nullptr;
  }

  if (!dstData)
  {
    size = 1;

    if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
    {
      if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
        size += dstSizes[0];
      else
        size += planeSize;
    }

    if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
      size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
    else
      size += (planeSize * 3);

    if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
      size++;

    dstData = malloc(size);

    if (!dstData)
      return nullptr;

    *pDstSize = size;
  }

  dstp = dstData;
  *dstp = FormatHeader; /* FormatHeader */
  dstp++;

  /* AlphaPlane */

  if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
  {
    if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
    {
      CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
      dstp += dstSizes[0];
    }
    else
    {
      CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
      dstp += planeSize;
    }
  }

  /* LumaOrRedPlane */

  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
  {
    CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
    dstp += dstSizes[1];
  }
  else
  {
    CopyMemory(dstp, context->planes[1], planeSize); /* Red */
    dstp += planeSize;
  }

  /* OrangeChromaOrGreenPlane */

  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
  {
    CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
    dstp += dstSizes[2];
  }
  else
  {
    CopyMemory(dstp, context->planes[2], planeSize); /* Green */
    dstp += planeSize;
  }

  /* GreenChromeOrBluePlane */

  if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
  {
    CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
    dstp += dstSizes[3];
  }
  else
  {
    CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
    dstp += planeSize;
  }

  /* Pad1 (1 byte) */

  if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
  {
    *dstp = 0;
    dstp++;
  }

  const intptr_t diff = (dstp - dstData);
  if ((diff < 0) || (diff > UINT32_MAX))
  {
    free(dstData);
    return nullptr;
  }
  size = (UINT32)diff;
  *pDstSize = size;
  return dstData;
}

BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
                                         UINT32 width, UINT32 height)
{
  if (!context)
    return FALSE;

  context->bgr = FALSE;
  const UINT32 maxWidth = PLANAR_ALIGN(width, 4);
  const UINT32 maxHeight = PLANAR_ALIGN(height, 4);
  UINT32 maxPlaneSize = 0;
  {
    const UINT64 tmp = 1ull * maxWidth * maxHeight;
    if (tmp > UINT32_MAX)
      return FALSE;
    maxPlaneSize = (UINT32)tmp;
  }

  if (maxWidth > UINT32_MAX / 4)
    return FALSE;
  const UINT32 nTempStep = maxWidth * 4ull;

  memset((void*)context->planes, 0, sizeof(context->planes));
  memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
  memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));

  void* tmp1 = nullptr;
  void* tmp2 = nullptr;
  void* tmp3 = nullptr;
  void* tmp4 = nullptr;
  if (maxPlaneSize > 0)
  {
    tmp1 = winpr_aligned_calloc(maxPlaneSize, 4, 32);
    tmp2 = winpr_aligned_calloc(maxPlaneSize, 6, 32);
    tmp3 = winpr_aligned_calloc(maxPlaneSize, 4, 32);
    tmp4 = winpr_aligned_calloc(maxPlaneSize, 4, 32);
    if (!tmp1 || !tmp2 || !tmp3 || !tmp4)
    {
      winpr_aligned_free(tmp1);
      winpr_aligned_free(tmp2);
      winpr_aligned_free(tmp3);
      winpr_aligned_free(tmp4);
      return FALSE;
    }
  }

  winpr_aligned_free(context->planesBuffer);
  context->planesBuffer = tmp1;

  winpr_aligned_free(context->pTempData);
  context->pTempData = tmp2;

  winpr_aligned_free(context->deltaPlanesBuffer);
  context->deltaPlanesBuffer = tmp3;

  winpr_aligned_free(context->rlePlanesBuffer);
  context->rlePlanesBuffer = tmp4;

  context->maxWidth = maxWidth;
  context->maxHeight = maxHeight;
  context->maxPlaneSize = maxPlaneSize;
  context->nTempStep = nTempStep;

  if (context->maxPlaneSize > 0)
  {
    context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
    context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
    context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
    context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
    context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
    context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
    context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
    context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
  }
  return TRUE;
}

BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
                                                         UINT32 maxHeight)
{
  BITMAP_PLANAR_CONTEXT* context =
      (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);

  if (!context)
    return nullptr;

  if (flags & PLANAR_FORMAT_HEADER_NA)
    context->AllowSkipAlpha = TRUE;

  if (flags & PLANAR_FORMAT_HEADER_RLE)
    context->AllowRunLengthEncoding = TRUE;

  if (flags & PLANAR_FORMAT_HEADER_CS)
    context->AllowColorSubsampling = TRUE;

  context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;

  if (context->ColorLossLevel)
    context->AllowDynamicColorFidelity = TRUE;

  if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
  {
    WINPR_PRAGMA_DIAG_PUSH
    WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
    freerdp_bitmap_planar_context_free(context);
    WINPR_PRAGMA_DIAG_POP
    return nullptr;
  }

  return context;
}

void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
{
  if (!context)
    return;

  winpr_aligned_free(context->pTempData);
  winpr_aligned_free(context->planesBuffer);
  winpr_aligned_free(context->deltaPlanesBuffer);
  winpr_aligned_free(context->rlePlanesBuffer);
  winpr_aligned_free(context);
}

void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
{
  WINPR_ASSERT(planar);
  planar->bgr = bgr;
}

void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
{
  WINPR_ASSERT(planar);
  planar->topdown = topdown;
}

Coverage Report

Created: 2026-10-28 06:50