/src/FreeRDP/channels/video/client/video_main.c

Source
/**
 * FreeRDP: A Remote Desktop Protocol Implementation
 * Video Optimized Remoting Virtual Channel Extension
 *
 * Copyright 2017 David Fort <contact@hardening-consulting.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <freerdp/config.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <winpr/crt.h>
#include <winpr/assert.h>
#include <winpr/cast.h>
#include <winpr/synch.h>
#include <winpr/print.h>
#include <winpr/stream.h>
#include <winpr/cmdline.h>
#include <winpr/collections.h>
#include <winpr/interlocked.h>
#include <winpr/sysinfo.h>

#include <freerdp/freerdp.h>
#include <freerdp/addin.h>
#include <freerdp/primitives.h>
#include <freerdp/client/channels.h>
#include <freerdp/client/geometry.h>
#include <freerdp/client/video.h>
#include <freerdp/channels/log.h>
#include <freerdp/codec/h264.h>
#include <freerdp/codec/yuv.h>
#include <freerdp/timer.h>

#define TAG CHANNELS_TAG("video.client")

#include "video_main.h"

typedef struct
{
  IWTSPlugin wtsPlugin;

  IWTSListener* controlListener;
  IWTSListener* dataListener;
  GENERIC_LISTENER_CALLBACK* control_callback;
  GENERIC_LISTENER_CALLBACK* data_callback;

  VideoClientContext* context;
  BOOL initialized;
  rdpContext* rdpcontext;
  BOOL connected;
  BOOL dataConnected;
} VIDEO_PLUGIN;

#define XF_VIDEO_UNLIMITED_RATE 31

static const BYTE MFVideoFormat_H264[] = { 'H',  '2',  '6',  '4',  0x00, 0x00, 0x10, 0x00,
                                         0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };

typedef struct
{
  BYTE PresentationId;
  UINT32 ScaledWidth;
  UINT32 ScaledHeight;

  UINT64 startTimeStamp;
  UINT64 publishOffset;
  wStream* currentSample;
  UINT64 lastPublishTime;
  UINT64 nextPublishTime;
  volatile LONG refCounter;
  H264_CONTEXT* h264;
  VideoSurface* surface;
  MAPPED_GEOMETRY* geometry;
  VideoClientContext* video;
} PresentationContext;

typedef struct
{
  BYTE PresentationId;
  UINT64 publishTime;
  UINT64 hnsDuration;
  MAPPED_GEOMETRY* geometry;
  UINT32 w, h;
  UINT32 scanline;
  BYTE* surfaceData;
} VideoFrame;

/** @brief private data for the channel */
struct s_VideoClientContextPriv
{
  VideoClientContext* video;
  GeometryClientContext* geometry;
  wQueue* frames;
  CRITICAL_SECTION framesLock;
  wBufferPool* surfacePool;
  UINT32 publishedFrames;
  UINT32 droppedFrames;
  UINT32 lastSentRate;
  UINT64 nextFeedbackTime;
  PresentationContext* currentPresentation;
  FreeRDP_TimerID timerID;
};

static void PresentationContext_unref(PresentationContext** presentation);
static void VideoClientContextPriv_free(VideoClientContextPriv* priv);

WINPR_ATTR_NODISCARD
static const char* video_command_name(BYTE cmd)
{
  switch (cmd)
  {
    case TSMM_START_PRESENTATION:
      return "start";
    case TSMM_STOP_PRESENTATION:
      return "stop";
    default:
      return "<unknown>";
  }
}

static void video_client_context_set_geometry(VideoClientContext* video,
                                              GeometryClientContext* geometry)
{
  WINPR_ASSERT(video);
  WINPR_ASSERT(video->priv);

  video->priv->geometry = geometry;
}

WINPR_ATTR_MALLOC(VideoClientContextPriv_free, 1)
static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
{
  WINPR_ASSERT(video);
  VideoClientContextPriv* ret = calloc(1, sizeof(*ret));
  if (!ret)
    return nullptr;

  ret->frames = Queue_New(TRUE, 10, 2);
  if (!ret->frames)
  {
    WLog_ERR(TAG, "unable to allocate frames queue");
    goto fail;
  }

  ret->surfacePool = BufferPool_New(FALSE, 0, 16);
  if (!ret->surfacePool)
  {
    WLog_ERR(TAG, "unable to create surface pool");
    goto fail;
  }

  if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
  {
    WLog_ERR(TAG, "unable to initialize frames lock");
    goto fail;
  }

  ret->video = video;

  /* don't set to unlimited so that we have the chance to send a feedback in
   * the first second (for servers that want feedback directly)
   */
  ret->lastSentRate = 30;
  return ret;

fail:
  VideoClientContextPriv_free(ret);
  return nullptr;
}

WINPR_ATTR_NODISCARD
static BOOL PresentationContext_ref(PresentationContext* presentation)
{
  WINPR_ASSERT(presentation);

  const LONG val = InterlockedIncrement(&presentation->refCounter);
  return val > 0;
}

static void PresentationContext_free(PresentationContext* presentation)
{
  if (!presentation)
    return;

  MAPPED_GEOMETRY* geometry = presentation->geometry;
  if (geometry)
  {
    geometry->MappedGeometryUpdate = nullptr;
    geometry->MappedGeometryClear = nullptr;
    geometry->custom = nullptr;
    mappedGeometryUnref(geometry);
  }

  h264_context_free(presentation->h264);
  Stream_Free(presentation->currentSample, TRUE);
  presentation->video->deleteSurface(presentation->video, presentation->surface);
  free(presentation);
}

WINPR_ATTR_MALLOC(PresentationContext_free, 1)
static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
                                                    UINT32 x, UINT32 y, UINT32 width, UINT32 height)
{
  if ((width == 0) || (height == 0))
  {
    WLog_ERR(TAG, "width==%" PRIu32 ", height=%" PRIu32, width, height);
    return nullptr;
  }
  const size_t s = 4ULL * width * height;

  WINPR_ASSERT(video);

  if (s > INT32_MAX)
    return nullptr;

  PresentationContext* ret = calloc(1, sizeof(*ret));
  if (!ret)
    return nullptr;

  ret->video = video;
  ret->PresentationId = PresentationId;

  ret->h264 = h264_context_new(FALSE);
  if (!ret->h264)
  {
    WLog_ERR(TAG, "unable to create a h264 context");
    goto fail;
  }

  VIDEO_PLUGIN* plugin = (VIDEO_PLUGIN*)video->handle;
  WINPR_ASSERT(plugin);
  WINPR_ASSERT(plugin->rdpcontext);
  if (!h264_context_set_option(ret->h264, H264_CONTEXT_OPTION_HW_ACCEL,
                               (UINT32)freerdp_settings_get_bool(plugin->rdpcontext->settings,
                                                                 FreeRDP_SoftwareGdi) == 0))
    goto fail;
  if (!h264_context_reset(ret->h264, width, height))
    goto fail;

  ret->currentSample = Stream_New(nullptr, 4096);
  if (!ret->currentSample)
  {
    WLog_ERR(TAG, "unable to create current packet stream");
    goto fail;
  }

  ret->surface = video->createSurface(video, x, y, width, height);
  if (!ret->surface)
  {
    WLog_ERR(TAG, "unable to create surface");
    goto fail;
  }

  if (!PresentationContext_ref(ret))
    goto fail;

  return ret;

fail:
  PresentationContext_free(ret);
  return nullptr;
}

static void PresentationContext_unref(PresentationContext** ppresentation)
{
  WINPR_ASSERT(ppresentation);

  PresentationContext* presentation = *ppresentation;
  if (!presentation)
    return;

  if (InterlockedDecrement(&presentation->refCounter) > 0)
    return;
  *ppresentation = nullptr;

  PresentationContext_free(presentation);
}

static void VideoFrame_free(VideoClientContextPriv* priv, VideoFrame* frame)
{
  WINPR_ASSERT(priv);
  if (!frame)
    return;

  mappedGeometryUnref(frame->geometry);

  BufferPool_Return(priv->surfacePool, frame->surfaceData);
  free(frame);
}

WINPR_ATTR_MALLOC(VideoFrame_free, 1)
static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
                                  MAPPED_GEOMETRY* geom)
{
  WINPR_ASSERT(priv);
  WINPR_ASSERT(presentation);
  WINPR_ASSERT(geom);

  const VideoSurface* surface = presentation->surface;
  WINPR_ASSERT(surface);

  VideoFrame* frame = calloc(1, sizeof(VideoFrame));
  if (!frame)
    goto fail;
  frame->PresentationId = presentation->PresentationId;

  mappedGeometryRef(geom);

  frame->publishTime = presentation->lastPublishTime;
  frame->geometry = geom;
  frame->w = surface->alignedWidth;
  frame->h = surface->alignedHeight;
  frame->scanline = surface->scanline;

  frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
  if (!frame->surfaceData)
    goto fail;

  return frame;

fail:
  VideoFrame_free(priv, frame);
  return nullptr;
}

void VideoClientContextPriv_free(VideoClientContextPriv* priv)
{
  if (!priv)
    return;

  EnterCriticalSection(&priv->framesLock);

  if (priv->frames)
  {
    while (Queue_Count(priv->frames))
    {
      VideoFrame* frame = Queue_Dequeue(priv->frames);
      if (frame)
        VideoFrame_free(priv, frame);
    }
  }

  Queue_Free(priv->frames);
  LeaveCriticalSection(&priv->framesLock);

  DeleteCriticalSection(&priv->framesLock);

  if (priv->currentPresentation)
    PresentationContext_unref(&priv->currentPresentation);

  BufferPool_Free(priv->surfacePool);
  free(priv);
}

WINPR_ATTR_NODISCARD
static UINT video_channel_write(VIDEO_PLUGIN* video, const BYTE* data, UINT32 length)
{
  WINPR_ASSERT(video);

  if (!video->control_callback || !video->control_callback->channel_callback)
    return ERROR_BAD_CONFIGURATION;
  IWTSVirtualChannel* channel = video->control_callback->channel_callback->channel;
  if (!channel || !channel->Write)
    return ERROR_BAD_CONFIGURATION;
  return channel->Write(channel, length, data, nullptr);
}

WINPR_ATTR_NODISCARD
static UINT video_control_send_presentation_response(VideoClientContext* context,
                                                     TSMM_PRESENTATION_RESPONSE* resp)
{
  BYTE buf[12] = WINPR_C_ARRAY_INIT;

  WINPR_ASSERT(context);
  WINPR_ASSERT(resp);

  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
  WINPR_ASSERT(video);

  wStream* s = Stream_New(buf, 12);
  if (!s)
    return CHANNEL_RC_NO_MEMORY;

  Stream_Write_UINT32(s, 12);                                     /* cbSize */
  Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
  Stream_Write_UINT8(s, resp->PresentationId);
  Stream_Zero(s, 3);
  Stream_SealLength(s);
  Stream_Free(s, FALSE);

  return video_channel_write(video, buf, sizeof(buf));
}

WINPR_ATTR_NODISCARD
static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
{
  WINPR_ASSERT(geometry);

  PresentationContext* presentation = (PresentationContext*)geometry->custom;
  WINPR_ASSERT(presentation);

  RDP_RECT* r = &geometry->geometry.boundingRect;
  WLog_DBG(TAG,
           "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
           ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
           "-%" PRId16 "x%" PRId16 ")",
           geometry->topLevelLeft, geometry->topLevelTop,
           geometry->topLevelRight - geometry->topLevelLeft,
           geometry->topLevelBottom - geometry->topLevelTop,

           geometry->left, geometry->top, geometry->right - geometry->left,
           geometry->bottom - geometry->top,

           r->x, r->y, r->width, r->height);

  WINPR_ASSERT(presentation->surface);
  if (geometry->topLevelLeft < 0)
  {
    WLog_ERR(TAG, "geometry->topLevelLeft=%d < 0", geometry->topLevelLeft);
    return FALSE;
  }
  if (geometry->left < 0)
  {
    WLog_ERR(TAG, "geometry->left=%d < 0", geometry->left);
    return FALSE;
  }
  presentation->surface->x = WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft) +
                             WINPR_ASSERTING_INT_CAST(uint32_t, geometry->left);

  if (geometry->topLevelTop < 0)
  {
    WLog_ERR(TAG, "geometry->topLevelTop=%d < 0", geometry->topLevelTop);
    return FALSE;
  }
  if (geometry->top < 0)
  {
    WLog_ERR(TAG, "geometry->top=%d < 0", geometry->top);
    return FALSE;
  }
  presentation->surface->y = WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop) +
                             WINPR_ASSERTING_INT_CAST(uint32_t, geometry->top);

  return TRUE;
}

WINPR_ATTR_NODISCARD
static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
{
  WINPR_ASSERT(geometry);

  PresentationContext* presentation = (PresentationContext*)geometry->custom;
  WINPR_ASSERT(presentation);

  mappedGeometryUnref(presentation->geometry);
  presentation->geometry = nullptr;
  return TRUE;
}

WINPR_ATTR_NODISCARD
static UINT video_PresentationRequest(VideoClientContext* video,
                                      const TSMM_PRESENTATION_REQUEST* req)
{
  UINT ret = CHANNEL_RC_OK;

  WINPR_ASSERT(video);
  WINPR_ASSERT(req);

  VideoClientContextPriv* priv = video->priv;
  WINPR_ASSERT(priv);

  EnterCriticalSection(&priv->framesLock);
  if (req->Command == TSMM_START_PRESENTATION)
  {
    MAPPED_GEOMETRY* geom = nullptr;
    TSMM_PRESENTATION_RESPONSE resp = WINPR_C_ARRAY_INIT;

    if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
    {
      WLog_ERR(TAG, "not a H264 video, ignoring request");
      goto fail;
    }

    if (priv->currentPresentation)
    {
      if (priv->currentPresentation->PresentationId == req->PresentationId)
      {
        WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
                 req->PresentationId);
        goto fail;
      }

      WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
      PresentationContext_unref(&priv->currentPresentation);
    }

    if (!priv->geometry)
    {
      WLog_ERR(TAG, "geometry channel not ready, ignoring request");
      goto fail;
    }

    geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
    if (!geom)
    {
      WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
      goto fail;
    }

    WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
    if ((geom->topLevelLeft < 0) || (geom->left < 0) || (geom->topLevelTop < 0) ||
        (geom->top < 0))
    {
      WLog_ERR(TAG,
               "geometry: topLevelLeft=%" PRId32 " < 0, left=%" PRId32
               " < 0, topLevelTop=%" PRId32 " < 0, top=%" PRId32 " < 0",
               geom->topLevelLeft, geom->left, geom->topLevelTop, geom->top);
      goto fail;
    }

    priv->currentPresentation = PresentationContext_new(
        video, req->PresentationId,
        WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
        WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
        req->SourceHeight);
    if (!priv->currentPresentation)
    {
      WLog_ERR(TAG, "unable to create presentation video");
      ret = CHANNEL_RC_NO_MEMORY;
      goto fail;
    }

    mappedGeometryRef(geom);
    priv->currentPresentation->geometry = geom;

    priv->currentPresentation->video = video;
    priv->currentPresentation->ScaledWidth = req->ScaledWidth;
    priv->currentPresentation->ScaledHeight = req->ScaledHeight;

    geom->custom = priv->currentPresentation;
    geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
    geom->MappedGeometryClear = video_onMappedGeometryClear;

    /* send back response */
    resp.PresentationId = req->PresentationId;
    ret = video_control_send_presentation_response(video, &resp);
  }
  else if (req->Command == TSMM_STOP_PRESENTATION)
  {
    WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
    if (!priv->currentPresentation)
    {
      WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
      goto fail;
    }

    priv->droppedFrames = 0;
    priv->publishedFrames = 0;
    PresentationContext_unref(&priv->currentPresentation);
  }

fail:
  LeaveCriticalSection(&priv->framesLock);
  return ret;
}

WINPR_ATTR_NODISCARD
static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
{
  TSMM_PRESENTATION_REQUEST req = WINPR_C_ARRAY_INIT;

  WINPR_ASSERT(context);
  WINPR_ASSERT(s);

  if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
    return ERROR_INVALID_DATA;

  Stream_Read_UINT8(s, req.PresentationId);
  Stream_Read_UINT8(s, req.Version);
  Stream_Read_UINT8(s, req.Command);
  Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */

  Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
  Stream_Seek_UINT16(s); /* reserved */

  Stream_Read_UINT32(s, req.SourceWidth);
  Stream_Read_UINT32(s, req.SourceHeight);
  Stream_Read_UINT32(s, req.ScaledWidth);
  Stream_Read_UINT32(s, req.ScaledHeight);
  if ((req.ScaledWidth == 0) || (req.SourceHeight == 0) || (req.ScaledWidth == 0) ||
      (req.ScaledHeight == 0))
  {
    WLog_ERR(TAG,
             "SourceWidth=%" PRIu32 ", SourceHeight=%" PRIu32 ", ScaledWidth=%" PRIu32
             ", ScaledHeight=%" PRIu32,
             req.SourceWidth, req.SourceHeight, req.ScaledWidth, req.ScaledHeight);
    return ERROR_INVALID_DATA;
  }
  Stream_Read_UINT64(s, req.hnsTimestampOffset);
  Stream_Read_UINT64(s, req.GeometryMappingId);
  Stream_Read(s, req.VideoSubtypeId, 16);

  Stream_Read_UINT32(s, req.cbExtra);

  if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
    return ERROR_INVALID_DATA;

  req.pExtraData = Stream_Pointer(s);

  WLog_DBG(TAG,
           "presentationReq: id:%" PRIu8 " version:%" PRIu8
           " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
           "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
           req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
           req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
           req.GeometryMappingId);

  return video_PresentationRequest(context, &req);
}

/**
 * Function description
 *
 * @return 0 on success, otherwise a Win32 error code
 */
WINPR_ATTR_NODISCARD
static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
{
  GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
  UINT ret = CHANNEL_RC_OK;
  UINT32 cbSize = 0;
  UINT32 packetType = 0;

  WINPR_ASSERT(callback);
  WINPR_ASSERT(s);

  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
  WINPR_ASSERT(video);

  VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
  WINPR_ASSERT(context);

  if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
    return ERROR_INVALID_DATA;

  Stream_Read_UINT32(s, cbSize);
  if (cbSize < 8)
  {
    WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
    return ERROR_INVALID_DATA;
  }
  if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
    return ERROR_INVALID_DATA;

  Stream_Read_UINT32(s, packetType);
  switch (packetType)
  {
    case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
      ret = video_read_tsmm_presentation_req(context, s);
      break;
    default:
      WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
      ret = ERROR_UNSUPPORTED_TYPE;
      break;
  }

  return ret;
}

static UINT video_control_send_client_notification(VideoClientContext* context,
                                                   const TSMM_CLIENT_NOTIFICATION* notif)
{
  BYTE buf[100] = WINPR_C_ARRAY_INIT;

  WINPR_ASSERT(context);
  WINPR_ASSERT(notif);

  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
  WINPR_ASSERT(video);

  wStream* s = Stream_New(buf, 32);
  if (!s)
    return CHANNEL_RC_NO_MEMORY;

  UINT32 cbSize = 16;
  Stream_Seek_UINT32(s);                                        /* cbSize */
  Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
  Stream_Write_UINT8(s, notif->PresentationId);
  Stream_Write_UINT8(s, notif->NotificationType);
  Stream_Zero(s, 2);
  if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
  {
    Stream_Write_UINT32(s, 16); /* cbData */

    /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
    Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
    Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
    Stream_Zero(s, 4ULL * 2ULL);

    cbSize += 4UL * 4UL;
  }
  else
  {
    Stream_Write_UINT32(s, 0); /* cbData */
  }

  Stream_SealLength(s);
  Stream_ResetPosition(s);
  Stream_Write_UINT32(s, cbSize);
  Stream_Free(s, FALSE);

  return video_channel_write(video, buf, cbSize);
}

static void video_timer(VideoClientContext* video, UINT64 now)
{
  VideoFrame* frame = nullptr;

  WINPR_ASSERT(video);

  VideoClientContextPriv* priv = video->priv;
  WINPR_ASSERT(priv);

  EnterCriticalSection(&priv->framesLock);
  PresentationContext* presentation = video->priv->currentPresentation;
  do
  {
    const VideoFrame* peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
    if (!peekFrame)
      break;

    if (peekFrame->publishTime > now)
      break;

    if (frame)
    {
      WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
      priv->droppedFrames++;
      VideoFrame_free(priv, frame);
    }
    frame = Queue_Dequeue(priv->frames);
  } while (1);

  if (frame)
  {
    if (presentation && (presentation->PresentationId == frame->PresentationId))
    {
      VideoSurface* surface = presentation->surface;
      const size_t frameSize = 1ull * frame->scanline * frame->h;
      const size_t surfaceSize = 1ull * surface->scanline * surface->alignedHeight;

      /* the presentation id is reused by the server across presentations of different
       * sizes, so a frame queued for a previous, larger presentation can outlive it in
       * the queue. copying it would write past the smaller surface->data buffer. */
      if (frameSize > surfaceSize)
        WLog_WARN(TAG, "dropping stale frame of %" PRIuz " bytes, surface holds %" PRIuz,
                  frameSize, surfaceSize);
      else
      {
        priv->publishedFrames++;
        memcpy(surface->data, frame->surfaceData, frameSize);

        WINPR_ASSERT(video->showSurface);
        if (!video->showSurface(video, surface, presentation->ScaledWidth,
                                presentation->ScaledHeight))
          WLog_WARN(TAG, "showSurface failed");
      }
    }
    VideoFrame_free(priv, frame);
  }

  if (priv->nextFeedbackTime < now)
  {
    /* we can compute some feedback only if we have some published frames and
     * a current presentation
     */
    if (priv->publishedFrames && priv->currentPresentation)
    {
      UINT32 computedRate = 0;

      if (!PresentationContext_ref(priv->currentPresentation))
        WLog_WARN(TAG, "PresentationContext_ref(priv->currentPresentation) failed");

      if (priv->droppedFrames)
      {
        /**
         * some dropped frames, looks like we're asking too many frames per seconds,
         * try lowering rate. We go directly from unlimited rate to 24 frames/seconds
         * otherwise we lower rate by 2 frames by seconds
         */
        if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
          computedRate = 24;
        else
        {
          computedRate = priv->lastSentRate - 2;
          if (!computedRate)
            computedRate = 2;
        }
      }
      else
      {
        /**
         * we treat all frames ok, so either ask the server to send more,
         * or stay unlimited
         */
        if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
          computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
        else
        {
          computedRate = priv->lastSentRate + 2;
          if (computedRate > XF_VIDEO_UNLIMITED_RATE)
            computedRate = XF_VIDEO_UNLIMITED_RATE;
        }
      }

      if (computedRate != priv->lastSentRate)
      {
        TSMM_CLIENT_NOTIFICATION notif = WINPR_C_ARRAY_INIT;

        WINPR_ASSERT(priv->currentPresentation);
        notif.PresentationId = priv->currentPresentation->PresentationId;
        notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
        if (computedRate == XF_VIDEO_UNLIMITED_RATE)
        {
          notif.FramerateOverride.Flags = 0x01;
          notif.FramerateOverride.DesiredFrameRate = 0x00;
        }
        else
        {
          notif.FramerateOverride.Flags = 0x02;
          notif.FramerateOverride.DesiredFrameRate = computedRate;
        }

        video_control_send_client_notification(video, &notif);
        priv->lastSentRate = computedRate;

        WLog_VRB(TAG,
                 "server notified with rate %" PRIu32 " published=%" PRIu32
                 " dropped=%" PRIu32,
                 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
      }

      PresentationContext_unref(&priv->currentPresentation);
    }

    priv->droppedFrames = 0;
    priv->publishedFrames = 0;
    priv->nextFeedbackTime = now + 1000;
  }
  LeaveCriticalSection(&priv->framesLock);
}

WINPR_ATTR_NODISCARD
static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
{
  int status = 0;
  UINT res = CHANNEL_RC_OK;

  WINPR_ASSERT(context);
  WINPR_ASSERT(data);

  VideoClientContextPriv* priv = context->priv;
  WINPR_ASSERT(priv);

  EnterCriticalSection(&priv->framesLock);
  PresentationContext* presentation = priv->currentPresentation;
  if (!presentation)
  {
    WLog_ERR(TAG, "no current presentation");
    LeaveCriticalSection(&priv->framesLock);
    return CHANNEL_RC_OK;
  }

  if (!PresentationContext_ref(presentation))
  {
    LeaveCriticalSection(&priv->framesLock);
    return ERROR_INTERNAL_ERROR;
  }

  if (presentation->PresentationId != data->PresentationId)
  {
    WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
             presentation->PresentationId, data->PresentationId);
    goto out;
  }

  if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
  {
    WLog_ERR(TAG, "unable to expand the current packet");
    res = CHANNEL_RC_NO_MEMORY;
    goto out;
  }

  Stream_Write(presentation->currentSample, data->pSample, data->cbSample);

  if (data->CurrentPacketIndex == data->PacketsInSample)
  {
    VideoSurface* surface = presentation->surface;
    H264_CONTEXT* h264 = presentation->h264;
    const UINT64 startTime = winpr_GetTickCount64NS();
    MAPPED_GEOMETRY* geom = presentation->geometry;
    if (!geom)
      goto out;

    const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
                              WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
    Stream_SealLength(presentation->currentSample);
    Stream_ResetPosition(presentation->currentSample);

    if (data->SampleNumber == 1)
    {
      presentation->lastPublishTime = startTime;
    }

    presentation->lastPublishTime += 100ull * data->hnsDuration;
    const size_t len = Stream_Length(presentation->currentSample);
    if (len > UINT32_MAX)
      goto out;

    BOOL enqueueResult = 0;
    VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
    if (!frame)
    {
      WLog_ERR(TAG, "unable to create frame");
      res = CHANNEL_RC_NO_MEMORY;
      goto out;
    }

    status = avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
                               frame->surfaceData, surface->format, surface->scanline,
                               surface->alignedWidth, surface->alignedHeight, &rect, 1);
    if (status < 0)
    {
      VideoFrame_free(priv, frame);
      goto out;
    }

    enqueueResult = Queue_Enqueue(priv->frames, frame);

    if (!enqueueResult)
    {
      WLog_ERR(TAG, "unable to enqueue frame");
      VideoFrame_free(priv, frame);
      res = CHANNEL_RC_NO_MEMORY;
      goto out;
    }

    // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
    WLog_DBG(TAG, "scheduling frame in %" PRIu64 " ns", (frame->publishTime - startTime));
  }

out:
  PresentationContext_unref(&priv->currentPresentation);
  LeaveCriticalSection(&priv->framesLock);

  return res;
}

WINPR_ATTR_NODISCARD
static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
{
  GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
  UINT32 cbSize = 0;
  UINT32 packetType = 0;
  TSMM_VIDEO_DATA data;

  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
  WINPR_ASSERT(video);

  VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;

  if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
    return ERROR_INVALID_DATA;

  Stream_Read_UINT32(s, cbSize);
  if (cbSize < 8)
  {
    WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
    return ERROR_INVALID_DATA;
  }

  if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
    return ERROR_INVALID_DATA;

  Stream_Read_UINT32(s, packetType);
  if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
  {
    WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
    return ERROR_INVALID_DATA;
  }

  if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
    return ERROR_INVALID_DATA;

  Stream_Read_UINT8(s, data.PresentationId);
  Stream_Read_UINT8(s, data.Version);
  Stream_Read_UINT8(s, data.Flags);
  Stream_Seek_UINT8(s); /* reserved */
  Stream_Read_UINT64(s, data.hnsTimestamp);
  Stream_Read_UINT64(s, data.hnsDuration);
  Stream_Read_UINT16(s, data.CurrentPacketIndex);
  Stream_Read_UINT16(s, data.PacketsInSample);
  Stream_Read_UINT32(s, data.SampleNumber);
  Stream_Read_UINT32(s, data.cbSample);
  if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
    return ERROR_INVALID_DATA;
  data.pSample = Stream_Pointer(s);

  /*
      WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
     duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
     cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
     data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
     data.cbSample);
  */

  return video_VideoData(context, &data);
}

/**
 * Function description
 *
 * @return 0 on success, otherwise a Win32 error code
 */
WINPR_ATTR_NODISCARD
static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
{
  if (pChannelCallback)
  {
    GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
    VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
    if (video && video->control_callback)
    {
      video->control_callback->channel_callback = nullptr;
    }
    if (video)
      video->connected = FALSE;
  }
  free(pChannelCallback);
  return CHANNEL_RC_OK;
}

WINPR_ATTR_NODISCARD
static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
{
  if (pChannelCallback)
  {
    GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
    VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
    if (video && video->data_callback)
    {
      video->data_callback->channel_callback = nullptr;
    }

    if (video)
      video->dataConnected = FALSE;
  }
  free(pChannelCallback);
  return CHANNEL_RC_OK;
}

/**
 * Function description
 *
 * @return 0 on success, otherwise a Win32 error code
 */
// NOLINTBEGIN(readability-non-const-parameter)
WINPR_ATTR_NODISCARD
static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
                                                    IWTSVirtualChannel* channel,
                                                    WINPR_ATTR_UNUSED BYTE* Data,
                                                    WINPR_ATTR_UNUSED BOOL* pbAccept,
                                                    IWTSVirtualChannelCallback** ppCallback)
// NOLINTEND(readability-non-const-parameter)
{
  GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
  WINPR_ASSERT(listener_callback);

  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
  WINPR_ASSERT(video);

  if (video->connected)
  {
    WLog_ERR(TAG, "Channel already connected, terminating.");
    return ERROR_DEVICE_ALREADY_ATTACHED;
  }

  GENERIC_CHANNEL_CALLBACK* callback =
      (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
  if (!callback)
  {
    WLog_ERR(TAG, "calloc failed!");
    return CHANNEL_RC_NO_MEMORY;
  }

  callback->iface.OnDataReceived = video_control_on_data_received;
  callback->iface.OnClose = video_control_on_close;
  callback->plugin = listener_callback->plugin;
  callback->channel_mgr = listener_callback->channel_mgr;
  callback->channel = channel;
  listener_callback->channel_callback = callback;

  *ppCallback = &callback->iface;
  video->connected = TRUE;

  return CHANNEL_RC_OK;
}

// NOLINTBEGIN(readability-non-const-parameter)
WINPR_ATTR_NODISCARD
static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
                                                 IWTSVirtualChannel* pChannel,
                                                 WINPR_ATTR_UNUSED BYTE* Data,
                                                 WINPR_ATTR_UNUSED BOOL* pbAccept,
                                                 IWTSVirtualChannelCallback** ppCallback)
// NOLINTEND(readability-non-const-parameter)
{
  GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
  WINPR_ASSERT(listener_callback);

  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
  WINPR_ASSERT(video);

  if (video->dataConnected)
  {
    WLog_ERR(TAG, "Channel already connected, terminating.");
    return ERROR_DEVICE_ALREADY_ATTACHED;
  }

  GENERIC_CHANNEL_CALLBACK* callback =
      (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
  if (!callback)
  {
    WLog_ERR(TAG, "calloc failed!");
    return CHANNEL_RC_NO_MEMORY;
  }

  callback->iface.OnDataReceived = video_data_on_data_received;
  callback->iface.OnClose = video_data_on_close;
  callback->plugin = listener_callback->plugin;
  callback->channel_mgr = listener_callback->channel_mgr;
  callback->channel = pChannel;
  listener_callback->channel_callback = callback;

  *ppCallback = &callback->iface;
  video->dataConnected = TRUE;

  return CHANNEL_RC_OK;
}

WINPR_ATTR_NODISCARD
static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
                         WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
                         uint64_t interval)
{
  VideoClientContext* video = userdata;
  if (!video)
    return 0;
  if (!video->timer)
    return 0;

  video->timer(video, timestamp);

  return interval;
}

/**
 * Function description
 *
 * @return 0 on success, otherwise a Win32 error code
 */
WINPR_ATTR_NODISCARD
static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
{
  UINT status = 0;
  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;

  if (video->initialized)
  {
    WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
    return ERROR_INVALID_DATA;
  }

  {
    GENERIC_LISTENER_CALLBACK* callback =
        (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
    if (!callback)
    {
      WLog_ERR(TAG, "calloc for control callback failed!");
      return CHANNEL_RC_NO_MEMORY;
    }

    callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
    callback->plugin = plugin;
    callback->channel_mgr = channelMgr;

    status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
                                        &callback->iface, &(video->controlListener));
    video->control_callback = callback;
    if (status != CHANNEL_RC_OK)
      return status;
  }
  video->controlListener->pInterface = video->wtsPlugin.pInterface;

  {
    GENERIC_LISTENER_CALLBACK* callback =
        (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
    if (!callback)
    {
      WLog_ERR(TAG, "calloc for data callback failed!");
      return CHANNEL_RC_NO_MEMORY;
    }

    callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
    callback->plugin = plugin;
    callback->channel_mgr = channelMgr;

    status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
                                        &callback->iface, &(video->dataListener));
    video->data_callback = callback;
    if (status == CHANNEL_RC_OK)
      video->dataListener->pInterface = video->wtsPlugin.pInterface;
  }

  if (status == CHANNEL_RC_OK)
    video->context->priv->timerID =
        freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
  video->initialized = video->context->priv->timerID != 0;
  if (!video->initialized)
    status = ERROR_INTERNAL_ERROR;
  return status;
}

/**
 * Function description
 *
 * @return 0 on success, otherwise a Win32 error code
 */
WINPR_ATTR_NODISCARD
static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
{
  VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
  if (!video)
    return CHANNEL_RC_INVALID_INSTANCE;

  if (video->context && video->context->priv)
    freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);

  if (video->control_callback)
  {
    IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
    if (mgr)
      IFCALL(mgr->DestroyListener, mgr, video->controlListener);
  }
  if (video->data_callback)
  {
    IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
    if (mgr)
      IFCALL(mgr->DestroyListener, mgr, video->dataListener);
  }

  if (video->context)
    VideoClientContextPriv_free(video->context->priv);

  free(video->control_callback);
  free(video->data_callback);
  free(video->wtsPlugin.pInterface);
  free(pPlugin);
  return CHANNEL_RC_OK;
}

/**
 * Channel Client Interface
 */
/**
 * Function description
 *
 * @return 0 on success, otherwise a Win32 error code
 */
WINPR_ATTR_NODISCARD
FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
{
  UINT error = ERROR_INTERNAL_ERROR;

  VIDEO_PLUGIN* videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
  if (!videoPlugin)
  {
    videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
    if (!videoPlugin)
    {
      WLog_ERR(TAG, "calloc failed!");
      return CHANNEL_RC_NO_MEMORY;
    }

    videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
    videoPlugin->wtsPlugin.Connected = nullptr;
    videoPlugin->wtsPlugin.Disconnected = nullptr;
    videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;

    VideoClientContext* videoContext =
        (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
    if (!videoContext)
    {
      WLog_ERR(TAG, "calloc failed!");
      free(videoPlugin);
      return CHANNEL_RC_NO_MEMORY;
    }

    VideoClientContextPriv* priv = VideoClientContextPriv_new(videoContext);
    if (!priv)
    {
      WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
      free(videoContext);
      free(videoPlugin);
      return CHANNEL_RC_NO_MEMORY;
    }

    videoContext->handle = (void*)videoPlugin;
    videoContext->priv = priv;
    videoContext->timer = video_timer;
    videoContext->setGeometry = video_client_context_set_geometry;

    videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
    videoPlugin->context = videoContext;
    videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
    if (videoPlugin->rdpcontext)
      error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
  }
  else
  {
    WLog_ERR(TAG, "could not get video Plugin.");
    return CHANNEL_RC_BAD_CHANNEL;
  }

  return error;
}

Coverage Report

Created: 2026-10-15 06:27