/src/gstreamer/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c

Source
/* GStreamer
 * Copyright (C) <2011> Stefan Kost <ensonic@users.sf.net>
 * Copyright (C) <2015> Luis de Bethencourt <luis@debethencourt.com>
 *
 * gstaudiovisualizer.h: base class for audio visualisation elements
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
/**
 * SECTION:gstaudiovisualizer
 * @title: GstAudioVisualizer
 * @short_description: Base class for visualizers.
 *
 * A baseclass for scopes (visualizers). It takes care of re-fitting the
 * audio-rate to video-rate and handles renegotiation (downstream video size
 * changes).
 *
 * It also provides several background shading effects. These effects are
 * applied to a previous picture before the `render()` implementation can draw a
 * new frame.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <string.h>

#include <gst/video/video.h>
#include <gst/video/gstvideometa.h>
#include <gst/video/gstvideopool.h>

#include "gstaudiovisualizer.h"
#include "pbutils-enumtypes.h"

GST_DEBUG_CATEGORY_STATIC (audio_visualizer_debug);
#define GST_CAT_DEFAULT (audio_visualizer_debug)

#define DEFAULT_SHADER GST_AUDIO_VISUALIZER_SHADER_FADE
#define DEFAULT_SHADE_AMOUNT   0x000a0a0a

enum
{
  PROP_0,
  PROP_SHADER,
  PROP_SHADE_AMOUNT
};

static GstBaseTransformClass *parent_class = NULL;
static gint private_offset = 0;

static void gst_audio_visualizer_class_init (GstAudioVisualizerClass * klass);
static void gst_audio_visualizer_init (GstAudioVisualizer * scope,
    GstAudioVisualizerClass * g_class);
static void gst_audio_visualizer_set_property (GObject * object,
    guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_audio_visualizer_get_property (GObject * object,
    guint prop_id, GValue * value, GParamSpec * pspec);
static void gst_audio_visualizer_dispose (GObject * object);

static gboolean gst_audio_visualizer_src_negotiate (GstAudioVisualizer * scope);
static gboolean gst_audio_visualizer_src_setcaps (GstAudioVisualizer *
    scope, GstCaps * caps);
static gboolean gst_audio_visualizer_sink_setcaps (GstAudioVisualizer *
    scope, GstCaps * caps);

static GstFlowReturn gst_audio_visualizer_chain (GstPad * pad,
    GstObject * parent, GstBuffer * buffer);

static gboolean gst_audio_visualizer_src_event (GstPad * pad,
    GstObject * parent, GstEvent * event);
static gboolean gst_audio_visualizer_sink_event (GstPad * pad,
    GstObject * parent, GstEvent * event);

static gboolean gst_audio_visualizer_src_query (GstPad * pad,
    GstObject * parent, GstQuery * query);

static GstStateChangeReturn gst_audio_visualizer_change_state (GstElement *
    element, GstStateChange transition);

static gboolean gst_audio_visualizer_do_bufferpool (GstAudioVisualizer * scope,
    GstCaps * outcaps);

static gboolean
default_decide_allocation (GstAudioVisualizer * scope, GstQuery * query);

static gboolean copy_metadata (GstAudioVisualizer * scope,
    GstBuffer * inbuf, GstBuffer * outbuf);

struct _GstAudioVisualizerPrivate
{
  gboolean negotiated;

  GstBufferPool *pool;
  gboolean pool_active;
  GstAllocator *allocator;
  GstAllocationParams params;
  GstQuery *query;

  /* pads */
  GstPad *srcpad, *sinkpad;

  GstAudioVisualizerShader shader_type;
  GstAudioVisualizerShaderFunc shader;
  guint32 shade_amount;

  GstAdapter *adapter;

  GstBuffer *inbuf;
  GstBuffer *tempbuf;
  GstVideoFrame tempframe;

  guint spf;                    /* samples per video frame */
  guint64 frame_duration;

  /* QoS stuff *//* with LOCK */
  gdouble proportion;
  GstClockTime earliest_time;

  guint dropped;                /* frames dropped / not dropped */
  guint processed;

  /* configuration mutex */
  GMutex config_lock;

  GstSegment segment;
};

/* shading functions */

/* we're only supporting GST_VIDEO_FORMAT_xRGB right now) */
#if G_BYTE_ORDER == G_LITTLE_ENDIAN

#define SHADE(_d, _s, _i, _r, _g, _b)                     \
G_STMT_START {                                            \
    _d[_i * 4 + 0] = (_s[_i * 4 + 0] > _b) ? _s[_i * 4 + 0] - _b : 0; \
    _d[_i * 4 + 1] = (_s[_i * 4 + 1] > _g) ? _s[_i * 4 + 1] - _g : 0; \
    _d[_i * 4 + 2] = (_s[_i * 4 + 2] > _r) ? _s[_i * 4 + 2] - _r : 0; \
    _d[_i * 4 + 3] = 0;                                       \
} G_STMT_END

#else /* G_BYTE_ORDER == G_LITTLE_ENDIAN */

#define SHADE(_d, _s, _i, _r, _g, _b)                     \
G_STMT_START {                                            \
    _d[_i * 4 + 0] = 0;                                       \
    _d[_i * 4 + 1] = (_s[_i * 4 + 1] > _r) ? _s[_i * 4 + 1] - _r : 0; \
    _d[_i * 4 + 2] = (_s[_i * 4 + 2] > _g) ? _s[_i * 4 + 2] - _g : 0; \
    _d[_i * 4 + 3] = (_s[_i * 4 + 3] > _b) ? _s[_i * 4 + 3] - _b : 0; \
} G_STMT_END

#endif

static void
shader_fade (GstAudioVisualizer * scope, const GstVideoFrame * sframe,
    GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  for (j = 0; j < height; j++) {
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    s += ss;
    d += ds;
  }
}

static void
shader_fade_and_move_up (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  for (j = 1; j < height; j++) {
    s += ss;
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    d += ds;
  }
}

static void
shader_fade_and_move_down (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  for (j = 1; j < height; j++) {
    d += ds;
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    s += ss;
  }
}

static void
shader_fade_and_move_left (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  width -= 1;
  s += 4;

  /* move to the left */
  for (j = 0; j < height; j++) {
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    d += ds;
    s += ss;
  }
}

static void
shader_fade_and_move_right (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  width -= 1;
  d += 4;

  /* move to the right */
  for (j = 0; j < height; j++) {
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    d += ds;
    s += ss;
  }
}

static void
shader_fade_and_move_horiz_out (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  /* move upper half up */
  for (j = 0; j < height / 2; j++) {
    s += ss;
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    d += ds;
  }

  /* rewind one stride */
  d -= ds;

  /* move lower half down */
  for (j = 0; j < height / 2; j++) {
    d += ds;
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    s += ss;
  }
}

static void
shader_fade_and_move_horiz_in (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *d;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  /* move upper half down */
  for (j = 0; j < height / 2; j++) {
    d += ds;
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    s += ss;
  }
  /* move lower half up */
  for (j = 0; j < height / 2; j++) {
    s += ss;
    for (i = 0; i < width; i++) {
      SHADE (d, s, i, r, g, b);
    }
    d += ds;
  }
}

static void
shader_fade_and_move_vert_out (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *s1, *d, *d1;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  for (j = 0; j < height; j++) {
    /* move left half to the left */
    s1 = s + 1;
    for (i = 0; i < width / 2; i++) {
      SHADE (d, s1, i, r, g, b);
    }
    /* move right half to the right */
    d1 = d + 1;
    for (; i < width - 1; i++) {
      SHADE (d1, s, i, r, g, b);
    }
    s += ss;
    d += ds;
  }
}

static void
shader_fade_and_move_vert_in (GstAudioVisualizer * scope,
    const GstVideoFrame * sframe, GstVideoFrame * dframe)
{
  guint i, j;
  guint r = (scope->priv->shade_amount >> 16) & 0xff;
  guint g = (scope->priv->shade_amount >> 8) & 0xff;
  guint b = (scope->priv->shade_amount >> 0) & 0xff;
  guint8 *s, *s1, *d, *d1;
  gint ss, ds, width, height;

  s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
  ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
  d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
  ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);

  width = GST_VIDEO_FRAME_WIDTH (sframe);
  height = GST_VIDEO_FRAME_HEIGHT (sframe);

  for (j = 0; j < height; j++) {
    /* move left half to the right */
    d1 = d + 1;
    for (i = 0; i < width / 2; i++) {
      SHADE (d1, s, i, r, g, b);
    }
    /* move right half to the left */
    s1 = s + 1;
    for (; i < width - 1; i++) {
      SHADE (d, s1, i, r, g, b);
    }
    s += ss;
    d += ds;
  }
}

static void
gst_audio_visualizer_change_shader (GstAudioVisualizer * scope)
{
  switch (scope->priv->shader_type) {
    case GST_AUDIO_VISUALIZER_SHADER_NONE:
      scope->priv->shader = NULL;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE:
      scope->priv->shader = shader_fade;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_UP:
      scope->priv->shader = shader_fade_and_move_up;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_DOWN:
      scope->priv->shader = shader_fade_and_move_down;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_LEFT:
      scope->priv->shader = shader_fade_and_move_left;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_RIGHT:
      scope->priv->shader = shader_fade_and_move_right;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_OUT:
      scope->priv->shader = shader_fade_and_move_horiz_out;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_IN:
      scope->priv->shader = shader_fade_and_move_horiz_in;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_OUT:
      scope->priv->shader = shader_fade_and_move_vert_out;
      break;
    case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_IN:
      scope->priv->shader = shader_fade_and_move_vert_in;
      break;
    default:
      GST_ERROR ("invalid shader function");
      scope->priv->shader = NULL;
      break;
  }
}

/* base class */

GType
gst_audio_visualizer_get_type (void)
{
  static gsize audio_visualizer_type = 0;

  if (g_once_init_enter (&audio_visualizer_type)) {
    static const GTypeInfo audio_visualizer_info = {
      sizeof (GstAudioVisualizerClass),
      NULL,
      NULL,
      (GClassInitFunc) gst_audio_visualizer_class_init,
      NULL,
      NULL,
      sizeof (GstAudioVisualizer),
      0,
      (GInstanceInitFunc) gst_audio_visualizer_init,
    };
    GType _type;

    /* TODO: rename when exporting it as a library */
    _type = g_type_register_static (GST_TYPE_ELEMENT,
        "GstAudioVisualizer", &audio_visualizer_info, G_TYPE_FLAG_ABSTRACT);

    private_offset =
        g_type_add_instance_private (_type, sizeof (GstAudioVisualizerPrivate));

    g_once_init_leave (&audio_visualizer_type, _type);
  }
  return (GType) audio_visualizer_type;
}

static inline GstAudioVisualizerPrivate *
gst_audio_visualizer_get_instance_private (GstAudioVisualizer * self)
{
  return (G_STRUCT_MEMBER_P (self, private_offset));
}

static void
gst_audio_visualizer_class_init (GstAudioVisualizerClass * klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;
  GstElementClass *element_class = (GstElementClass *) klass;

  if (private_offset != 0)
    g_type_class_adjust_private_offset (klass, &private_offset);

  parent_class = g_type_class_peek_parent (klass);

  GST_DEBUG_CATEGORY_INIT (audio_visualizer_debug,
      "baseaudiovisualizer-libvisual", 0,
      "scope audio visualisation base class");

  gobject_class->set_property = gst_audio_visualizer_set_property;
  gobject_class->get_property = gst_audio_visualizer_get_property;
  gobject_class->dispose = gst_audio_visualizer_dispose;

  element_class->change_state =
      GST_DEBUG_FUNCPTR (gst_audio_visualizer_change_state);

  klass->decide_allocation = GST_DEBUG_FUNCPTR (default_decide_allocation);

  g_object_class_install_property (gobject_class, PROP_SHADER,
      g_param_spec_enum ("shader", "shader type",
          "Shader function to apply on each frame",
          GST_TYPE_AUDIO_VISUALIZER_SHADER, DEFAULT_SHADER,
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
  g_object_class_install_property (gobject_class, PROP_SHADE_AMOUNT,
      g_param_spec_uint ("shade-amount", "shade amount",
          "Shading color to use (big-endian ARGB)", 0, G_MAXUINT32,
          DEFAULT_SHADE_AMOUNT,
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
}

static void
gst_audio_visualizer_init (GstAudioVisualizer * scope,
    GstAudioVisualizerClass * g_class)
{
  GstPadTemplate *pad_template;

  scope->priv = gst_audio_visualizer_get_instance_private (scope);

  /* create the sink and src pads */
  pad_template =
      gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "sink");
  g_return_if_fail (pad_template != NULL);
  scope->priv->sinkpad = gst_pad_new_from_template (pad_template, "sink");
  gst_pad_set_chain_function (scope->priv->sinkpad,
      GST_DEBUG_FUNCPTR (gst_audio_visualizer_chain));
  gst_pad_set_event_function (scope->priv->sinkpad,
      GST_DEBUG_FUNCPTR (gst_audio_visualizer_sink_event));
  gst_element_add_pad (GST_ELEMENT (scope), scope->priv->sinkpad);

  pad_template =
      gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "src");
  g_return_if_fail (pad_template != NULL);
  scope->priv->srcpad = gst_pad_new_from_template (pad_template, "src");
  gst_pad_set_event_function (scope->priv->srcpad,
      GST_DEBUG_FUNCPTR (gst_audio_visualizer_src_event));
  gst_pad_set_query_function (scope->priv->srcpad,
      GST_DEBUG_FUNCPTR (gst_audio_visualizer_src_query));
  gst_element_add_pad (GST_ELEMENT (scope), scope->priv->srcpad);

  scope->priv->adapter = gst_adapter_new ();
  scope->priv->inbuf = gst_buffer_new ();

  /* properties */
  scope->priv->shader_type = DEFAULT_SHADER;
  gst_audio_visualizer_change_shader (scope);
  scope->priv->shade_amount = DEFAULT_SHADE_AMOUNT;

  /* reset the initial video state */
  gst_video_info_init (&scope->vinfo);
  scope->priv->frame_duration = GST_CLOCK_TIME_NONE;

  /* reset the initial state */
  gst_audio_info_init (&scope->ainfo);
  gst_video_info_init (&scope->vinfo);

  g_mutex_init (&scope->priv->config_lock);
}

static void
gst_audio_visualizer_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);

  switch (prop_id) {
    case PROP_SHADER:
      scope->priv->shader_type = g_value_get_enum (value);
      gst_audio_visualizer_change_shader (scope);
      break;
    case PROP_SHADE_AMOUNT:
      scope->priv->shade_amount = g_value_get_uint (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_audio_visualizer_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);

  switch (prop_id) {
    case PROP_SHADER:
      g_value_set_enum (value, scope->priv->shader_type);
      break;
    case PROP_SHADE_AMOUNT:
      g_value_set_uint (value, scope->priv->shade_amount);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_audio_visualizer_dispose (GObject * object)
{
  GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);

  if (scope->priv->adapter) {
    g_object_unref (scope->priv->adapter);
    scope->priv->adapter = NULL;
  }
  if (scope->priv->inbuf) {
    gst_buffer_unref (scope->priv->inbuf);
    scope->priv->inbuf = NULL;
  }
  if (scope->priv->tempbuf) {
    gst_video_frame_unmap (&scope->priv->tempframe);
    gst_buffer_unref (scope->priv->tempbuf);
    scope->priv->tempbuf = NULL;
  }
  if (scope->priv->config_lock.p) {
    g_mutex_clear (&scope->priv->config_lock);
    scope->priv->config_lock.p = NULL;
  }
  G_OBJECT_CLASS (parent_class)->dispose (object);
}

static void
gst_audio_visualizer_reset (GstAudioVisualizer * scope)
{
  gst_adapter_clear (scope->priv->adapter);
  gst_segment_init (&scope->priv->segment, GST_FORMAT_UNDEFINED);

  GST_OBJECT_LOCK (scope);
  scope->priv->proportion = 1.0;
  scope->priv->earliest_time = -1;
  scope->priv->dropped = 0;
  scope->priv->processed = 0;
  GST_OBJECT_UNLOCK (scope);
}

static gboolean
gst_audio_visualizer_sink_setcaps (GstAudioVisualizer * scope, GstCaps * caps)
{
  GstAudioInfo info;

  if (!gst_audio_info_from_caps (&info, caps))
    goto wrong_caps;

  scope->ainfo = info;

  GST_DEBUG_OBJECT (scope, "audio: channels %d, rate %d",
      GST_AUDIO_INFO_CHANNELS (&info), GST_AUDIO_INFO_RATE (&info));

  if (!gst_audio_visualizer_src_negotiate (scope)) {
    goto not_negotiated;
  }

  return TRUE;

  /* Errors */
wrong_caps:
  {
    GST_WARNING_OBJECT (scope, "could not parse caps");
    return FALSE;
  }
not_negotiated:
  {
    GST_WARNING_OBJECT (scope, "failed to negotiate");
    return FALSE;
  }
}

static gboolean
gst_audio_visualizer_src_setcaps (GstAudioVisualizer * scope, GstCaps * caps)
{
  GstVideoInfo info;
  GstAudioVisualizerClass *klass;
  gboolean res;

  if (!gst_video_info_from_caps (&info, caps))
    goto wrong_caps;

  klass = GST_AUDIO_VISUALIZER_CLASS (G_OBJECT_GET_CLASS (scope));

  scope->vinfo = info;

  scope->priv->frame_duration = gst_util_uint64_scale_int (GST_SECOND,
      GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info));
  scope->priv->spf =
      gst_util_uint64_scale_int (GST_AUDIO_INFO_RATE (&scope->ainfo),
      GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info));
  scope->req_spf = scope->priv->spf;

  if (scope->priv->tempbuf) {
    gst_video_frame_unmap (&scope->priv->tempframe);
    gst_buffer_unref (scope->priv->tempbuf);
  }
  scope->priv->tempbuf = gst_buffer_new_wrapped (g_malloc0 (scope->vinfo.size),
      scope->vinfo.size);
  gst_video_frame_map (&scope->priv->tempframe, &scope->vinfo,
      scope->priv->tempbuf, GST_MAP_READWRITE);

  if (klass->setup && !klass->setup (scope))
    goto setup_failed;

  GST_DEBUG_OBJECT (scope, "video: dimension %dx%d, framerate %d/%d",
      GST_VIDEO_INFO_WIDTH (&info), GST_VIDEO_INFO_HEIGHT (&info),
      GST_VIDEO_INFO_FPS_N (&info), GST_VIDEO_INFO_FPS_D (&info));
  GST_DEBUG_OBJECT (scope, "blocks: spf %u, req_spf %u",
      scope->priv->spf, scope->req_spf);

  gst_pad_set_caps (scope->priv->srcpad, caps);

  /* find a pool for the negotiated caps now */
  res = gst_audio_visualizer_do_bufferpool (scope, caps);
  gst_caps_unref (caps);

  return res;

  /* ERRORS */
wrong_caps:
  {
    gst_caps_unref (caps);
    GST_DEBUG_OBJECT (scope, "error parsing caps");
    return FALSE;
  }

setup_failed:
  {
    GST_WARNING_OBJECT (scope, "failed to set up");
    return FALSE;
  }
}

static gboolean
gst_audio_visualizer_src_negotiate (GstAudioVisualizer * scope)
{
  GstCaps *othercaps, *target;
  GstStructure *structure;
  GstCaps *templ;
  gboolean ret;

  templ = gst_pad_get_pad_template_caps (scope->priv->srcpad);

  GST_DEBUG_OBJECT (scope, "performing negotiation");

  /* see what the peer can do */
  othercaps = gst_pad_peer_query_caps (scope->priv->srcpad, NULL);
  if (othercaps) {
    target = gst_caps_intersect (othercaps, templ);
    gst_caps_unref (othercaps);
    gst_caps_unref (templ);

    if (gst_caps_is_empty (target))
      goto no_format;

    target = gst_caps_truncate (target);
  } else {
    target = templ;
  }

  target = gst_caps_make_writable (target);
  structure = gst_caps_get_structure (target, 0);
  gst_structure_fixate_field_nearest_int (structure, "width", 320);
  gst_structure_fixate_field_nearest_int (structure, "height", 200);
  gst_structure_fixate_field_nearest_fraction (structure, "framerate", 25, 1);
  if (gst_structure_has_field (structure, "pixel-aspect-ratio"))
    gst_structure_fixate_field_nearest_fraction (structure,
        "pixel-aspect-ratio", 1, 1);

  target = gst_caps_fixate (target);

  GST_DEBUG_OBJECT (scope, "final caps are %" GST_PTR_FORMAT, target);

  ret = gst_audio_visualizer_src_setcaps (scope, target);

  return ret;

no_format:
  {
    gst_caps_unref (target);
    return FALSE;
  }
}

/* takes ownership of the pool, allocator and query */
static gboolean
gst_audio_visualizer_set_allocation (GstAudioVisualizer * scope,
    GstBufferPool * pool, GstAllocator * allocator,
    const GstAllocationParams * params, GstQuery * query)
{
  GstAllocator *oldalloc;
  GstBufferPool *oldpool;
  GstQuery *oldquery;
  GstAudioVisualizerPrivate *priv = scope->priv;

  GST_OBJECT_LOCK (scope);
  oldpool = priv->pool;
  priv->pool = pool;
  priv->pool_active = FALSE;

  oldalloc = priv->allocator;
  priv->allocator = allocator;

  oldquery = priv->query;
  priv->query = query;

  if (params)
    priv->params = *params;
  else
    gst_allocation_params_init (&priv->params);
  GST_OBJECT_UNLOCK (scope);

  if (oldpool) {
    GST_DEBUG_OBJECT (scope, "deactivating old pool %p", oldpool);
    gst_buffer_pool_set_active (oldpool, FALSE);
    gst_object_unref (oldpool);
  }
  if (oldalloc) {
    gst_object_unref (oldalloc);
  }
  if (oldquery) {
    gst_query_unref (oldquery);
  }
  return TRUE;
}

static gboolean
gst_audio_visualizer_do_bufferpool (GstAudioVisualizer * scope,
    GstCaps * outcaps)
{
  GstQuery *query;
  gboolean result = TRUE;
  GstBufferPool *pool = NULL;
  GstAudioVisualizerClass *klass;
  GstAllocator *allocator;
  GstAllocationParams params;

  /* not passthrough, we need to allocate */
  /* find a pool for the negotiated caps now */
  GST_DEBUG_OBJECT (scope, "doing allocation query");
  query = gst_query_new_allocation (outcaps, TRUE);

  if (!gst_pad_peer_query (scope->priv->srcpad, query)) {
    /* not a problem, we use the query defaults */
    GST_DEBUG_OBJECT (scope, "allocation query failed");
  }

  klass = GST_AUDIO_VISUALIZER_GET_CLASS (scope);

  GST_DEBUG_OBJECT (scope, "calling decide_allocation");
  g_assert (klass->decide_allocation != NULL);
  result = klass->decide_allocation (scope, query);

  GST_DEBUG_OBJECT (scope, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result,
      query);

  if (!result)
    goto no_decide_allocation;

  /* we got configuration from our peer or the decide_allocation method,
   * parse them */
  if (gst_query_get_n_allocation_params (query) > 0) {
    gst_query_parse_nth_allocation_param (query, 0, &allocator, &params);
  } else {
    allocator = NULL;
    gst_allocation_params_init (&params);
  }

  if (gst_query_get_n_allocation_pools (query) > 0)
    gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);

  /* now store */
  result =
      gst_audio_visualizer_set_allocation (scope, pool, allocator, &params,
      query);

  return result;

  /* Errors */
no_decide_allocation:
  {
    GST_WARNING_OBJECT (scope, "Subclass failed to decide allocation");
    gst_query_unref (query);

    return result;
  }
}

static gboolean
default_decide_allocation (GstAudioVisualizer * scope, GstQuery * query)
{
  GstCaps *outcaps;
  GstBufferPool *pool;
  guint size, min, max;
  GstAllocator *allocator;
  GstAllocationParams params;
  GstStructure *config;
  gboolean update_allocator;
  gboolean update_pool;

  gst_query_parse_allocation (query, &outcaps, NULL);

  /* we got configuration from our peer or the decide_allocation method,
   * parse them */
  if (gst_query_get_n_allocation_params (query) > 0) {
    /* try the allocator */
    gst_query_parse_nth_allocation_param (query, 0, &allocator, &params);
    update_allocator = TRUE;
  } else {
    allocator = NULL;
    gst_allocation_params_init (&params);
    update_allocator = FALSE;
  }

  if (gst_query_get_n_allocation_pools (query) > 0) {
    gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
    update_pool = TRUE;
  } else {
    pool = NULL;
    size = GST_VIDEO_INFO_SIZE (&scope->vinfo);
    min = max = 0;
    update_pool = FALSE;
  }

  if (pool == NULL) {
    /* we did not get a pool, make one ourselves then */
    pool = gst_video_buffer_pool_new ();
    {
      gchar *name = g_strdup_printf ("%s-pool", GST_OBJECT_NAME (scope));
      g_object_set (pool, "name", name, NULL);
      g_free (name);
    }
  }

  config = gst_buffer_pool_get_config (pool);
  gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
  gst_buffer_pool_config_set_allocator (config, allocator, &params);
  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
  gst_buffer_pool_set_config (pool, config);

  if (update_allocator)
    gst_query_set_nth_allocation_param (query, 0, allocator, &params);
  else
    gst_query_add_allocation_param (query, allocator, &params);

  if (allocator)
    gst_object_unref (allocator);

  if (update_pool)
    gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
  else
    gst_query_add_allocation_pool (query, pool, size, min, max);

  if (pool)
    gst_object_unref (pool);

  return TRUE;
}

static GstFlowReturn
default_prepare_output_buffer (GstAudioVisualizer * scope, GstBuffer ** outbuf)
{
  GstAudioVisualizerPrivate *priv;

  priv = scope->priv;

  g_assert (priv->pool != NULL);

  /* we can't reuse the input buffer */
  if (!priv->pool_active) {
    GST_DEBUG_OBJECT (scope, "setting pool %p active", priv->pool);
    if (!gst_buffer_pool_set_active (priv->pool, TRUE))
      goto activate_failed;
    priv->pool_active = TRUE;
  }
  GST_DEBUG_OBJECT (scope, "using pool alloc");

  return gst_buffer_pool_acquire_buffer (priv->pool, outbuf, NULL);

  /* ERRORS */
activate_failed:
  {
    GST_ELEMENT_ERROR (scope, RESOURCE, SETTINGS,
        ("failed to activate bufferpool"), ("failed to activate bufferpool"));
    return GST_FLOW_ERROR;
  }
}

static GstFlowReturn
gst_audio_visualizer_chain (GstPad * pad, GstObject * parent,
    GstBuffer * buffer)
{
  GstFlowReturn ret = GST_FLOW_OK;
  GstAudioVisualizer *scope;
  GstAudioVisualizerClass *klass;
  GstBuffer *inbuf, *databuf;
  guint64 dist, ts;
  guint avail, sbpf;
  gint bpf, rate;

  scope = GST_AUDIO_VISUALIZER (parent);
  klass = GST_AUDIO_VISUALIZER_CLASS (G_OBJECT_GET_CLASS (scope));

  GST_LOG_OBJECT (scope, "chainfunc called");

  /* resync on DISCONT */
  if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT)) {
    gst_adapter_clear (scope->priv->adapter);
  }

  /* Make sure have an output format */
  if (gst_pad_check_reconfigure (scope->priv->srcpad)) {
    if (!gst_audio_visualizer_src_negotiate (scope)) {
      gst_pad_mark_reconfigure (scope->priv->srcpad);
      goto not_negotiated;
    }
  }

  rate = GST_AUDIO_INFO_RATE (&scope->ainfo);
  bpf = GST_AUDIO_INFO_BPF (&scope->ainfo);

  if (bpf == 0) {
    goto not_negotiated;
  }

  g_mutex_lock (&scope->priv->config_lock);

  inbuf = scope->priv->inbuf;

  gst_adapter_push (scope->priv->adapter, buffer);

  /* this is what we want */
  sbpf = scope->req_spf * bpf;

  /* this is what we have */
  avail = gst_adapter_available (scope->priv->adapter);
  GST_LOG_OBJECT (scope, "avail: %u, bpf: %u", avail, sbpf);
  while (avail >= sbpf) {
    GstBuffer *outbuf;
    GstVideoFrame outframe;

    /* get timestamp of the current adapter content */
    ts = gst_adapter_prev_pts (scope->priv->adapter, &dist);
    if (GST_CLOCK_TIME_IS_VALID (ts)) {
      /* convert bytes to time */
      ts += gst_util_uint64_scale_int (dist, GST_SECOND, rate * bpf);
    }

    /* check for QoS, don't compute buffers that are known to be late */
    if (GST_CLOCK_TIME_IS_VALID (ts)) {
      GstClockTime earliest_time;
      gdouble proportion;
      gint64 qostime;

      qostime =
          gst_segment_to_running_time (&scope->priv->segment,
          GST_FORMAT_TIME, ts) + scope->priv->frame_duration;

      GST_OBJECT_LOCK (scope);
      earliest_time = scope->priv->earliest_time;
      proportion = scope->priv->proportion;
      GST_OBJECT_UNLOCK (scope);

      if (GST_CLOCK_TIME_IS_VALID (earliest_time) && qostime <= earliest_time) {
        GstClockTime stream_time, jitter;
        GstMessage *qos_msg;

        GST_DEBUG_OBJECT (scope,
            "QoS: skip ts: %" GST_TIME_FORMAT ", earliest: %" GST_TIME_FORMAT,
            GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time));

        ++scope->priv->dropped;
        stream_time = gst_segment_to_stream_time (&scope->priv->segment,
            GST_FORMAT_TIME, ts);
        jitter = GST_CLOCK_DIFF (qostime, earliest_time);
        qos_msg = gst_message_new_qos (GST_OBJECT (scope), FALSE, qostime,
            stream_time, ts, GST_BUFFER_DURATION (buffer));
        gst_message_set_qos_values (qos_msg, jitter, proportion, 1000000);
        gst_message_set_qos_stats (qos_msg, GST_FORMAT_BUFFERS,
            scope->priv->processed, scope->priv->dropped);
        gst_element_post_message (GST_ELEMENT (scope), qos_msg);

        goto skip;
      }
    }

    ++scope->priv->processed;

    g_mutex_unlock (&scope->priv->config_lock);
    ret = default_prepare_output_buffer (scope, &outbuf);
    g_mutex_lock (&scope->priv->config_lock);
    /* recheck as the value could have changed */
    sbpf = scope->req_spf * bpf;

    /* no buffer allocated, we don't care why. */
    if (ret != GST_FLOW_OK)
      break;

    /* sync controlled properties */
    if (GST_CLOCK_TIME_IS_VALID (ts))
      gst_object_sync_values (GST_OBJECT (scope), ts);

    GST_BUFFER_PTS (outbuf) = ts;
    GST_BUFFER_DURATION (outbuf) = scope->priv->frame_duration;

    /* this can fail as the data size we need could have changed */
    if (!(databuf = gst_adapter_get_buffer (scope->priv->adapter, sbpf)))
      break;

    if (!copy_metadata (scope, databuf, outbuf)) {
      GST_ELEMENT_WARNING (scope, STREAM, NOT_IMPLEMENTED,
          ("could not copy metadata"), (NULL));
    }

    gst_video_frame_map (&outframe, &scope->vinfo, outbuf,
        GST_MAP_READWRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF);

    if (scope->priv->shader) {
      gst_video_frame_copy (&outframe, &scope->priv->tempframe);
    } else {
      /* gst_video_frame_clear() or is output frame already cleared */
      gint i;

      for (i = 0; i < scope->vinfo.finfo->n_planes; i++) {
        memset (outframe.data[i], 0, outframe.map[i].size);
      }
    }

    gst_buffer_remove_all_memory (inbuf);
    gst_buffer_copy_into (inbuf, databuf, GST_BUFFER_COPY_MEMORY, 0, sbpf);
    gst_buffer_unref (databuf);

    /* call class->render() vmethod */
    if (klass->render) {
      if (!klass->render (scope, inbuf, &outframe)) {
        ret = GST_FLOW_ERROR;
        gst_video_frame_unmap (&outframe);
        break;
      } else {
        /* run various post processing (shading and geometric transformation) */
        /* FIXME: SHADER assumes 32bpp */
        if (scope->priv->shader &&
            GST_VIDEO_INFO_COMP_PSTRIDE (&scope->vinfo, 0) == 4) {
          scope->priv->shader (scope, &outframe, &scope->priv->tempframe);
        }
      }
    }
    gst_video_frame_unmap (&outframe);

    g_mutex_unlock (&scope->priv->config_lock);
    ret = gst_pad_push (scope->priv->srcpad, outbuf);
    outbuf = NULL;
    g_mutex_lock (&scope->priv->config_lock);

  skip:
    /* recheck as the value could have changed */
    sbpf = scope->req_spf * bpf;
    GST_LOG_OBJECT (scope, "avail: %u, bpf: %u", avail, sbpf);
    /* we want to take less or more, depending on spf : req_spf */
    if (avail - sbpf >= sbpf) {
      gst_adapter_flush (scope->priv->adapter, sbpf);
    } else if (avail >= sbpf) {
      /* just flush a bit and stop */
      gst_adapter_flush (scope->priv->adapter, (avail - sbpf));
      break;
    }
    avail = gst_adapter_available (scope->priv->adapter);

    if (ret != GST_FLOW_OK)
      break;
  }

  g_mutex_unlock (&scope->priv->config_lock);

  return ret;

  /* ERRORS */
not_negotiated:
  {
    GST_DEBUG_OBJECT (scope, "Failed to renegotiate");
    gst_buffer_unref (buffer);
    return GST_FLOW_NOT_NEGOTIATED;
  }
}

typedef struct
{
  GstAudioVisualizer *scope;
  GstBuffer *outbuf;
  GHashTable *ref_ts_metas;
} GatherCopyRefTsMetasData;

static gboolean
gather_ref_timestamp_metas (GstBuffer * inbuf, GstMeta ** meta,
    GatherCopyRefTsMetasData * data)
{
  GstAudioVisualizer *scope = data->scope;
  const GstMetaInfo *info = (*meta)->info;
  const GstReferenceTimestampMeta *ref_ts_meta;
  GstStructure *caps_structure;
  const gchar *caps_name;

  /* drop metas with tags for now */
  if (gst_meta_api_type_get_tags (info->api)) {
    GST_TRACE_OBJECT (scope, "meta has tags %s", g_type_name (info->api));
    goto not_copying;
  }

  if (info->api != GST_REFERENCE_TIMESTAMP_META_API_TYPE) {
    GstMetaTransformCopy copy_data = { FALSE, 0, -1 };
    if (info->transform_func) {
      GST_DEBUG_OBJECT (scope, "copying metadata %s", g_type_name (info->api));
      info->transform_func (data->outbuf, *meta, inbuf,
          _gst_meta_transform_copy, &copy_data);
    } else {
      GST_LOG_OBJECT (scope, "couldn't copy metadata %s",
          g_type_name (info->api));
    }

    return TRUE;
  }

  ref_ts_meta = (const GstReferenceTimestampMeta *) *meta;
  if (!ref_ts_meta->reference) {
    goto not_copying;
  }

  caps_structure = gst_caps_get_structure (ref_ts_meta->reference, 0);
  if (!caps_structure) {
    goto not_copying;
  }

  caps_name = gst_structure_get_name (caps_structure);
  if (!caps_name) {
    goto not_copying;
  }

  GST_LOG_OBJECT (scope,
      "found ref ts metadata %" GST_PTR_FORMAT ", ts %" GST_TIME_FORMAT
      ", duration %" GST_TIME_FORMAT, ref_ts_meta->reference,
      GST_TIME_ARGS (ref_ts_meta->timestamp),
      GST_TIME_ARGS (ref_ts_meta->duration));

  /* keeping the last reference timestamp meta with this reference name */
  g_hash_table_replace (data->ref_ts_metas, (gpointer) caps_name, (gpointer)
      ref_ts_meta);

  return TRUE;

not_copying:
  {
    GST_LOG_OBJECT (scope, "not copying metadata %s", g_type_name (info->api));
    return TRUE;
  }
}

static void
copy_gathered_ref_ts_metas_to_output (const char *_caps_name,
    GstReferenceTimestampMeta * ref_ts_meta, GatherCopyRefTsMetasData * data)
{

  GST_DEBUG_OBJECT (data->scope,
      "copying ref ts metadata %" GST_PTR_FORMAT ", ts %" GST_TIME_FORMAT
      ", duration %" GST_TIME_FORMAT, ref_ts_meta->reference,
      GST_TIME_ARGS (ref_ts_meta->timestamp),
      GST_TIME_ARGS (ref_ts_meta->duration));

  gst_buffer_add_reference_timestamp_meta (data->outbuf, ref_ts_meta->reference,
      ref_ts_meta->timestamp, ref_ts_meta->duration);
}

static gboolean
copy_metadata (GstAudioVisualizer * scope,
    GstBuffer * inbuf, GstBuffer * outbuf)
{
  GatherCopyRefTsMetasData data;

  GST_LOG_OBJECT (scope, "copying metadata");

  /* this should not happen, buffers allocated from a pool or with
   * new_allocate should always be writable. */
  if (!gst_buffer_is_writable (outbuf)) {
    goto not_writable;
  }

  data.scope = scope;
  data.outbuf = outbuf;
  data.ref_ts_metas = g_hash_table_new_full (g_str_hash, g_str_equal,
      NULL, NULL);

  gst_buffer_foreach_meta (inbuf,
      (GstBufferForeachMetaFunc) gather_ref_timestamp_metas, &data);
  g_hash_table_foreach (data.ref_ts_metas,
      (GHFunc) copy_gathered_ref_ts_metas_to_output, &data);

  g_hash_table_unref (data.ref_ts_metas);

  return TRUE;

not_writable:
  {
    GST_WARNING_OBJECT (scope, "buffer %p not writable", outbuf);
    return FALSE;
  }
}

static gboolean
gst_audio_visualizer_src_event (GstPad * pad, GstObject * parent,
    GstEvent * event)
{
  gboolean res;
  GstAudioVisualizer *scope;

  scope = GST_AUDIO_VISUALIZER (parent);

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_QOS:
    {
      gdouble proportion;
      GstClockTimeDiff diff;
      GstClockTime timestamp;

      gst_event_parse_qos (event, NULL, &proportion, &diff, &timestamp);

      /* save stuff for the _chain() function */
      GST_OBJECT_LOCK (scope);
      scope->priv->proportion = proportion;
      if (diff >= 0)
        /* we're late, this is a good estimate for next displayable
         * frame (see part-qos.txt) */
        scope->priv->earliest_time = timestamp + MIN (2 * diff, GST_SECOND) +
            scope->priv->frame_duration;
      else
        scope->priv->earliest_time = timestamp + diff;
      GST_OBJECT_UNLOCK (scope);

      res = gst_pad_push_event (scope->priv->sinkpad, event);
      break;
    }
    case GST_EVENT_RECONFIGURE:
      /* don't forward */
      gst_event_unref (event);
      res = TRUE;
      break;
    default:
      res = gst_pad_event_default (pad, parent, event);
      break;
  }

  return res;
}

static gboolean
gst_audio_visualizer_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event)
{
  gboolean res;
  GstAudioVisualizer *scope;

  scope = GST_AUDIO_VISUALIZER (parent);

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_CAPS:
    {
      GstCaps *caps;

      gst_event_parse_caps (event, &caps);
      res = gst_audio_visualizer_sink_setcaps (scope, caps);
      gst_event_unref (event);
      break;
    }
    case GST_EVENT_FLUSH_STOP:
      gst_audio_visualizer_reset (scope);
      res = gst_pad_push_event (scope->priv->srcpad, event);
      break;
    case GST_EVENT_SEGMENT:
    {
      /* the newsegment values are used to clip the input samples
       * and to convert the incoming timestamps to running time so
       * we can do QoS */
      gst_event_copy_segment (event, &scope->priv->segment);

      res = gst_pad_push_event (scope->priv->srcpad, event);
      break;
    }
    default:
      res = gst_pad_event_default (pad, parent, event);
      break;
  }

  return res;
}

static gboolean
gst_audio_visualizer_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query)
{
  gboolean res = FALSE;
  GstAudioVisualizer *scope;

  scope = GST_AUDIO_VISUALIZER (parent);

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_LATENCY:
    {
      /* We need to send the query upstream and add the returned latency to our
       * own */
      GstClockTime min_latency, max_latency;
      gboolean us_live;
      GstClockTime our_latency;
      guint max_samples;
      gint rate = GST_AUDIO_INFO_RATE (&scope->ainfo);

      if (rate == 0)
        break;

      if ((res = gst_pad_peer_query (scope->priv->sinkpad, query))) {
        gst_query_parse_latency (query, &us_live, &min_latency, &max_latency);

        GST_DEBUG_OBJECT (scope, "Peer latency: min %"
            GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
            GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));

        /* the max samples we must buffer buffer */
        max_samples = MAX (scope->req_spf, scope->priv->spf);
        our_latency = gst_util_uint64_scale_int (max_samples, GST_SECOND, rate);

        GST_DEBUG_OBJECT (scope, "Our latency: %" GST_TIME_FORMAT,
            GST_TIME_ARGS (our_latency));

        /* we add some latency but only if we need to buffer more than what
         * upstream gives us */
        min_latency += our_latency;
        if (max_latency != -1)
          max_latency += our_latency;

        GST_DEBUG_OBJECT (scope, "Calculated total latency : min %"
            GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
            GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));

        gst_query_set_latency (query, TRUE, min_latency, max_latency);
      }
      break;
    }
    default:
      res = gst_pad_query_default (pad, parent, query);
      break;
  }

  return res;
}

static GstStateChangeReturn
gst_audio_visualizer_change_state (GstElement * element,
    GstStateChange transition)
{
  GstStateChangeReturn ret;
  GstAudioVisualizer *scope;

  scope = GST_AUDIO_VISUALIZER (element);

  switch (transition) {
    case GST_STATE_CHANGE_READY_TO_PAUSED:
      gst_audio_visualizer_reset (scope);
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);

  switch (transition) {
    case GST_STATE_CHANGE_PAUSED_TO_READY:
      gst_audio_visualizer_set_allocation (scope, NULL, NULL, NULL, NULL);
      break;
    case GST_STATE_CHANGE_READY_TO_NULL:
      break;
    default:
      break;
  }

  return ret;
}

Coverage Report

Created: 2026-07-12 07:12