/src/serenity/Userland/Libraries/LibMedia/Video/VP9/Context.h

Source
/*
 * Copyright (c) 2021, Hunter Salyer <thefalsehonesty@gmail.com>
 * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#pragma once

#include <AK/Array.h>
#include <AK/BitStream.h>
#include <AK/Error.h>
#include <AK/FixedArray.h>
#include <AK/MemoryStream.h>
#include <LibGfx/Size.h>
#include <LibMedia/Color/CodingIndependentCodePoints.h>
#include <LibMedia/DecoderError.h>
#include <LibMedia/Subsampling.h>

#include "BooleanDecoder.h"
#include "ContextStorage.h"
#include "Enums.h"
#include "LookupTables.h"
#include "MotionVector.h"
#include "SyntaxElementCounter.h"
#include "Utilities.h"

namespace Media::Video::VP9 {

enum class FrameShowMode {
    CreateAndShowNewFrame,
    ShowExistingFrame,
    DoNotShowFrame,
};

struct Quantizers {
    u16 y_ac_quantizer { 0 };
    u16 uv_ac_quantizer { 0 };

    u16 y_dc_quantizer { 0 };
    u16 uv_dc_quantizer { 0 };
};

struct FrameContext {
public:
    static ErrorOr<FrameContext> create(ReadonlyBytes data,
        Vector2D<FrameBlockContext>& contexts)
    {
        return FrameContext(
            data,
            TRY(try_make<FixedMemoryStream>(data)),
            TRY(try_make<SyntaxElementCounter>()),
            contexts);
    }

    FrameContext(FrameContext const&) = delete;
    FrameContext(FrameContext&&) = default;

    ReadonlyBytes stream_data;
    NonnullOwnPtr<FixedMemoryStream> stream;
    BigEndianInputBitStream bit_stream;

    DecoderErrorOr<BooleanDecoder> create_range_decoder(size_t size)
    {
        if (size > stream->remaining())
            return DecoderError::corrupted("Range decoder size invalid"sv);

        auto compressed_header_data = ReadonlyBytes(stream_data.data() + stream->offset(), size);

        // 9.2.1: The Boolean decoding process specified in section 9.2.2 is invoked to read a marker syntax element from the
        //        bitstream. It is a requirement of bitstream conformance that the value read is equal to 0.
        auto decoder = DECODER_TRY(DecoderErrorCategory::Corrupted, BooleanDecoder::initialize(compressed_header_data));
        if (decoder.read_bool(128))
            return DecoderError::corrupted("Range decoder marker was non-zero"sv);

        DECODER_TRY(DecoderErrorCategory::Corrupted, bit_stream.discard(size));
        return decoder;
    }

    NonnullOwnPtr<SyntaxElementCounter> counter;

    u8 profile { 0 };

    FrameType type { FrameType::KeyFrame };
    bool is_inter_predicted() const { return type == FrameType::InterFrame; }

    bool error_resilient_mode { false };
    bool parallel_decoding_mode { false };
    bool should_replace_probability_context { false };

    bool shows_a_frame() const { return m_frame_show_mode != FrameShowMode::DoNotShowFrame; }
    bool shows_a_new_frame() const { return m_frame_show_mode == FrameShowMode::CreateAndShowNewFrame; }
    bool shows_existing_frame() const { return m_frame_show_mode == FrameShowMode::ShowExistingFrame; }
    void set_frame_hidden() { m_frame_show_mode = FrameShowMode::DoNotShowFrame; }
    void set_existing_frame_to_show(u8 index)
    {
        m_frame_show_mode = FrameShowMode::ShowExistingFrame;
        m_existing_frame_index = index;
    }
    u8 existing_frame_index() const { return m_existing_frame_index; }

    bool use_previous_frame_motion_vectors { false };

    ColorConfig color_config {};

    u8 reference_frames_to_update_flags { 0 };
    bool should_update_reference_frame_at_index(u8 index) const { return (reference_frames_to_update_flags & (1 << index)) != 0; }

    u8 probability_context_index { 0 };

    Gfx::Size<u32> size() const { return m_size; }
    ErrorOr<void> set_size(Gfx::Size<u32> size)
    {
        m_size = size;

        // From spec, compute_image_size( )
        m_rows = pixels_to_blocks(size.height() + 7u);
        m_columns = pixels_to_blocks(size.width() + 7u);
        return m_block_contexts.try_resize(m_rows, m_columns);
    }
    u32 rows() const { return m_rows; }
    u32 columns() const { return m_columns; }
    u32 superblock_rows() const { return blocks_ceiled_to_superblocks(rows()); }
    u32 superblock_columns() const { return blocks_ceiled_to_superblocks(columns()); }
    // Calculates the output size for each plane in the frame.
    Gfx::Size<u32> decoded_size(bool uv) const
    {
        if (uv) {
            return {
                Subsampling::subsampled_size(color_config.subsampling_y, blocks_to_pixels(columns())),
                Subsampling::subsampled_size(color_config.subsampling_y, blocks_to_pixels(rows())),
            };
        }
        return {
            blocks_to_pixels(columns()),
            blocks_to_pixels(rows()),
        };
    }

    Vector2D<FrameBlockContext> const& block_contexts() const { return m_block_contexts; }

    Gfx::Size<u32> render_size { 0, 0 };
    Gfx::Size<u16> log2_of_tile_counts { 0, 0 };

    // This group of fields is only needed for inter-predicted frames.
    Array<u8, 3> reference_frame_indices;
    Array<bool, ReferenceFrameType::LastFrame + 3> reference_frame_sign_biases;
    bool high_precision_motion_vectors_allowed { false };
    InterpolationFilter interpolation_filter { InterpolationFilter::Switchable };

    u8 loop_filter_level { 0 };
    u8 loop_filter_sharpness { 0 };
    bool loop_filter_delta_enabled { false };
    Array<i8, MAX_REF_FRAMES> loop_filter_reference_deltas;
    Array<i8, 2> loop_filter_mode_deltas;

    // Set based on quantization_params( ) in the spec.
    bool lossless { false };
    Array<Quantizers, MAX_SEGMENTS> segment_quantizers;

    bool segmentation_enabled { false };
    // Note: We can use Optional<Array<...>> for these tree probabilities, but unfortunately it seems to have measurable performance overhead.
    bool use_full_segment_id_tree { false };
    Array<u8, 7> full_segment_id_tree_probabilities;
    bool use_predicted_segment_id_tree { false };
    Array<u8, 3> predicted_segment_id_tree_probabilities;
    bool should_use_absolute_segment_base_quantizer { false };
    SegmentationFeatures segmentation_features;
    SegmentFeatureStatus get_segment_feature(u8 segment_id, SegmentFeature feature) const
    {
        return segmentation_features[segment_id][to_underlying(feature)];
    }

    u16 header_size_in_bytes { 0 };

    TransformMode transform_mode;

    // This group also is only needed for inter-predicted frames.
    ReferenceMode reference_mode;
    ReferenceFrameType fixed_reference_type;
    ReferenceFramePair variable_reference_types;

private:
    friend struct TileContext;

    FrameContext(ReadonlyBytes data,
        NonnullOwnPtr<FixedMemoryStream> stream,
        NonnullOwnPtr<SyntaxElementCounter> counter,
        Vector2D<FrameBlockContext>& contexts)
        : stream_data(data)
        , stream(move(stream))
        , bit_stream(MaybeOwned<Stream>(*this->stream))
        , counter(move(counter))
        , m_block_contexts(contexts)
    {
    }

    FrameShowMode m_frame_show_mode { FrameShowMode::CreateAndShowNewFrame };
    u8 m_existing_frame_index { 0 };

    Gfx::Size<u32> m_size { 0, 0 };
    u32 m_rows { 0 };
    u32 m_columns { 0 };
    // FIXME: From spec: NOTE – We are using a 2D array to store the SubModes for clarity. It is possible to reduce memory
    //        consumption by only storing one intra mode for each 8x8 horizontal and vertical position, i.e. to use two 1D
    //        arrays instead.
    //        I think should also apply to other fields that are only accessed relative to the current block. Worth looking
    //        into how much of this context needs to be stored for the whole frame vs a row or column from the current tile.
    Vector2D<FrameBlockContext>& m_block_contexts;
};

static ErrorOr<NonZeroTokens> create_non_zero_tokens(u32 size_in_sub_blocks, bool subsampling)
{
    return NonZeroTokens {
        TRY(FixedArray<bool>::create(size_in_sub_blocks)),
        TRY(FixedArray<bool>::create(size_in_sub_blocks >>= subsampling)),
        TRY(FixedArray<bool>::create(size_in_sub_blocks)),
    };
}

template<typename T>
static Span<T> safe_slice(Span<T> span, u32 start, u32 size)
{
    return span.slice(start, min(size, span.size() - start));
}

static NonZeroTokensView create_non_zero_tokens_view(NonZeroTokensView non_zero_tokens, u32 start_in_sub_blocks, u32 size_in_sub_blocks, bool subsampling)
{
    NonZeroTokensView result;
    // Y plane
    result[0] = safe_slice(non_zero_tokens[0], start_in_sub_blocks, size_in_sub_blocks);
    // UV planes
    start_in_sub_blocks >>= subsampling;
    size_in_sub_blocks >>= subsampling;
    result[1] = safe_slice(non_zero_tokens[1], start_in_sub_blocks, size_in_sub_blocks);
    result[2] = safe_slice(non_zero_tokens[2], start_in_sub_blocks, size_in_sub_blocks);
    return result;
}

static NonZeroTokensView create_non_zero_tokens_view(NonZeroTokens& non_zero_tokens, u32 start_in_sub_blocks, u32 size_in_sub_blocks, bool subsampling)
{
    return create_non_zero_tokens_view({ non_zero_tokens[0], non_zero_tokens[1], non_zero_tokens[2] }, start_in_sub_blocks, size_in_sub_blocks, subsampling);
}

struct TileContext {
public:
    static DecoderErrorOr<TileContext> try_create(FrameContext& frame_context, u32 tile_size, u32 rows_start, u32 rows_end, u32 columns_start, u32 columns_end, PartitionContextView above_partition_context, NonZeroTokensView above_non_zero_tokens, SegmentationPredictionContextView above_segmentation_ids)
    {
        auto width = columns_end - columns_start;
        auto height = rows_end - rows_start;
        auto context_view = frame_context.m_block_contexts.view(rows_start, columns_start, height, width);

        return TileContext {
            frame_context,
            TRY(frame_context.create_range_decoder(tile_size)),
            DECODER_TRY_ALLOC(try_make<SyntaxElementCounter>()),
            rows_start,
            rows_end,
            columns_start,
            columns_end,
            context_view,
            above_partition_context,
            above_non_zero_tokens,
            above_segmentation_ids,
            DECODER_TRY_ALLOC(PartitionContext::create(superblocks_to_blocks(blocks_ceiled_to_superblocks(height)))),
            DECODER_TRY_ALLOC(create_non_zero_tokens(blocks_to_sub_blocks(height), frame_context.color_config.subsampling_y)),
            DECODER_TRY_ALLOC(SegmentationPredictionContext::create(height)),
        };
    }

    Vector2D<FrameBlockContext> const& frame_block_contexts() const { return frame_context.block_contexts(); }

    FrameContext const& frame_context;
    BooleanDecoder decoder;
    NonnullOwnPtr<SyntaxElementCounter> counter;
    u32 rows_start { 0 };
    u32 rows_end { 0 };
    u32 columns_start { 0 };
    u32 columns_end { 0 };
    u32 rows() const { return rows_end - rows_start; }
    u32 columns() const { return columns_end - columns_start; }
    Vector2DView<FrameBlockContext> block_contexts_view;

    PartitionContextView above_partition_context;
    NonZeroTokensView above_non_zero_tokens;
    SegmentationPredictionContextView above_segmentation_ids;
    PartitionContext left_partition_context;
    NonZeroTokens left_non_zero_tokens;
    SegmentationPredictionContext left_segmentation_ids;
};

struct BlockContext {
    static BlockContext create(TileContext& tile_context, u32 row, u32 column, BlockSubsize size)
    {
        auto contexts_view = tile_context.block_contexts_view.view(
            row - tile_context.rows_start,
            column - tile_context.columns_start,
            min<u32>(num_8x8_blocks_high_lookup[size], tile_context.frame_context.rows() - row),
            min<u32>(num_8x8_blocks_wide_lookup[size], tile_context.frame_context.columns() - column));

        auto size_in_blocks = block_size_to_blocks(size);
        auto size_in_sub_blocks = block_size_to_sub_blocks(get_subsampled_block_size(size, false, false));

        return BlockContext {
            .frame_context = tile_context.frame_context,
            .tile_context = tile_context,
            .decoder = tile_context.decoder,
            .counter = *tile_context.counter,
            .row = row,
            .column = column,
            .size = size,
            .contexts_view = contexts_view,
            .above_non_zero_tokens = create_non_zero_tokens_view(tile_context.above_non_zero_tokens, blocks_to_sub_blocks(column - tile_context.columns_start), size_in_sub_blocks.width(), tile_context.frame_context.color_config.subsampling_x),
            .above_segmentation_ids = safe_slice(tile_context.above_segmentation_ids, column - tile_context.columns_start, size_in_blocks.width()),
            .left_non_zero_tokens = create_non_zero_tokens_view(tile_context.left_non_zero_tokens, blocks_to_sub_blocks(row - tile_context.rows_start), size_in_sub_blocks.height(), tile_context.frame_context.color_config.subsampling_y),
            .left_segmentation_ids = safe_slice(tile_context.left_segmentation_ids.span(), row - tile_context.rows_start, size_in_blocks.height()),
        };
    }

    Vector2D<FrameBlockContext> const& frame_block_contexts() const { return frame_context.block_contexts(); }

    FrameContext const& frame_context;
    TileContext const& tile_context;
    BooleanDecoder& decoder;
    SyntaxElementCounter& counter;
    u32 row { 0 };
    u32 column { 0 };
    BlockSubsize size;
    Gfx::Size<u8> get_size_in_sub_blocks() const
    {
        return block_size_to_sub_blocks(size);
    }

    Vector2DView<FrameBlockContext> contexts_view;

    u8 segment_id { 0 };
    bool should_skip_residuals { false };

    TransformSize transform_size { Transform_4x4 };

    ReferenceFramePair reference_frame_types {};
    bool is_inter_predicted() const { return reference_frame_types.primary != ReferenceFrameType::None; }
    bool is_compound() const { return reference_frame_types.secondary != ReferenceFrameType::None; }

    Array<PredictionMode, 4> sub_block_prediction_modes {};
    PredictionMode y_prediction_mode() const { return sub_block_prediction_modes.last(); }
    PredictionMode& y_prediction_mode() { return sub_block_prediction_modes.last(); }
    PredictionMode uv_prediction_mode { 0 };

    InterpolationFilter interpolation_filter { EightTap };
    Array<MotionVectorPair, 4> sub_block_motion_vectors {};

    Array<i32, 1024> residual_tokens {};

    // Indexed by ReferenceFrame enum.
    Array<u8, 4> mode_context {};

    NonZeroTokensView above_non_zero_tokens;
    SegmentationPredictionContextView above_segmentation_ids;
    NonZeroTokensView left_non_zero_tokens;
    SegmentationPredictionContextView left_segmentation_ids;

    SegmentFeatureStatus get_segment_feature(SegmentFeature feature) const
    {
        return frame_context.get_segment_feature(segment_id, feature);
    }
};

struct BlockMotionVectorCandidateSet {
    MotionVector near_vector;
    MotionVector nearest_vector;
    MotionVector best_vector;
};

struct MotionVectorCandidate {
    ReferenceFrameType type;
    MotionVector vector;
};

}

Coverage Report

Created: 2026-09-26 06:49