// Copyright 2012 Google Inc. All Rights Reserved.

//

// Use of this source code is governed by a BSD-style license

// that can be found in the COPYING file in the root of the source

// tree. An additional intellectual property rights grant can be found

// in the file PATENTS. All contributing project authors may

// be found in the AUTHORS file in the root of the source tree.

// -----------------------------------------------------------------------------

//

// main entry for the lossless encoder.

//

// Author: Vikas Arora (vikaas.arora@gmail.com)

//

#include <assert.h>

#include <stdlib.h>

#include <string.h>

#include "src/dec/common_dec.h"

#include "src/dsp/lossless.h"

#include "src/dsp/lossless_common.h"

#include "src/enc/backward_references_enc.h"

#include "src/enc/histogram_enc.h"

#include "src/enc/vp8i_enc.h"

#include "src/enc/vp8li_enc.h"

#include "src/utils/bit_writer_utils.h"

#include "src/utils/huffman_encode_utils.h"

#include "src/utils/palette.h"

#include "src/utils/thread_utils.h"

#include "src/utils/utils.h"

#include "src/webp/encode.h"

#include "src/webp/format_constants.h"

#include "src/webp/types.h"

#define MAX_HUFFMAN_BITS (MIN_HUFFMAN_BITS + (1 << NUM_HUFFMAN_BITS) - 1)

// Empirical value for which it becomes too computationally expensive to

// compute the best predictor image.

#define MAX_PREDICTOR_IMAGE_SIZE (1 << 14)

// -----------------------------------------------------------------------------

// Palette

// These five modes are evaluated and their respective entropy is computed.

typedef enum {

  kDirect = 0,

  kSpatial = 1,

  kSubGreen = 2,

  kSpatialSubGreen = 3,

  kPalette = 4,

  kPaletteAndSpatial = 5,

  kNumEntropyIx = 6

} EntropyIx;

typedef enum {

  kHistoAlpha = 0,

  kHistoAlphaPred,

  kHistoGreen,

  kHistoGreenPred,

  kHistoRed,

  kHistoRedPred,

  kHistoBlue,

  kHistoBluePred,

  kHistoRedSubGreen,

  kHistoRedPredSubGreen,

  kHistoBlueSubGreen,

  kHistoBluePredSubGreen,

  kHistoPalette,

  kHistoTotal  // Must be last.

} HistoIx;

#define NUM_BUCKETS 256

typedef uint32_t HistogramBuckets[NUM_BUCKETS];

// Keeping track of histograms, indexed by HistoIx.

// Ideally, this would just be a struct with meaningful fields, but the

// calculation of `entropy_comp` uses the index. One refactoring at a time :)

typedef struct {

  HistogramBuckets category[kHistoTotal];

} Histograms;

static void AddSingleSubGreen(uint32_t p, HistogramBuckets r,

                              HistogramBuckets b) {

  const int green = (int)p >> 8;  // The upper bits are masked away later.

  ++r[(((int)p >> 16) - green) & 0xff];

  ++b[(((int)p >> 0) - green) & 0xff];

}

static void AddSingle(uint32_t p, HistogramBuckets a, HistogramBuckets r,

                      HistogramBuckets g, HistogramBuckets b) {

  ++a[(p >> 24) & 0xff];

  ++r[(p >> 16) & 0xff];

  ++g[(p >> 8) & 0xff];

  ++b[(p >> 0) & 0xff];

}

static WEBP_INLINE uint8_t HashPix(uint32_t pix) {

  // Note that masking with 0xffffffffu is for preventing an

  // 'unsigned int overflow' warning. Doesn't impact the compiled code.

  return ((((uint64_t)pix + (pix >> 19)) * 0x39c5fba7ull) & 0xffffffffu) >> 24;

}

static int AnalyzeEntropy(const uint32_t* argb, int width, int height,

                          int argb_stride, int use_palette, int palette_size,

                          int transform_bits, EntropyIx* const min_entropy_ix,

                          int* const red_and_blue_always_zero) {

  Histograms* histo;

  if (use_palette && palette_size <= 16) {

    // In the case of small palettes, we pack 2, 4 or 8 pixels together. In

    // practice, small palettes are better than any other transform.

    *min_entropy_ix = kPalette;

    *red_and_blue_always_zero = 1;

    return 1;

  }

  histo = (Histograms*)WebPSafeCalloc(1, sizeof(*histo));

  if (histo != NULL) {

    int i, x, y;

    const uint32_t* prev_row = NULL;

    const uint32_t* curr_row = argb;

    uint32_t pix_prev = argb[0];  // Skip the first pixel.

    for (y = 0; y < height; ++y) {

      for (x = 0; x < width; ++x) {

        const uint32_t pix = curr_row[x];

        const uint32_t pix_diff = VP8LSubPixels(pix, pix_prev);

        pix_prev = pix;

        if ((pix_diff == 0) || (prev_row != NULL && pix == prev_row[x])) {

          continue;

        }

        AddSingle(pix, histo->category[kHistoAlpha], histo->category[kHistoRed],

                  histo->category[kHistoGreen], histo->category[kHistoBlue]);

        AddSingle(pix_diff, histo->category[kHistoAlphaPred],

                  histo->category[kHistoRedPred],

                  histo->category[kHistoGreenPred],

                  histo->category[kHistoBluePred]);

        AddSingleSubGreen(pix, histo->category[kHistoRedSubGreen],

                          histo->category[kHistoBlueSubGreen]);

        AddSingleSubGreen(pix_diff, histo->category[kHistoRedPredSubGreen],

                          histo->category[kHistoBluePredSubGreen]);

        {

          // Approximate the palette by the entropy of the multiplicative hash.

          const uint8_t hash = HashPix(pix);

          ++histo->category[kHistoPalette][hash];

        }

      }

      prev_row = curr_row;

      curr_row += argb_stride;

    }

    {

      uint64_t entropy_comp[kHistoTotal];

      uint64_t entropy[kNumEntropyIx];

      int k;

      int last_mode_to_analyze = use_palette ? kPalette : kSpatialSubGreen;

      int j;

      // Let's add one zero to the predicted histograms. The zeros are removed

      // too efficiently by the pix_diff == 0 comparison, at least one of the

      // zeros is likely to exist.

      ++histo->category[kHistoRedPredSubGreen][0];

      ++histo->category[kHistoBluePredSubGreen][0];

      ++histo->category[kHistoRedPred][0];

      ++histo->category[kHistoGreenPred][0];

      ++histo->category[kHistoBluePred][0];

      ++histo->category[kHistoAlphaPred][0];

      for (j = 0; j < kHistoTotal; ++j) {

        entropy_comp[j] = VP8LBitsEntropy(histo->category[j], NUM_BUCKETS);

      }

      entropy[kDirect] = entropy_comp[kHistoAlpha] + entropy_comp[kHistoRed] +

                         entropy_comp[kHistoGreen] + entropy_comp[kHistoBlue];

      entropy[kSpatial] =

          entropy_comp[kHistoAlphaPred] + entropy_comp[kHistoRedPred] +

          entropy_comp[kHistoGreenPred] + entropy_comp[kHistoBluePred];

      entropy[kSubGreen] =

          entropy_comp[kHistoAlpha] + entropy_comp[kHistoRedSubGreen] +

          entropy_comp[kHistoGreen] + entropy_comp[kHistoBlueSubGreen];

      entropy[kSpatialSubGreen] =

          entropy_comp[kHistoAlphaPred] + entropy_comp[kHistoRedPredSubGreen] +

          entropy_comp[kHistoGreenPred] + entropy_comp[kHistoBluePredSubGreen];

      entropy[kPalette] = entropy_comp[kHistoPalette];

      // When including transforms, there is an overhead in bits from

      // storing them. This overhead is small but matters for small images.

      // For spatial, there are 14 transformations.

      entropy[kSpatial] += (uint64_t)VP8LSubSampleSize(width, transform_bits) *

                           VP8LSubSampleSize(height, transform_bits) *

                           VP8LFastLog2(14);

      // For color transforms: 24 as only 3 channels are considered in a

      // ColorTransformElement.

      entropy[kSpatialSubGreen] +=

          (uint64_t)VP8LSubSampleSize(width, transform_bits) *

          VP8LSubSampleSize(height, transform_bits) * VP8LFastLog2(24);

      // For palettes, add the cost of storing the palette.

      // We empirically estimate the cost of a compressed entry as 8 bits.

      // The palette is differential-coded when compressed hence a much

      // lower cost than sizeof(uint32_t)*8.

      entropy[kPalette] += (palette_size * 8ull) << LOG_2_PRECISION_BITS;

      *min_entropy_ix = kDirect;

      for (k = kDirect + 1; k <= last_mode_to_analyze; ++k) {

        if (entropy[*min_entropy_ix] > entropy[k]) {

          *min_entropy_ix = (EntropyIx)k;

        }

      }

      assert((int)*min_entropy_ix <= last_mode_to_analyze);

      *red_and_blue_always_zero = 1;

      // Let's check if the histogram of the chosen entropy mode has

      // non-zero red and blue values. If all are zero, we can later skip

      // the cross color optimization.

      {

        static const uint8_t kHistoPairs[5][2] = {

            {kHistoRed, kHistoBlue},

            {kHistoRedPred, kHistoBluePred},

            {kHistoRedSubGreen, kHistoBlueSubGreen},

            {kHistoRedPredSubGreen, kHistoBluePredSubGreen},

            {kHistoRed, kHistoBlue}};

        const HistogramBuckets* const red_histo =

            &histo->category[kHistoPairs[*min_entropy_ix][0]];

        const HistogramBuckets* const blue_histo =

            &histo->category[kHistoPairs[*min_entropy_ix][1]];

        for (i = 1; i < NUM_BUCKETS; ++i) {

          if (((*red_histo)[i] | (*blue_histo)[i]) != 0) {

            *red_and_blue_always_zero = 0;

            break;

          }

        }

      }

    }

    WebPSafeFree(histo);

    return 1;

  } else {

    return 0;

  }

}

// Clamp histogram and transform bits.

static int ClampBits(int width, int height, int bits, int min_bits,

                     int max_bits, int image_size_max) {

  int image_size;

  bits = (bits < min_bits) ? min_bits : (bits > max_bits) ? max_bits : bits;

  image_size = VP8LSubSampleSize(width, bits) * VP8LSubSampleSize(height, bits);

  while (bits < max_bits && image_size > image_size_max) {

    ++bits;

    image_size =

        VP8LSubSampleSize(width, bits) * VP8LSubSampleSize(height, bits);

  }

  // In case the bits reduce the image too much, choose the smallest value

  // setting the histogram image size to 1.

  while (bits > min_bits && image_size == 1) {

    image_size = VP8LSubSampleSize(width, bits - 1) *

                 VP8LSubSampleSize(height, bits - 1);

    if (image_size != 1) break;

    --bits;

  }

  return bits;

}

static int GetHistoBits(int method, int use_palette, int width, int height) {

  // Make tile size a function of encoding method (Range: 0 to 6).

  const int histo_bits = (use_palette ? 9 : 7) - method;

  return ClampBits(width, height, histo_bits, MIN_HUFFMAN_BITS,

                   MAX_HUFFMAN_BITS, MAX_HUFF_IMAGE_SIZE);

}

static int GetTransformBits(int method, int histo_bits) {

  const int max_transform_bits = (method < 4) ? 6 : (method > 4) ? 4 : 5;

  const int res =

      (histo_bits > max_transform_bits) ? max_transform_bits : histo_bits;

  assert(res <= MAX_TRANSFORM_BITS);

  return res;

}

// Set of parameters to be used in each iteration of the cruncher.

#define CRUNCH_SUBCONFIGS_MAX 2

typedef struct {

  int lz77;

  int do_no_cache;

} CrunchSubConfig;

typedef struct {

  int entropy_idx;

  PaletteSorting palette_sorting_type;

  CrunchSubConfig sub_configs[CRUNCH_SUBCONFIGS_MAX];

  int sub_configs_size;

} CrunchConfig;

// +2 because we add a palette sorting configuration for kPalette and

// kPaletteAndSpatial.

#define CRUNCH_CONFIGS_MAX (kNumEntropyIx + 2 * kPaletteSortingNum)

static int EncoderAnalyze(VP8LEncoder* const enc,

                          CrunchConfig crunch_configs[CRUNCH_CONFIGS_MAX],

                          int* const crunch_configs_size,

                          int* const red_and_blue_always_zero) {

  const WebPPicture* const pic = enc->pic;

  const int width = pic->width;

  const int height = pic->height;

  const WebPConfig* const config = enc->config;

  const int method = config->method;

  const int low_effort = (config->method == 0);

  int i;

  int use_palette, transform_bits;

  int n_lz77s;

  // If set to 0, analyze the cache with the computed cache value. If 1, also

  // analyze with no-cache.

  int do_no_cache = 0;

  assert(pic != NULL && pic->argb != NULL);

  // Check whether a palette is possible.

  enc->palette_size = GetColorPalette(pic, enc->palette_sorted);

  use_palette = (enc->palette_size <= MAX_PALETTE_SIZE);

  if (!use_palette) {

    enc->palette_size = 0;

  }

  // Empirical bit sizes.

  enc->histo_bits = GetHistoBits(method, use_palette, pic->width, pic->height);

  transform_bits = GetTransformBits(method, enc->histo_bits);

  enc->predictor_transform_bits = transform_bits;

  enc->cross_color_transform_bits = transform_bits;

  if (low_effort) {

    // AnalyzeEntropy is somewhat slow.

    crunch_configs[0].entropy_idx = use_palette ? kPalette : kSpatialSubGreen;

    crunch_configs[0].palette_sorting_type =

        use_palette ? kSortedDefault : kUnusedPalette;

    n_lz77s = 1;

    *crunch_configs_size = 1;

  } else {

    EntropyIx min_entropy_ix;

    // Try out multiple LZ77 on images with few colors.

    n_lz77s = (enc->palette_size > 0 && enc->palette_size <= 16) ? 2 : 1;

    if (!AnalyzeEntropy(pic->argb, width, height, pic->argb_stride, use_palette,

                        enc->palette_size, transform_bits, &min_entropy_ix,

                        red_and_blue_always_zero)) {

      return 0;

    }

    if (method == 6 && config->quality == 100) {

      do_no_cache = 1;

      // Go brute force on all transforms.

      *crunch_configs_size = 0;

      for (i = 0; i < kNumEntropyIx; ++i) {

        // We can only apply kPalette or kPaletteAndSpatial if we can indeed use

        // a palette.

        if ((i != kPalette && i != kPaletteAndSpatial) || use_palette) {

          assert(*crunch_configs_size < CRUNCH_CONFIGS_MAX);

          if (use_palette && (i == kPalette || i == kPaletteAndSpatial)) {

            int sorting_method;

            for (sorting_method = 0; sorting_method < kPaletteSortingNum;

                 ++sorting_method) {

              const PaletteSorting typed_sorting_method =

                  (PaletteSorting)sorting_method;

              // TODO(vrabaud) kSortedDefault should be tested. It is omitted

              // for now for backward compatibility.

              if (typed_sorting_method == kUnusedPalette ||

                  typed_sorting_method == kSortedDefault) {

                continue;

              }

              crunch_configs[(*crunch_configs_size)].entropy_idx = i;

              crunch_configs[(*crunch_configs_size)].palette_sorting_type =

                  typed_sorting_method;

              ++*crunch_configs_size;

            }

          } else {

            crunch_configs[(*crunch_configs_size)].entropy_idx = i;

            crunch_configs[(*crunch_configs_size)].palette_sorting_type =

                kUnusedPalette;

            ++*crunch_configs_size;

          }

        }

      }

    } else {

      // Only choose the guessed best transform.

      *crunch_configs_size = 1;

      crunch_configs[0].entropy_idx = min_entropy_ix;

      crunch_configs[0].palette_sorting_type =

          use_palette ? kMinimizeDelta : kUnusedPalette;

      if (config->quality >= 75 && method == 5) {

        // Test with and without color cache.

        do_no_cache = 1;

        // If we have a palette, also check in combination with spatial.

        if (min_entropy_ix == kPalette) {

          *crunch_configs_size = 2;

          crunch_configs[1].entropy_idx = kPaletteAndSpatial;

          crunch_configs[1].palette_sorting_type = kMinimizeDelta;

        }

      }

    }

  }

  // Fill in the different LZ77s.

  assert(n_lz77s <= CRUNCH_SUBCONFIGS_MAX);

  for (i = 0; i < *crunch_configs_size; ++i) {

    int j;

    for (j = 0; j < n_lz77s; ++j) {

      assert(j < CRUNCH_SUBCONFIGS_MAX);

      crunch_configs[i].sub_configs[j].lz77 =

          (j == 0) ? kLZ77Standard | kLZ77RLE : kLZ77Box;

      crunch_configs[i].sub_configs[j].do_no_cache = do_no_cache;

    }

    crunch_configs[i].sub_configs_size = n_lz77s;

  }

  return 1;

}

static int EncoderInit(VP8LEncoder* const enc) {

  const WebPPicture* const pic = enc->pic;

  const int width = pic->width;

  const int height = pic->height;

  const int pix_cnt = width * height;

  // we round the block size up, so we're guaranteed to have

  // at most MAX_REFS_BLOCK_PER_IMAGE blocks used:

  const int refs_block_size = (pix_cnt - 1) / MAX_REFS_BLOCK_PER_IMAGE + 1;

  int i;

  if (!VP8LHashChainInit(&enc->hash_chain, pix_cnt)) return 0;

  for (i = 0; i < 4; ++i) VP8LBackwardRefsInit(&enc->refs[i], refs_block_size);

  return 1;

}

// Returns false in case of memory error.

static int GetHuffBitLengthsAndCodes(

    const VP8LHistogramSet* const histogram_image,

    HuffmanTreeCode* const huffman_codes) {

  int i, k;

  int ok = 0;

  uint64_t total_length_size = 0;

  uint8_t* mem_buf = NULL;

  const int histogram_image_size = histogram_image->size;

  int max_num_symbols = 0;

  uint8_t* buf_rle = NULL;

  HuffmanTree* huff_tree = NULL;

  // Iterate over all histograms and get the aggregate number of codes used.

  for (i = 0; i < histogram_image_size; ++i) {

    const VP8LHistogram* const histo = histogram_image->histograms[i];

    HuffmanTreeCode* const codes = &huffman_codes[5 * i];

    assert(histo != NULL);

    for (k = 0; k < 5; ++k) {

      const int num_symbols =

          (k == 0)   ? VP8LHistogramNumCodes(histo->palette_code_bits)

          : (k == 4) ? NUM_DISTANCE_CODES

                     : 256;

      codes[k].num_symbols = num_symbols;

      total_length_size += num_symbols;

    }

  }

  // Allocate and Set Huffman codes.

  {

    uint16_t* codes;

    uint8_t* lengths;

    mem_buf = (uint8_t*)WebPSafeCalloc(total_length_size,

                                       sizeof(*lengths) + sizeof(*codes));

    if (mem_buf == NULL) goto End;

    codes = (uint16_t*)mem_buf;

    lengths = (uint8_t*)&codes[total_length_size];

    for (i = 0; i < 5 * histogram_image_size; ++i) {

      const int bit_length = huffman_codes[i].num_symbols;

      huffman_codes[i].codes = codes;

      huffman_codes[i].code_lengths = lengths;

      codes += bit_length;

      lengths += bit_length;

      if (max_num_symbols < bit_length) {

        max_num_symbols = bit_length;

      }

    }

  }

  buf_rle = (uint8_t*)WebPSafeMalloc(1ULL, max_num_symbols);

  huff_tree =

      (HuffmanTree*)WebPSafeMalloc(3ULL * max_num_symbols, sizeof(*huff_tree));

  if (buf_rle == NULL || huff_tree == NULL) goto End;

  // Create Huffman trees.

  for (i = 0; i < histogram_image_size; ++i) {

    HuffmanTreeCode* const codes = &huffman_codes[5 * i];

    VP8LHistogram* const histo = histogram_image->histograms[i];

    VP8LCreateHuffmanTree(histo->literal, 15, buf_rle, huff_tree, codes + 0);

    VP8LCreateHuffmanTree(histo->red, 15, buf_rle, huff_tree, codes + 1);

    VP8LCreateHuffmanTree(histo->blue, 15, buf_rle, huff_tree, codes + 2);

    VP8LCreateHuffmanTree(histo->alpha, 15, buf_rle, huff_tree, codes + 3);

    VP8LCreateHuffmanTree(histo->distance, 15, buf_rle, huff_tree, codes + 4);

  }

  ok = 1;

End:

  WebPSafeFree(huff_tree);

  WebPSafeFree(buf_rle);

  if (!ok) {

    WebPSafeFree(mem_buf);

    memset(huffman_codes, 0, 5 * histogram_image_size * sizeof(*huffman_codes));

  }

  return ok;

}

static void StoreHuffmanTreeOfHuffmanTreeToBitMask(

    VP8LBitWriter* const bw, const uint8_t* code_length_bitdepth) {

  // RFC 1951 will calm you down if you are worried about this funny sequence.

  // This sequence is tuned from that, but more weighted for lower symbol count,

  // and more spiking histograms.

  static const uint8_t kStorageOrder[CODE_LENGTH_CODES] = {

      17, 18, 0, 1, 2, 3, 4, 5, 16, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};

  int i;

  // Throw away trailing zeros:

  int codes_to_store = CODE_LENGTH_CODES;

  for (; codes_to_store > 4; --codes_to_store) {

    if (code_length_bitdepth[kStorageOrder[codes_to_store - 1]] != 0) {

      break;

    }

  }

  VP8LPutBits(bw, codes_to_store - 4, 4);

  for (i = 0; i < codes_to_store; ++i) {

    VP8LPutBits(bw, code_length_bitdepth[kStorageOrder[i]], 3);

  }

}

static void ClearHuffmanTreeIfOnlyOneSymbol(

    HuffmanTreeCode* const huffman_code) {

  int k;

  int count = 0;

  for (k = 0; k < huffman_code->num_symbols; ++k) {

    if (huffman_code->code_lengths[k] != 0) {

      ++count;

      if (count > 1) return;

    }

  }

  for (k = 0; k < huffman_code->num_symbols; ++k) {

    huffman_code->code_lengths[k] = 0;

    huffman_code->codes[k] = 0;

  }

}

static void StoreHuffmanTreeToBitMask(

    VP8LBitWriter* const bw, const HuffmanTreeToken* const tokens,

    const int num_tokens, const HuffmanTreeCode* const huffman_code) {

  int i;

  for (i = 0; i < num_tokens; ++i) {

    const int ix = tokens[i].code;

    const int extra_bits = tokens[i].extra_bits;

    VP8LPutBits(bw, huffman_code->codes[ix], huffman_code->code_lengths[ix]);

    switch (ix) {

      case 16:

        VP8LPutBits(bw, extra_bits, 2);

        break;

      case 17:

        VP8LPutBits(bw, extra_bits, 3);

        break;

      case 18:

        VP8LPutBits(bw, extra_bits, 7);

        break;

    }

  }

}

// 'huff_tree' and 'tokens' are pre-alloacted buffers.

static void StoreFullHuffmanCode(VP8LBitWriter* const bw,

                                 HuffmanTree* const huff_tree,

                                 HuffmanTreeToken* const tokens,

                                 const HuffmanTreeCode* const tree) {

  uint8_t code_length_bitdepth[CODE_LENGTH_CODES] = {0};

  uint16_t code_length_bitdepth_symbols[CODE_LENGTH_CODES] = {0};

  const int max_tokens = tree->num_symbols;

  int num_tokens;

  HuffmanTreeCode huffman_code;

  huffman_code.num_symbols = CODE_LENGTH_CODES;

  huffman_code.code_lengths = code_length_bitdepth;

  huffman_code.codes = code_length_bitdepth_symbols;

  VP8LPutBits(bw, 0, 1);

  num_tokens = VP8LCreateCompressedHuffmanTree(tree, tokens, max_tokens);

  {

    uint32_t histogram[CODE_LENGTH_CODES] = {0};

    uint8_t buf_rle[CODE_LENGTH_CODES] = {0};

    int i;

    for (i = 0; i < num_tokens; ++i) {

      ++histogram[tokens[i].code];

    }

    VP8LCreateHuffmanTree(histogram, 7, buf_rle, huff_tree, &huffman_code);

  }

  StoreHuffmanTreeOfHuffmanTreeToBitMask(bw, code_length_bitdepth);

  ClearHuffmanTreeIfOnlyOneSymbol(&huffman_code);

  {

    int trailing_zero_bits = 0;

    int trimmed_length = num_tokens;

    int write_trimmed_length;

    int length;

    int i = num_tokens;

    while (i-- > 0) {

      const int ix = tokens[i].code;

      if (ix == 0 || ix == 17 || ix == 18) {

        --trimmed_length;  // discount trailing zeros

        trailing_zero_bits += code_length_bitdepth[ix];

        if (ix == 17) {

          trailing_zero_bits += 3;

        } else if (ix == 18) {

          trailing_zero_bits += 7;

        }

      } else {

        break;

      }

    }

    write_trimmed_length = (trimmed_length > 1 && trailing_zero_bits > 12);

    length = write_trimmed_length ? trimmed_length : num_tokens;

    VP8LPutBits(bw, write_trimmed_length, 1);

    if (write_trimmed_length) {

      if (trimmed_length == 2) {

        VP8LPutBits(bw, 0, 3 + 2);  // nbitpairs=1, trimmed_length=2

      } else {

        const int nbits = BitsLog2Floor(trimmed_length - 2);

        const int nbitpairs = nbits / 2 + 1;

        assert(trimmed_length > 2);

        assert(nbitpairs - 1 < 8);

        VP8LPutBits(bw, nbitpairs - 1, 3);

        VP8LPutBits(bw, trimmed_length - 2, nbitpairs * 2);

      }

    }

    StoreHuffmanTreeToBitMask(bw, tokens, length, &huffman_code);

  }

}

// 'huff_tree' and 'tokens' are pre-alloacted buffers.

static void StoreHuffmanCode(VP8LBitWriter* const bw,

                             HuffmanTree* const huff_tree,

                             HuffmanTreeToken* const tokens,

                             const HuffmanTreeCode* const huffman_code) {

  int i;

  int count = 0;

  int symbols[2] = {0, 0};

  const int kMaxBits = 8;

  const int kMaxSymbol = 1 << kMaxBits;

  // Check whether it's a small tree.

  for (i = 0; i < huffman_code->num_symbols && count < 3; ++i) {

    if (huffman_code->code_lengths[i] != 0) {

      if (count < 2) symbols[count] = i;

      ++count;

    }

  }

  if (count == 0) {  // emit minimal tree for empty cases

    // bits: small tree marker: 1, count-1: 0, large 8-bit code: 0, code: 0

    VP8LPutBits(bw, 0x01, 4);

  } else if (count <= 2 && symbols[0] < kMaxSymbol && symbols[1] < kMaxSymbol) {

    VP8LPutBits(bw, 1, 1);  // Small tree marker to encode 1 or 2 symbols.

    VP8LPutBits(bw, count - 1, 1);

    if (symbols[0] <= 1) {

      VP8LPutBits(bw, 0, 1);  // Code bit for small (1 bit) symbol value.

      VP8LPutBits(bw, symbols[0], 1);

    } else {

      VP8LPutBits(bw, 1, 1);

      VP8LPutBits(bw, symbols[0], 8);

    }

    if (count == 2) {

      VP8LPutBits(bw, symbols[1], 8);

    }

  } else {

    StoreFullHuffmanCode(bw, huff_tree, tokens, huffman_code);

  }

}

static WEBP_INLINE void WriteHuffmanCode(VP8LBitWriter* const bw,

                                         const HuffmanTreeCode* const code,

                                         int code_index) {

  const int depth = code->code_lengths[code_index];

  const int symbol = code->codes[code_index];

  VP8LPutBits(bw, symbol, depth);

}

static WEBP_INLINE void WriteHuffmanCodeWithExtraBits(

    VP8LBitWriter* const bw, const HuffmanTreeCode* const code, int code_index,

    int bits, int n_bits) {

  const int depth = code->code_lengths[code_index];

  const int symbol = code->codes[code_index];

  VP8LPutBits(bw, (bits << depth) | symbol, depth + n_bits);

}

static int StoreImageToBitMask(VP8LBitWriter* const bw, int width,

                               int histo_bits,

                               const VP8LBackwardRefs* const refs,

                               const uint32_t* histogram_symbols,

                               const HuffmanTreeCode* const huffman_codes,

                               const WebPPicture* const pic) {

  const int histo_xsize = histo_bits ? VP8LSubSampleSize(width, histo_bits) : 1;

  const int tile_mask = (histo_bits == 0) ? 0 : -(1 << histo_bits);

  // x and y trace the position in the image.

  int x = 0;

  int y = 0;

  int tile_x = x & tile_mask;

  int tile_y = y & tile_mask;

  int histogram_ix = (histogram_symbols[0] >> 8) & 0xffff;

  const HuffmanTreeCode* codes = huffman_codes + 5 * histogram_ix;

  VP8LRefsCursor c = VP8LRefsCursorInit(refs);

  while (VP8LRefsCursorOk(&c)) {

    const PixOrCopy* const v = c.cur_pos;

    if ((tile_x != (x & tile_mask)) || (tile_y != (y & tile_mask))) {

      tile_x = x & tile_mask;

      tile_y = y & tile_mask;

      histogram_ix = (histogram_symbols[(y >> histo_bits) * histo_xsize +

                                        (x >> histo_bits)] >>

                      8) &

                     0xffff;

      codes = huffman_codes + 5 * histogram_ix;

    }

    if (PixOrCopyIsLiteral(v)) {

      static const uint8_t order[] = {1, 2, 0, 3};

      int k;

      for (k = 0; k < 4; ++k) {

        const int code = PixOrCopyLiteral(v, order[k]);

        WriteHuffmanCode(bw, codes + k, code);

      }

    } else if (PixOrCopyIsCacheIdx(v)) {

      const int code = PixOrCopyCacheIdx(v);

      const int literal_ix = 256 + NUM_LENGTH_CODES + code;

      WriteHuffmanCode(bw, codes, literal_ix);

    } else {

      int bits, n_bits;

      int code;

      const int distance = PixOrCopyDistance(v);

      VP8LPrefixEncode(v->len, &code, &n_bits, &bits);

      WriteHuffmanCodeWithExtraBits(bw, codes, 256 + code, bits, n_bits);

      // Don't write the distance with the extra bits code since

      // the distance can be up to 18 bits of extra bits, and the prefix

      // 15 bits, totaling to 33, and our PutBits only supports up to 32 bits.

      VP8LPrefixEncode(distance, &code, &n_bits, &bits);

      WriteHuffmanCode(bw, codes + 4, code);

      VP8LPutBits(bw, bits, n_bits);

    }

    x += PixOrCopyLength(v);

    while (x >= width) {

      x -= width;

      ++y;

    }

    VP8LRefsCursorNext(&c);

  }

  if (bw->error) {

    return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);

  }

  return 1;

}

// Special case of EncodeImageInternal() for cache-bits=0, histo_bits=31.

// pic and percent are for progress.

static int EncodeImageNoHuffman(VP8LBitWriter* const bw,

                                const uint32_t* const argb,

                                VP8LHashChain* const hash_chain,

                                VP8LBackwardRefs* const refs_array, int width,

                                int height, int quality, int low_effort,

                                const WebPPicture* const pic, int percent_range,

                                int* const percent) {

  int i;

  int max_tokens = 0;

  VP8LBackwardRefs* refs;

  HuffmanTreeToken* tokens = NULL;

  HuffmanTreeCode huffman_codes[5] = {{0, NULL, NULL}};

  const uint32_t histogram_symbols[1] = {0};  // only one tree, one symbol

  int cache_bits = 0;

  VP8LHistogramSet* histogram_image = NULL;

  HuffmanTree* const huff_tree = (HuffmanTree*)WebPSafeMalloc(

      3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree));

  if (huff_tree == NULL) {

    WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);

    goto Error;

  }

  // Calculate backward references from ARGB image.

  if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, low_effort,

                         pic, percent_range / 2, percent)) {

    goto Error;

  }

  if (!VP8LGetBackwardReferences(width, height, argb, quality, /*low_effort=*/0,

                                 kLZ77Standard | kLZ77RLE, cache_bits,

                                 /*do_no_cache=*/0, hash_chain, refs_array,

                                 &cache_bits, pic,

                                 percent_range - percent_range / 2, percent)) {

    goto Error;

  }

  refs = &refs_array[0];

  histogram_image = VP8LAllocateHistogramSet(1, cache_bits);

  if (histogram_image == NULL) {

    WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);

    goto Error;

  }

  VP8LHistogramSetClear(histogram_image);

  // Build histogram image and symbols from backward references.

  VP8LHistogramStoreRefs(refs, /*distance_modifier=*/NULL,

                         /*distance_modifier_arg0=*/0,

                         histogram_image->histograms[0]);

  // Create Huffman bit lengths and codes for each histogram image.

  assert(histogram_image->size == 1);

  if (!GetHuffBitLengthsAndCodes(histogram_image, huffman_codes)) {

    WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);

    goto Error;

  }

  // No color cache, no Huffman image.

  VP8LPutBits(bw, 0, 1);

  // Find maximum number of symbols for the huffman tree-set.

  for (i = 0; i < 5; ++i) {

    HuffmanTreeCode* const codes = &huffman_codes[i];

    if (max_tokens < codes->num_symbols) {

      max_tokens = codes->num_symbols;

    }

  }

  tokens = (HuffmanTreeToken*)WebPSafeMalloc(max_tokens, sizeof(*tokens));

  if (tokens == NULL) {

    WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);

    goto Error;

  }

  // Store Huffman codes.

  for (i = 0; i < 5; ++i) {

    HuffmanTreeCode* const codes = &huffman_codes[i];

    StoreHuffmanCode(bw, huff_tree, tokens, codes);

    ClearHuffmanTreeIfOnlyOneSymbol(codes);

  }

  // Store actual literals.

  if (!StoreImageToBitMask(bw, width, 0, refs, histogram_symbols, huffman_codes,

                           pic)) {

    goto Error;

  }

Error:

  WebPSafeFree(tokens);

  WebPSafeFree(huff_tree);

  VP8LFreeHistogramSet(histogram_image);

  WebPSafeFree(huffman_codes[0].codes);

  return (pic->error_code == VP8_ENC_OK);

}

// pic and percent are for progress.

static int EncodeImageInternal(

    VP8LBitWriter* const bw, const uint32_t* const argb,

    VP8LHashChain* const hash_chain, VP8LBackwardRefs refs_array[4], int width,

    int height, int quality, int low_effort, const CrunchConfig* const config,

    int* cache_bits, int histogram_bits_in, size_t init_byte_position,

    int* const hdr_size, int* const data_size, const WebPPicture* const pic,

    int percent_range, int* const percent) {

  const uint32_t histogram_image_xysize =

      VP8LSubSampleSize(width, histogram_bits_in) *

      VP8LSubSampleSize(height, histogram_bits_in);

  int remaining_percent = percent_range;

  int percent_start = *percent;

  VP8LHistogramSet* histogram_image = NULL;

  VP8LHistogram* tmp_histo = NULL;

  uint32_t i, histogram_image_size = 0;

  size_t bit_array_size = 0;

  HuffmanTree* const huff_tree = (HuffmanTree*)WebPSafeMalloc(

      3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree));

  HuffmanTreeToken* tokens = NULL;

  HuffmanTreeCode* huffman_codes = NULL;

  uint32_t* const histogram_argb = (uint32_t*)WebPSafeMalloc(

      histogram_image_xysize, sizeof(*histogram_argb));

  int sub_configs_idx;

  int cache_bits_init, write_histogram_image;

  VP8LBitWriter bw_init = *bw, bw_best;

  int hdr_size_tmp;

  VP8LHashChain hash_chain_histogram;  // histogram image hash chain

  size_t bw_size_best = ~(size_t)0;

  assert(histogram_bits_in >= MIN_HUFFMAN_BITS);

  assert(histogram_bits_in <= MAX_HUFFMAN_BITS);

  assert(hdr_size != NULL);

  assert(data_size != NULL);

  memset(&hash_chain_histogram, 0, sizeof(hash_chain_histogram));

  if (!VP8LBitWriterInit(&bw_best, 0)) {

    WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);