Line data    Source code

       1             : // Copyright 2014 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #ifndef V8_UNICODE_DECODER_H_
       6             : #define V8_UNICODE_DECODER_H_
       7             : 
       8             : #include <sys/types.h>
       9             : #include "src/globals.h"
      10             : #include "src/utils.h"
      11             : 
      12             : namespace unibrow {
      13             : 
      14             : class V8_EXPORT_PRIVATE Utf8DecoderBase {
      15             :  public:
      16             :   // Initialization done in subclass.
      17             :   inline Utf8DecoderBase();
      18             :   inline Utf8DecoderBase(uint16_t* buffer, size_t buffer_length,
      19             :                          const uint8_t* stream, size_t stream_length);
      20             :   inline size_t Utf16Length() const { return utf16_length_; }
      21             : 
      22             :  protected:
      23             :   // This reads all characters and sets the utf16_length_.
      24             :   // The first buffer_length utf16 chars are cached in the buffer.
      25             :   void Reset(uint16_t* buffer, size_t buffer_length, const uint8_t* stream,
      26             :              size_t stream_length);
      27             :   static void WriteUtf16Slow(const uint8_t* stream, size_t stream_length,
      28             :                              uint16_t* data, size_t length);
      29             :   const uint8_t* unbuffered_start_;
      30             :   size_t unbuffered_length_;
      31             :   size_t utf16_length_;
      32             :   bool last_byte_of_buffer_unused_;
      33             : 
      34             :  private:
      35             :   DISALLOW_COPY_AND_ASSIGN(Utf8DecoderBase);
      36             : };
      37             : 
      38             : template <size_t kBufferSize>
      39             : class Utf8Decoder : public Utf8DecoderBase {
      40             :  public:
      41             :   inline Utf8Decoder() {}
      42             :   inline Utf8Decoder(const char* stream, size_t length);
      43             :   inline void Reset(const char* stream, size_t length);
      44             :   inline size_t WriteUtf16(uint16_t* data, size_t length) const;
      45             : 
      46             :  private:
      47             :   uint16_t buffer_[kBufferSize];
      48             : };
      49             : 
      50             : Utf8DecoderBase::Utf8DecoderBase()
      51             :     : unbuffered_start_(nullptr),
      52             :       unbuffered_length_(0),
      53             :       utf16_length_(0),
      54     1064573 :       last_byte_of_buffer_unused_(false) {}
      55             : 
      56             : Utf8DecoderBase::Utf8DecoderBase(uint16_t* buffer, size_t buffer_length,
      57             :                                  const uint8_t* stream, size_t stream_length) {
      58             :   Reset(buffer, buffer_length, stream, stream_length);
      59             : }
      60             : 
      61             : 
      62             : template <size_t kBufferSize>
      63             : Utf8Decoder<kBufferSize>::Utf8Decoder(const char* stream, size_t length)
      64             :     : Utf8DecoderBase(buffer_, kBufferSize,
      65             :                       reinterpret_cast<const uint8_t*>(stream), length) {}
      66             : 
      67             : 
      68             : template <size_t kBufferSize>
      69             : void Utf8Decoder<kBufferSize>::Reset(const char* stream, size_t length) {
      70       10618 :   Utf8DecoderBase::Reset(buffer_, kBufferSize,
      71        5953 :                          reinterpret_cast<const uint8_t*>(stream), length);
      72             : }
      73             : 
      74             : 
      75             : template <size_t kBufferSize>
      76        5949 : size_t Utf8Decoder<kBufferSize>::WriteUtf16(uint16_t* data,
      77             :                                             size_t length) const {
      78             :   DCHECK_GT(length, 0);
      79        5949 :   if (length > utf16_length_) length = utf16_length_;
      80             :   // memcpy everything in buffer.
      81             :   size_t buffer_length =
      82        5949 :       last_byte_of_buffer_unused_ ? kBufferSize - 1 : kBufferSize;
      83        5949 :   size_t memcpy_length = length <= buffer_length ? length : buffer_length;
      84        5949 :   v8::internal::MemCopy(data, buffer_, memcpy_length * sizeof(uint16_t));
      85        5949 :   if (length <= buffer_length) return length;
      86             :   DCHECK_NOT_NULL(unbuffered_start_);
      87             :   // Copy the rest the slow way.
      88         349 :   WriteUtf16Slow(unbuffered_start_, unbuffered_length_, data + buffer_length,
      89             :                  length - buffer_length);
      90         349 :   return length;
      91             : }
      92             : 
      93             : class Latin1 {
      94             :  public:
      95             :   static const unsigned kMaxChar = 0xff;
      96             :   // Returns 0 if character does not convert to single latin-1 character
      97             :   // or if the character doesn't not convert back to latin-1 via inverse
      98             :   // operation (upper to lower, etc).
      99             :   static inline uint16_t ConvertNonLatin1ToLatin1(uint16_t);
     100             : };
     101             : 
     102             : 
     103             : uint16_t Latin1::ConvertNonLatin1ToLatin1(uint16_t c) {
     104             :   DCHECK_GT(c, Latin1::kMaxChar);
     105         118 :   switch (c) {
     106             :     // This are equivalent characters in unicode.
     107             :     case 0x39c:
     108             :     case 0x3bc:
     109             :       return 0xb5;
     110             :     // This is an uppercase of a Latin-1 character
     111             :     // outside of Latin-1.
     112             :     case 0x178:
     113             :       return 0xff;
     114             :   }
     115             :   return 0;
     116             : }
     117             : 
     118             : 
     119             : }  // namespace unibrow
     120             : 
     121             : #endif  // V8_UNICODE_DECODER_H_