Fix input and output to handle UTF16 surrogate pairs.

src/unicode.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 96 matching lines @@
  public:
   inline Buffer(Data data, unsigned length) : data_(data), length_(length) { }
   inline Buffer() : data_(0), length_(0) { }
   Data data() { return data_; }
   unsigned length() { return length_; }
  private:
   Data data_;
   unsigned length_;
 };
 
+
+class Utf16 {

[rossberg, 2012/03/07 13:32:47]

Nit: this doesn't quite fit into the above Utf8 section comment.

[Erik Corry, 2012/03/11 19:29:22]

Done.

+ public:
+  static inline bool IsLeadSurrogate(uchar code) {
+    return (code & 0xfc00) == 0xd800;
+  }
+  static inline bool IsTrailSurrogate(uchar code) {
+    return (code & 0xfc00) == 0xdc00;
+  }
+  static inline int CombineSurrogatePair(uchar lead, uchar trail) {

[rossberg, 2012/03/07 13:32:47]

Isn't int32_t more accurate as result type?

[Erik Corry, 2012/03/11 19:29:22]

Done.

+    return 0x10000 + ((lead & 0x3ff) << 10) + (trail & 0x3ff);
+  }
+  static const uchar kMaxNonSurrogateCharCode = 0xffff;
+  static inline uchar LeadSurrogate(int char_code) {

[rossberg, 2012/03/07 13:32:47]

Similar here (and below), isn't char_code an int32?

[Erik Corry, 2012/03/11 19:29:22]

Done.

+    return 0xd800 + (((char_code - 0x10000) >> 10) & 0x3ff);
+  }
+  static inline uchar TrailSurrogate(int char_code) {
+    return 0xdc00 + (char_code & 0x3ff);
+  }
+};
+
+
 class Utf8 {
  public:
-  static inline uchar Length(uchar chr);
-  static inline unsigned Encode(char* out, uchar c);
+  static inline uchar Length(uchar chr, int previous);
+  static inline unsigned Encode(
+      char* out, uchar c, int previous);
   static const byte* ReadBlock(Buffer<const char*> str, byte* buffer,
       unsigned capacity, unsigned* chars_read, unsigned* offset);
   static uchar CalculateValue(const byte* str,
                               unsigned length,
                               unsigned* cursor);
   static const uchar kBadChar = 0xFFFD;
   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
   static const unsigned kMaxTwoByteChar   = 0x7ff;
   static const unsigned kMaxThreeByteChar = 0xffff;
   static const unsigned kMaxFourByteChar  = 0x1fffff;
 
+  static const int kNoPreviousCharacter = -1;
+
+  // A single surrogate is coded as a 3 byte UTF-8 sequence, but two together
+  // that match are coded as a 4 byte UTF-8 sequence.
+  static const unsigned kBytesSavedByCombiningSurrogates = 2;
+  static const unsigned kSizeOfUnmatchedSurrogate = 3;
+
  private:
   template <unsigned s> friend class Utf8InputBuffer;
   friend class Test;
   static inline uchar ValueOf(const byte* str,
                               unsigned length,
                               unsigned* cursor);
 };
 
 // --- C h a r a c t e r   S t r e a m ---
 
 class CharacterStream {
  public:
   inline uchar GetNext();
   inline bool has_more() { return remaining_ != 0; }
   // Note that default implementation is not efficient.
   virtual void Seek(unsigned);
   unsigned Length();
+  unsigned Utf16Length();
   virtual ~CharacterStream() { }
   static inline bool EncodeCharacter(uchar c, byte* buffer, unsigned capacity,
       unsigned& offset);
   static inline bool EncodeAsciiCharacter(uchar c, byte* buffer,
       unsigned capacity, unsigned& offset);
   static inline bool EncodeNonAsciiCharacter(uchar c, byte* buffer,
       unsigned capacity, unsigned& offset);
   static inline uchar DecodeCharacter(const byte* buffer, unsigned* offset);
   virtual void Rewind() = 0;
+
  protected:
   virtual void FillBuffer() = 0;
   // The number of characters left in the current buffer
   unsigned remaining_;
   // The current offset within the buffer
   unsigned cursor_;
   // The buffer containing the decoded characters.
   const byte* buffer_;
 };
 
@@ skipping 102 matching lines @@
   static const int kMaxWidth = 1;
   static int Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr);
 };
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_H_