Fix input and output to handle UTF16 surrogate pairs.

src/preparser-api.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 28 matching lines @@
 #include "list.h"
 #include "hashmap.h"
 #include "preparse-data-format.h"
 #include "preparse-data.h"
 #include "preparser.h"
 
 namespace v8 {
 namespace internal {
 
 // UTF16Buffer based on a v8::UnicodeInputStream.
-class InputStreamUTF16Buffer : public UC16CharacterStream {
+class InputStreamUtf16Buffer : public Utf16CharacterStream {
  public:
-  /* The InputStreamUTF16Buffer maintains an internal buffer
-   * that is filled in chunks from the UC16CharacterStream.
+  /* The InputStreamUtf16Buffer maintains an internal buffer
+   * that is filled in chunks from the Utf16CharacterStream.
    * It also maintains unlimited pushback capability, but optimized
    * for small pushbacks.
    * The pushback_buffer_ pointer points to the limit of pushbacks
    * in the current buffer. There is room for a few pushback'ed chars before
    * the buffer containing the most recently read chunk. If this is overflowed,
    * an external buffer is allocated/reused to hold further pushbacks, and
    * pushback_buffer_ and buffer_cursor_/buffer_end_ now points to the
    * new buffer. When this buffer is read to the end again, the cursor is
    * switched back to the internal buffer
    */
-  explicit InputStreamUTF16Buffer(v8::UnicodeInputStream* stream)
-      : UC16CharacterStream(),
+  explicit InputStreamUtf16Buffer(v8::UnicodeInputStream* stream)
+      : Utf16CharacterStream(),
         stream_(stream),
         pushback_buffer_(buffer_),
         pushback_buffer_end_cache_(NULL),
         pushback_buffer_backing_(NULL),
         pushback_buffer_backing_size_(0) {
     buffer_cursor_ = buffer_end_ = buffer_ + kPushBackSize;
   }
 
-  virtual ~InputStreamUTF16Buffer() {
+  virtual ~InputStreamUtf16Buffer() {
     if (pushback_buffer_backing_ != NULL) {
       DeleteArray(pushback_buffer_backing_);
     }
   }
 
   virtual void PushBack(uc32 ch) {
     ASSERT(pos_ > 0);
     if (ch == kEndOfInput) {
       pos_--;
       return;
@@ skipping 36 matching lines @@
       buffer_cursor_ = buffer_;
       buffer_end_ = pushback_buffer_end_cache_;
       pushback_buffer_end_cache_ = NULL;
       return buffer_end_ > buffer_cursor_;
     }
     // Copy the top of the buffer into the pushback area.
     int32_t value;
     uc16* buffer_start = buffer_ + kPushBackSize;
     buffer_cursor_ = buffer_end_ = buffer_start;
     while ((value = stream_->Next()) >= 0) {
-      if (value > static_cast<int32_t>(unibrow::Utf8::kMaxThreeByteChar)) {
-        value = unibrow::Utf8::kBadChar;
+      if (value >
+          static_cast<int32_t>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
+        buffer_start[buffer_end_++ - buffer_start] =
+            unibrow::Utf16::LeadSurrogate(value);
+        buffer_start[buffer_end_++ - buffer_start] =
+            unibrow::Utf16::TrailSurrogate(value);
+      } else {
+        // buffer_end_ is a const pointer, but buffer_ is writable.
+        buffer_start[buffer_end_++ - buffer_start] = static_cast<uc16>(value);
       }
-      // buffer_end_ is a const pointer, but buffer_ is writable.
-      buffer_start[buffer_end_++ - buffer_start] = static_cast<uc16>(value);
-      if (buffer_end_ == buffer_ + kPushBackSize + kBufferSize) break;
+      // Stop one before the end of the buffer in case we get a surrogate pair.
+      if (buffer_end_ <= buffer_ + 1 + kPushBackSize + kBufferSize) break;
     }
     return buffer_end_ > buffer_start;
   }
 
   virtual unsigned SlowSeekForward(unsigned pos) {
     // Seeking in the input is not used by preparsing.
     // It's only used by the real parser based on preparser data.
     UNIMPLEMENTED();
     return 0;
   }
@@ skipping 26 matching lines @@
 
 bool EnableSlowAsserts() { return true; }
 
 }  // namespace internal.
 
 
 UnicodeInputStream::~UnicodeInputStream() { }
 
 
 PreParserData Preparse(UnicodeInputStream* input, size_t max_stack) {
-  internal::InputStreamUTF16Buffer buffer(input);
+  internal::InputStreamUtf16Buffer buffer(input);
   uintptr_t stack_limit = reinterpret_cast<uintptr_t>(&buffer) - max_stack;
   internal::UnicodeCache unicode_cache;
   internal::Scanner scanner(&unicode_cache);
   scanner.Initialize(&buffer);
   internal::CompleteParserRecorder recorder;
   preparser::PreParser::PreParseResult result =
       preparser::PreParser::PreParseProgram(&scanner,
                                             &recorder,
                                             internal::kAllowLazy,
                                             stack_limit);
   if (result == preparser::PreParser::kPreParseStackOverflow) {
     return PreParserData::StackOverflow();
   }
   internal::Vector<unsigned> pre_data = recorder.ExtractData();
   size_t size = pre_data.length() * sizeof(pre_data[0]);
   unsigned char* data = reinterpret_cast<unsigned char*>(pre_data.start());
   return PreParserData(size, data);
 }
 
 }  // namespace v8.
 
 
 // Used by ASSERT macros and other immediate exits.
 extern "C" void V8_Fatal(const char* file, int line, const char* format, ...) {
   exit(EXIT_FAILURE);
 }