Fix input and output to handle UTF16 surrogate pairs.

test/cctest/test-parsing.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 45 matching lines @@
 
   KeywordToken key_token;
   i::UnicodeCache unicode_cache;
   i::byte buffer[32];
   for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) {
     const i::byte* keyword =
         reinterpret_cast<const i::byte*>(key_token.keyword);
     int length = i::StrLength(key_token.keyword);
     CHECK(static_cast<int>(sizeof(buffer)) >= length);
     {
-      i::Utf8ToUC16CharacterStream stream(keyword, length);
+      i::Utf8ToUtf16CharacterStream stream(keyword, length);
       i::Scanner scanner(&unicode_cache);
       // The scanner should parse Harmony keywords for this test.
       scanner.SetHarmonyScoping(true);
       scanner.SetHarmonyModules(true);
       scanner.Initialize(&stream);
       CHECK_EQ(key_token.token, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Removing characters will make keyword matching fail.
     {
-      i::Utf8ToUC16CharacterStream stream(keyword, length - 1);
+      i::Utf8ToUtf16CharacterStream stream(keyword, length - 1);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Adding characters will make keyword matching fail.
     static const char chars_to_append[] = { 'z', '0', '_' };
     for (int j = 0; j < static_cast<int>(ARRAY_SIZE(chars_to_append)); ++j) {
       memmove(buffer, keyword, length);
       buffer[length] = chars_to_append[j];
-      i::Utf8ToUC16CharacterStream stream(buffer, length + 1);
+      i::Utf8ToUtf16CharacterStream stream(buffer, length + 1);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Replacing characters will make keyword matching fail.
     {
       memmove(buffer, keyword, length);
       buffer[length - 1] = '_';
-      i::Utf8ToUC16CharacterStream stream(buffer, length);
+      i::Utf8ToUtf16CharacterStream stream(buffer, length);
       i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
   }
 }
 
 
 TEST(ScanHTMLEndComments) {
@@ skipping 139 matching lines @@
       "var x = 42;",
       "function foo(x, y) { return x + y; }",
       "%ArgleBargle(glop);",
       "var x = new new Function('this.x = 42');",
       NULL
   };
 
   uintptr_t stack_limit = i::Isolate::Current()->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     const char* program = programs[i];
-    i::Utf8ToUC16CharacterStream stream(
+    i::Utf8ToUtf16CharacterStream stream(
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
     i::Scanner scanner(i::Isolate::Current()->unicode_cache());
     scanner.Initialize(&stream);
 
     int flags = i::kAllowLazy | i::kAllowNativesSyntax;
     v8::preparser::PreParser::PreParseResult result =
         v8::preparser::PreParser::PreParseProgram(&scanner,
                                                   &log,
@@ skipping 15 matching lines @@
 
   const char* programs[] = {
       "%ArgleBargle(glop);",
       "var x = %_IsSmi(42);",
       NULL
   };
 
   uintptr_t stack_limit = i::Isolate::Current()->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     const char* program = programs[i];
-    i::Utf8ToUC16CharacterStream stream(
+    i::Utf8ToUtf16CharacterStream stream(
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
     i::Scanner scanner(i::Isolate::Current()->unicode_cache());
     scanner.Initialize(&stream);
 
     // Flags don't allow natives syntax.
     v8::preparser::PreParser::PreParseResult result =
         v8::preparser::PreParser::PreParseProgram(&scanner,
                                                   &log,
@@ skipping 14 matching lines @@
   i::Isolate::Current()->stack_guard()->SetStackLimit(
       reinterpret_cast<uintptr_t>(&marker) - 128 * 1024);
 
   const char* program = "var x = 'something';\n"
                         "escape: function() {}";
   // Fails parsing expecting an identifier after "function".
   // Before fix, didn't check *ok after Expect(Token::Identifier, ok),
   // and then used the invalid currently scanned literal. This always
   // failed in debug mode, and sometimes crashed in release mode.
 
-  i::Utf8ToUC16CharacterStream stream(reinterpret_cast<const i::byte*>(program),
-                                      static_cast<unsigned>(strlen(program)));
+  i::Utf8ToUtf16CharacterStream stream(
+      reinterpret_cast<const i::byte*>(program),
+      static_cast<unsigned>(strlen(program)));
   i::ScriptDataImpl* data =
       i::ParserApi::PreParse(&stream, NULL, false);
   CHECK(data->HasError());
   delete data;
 }
 
 
 TEST(Regress928) {
   v8::V8::Initialize();
 
@@ skipping 44 matching lines @@
       reinterpret_cast<uintptr_t>(&marker) - 128 * 1024);
 
   size_t kProgramSize = 1024 * 1024;
   i::SmartArrayPointer<char> program(
       reinterpret_cast<char*>(malloc(kProgramSize + 1)));
   memset(*program, '(', kProgramSize);
   program[kProgramSize] = '\0';
 
   uintptr_t stack_limit = i::Isolate::Current()->stack_guard()->real_climit();
 
-  i::Utf8ToUC16CharacterStream stream(
+  i::Utf8ToUtf16CharacterStream stream(
       reinterpret_cast<const i::byte*>(*program),
       static_cast<unsigned>(kProgramSize));
   i::CompleteParserRecorder log;
   i::Scanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(&stream);
 
 
   v8::preparser::PreParser::PreParseResult result =
       v8::preparser::PreParser::PreParseProgram(&scanner,
                                                 &log,
@@ skipping 36 matching lines @@
   for (unsigned i = 0; i < length; i++) {
     uc16_buffer[i] = static_cast<i::uc16>(ascii_source[i]);
   }
   i::Vector<const char> ascii_vector(ascii_source, static_cast<int>(length));
   i::Handle<i::String> ascii_string(
       FACTORY->NewStringFromAscii(ascii_vector));
   TestExternalResource resource(*uc16_buffer, length);
   i::Handle<i::String> uc16_string(
       FACTORY->NewExternalStringFromTwoByte(&resource));
 
-  i::ExternalTwoByteStringUC16CharacterStream uc16_stream(
+  i::ExternalTwoByteStringUtf16CharacterStream uc16_stream(
       i::Handle<i::ExternalTwoByteString>::cast(uc16_string), start, end);
-  i::GenericStringUC16CharacterStream string_stream(ascii_string, start, end);
-  i::Utf8ToUC16CharacterStream utf8_stream(
+  i::GenericStringUtf16CharacterStream string_stream(ascii_string, start, end);
+  i::Utf8ToUtf16CharacterStream utf8_stream(
       reinterpret_cast<const i::byte*>(ascii_source), end);
   utf8_stream.SeekForward(start);
 
   unsigned i = start;
   while (i < end) {
     // Read streams one char at a time
     CHECK_EQU(i, uc16_stream.pos());
     CHECK_EQU(i, string_stream.pos());
     CHECK_EQU(i, utf8_stream.pos());
     int32_t c0 = ascii_source[i];
@@ skipping 102 matching lines @@
   static const int kAllUtf8CharsSize =
       (unibrow::Utf8::kMaxOneByteChar + 1) +
       (unibrow::Utf8::kMaxTwoByteChar - unibrow::Utf8::kMaxOneByteChar) * 2 +
       (unibrow::Utf8::kMaxThreeByteChar - unibrow::Utf8::kMaxTwoByteChar) * 3;
   static const unsigned kAllUtf8CharsSizeU =
       static_cast<unsigned>(kAllUtf8CharsSize);
 
   char buffer[kAllUtf8CharsSizeU];
   unsigned cursor = 0;
   for (int i = 0; i <= kMaxUC16Char; i++) {
-    cursor += unibrow::Utf8::Encode(buffer + cursor, i);
+    cursor += unibrow::Utf8::Encode(buffer + cursor,
+                                    i,
+                                    unibrow::Utf16::kNoPreviousCharacter);
   }
   ASSERT(cursor == kAllUtf8CharsSizeU);
 
-  i::Utf8ToUC16CharacterStream stream(reinterpret_cast<const i::byte*>(buffer),
-                                      kAllUtf8CharsSizeU);
+  i::Utf8ToUtf16CharacterStream stream(reinterpret_cast<const i::byte*>(buffer),
+                                       kAllUtf8CharsSizeU);
   for (int i = 0; i <= kMaxUC16Char; i++) {
     CHECK_EQU(i, stream.pos());
     int32_t c = stream.Advance();
     CHECK_EQ(i, c);
     CHECK_EQU(i + 1, stream.pos());
   }
   for (int i = kMaxUC16Char; i >= 0; i--) {
     CHECK_EQU(i + 1, stream.pos());
     stream.PushBack(i);
     CHECK_EQU(i, stream.pos());
   }
   int i = 0;
   while (stream.pos() < kMaxUC16CharU) {
     CHECK_EQU(i, stream.pos());
     unsigned progress = stream.SeekForward(12);
     i += progress;
     int32_t c = stream.Advance();
     if (i <= kMaxUC16Char) {
       CHECK_EQ(i, c);
     } else {
       CHECK_EQ(-1, c);
     }
     i += 1;
     CHECK_EQU(i, stream.pos());
   }
 }
 
 #undef CHECK_EQU
 
-void TestStreamScanner(i::UC16CharacterStream* stream,
+void TestStreamScanner(i::Utf16CharacterStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
   i::Scanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(stream);
 
   int i = 0;
   do {
     i::Token::Value expected = expected_tokens[i];
     i::Token::Value actual = scanner.Next();
     CHECK_EQ(i::Token::String(expected), i::Token::String(actual));
     if (scanner.location().end_pos == skip_pos) {
       scanner.SeekForward(skip_to);
     }
     i++;
   } while (expected_tokens[i] != i::Token::ILLEGAL);
 }
 
 TEST(StreamScanner) {
   v8::V8::Initialize();
 
   const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib";
-  i::Utf8ToUC16CharacterStream stream1(reinterpret_cast<const i::byte*>(str1),
-                                       static_cast<unsigned>(strlen(str1)));
+  i::Utf8ToUtf16CharacterStream stream1(reinterpret_cast<const i::byte*>(str1),
+                                        static_cast<unsigned>(strlen(str1)));
   i::Token::Value expectations1[] = {
       i::Token::LBRACE,
       i::Token::IDENTIFIER,
       i::Token::IDENTIFIER,
       i::Token::FOR,
       i::Token::COLON,
       i::Token::MUL,
       i::Token::DIV,
       i::Token::LT,
       i::Token::SUB,
       i::Token::IDENTIFIER,
       i::Token::EOS,
       i::Token::ILLEGAL
   };
   TestStreamScanner(&stream1, expectations1, 0, 0);
 
   const char* str2 = "case default const {THIS\nPART\nSKIPPED} do";
-  i::Utf8ToUC16CharacterStream stream2(reinterpret_cast<const i::byte*>(str2),
-                                       static_cast<unsigned>(strlen(str2)));
+  i::Utf8ToUtf16CharacterStream stream2(reinterpret_cast<const i::byte*>(str2),
+                                        static_cast<unsigned>(strlen(str2)));
   i::Token::Value expectations2[] = {
       i::Token::CASE,
       i::Token::DEFAULT,
       i::Token::CONST,
       i::Token::LBRACE,
       // Skipped part here
       i::Token::RBRACE,
       i::Token::DO,
       i::Token::EOS,
       i::Token::ILLEGAL
   };
   ASSERT_EQ('{', str2[19]);
   ASSERT_EQ('}', str2[37]);
   TestStreamScanner(&stream2, expectations2, 20, 37);
 
   const char* str3 = "{}}}}";
   i::Token::Value expectations3[] = {
       i::Token::LBRACE,
       i::Token::RBRACE,
       i::Token::RBRACE,
       i::Token::RBRACE,
       i::Token::RBRACE,
       i::Token::EOS,
       i::Token::ILLEGAL
   };
   // Skip zero-four RBRACEs.
   for (int i = 0; i <= 4; i++) {
      expectations3[6 - i] = i::Token::ILLEGAL;
      expectations3[5 - i] = i::Token::EOS;
-     i::Utf8ToUC16CharacterStream stream3(
+     i::Utf8ToUtf16CharacterStream stream3(
          reinterpret_cast<const i::byte*>(str3),
          static_cast<unsigned>(strlen(str3)));
      TestStreamScanner(&stream3, expectations3, 1, 1 + i);
   }
 }
 
 
 void TestScanRegExp(const char* re_source, const char* expected) {
-  i::Utf8ToUC16CharacterStream stream(
+  i::Utf8ToUtf16CharacterStream stream(
        reinterpret_cast<const i::byte*>(re_source),
        static_cast<unsigned>(strlen(re_source)));
   i::Scanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(&stream);
 
   i::Token::Value start = scanner.peek();
   CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
   CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV));
   scanner.Next();  // Current token is now the regexp literal.
   CHECK(scanner.is_literal_ascii());
@@ skipping 122 matching lines @@
       "    block;\n"
       "  }", "\n"
       "  more;", i::BLOCK_SCOPE, i::EXTENDED_MODE },
     { "  for ", "(let x in {}) statement;", " more;",
       i::BLOCK_SCOPE, i::EXTENDED_MODE },
     { "  for ", "(let x in {}) statement", "\n"
       "  more;", i::BLOCK_SCOPE, i::EXTENDED_MODE },
     { "  for ", "(let x in {})\n"
       "    statement;", "\n"
       "  more;", i::BLOCK_SCOPE, i::EXTENDED_MODE },
+    // Check that 6-byte and 4-byte encodings of UTF-8 strings do not throw
+    // the preparser off in terms of byte offsets.
+    { "  'foo\355\240\201\355\260\211';\n"
+      "  (function fun", "(a,b) { infunction; }", ")();",
+      i::FUNCTION_SCOPE, i::CLASSIC_MODE },
+    { "  'foo\360\220\220\212';\n"
+      "  (function fun", "(a,b) { infunction; }", ")();",
+      i::FUNCTION_SCOPE, i::CLASSIC_MODE },
+    { "  'foo';\n"
+      "  (function fun", "(a,b) { 'bar\355\240\201\355\213'; }", ")();",
+      i::FUNCTION_SCOPE, i::CLASSIC_MODE },
+    { "  'foo';\n"
+      "  (function fun", "(a,b) { 'bar\360\220\220\214'; }", ")();",
+      i::FUNCTION_SCOPE, i::CLASSIC_MODE },
     { NULL, NULL, NULL, i::EVAL_SCOPE, i::CLASSIC_MODE }
   };
 
   v8::HandleScope handles;
   v8::Persistent<v8::Context> context = v8::Context::New();
   v8::Context::Scope context_scope(context);
 
   int marker;
   i::Isolate::Current()->stack_guard()->SetStackLimit(
       reinterpret_cast<uintptr_t>(&marker) - 128 * 1024);
@@ skipping 39 matching lines @@
 }
 
 
 void TestParserSync(i::Handle<i::String> source, int flags) {
   uintptr_t stack_limit = i::Isolate::Current()->stack_guard()->real_climit();
   bool harmony_scoping = ((i::kLanguageModeMask & flags) == i::EXTENDED_MODE);
 
   // Preparse the data.
   i::CompleteParserRecorder log;
   i::Scanner scanner(i::Isolate::Current()->unicode_cache());
-  i::GenericStringUC16CharacterStream stream(source, 0, source->length());
+  i::GenericStringUtf16CharacterStream stream(source, 0, source->length());
   scanner.SetHarmonyScoping(harmony_scoping);
   scanner.Initialize(&stream);
   v8::preparser::PreParser::PreParseResult result =
       v8::preparser::PreParser::PreParseProgram(
           &scanner, &log, flags, stack_limit);
   CHECK_EQ(v8::preparser::PreParser::kPreParseSuccess, result);
   i::ScriptDataImpl data(log.ExtractData());
 
   // Parse the data
   i::Handle<i::Script> script = FACTORY->NewScript(source);
@@ skipping 168 matching lines @@
             termination_data[k],
             context_data[i][1]);
         CHECK(length == kProgramSize);
         i::Handle<i::String> source =
             FACTORY->NewStringFromAscii(i::CStrVector(program.start()));
         TestParserSyncWithFlags(source);
       }
     }
   }
 }