Use utf8 instead of ascii in baseline scanner.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 28 matching lines @@
 #include "platform.h"
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 #include "scanner.h"
 #include "lexer.h"
 
 using namespace v8::internal;
 
-Handle<String> ReadFile(const char* name, Isolate* isolate) {
+const byte* ReadFile(const char* name, Isolate* isolate, int* size) {
   FILE* file = fopen(name, "rb");
-  if (file == NULL) return Handle<String>();
+  *size = 0;
+  if (file == NULL) return NULL;
 
   fseek(file, 0, SEEK_END);
-  int size = ftell(file);
+  *size = ftell(file);
   rewind(file);
 
-  uint8_t* chars = new uint8_t[size + 1];
-  chars[size] = '\0';
-  for (int i = 0; i < size;) {
-    int read = static_cast<int>(fread(&chars[i], 1, size - i, file));
+  byte* chars = new byte[*size + 1];
+  chars[*size] = 0;
+  for (int i = 0; i < *size;) {
+    int read = static_cast<int>(fread(&chars[i], 1, *size - i, file));
     i += read;
   }
   fclose(file);
-  Handle<String> result = isolate->factory()->NewStringFromOneByte(
-      Vector<const uint8_t>(chars, size));
-  delete[] chars;
-  return result;
+  return chars;
 }
 
 
 class BaselineScanner {
  public:
   BaselineScanner(const char* fname, Isolate* isolate) {
-    Handle<String> source = ReadFile(fname, isolate);
+    int length = 0;
+    source_ = ReadFile(fname, isolate, &length);
     unicode_cache_ = new UnicodeCache();
     scanner_ = new Scanner(unicode_cache_);
-    stream_ = new GenericStringUtf16CharacterStream(
-                      source, 0, source->length());
+    stream_ = new Utf8ToUtf16CharacterStream(source_, length);
     scanner_->Initialize(stream_);
   }
 
   ~BaselineScanner() {
     delete scanner_;
     delete stream_;
     delete unicode_cache_;
+    delete source_;
   }
 
   Token::Value Next(int* beg_pos, int* end_pos) {
     Token::Value res = scanner_->Next();
     *beg_pos = scanner_->location().beg_pos;
     *end_pos = scanner_->location().end_pos;
     return res;
   }
 
  private:
   UnicodeCache* unicode_cache_;
   Scanner* scanner_;
-  GenericStringUtf16CharacterStream* stream_;
+  const byte* source_;
+  Utf8ToUtf16CharacterStream* stream_;
 };
 
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   v8::Isolate* isolate = v8::Isolate::GetCurrent();
   {
     v8::HandleScope handle_scope(isolate);
     v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
@@ skipping 24 matching lines @@
                  Token::Name(actual_token),
                  actual_beg, actual_end);
           return 1;
         }
       } while (actual_token != Token::EOS);
     }
   }
   v8::V8::Dispose();
   return 0;
 }