Add early exit on EOS to lexer.

src/lexer/lexer.re

 #include <fcntl.h>
 #include <stdio.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 
 // TODO:
 // - SpiderMonkey compatibility hack: "  --> something" is treated
 //   as a single line comment.
 // - An identifier cannot start immediately after a number.
@@ skipping 30 matching lines @@
 #endif
 
 #endif //  defined(WIN32)
 
 #include "lexer.h"
 
 using namespace v8::internal;
 
 #define PUSH_TOKEN(T) { send(T); SKIP(); }
 #define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
+#define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
 #define PUSH_LINE_TERMINATOR() { SKIP(); }
 #define TERMINATE_ILLEGAL() { return 1; }
 
 class PushScanner {
 
 public:
   PushScanner(ExperimentalScanner* sink):
       eof_(false),
       state_(-1),
       condition_(kConditionNormal),
@@ skipping 57 matching lines @@
     //  ^          ^       ^        ^       ^           ^
     //  buffer_       start_     marker_     cursor_    limit_         buffer_end_
     //
     //  We need to stretch the buffer_ and concatenate the new chunk of input to it
 
     size_t used = limit_ - buffer_;
     size_t needed = used + input_size;
     size_t allocated = buffer_end_ - buffer_;
     if(allocated < needed) {
       size_t limit__offset = limit_ - buffer_;
-      size_t start__offset = start_ - buffer_;
+      size_t start_offset = start_ - buffer_;
       size_t marker__offset = marker_ - buffer_;
       size_t cursor__offset = cursor_ - buffer_;
 
       buffer_ = (uint8_t*)realloc(buffer_, needed);
       buffer_end_ = needed + buffer_;
 
       marker_ = marker__offset + buffer_;
       cursor_ = cursor__offset + buffer_;
-      start_ = buffer_ + start__offset;
+      start_ = buffer_ + start_offset;
       limit_ = limit__offset + buffer_;
     }
     memcpy(limit_, input, input_size);
     limit_ += input_size;
 
     // The scanner start_s here
     #define YYLIMIT     limit_
     #define YYCURSOR    cursor_
     #define YYMARKER    marker_
     #define YYCTYPE     uint8_t
@@ skipping 138 matching lines @@
     <Normal> ","           { PUSH_TOKEN(Token::COMMA); }
 
     <Normal> line_terminator+ { PUSH_LINE_TERMINATOR(); }
     <Normal> whitespace       { SKIP(); }
 
     <Normal> ["]           :=> DoubleQuoteString
     <Normal> [']           :=> SingleQuoteString
 
     <Normal> identifier_start_    :=> Identifier
 
-    <Normal> eof           { PUSH_TOKEN(Token::EOS); return 1; }
+    <Normal> eof           { PUSH_EOF_AND_RETURN();}
     <Normal> any           { TERMINATE_ILLEGAL(); }
 
     <DoubleQuoteString> "\\\""  { goto yy0; }
     <DoubleQuoteString> '"'     { PUSH_TOKEN(Token::STRING);}
     <DoubleQuoteString> any     { goto yy0; }
 
     <SingleQuoteString> "\\'"   { goto yy0; }
     <SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING);}
     <SingleQuoteString> any     { goto yy0; }
 
     <Identifier> identifier_char+  { goto yy0; }
     <Identifier> any               { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
 
     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> eof             { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> any             { goto yy0; }
 
     <MultiLineComment> [*][//]  { PUSH_LINE_TERMINATOR();}
     <MultiLineComment> eof      { TERMINATE_ILLEGAL(); }
     <MultiLineComment> any      { goto yy0; }
 
     <HtmlComment> eof        { TERMINATE_ILLEGAL(); }
     <HtmlComment> "-->"      { PUSH_LINE_TERMINATOR();}
     <HtmlComment> any        { goto yy0; }
     */
 
  fill:
-    int unfinished_size = cursor_-start_;
+    int unfinished_size = cursor_ - start_;
     if (FLAG_trace_lexer) {
       printf(
         "scanner needs a refill. Exiting for now with:\n"
         "  saved fill state_ = %d\n"
         "  unfinished token size = %d\n",
         state_,
         unfinished_size
       );
       if(0 < unfinished_size && start_ < limit_) {
         printf("  unfinished token is: ");
-        fwrite(start_, 1, cursor_-start_, stdout);
+        fwrite(start_, 1, cursor_ - start_, stdout);
         putchar('\n');
       }
       putchar('\n');
     }
 
     if (eof_) goto start_;
 
     //  Once we get here, we can get rid of
     //  everything before start_ and after limit_.
 
     if (buffer_ < start_) {
-      size_t start__offset = start_ - buffer_;
+      size_t start_offset = start_ - buffer_;
       memmove(buffer_, start_, limit_ - start_);
-      marker_ -= start__offset;
-      cursor_ -= start__offset;
-      limit_ -= start__offset;
-      start_ -= start__offset;
-      real_start_ += start__offset;
+      marker_ -= start_offset;
+      cursor_ -= start_offset;
+      limit_ -= start_offset;
+      start_ -= start_offset;
+      real_start_ += start_offset;
     }
     return 0;
   }
 
  private:
   bool eof_;
   int32_t state_;
   int32_t condition_;
 
   uint8_t* limit_;
@@ skipping 49 matching lines @@
 }
 
 
 void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
   if (token == Token::EOS) end--;
   token_[fetched_] = token;
   beg_[fetched_] = beg;
   end_[fetched_] = end;
   fetched_++;
 }