Fix fetching tokens in lexer.

src/lexer/lexer.re

 #include <fcntl.h>
 #include <stdio.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 
 // TODO:
 // - SpiderMonkey compatibility hack: "  --> something" is treated
 //   as a single line comment.
 // - An identifier cannot start immediately after a number.
@@ skipping 325 matching lines @@
 ExperimentalScanner::~ExperimentalScanner() {
   fclose(file_);
 }
 
 
 void ExperimentalScanner::FillTokens() {
   current_ = 0;
   fetched_ = 0;
   uint8_t chars[BUFFER_SIZE];
   int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
-  scanner_->push(chars, n);
+  for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
+  scanner_->push(chars, BUFFER_SIZE);
 }
 
 
 Token::Value ExperimentalScanner::Next(int* beg_pos, int* end_pos) {
-  if (current_ == fetched_) {
+  while (current_ == fetched_) {
     FillTokens();
   }
   *beg_pos = beg_[current_];
   *end_pos = end_[current_];
   Token::Value res = token_[current_];
   if (token_[current_] != Token::Token::EOS &&
       token_[current_] != Token::ILLEGAL) {
     current_++;
   }
   return res;
 }
 
 
 void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
   if (token == Token::EOS) end--;
   token_[fetched_] = token;
   beg_[fetched_] = beg;
   end_[fetched_] = end;
   fetched_++;
 }