Prepare lexer-shell for benchmarking: process multiple input files, adjust output.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 199 matching lines @@
   limit_ += input_size;
 
 #define SKIP()                { start_ = cursor_; YYSETCONDITION(kConditionNormal); goto yyc_Normal; }
 #define YYFILL(n)             { goto fill;        }
 
 #define YYGETSTATE()          state_
 #define YYSETSTATE(x)         { state_ = (x); }
 
 #define YYGETCONDITION()      condition_
 #define YYSETCONDITION(x)     { condition_ = (x); }
+#define YYDEBUG(state, current) {printf("%d: |%c| (%d)\n", state, current, (int)(current));}
 
 start_:
   if (FLAG_trace_lexer) {
     printf("Starting a round; state_: %d, condition_: %d\n", state_, condition_);
   }
 
   /*!re2c
     re2c:indent:top = 1;
     re2c:yych:conversion = 0;
     re2c:condenumprefix = kCondition;
@@ skipping 201 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }