Reject non-ASCII identifiers.

dart/lib/compiler/implementation/scanner/scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
@@ skipping 238 matching lines @@
     // TODO(ahe): Would a range check be faster?
     if (next === $1 || next === $2 || next === $3 || next === $4 ||  next === $5
         || next === $6 || next === $7 || next === $8 || next === $9) {
       return tokenizeNumber(next);
     }
 
     if (next === $EOF) {
       return $EOF;
     }
     if (next < 0x1f) {
-      throw new MalformedInputException("illegal character $next", charOffset);
+      return error(new SourceString("unexpected character $next"));
     }
 
     // The following are non-ASCII characters.
 
     if (next === $NBSP) {
       appendWhiteSpace(next);
       return advance();
     }
 
     return tokenizeIdentifier(next, byteOffset, true);
@@ skipping 215 matching lines @@
     int start = byteOffset - 1;
     bool hasDigits = false;
     while (true) {
       next = advance();
       if (($0 <= next && next <= $9)
           || ($A <= next && next <= $F)
           || ($a <= next && next <= $f)) {
         hasDigits = true;
       } else {
         if (!hasDigits) {
-          throw new MalformedInputException("hex digit expected", charOffset);
+          return error(const SourceString("hex digit expected"));
         }
         appendByteStringToken(HEXADECIMAL_INFO, asciiString(start, 0));
         return next;
       }
     }
   }
 
   int tokenizeDotsOrNumber(int next) {
     int start = byteOffset;
     next = advance();
@@ skipping 43 matching lines @@
   int tokenizeExponent(int next) {
     if (next === $PLUS || next === $MINUS) {
       next = advance();
     }
     bool hasDigits = false;
     while (true) {
       if ($0 <= next && next <= $9) {
         hasDigits = true;
       } else {
         if (!hasDigits) {
-          throw new MalformedInputException("digit expected", charOffset);
+          return error(const SourceString("digit expected"));
         }
         return next;
       }
       next = advance();
     }
   }
 
   int tokenizeSlashOrComment(int next) {
     next = advance();
     if ($STAR === next) {
@@ skipping 101 matching lines @@
     }
 
     while (true) {
       if (($a <= next && next <= $z) ||
           ($A <= next && next <= $Z) ||
           ($0 <= next && next <= $9) ||
           next === $_ ||
           (next === $$ && allowDollar)) {
         isDynamicBuiltIn = false;
         next = advance();
-      } else if (next < 128) {
+      } else if ((next < 128) || (next === $NBSP)) {
         // Identifier ends here.
         if (start == byteOffset) {
-          throw new MalformedInputException("expected identifier not found",
-                                            charOffset);
-        }
-        if (isDynamicBuiltIn) {
+          return error(const SourceString("expected identifier"));
+        } else if (isDynamicBuiltIn) {
           appendKeywordToken(Keyword.DYNAMIC);
         } else if (isAscii) {
           appendByteStringToken(IDENTIFIER_INFO, asciiString(start, 0));
         } else {
-          appendByteStringToken(IDENTIFIER_INFO, utf8String(start, -1));
+          appendByteStringToken(BAD_INPUT_INFO, utf8String(start, -1));
         }
         return next;
       } else {
         isDynamicBuiltIn = false;
         int nonAsciiStart = byteOffset;
         do {
           next = nextByte();
+          if (next === $NBSP) break;
         } while (next > 127);
         String string = utf8String(nonAsciiStart, -1).slowToString();
         isAscii = false;
         int byteLength = nonAsciiStart - byteOffset;
         addToCharOffset(string.length - byteLength);
       }
     }
   }
 
   int tokenizeAtOrRawString(int next) {
@@ skipping 37 matching lines @@
   int tokenizeSingleLineString(int next, int quoteChar, int start) {
     while (next !== quoteChar) {
       if (next === $BACKSLASH) {
         next = advance();
       } else if (next === $$) {
         next = tokenizeStringInterpolation(start);
         start = byteOffset;
         continue;
       }
       if (next <= $CR && (next === $LF || next === $CR || next === $EOF)) {
-        throw new MalformedInputException("unterminated string literal",
-                                          charOffset);
+        return error(const SourceString("unterminated string literal"));
       }
       next = advance();
     }
     appendByteStringToken(STRING_INFO, utf8String(start, 0));
     return advance();
   }
 
   int tokenizeStringInterpolation(int start) {
     appendByteStringToken(STRING_INFO, utf8String(start, -1));
     beginToken(); // $ starts here.
@@ skipping 26 matching lines @@
     return next;
   }
 
   int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
     next = advance();
     while (next != $EOF) {
       if (next === quoteChar) {
         appendByteStringToken(STRING_INFO, utf8String(start, 0));
         return advance();
       } else if (next === $LF || next === $CR) {
-        throw new MalformedInputException("unterminated string literal",
-                                          charOffset);
+        return error(const SourceString("unterminated string literal"));
       }
       next = advance();
     }
-    throw new MalformedInputException("unterminated string literal",
-                                      charOffset);
+    return error(const SourceString("unterminated string literal"));
   }
 
   int tokenizeMultiLineRawString(int quoteChar, int start) {
     int next = advance();
     outer: while (next !== $EOF) {
       while (next !== quoteChar) {
         next = advance();
         if (next === $EOF) break outer;
       }
       next = advance();
       if (next === quoteChar) {
         next = advance();
         if (next === quoteChar) {
           appendByteStringToken(STRING_INFO, utf8String(start, 0));
           return advance();
         }
       }
     }
-    throw new MalformedInputException("unterminated string literal",
-                                      charOffset);
+    return error(const SourceString("unterminated string literal"));
   }
 
   int tokenizeMultiLineString(int quoteChar, int start, bool raw) {
     if (raw) return tokenizeMultiLineRawString(quoteChar, start);
     int next = advance();
     while (next !== $EOF) {
       if (next === $$) {
         next = tokenizeStringInterpolation(start);
         start = byteOffset;
         continue;
       }
       if (next === quoteChar) {
         next = advance();
         if (next === quoteChar) {
           next = advance();
           if (next === quoteChar) {
             appendByteStringToken(STRING_INFO, utf8String(start, 0));
             return advance();
           }
         }
         continue;
       }
       if (next === $BACKSLASH) {
         next = advance();
         if (next === $EOF) break;
       }
       next = advance();
     }
-    throw new MalformedInputException("unterminated string literal",
-                                      charOffset);
+    return error(const SourceString("unterminated string literal"));
+  }
+
+  int error(SourceString message) {
+    appendByteStringToken(BAD_INPUT_INFO, message);
+    return advance(); // Ensure progress.
   }
 }
-
-class MalformedInputException {
-  final String message;
-  final position;
-  MalformedInputException(this.message, this.position);
-  toString() => message;
-}