Represent tokens as a compressed stream instead of an array.

vm/raw_object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_RAW_OBJECT_H_
 #define VM_RAW_OBJECT_H_
 
 #include "platform/assert.h"
 #include "vm/globals.h"
 #include "vm/token.h"
@@ skipping 608 matching lines @@
 };
 
 
 class RawTokenStream : public RawObject {
   RAW_HEAP_OBJECT_IMPLEMENTATION(TokenStream);
 
   RawObject** from() {
     return reinterpret_cast<RawObject**>(&ptr()->private_key_);
   }
   RawString* private_key_;  // Key used for private identifiers.
-  RawSmi* length_;  // Number of tokens.
-
+  RawSmi* length_;  // Length of token stream.
+  RawArray* token_objects_;
+  RawObject** to() {
+    return reinterpret_cast<RawObject**>(&ptr()->token_objects_);
+  }
   // Variable length data follows here.
-  RawObject* data_[0];
-  RawObject** to(intptr_t length) {
-    return reinterpret_cast<RawObject**>(&ptr()->data_[length - 1]);
-  }
+  uint8_t data_[0];
 
   friend class SnapshotReader;
 };
 
 
 class RawScript : public RawObject {
  public:
   enum Kind {
     kScript = 0,
     kLibrary,
@@ skipping 870 matching lines @@
          kExternalUint64Array == kByteArray + 18 &&
          kExternalFloat32Array == kByteArray + 19 &&
          kExternalFloat64Array == kByteArray + 20 &&
          kClosure == kByteArray + 21);
   return (index >= kByteArray && index <= kClosure);
 }
 
 }  // namespace dart
 
 #endif  // VM_RAW_OBJECT_H_