Represent tokens as a compressed stream instead of an array.

vm/raw_object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/raw_object.h"
 
 #include "vm/class_table.h"
 #include "vm/freelist.h"
 #include "vm/isolate.h"
 #include "vm/object.h"
@@ skipping 42 matching lines @@
 
   RawClass* raw_class = isolate->class_table()->At(GetClassId());
   intptr_t instance_size = raw_class->ptr()->instance_size_;
   ObjectKind instance_kind = raw_class->ptr()->instance_kind_;
 
   if (instance_size == 0) {
     switch (instance_kind) {
       case kTokenStream: {
         const RawTokenStream* raw_tokens =
             reinterpret_cast<const RawTokenStream*>(this);
-        intptr_t tokens_length = Smi::Value(raw_tokens->ptr()->length_);
+        intptr_t tokens_length = raw_tokens->ptr()->length_;
         instance_size = TokenStream::InstanceSize(tokens_length);
         break;
       }
       case kCode: {
         const RawCode* raw_code = reinterpret_cast<const RawCode*>(this);
         intptr_t pointer_offsets_length =
             raw_code->ptr()->pointer_offsets_length_;
         instance_size = Code::InstanceSize(pointer_offsets_length);
         break;
       }
@@ skipping 323 matching lines @@
 
 intptr_t RawLiteralToken::VisitLiteralTokenPointers(
     RawLiteralToken* raw_obj, ObjectPointerVisitor* visitor) {
   visitor->VisitPointers(raw_obj->from(), raw_obj->to());
   return LiteralToken::InstanceSize();
 }
 
 
 intptr_t RawTokenStream::VisitTokenStreamPointers(
     RawTokenStream* raw_obj, ObjectPointerVisitor* visitor) {
-  intptr_t length = Smi::Value(raw_obj->ptr()->length_);
-  visitor->VisitPointers(raw_obj->from(), raw_obj->to(length));
+  intptr_t length = raw_obj->ptr()->length_;
+  visitor->VisitPointers(raw_obj->from(), raw_obj->to());
   return TokenStream::InstanceSize(length);
 }
 
 
 intptr_t RawScript::VisitScriptPointers(RawScript* raw_obj,
                                         ObjectPointerVisitor* visitor) {
   visitor->VisitPointers(raw_obj->from(), raw_obj->to());
   return Script::InstanceSize();
 }
 
@@ skipping 534 matching lines @@
 intptr_t RawJSRegExp::VisitJSRegExpPointers(RawJSRegExp* raw_obj,
                                             ObjectPointerVisitor* visitor) {
   // Make sure that we got here with the tagged pointer as this.
   ASSERT(raw_obj->IsHeapObject());
   intptr_t length = Smi::Value(raw_obj->ptr()->data_length_);
   visitor->VisitPointers(raw_obj->from(), raw_obj->to());
   return JSRegExp::InstanceSize(length);
 }
 
 }  // namespace dart