Represent tokens as a compressed stream instead of an array.

vm/raw_object_snapshot.cc

Index: vm/raw_object_snapshot.cc
===================================================================
--- vm/raw_object_snapshot.cc	(revision 9303)
+++ vm/raw_object_snapshot.cc	(working copy)
@@ -593,7 +593,7 @@
   ASSERT(kind != Snapshot::kMessage && !RawObject::IsCreatedFromSnapshot(tags));
 
   // Read the length so that we can determine number of tokens to read.
-  intptr_t len = reader->ReadSmiValue();
+  intptr_t len = reader->ReadIntptrValue();
 
   // Create the token stream object.
   TokenStream& token_stream = TokenStream::ZoneHandle(
@@ -603,11 +603,13 @@
   // Set the object tags.
   token_stream.set_tags(tags);
 
-  // Read the token stream into the TokenStream.
-  for (intptr_t i = 0; i < len; i++) {
-    *reader->ObjectHandle() = reader->ReadObjectImpl();
-    token_stream.SetTokenAt(i, *reader->ObjectHandle());
-  }
+  // Read the stream of tokens into the TokenStream object.
+  reader->ReadBytes(reinterpret_cast<uint8_t*>(token_stream.EntryAddr(0)), len);
+
+  // Read in the literal/identifier token array.
+  *(reader->TokensHandle()) ^= reader->ReadObjectImpl();
+  token_stream.SetTokenObjects(*(reader->TokensHandle()));
+
   return token_stream.raw();
 }
 
@@ -626,14 +628,13 @@
   writer->WriteObjectHeader(Object::kTokenStreamClass,
                             writer->GetObjectTags(this));
 
-  // Write out the length field.
-  writer->Write<RawObject*>(ptr()->length_);
+  // Write out the length field and the token stream.
+  intptr_t len = ptr()->length_;
+  writer->Write<intptr_t>(len);
+  writer->WriteBytes(reinterpret_cast<uint8_t*>(ptr()->data_), len);
 
-  // Write out the token stream (token kind and literal).
-  intptr_t len = Smi::Value(ptr()->length_);
-  for (intptr_t i = 0; i < TokenStream::StreamLength(len); i++) {
-    writer->WriteObjectImpl(ptr()->data_[i]);
-  }
+  // Write out the literal/identifier token array.
+  writer->WriteObjectImpl(ptr()->token_objects_);
 }