Represent tokens as a compressed stream instead of an array.

vm/datastream.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_DATASTREAM_H_
 #define VM_DATASTREAM_H_
 
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/allocation.h"
 #include "vm/globals.h"
 
 namespace dart {
 
 static const int8_t kDataBitsPerByte = 7;
 static const int8_t kByteMask = (1 << kDataBitsPerByte) - 1;
 static const int8_t kMaxUnsignedDataPerByte = kByteMask;
 static const int8_t kMinDataPerByte = -(1 << (kDataBitsPerByte - 1));
 static const int8_t kMaxDataPerByte = (~kMinDataPerByte & kByteMask);
 static const uint8_t kEndByteMarker = (255 - kMaxDataPerByte);
+static const uint8_t kEndUnsignedByteMarker = (255 - kMaxUnsignedDataPerByte);
 
 typedef uint8_t* (*ReAlloc)(uint8_t* ptr, intptr_t old_size, intptr_t new_size);
 
 // Stream for reading various types from a buffer.
 class ReadStream : public ValueObject {
  public:
   ReadStream(const uint8_t* buffer, intptr_t size) : buffer_(buffer),
                                                      current_(buffer),
                                                      end_(buffer + size)  {}
 
@@ skipping 25 matching lines @@
   };
 
   template<typename T>
   class Raw<8, T> {
    public:
     static T Read(ReadStream* st) {
       return bit_cast<T>(st->Read<int64_t>());
     }
   };
 
+  // Reads 'len' bytes from the stream.
   void ReadBytes(uint8_t* addr, intptr_t len) {
     ASSERT((end_ - current_) >= len);
     memmove(addr, current_, len);
     current_ += len;
   }
 
+  intptr_t ReadUnsigned() {
+    return Read<intptr_t>(kEndUnsignedByteMarker);
+  }
+
  private:
   template<typename T>
   T Read() {
+    return Read<T>(kEndByteMarker);
+  }
+
+  template<typename T>
+  T Read(uint8_t end_byte_marker) {
     uint8_t b = ReadByte();
     if (b > kMaxUnsignedDataPerByte) {
-      return static_cast<T>(b) - kEndByteMarker;
+      return static_cast<T>(b) - end_byte_marker;
     }
     T r = 0;
     uint8_t s = 0;
     do {
       r |= static_cast<T>(b) << s;
       s += kDataBitsPerByte;
       b = ReadByte();
     } while (b <= kMaxUnsignedDataPerByte);
-    return r | ((static_cast<T>(b) - kEndByteMarker) << s);
+    return r | ((static_cast<T>(b) - end_byte_marker) << s);
   }
 
   uint8_t ReadByte() {
     ASSERT(current_ < end_);
     return *current_++;
   }
 
  private:
   const uint8_t* buffer_;
   const uint8_t* current_;
@@ skipping 58 matching lines @@
   };
 
   template<typename T>
   class Raw<8, T> {
    public:
     static void Write(WriteStream* st, T value) {
       st->Write<int64_t>(bit_cast<int64_t>(value));
     }
   };
 
+  void WriteUnsigned(intptr_t value) {
+    ASSERT((value >= 0) && (value <= kIntptrMax));
+    while (value > kMaxUnsignedDataPerByte) {
+      WriteByte(static_cast<uint8_t>(value & kByteMask));
+      value = value >> kDataBitsPerByte;
+    }
+    WriteByte(static_cast<uint8_t>(value + kEndUnsignedByteMarker));
+  }
+
   void WriteBytes(const uint8_t* addr, intptr_t len) {
     if ((end_ - current_) < len) {
       Resize(len);
     }
     ASSERT((end_ - current_) >= len);
     memmove(current_, addr, len);
     current_ += len;
   }
 
  private:
@@ skipping 36 matching lines @@
   uint8_t* current_;
   intptr_t current_size_;
   ReAlloc alloc_;
 
   DISALLOW_COPY_AND_ASSIGN(WriteStream);
 };
 
 }  // namespace dart
 
 #endif  // VM_DATASTREAM_H_