Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(30)

Side by Side Diff: vm/object.cc

Issue 10914050: Use external byte arrays for token stream, this moves the token stream out of the isolate heap. For… (Closed) Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/runtime/
Patch Set: Created 8 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « vm/object.h ('k') | vm/raw_object.h » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 #include "vm/object.h" 5 #include "vm/object.h"
6 6
7 #include "include/dart_api.h" 7 #include "include/dart_api.h"
8 #include "platform/assert.h" 8 #include "platform/assert.h"
9 #include "vm/assembler.h" 9 #include "vm/assembler.h"
10 #include "vm/bigint_operations.h" 10 #include "vm/bigint_operations.h"
(...skipping 4770 matching lines...) Expand 10 before | Expand all | Expand 10 after
4781 RawArray* TokenStream::TokenObjects() const { 4781 RawArray* TokenStream::TokenObjects() const {
4782 return raw_ptr()->token_objects_; 4782 return raw_ptr()->token_objects_;
4783 } 4783 }
4784 4784
4785 4785
4786 void TokenStream::SetTokenObjects(const Array& value) const { 4786 void TokenStream::SetTokenObjects(const Array& value) const {
4787 StorePointer(&raw_ptr()->token_objects_, value.raw()); 4787 StorePointer(&raw_ptr()->token_objects_, value.raw());
4788 } 4788 }
4789 4789
4790 4790
4791 void TokenStream::SetLength(intptr_t value) const { 4791 RawExternalUint8Array* TokenStream::GetStream() const {
4792 raw_ptr()->length_ = Smi::New(value); 4792 return raw_ptr()->stream_;
4793 }
4794
4795
4796 void TokenStream::SetStream(const ExternalUint8Array& value) const {
4797 StorePointer(&raw_ptr()->stream_, value.raw());
4798 }
4799
4800
4801 void TokenStream::DataFinalizer(void *peer) {
4802 ASSERT(peer != NULL);
4803 ::free(peer);
4793 } 4804 }
4794 4805
4795 4806
4796 RawString* TokenStream::PrivateKey() const { 4807 RawString* TokenStream::PrivateKey() const {
4797 return raw_ptr()->private_key_; 4808 return raw_ptr()->private_key_;
4798 } 4809 }
4799 4810
4800 4811
4801 void TokenStream::SetPrivateKey(const String& value) const { 4812 void TokenStream::SetPrivateKey(const String& value) const {
4802 StorePointer(&raw_ptr()->private_key_, value.raw()); 4813 StorePointer(&raw_ptr()->private_key_, value.raw());
4803 } 4814 }
4804 4815
4805 4816
4806 RawString* TokenStream::GenerateSource() const { 4817 RawString* TokenStream::GenerateSource() const {
4807 Iterator iterator(*this, 0); 4818 Iterator iterator(*this, 0);
4819 const ExternalUint8Array& data = ExternalUint8Array::Handle(GetStream());
4808 const GrowableObjectArray& literals = 4820 const GrowableObjectArray& literals =
4809 GrowableObjectArray::Handle(GrowableObjectArray::New(Length())); 4821 GrowableObjectArray::Handle(GrowableObjectArray::New(data.Length()));
4810 const String& private_key = String::Handle(PrivateKey()); 4822 const String& private_key = String::Handle(PrivateKey());
4811 intptr_t private_len = private_key.Length(); 4823 intptr_t private_len = private_key.Length();
4812 4824
4813 String& blank = String::Handle(String::New(" ")); 4825 String& blank = String::Handle(String::New(" "));
4814 String& newline = String::Handle(String::New("\n")); 4826 String& newline = String::Handle(String::New("\n"));
4815 String& two_newlines = String::Handle(String::New("\n\n")); 4827 String& two_newlines = String::Handle(String::New("\n\n"));
4816 String& double_quotes = String::Handle(String::New("\"")); 4828 String& double_quotes = String::Handle(String::New("\""));
4817 String& dollar = String::Handle(String::New("$")); 4829 String& dollar = String::Handle(String::New("$"));
4818 String& two_spaces = String::Handle(String::New(" ")); 4830 String& two_spaces = String::Handle(String::New(" "));
4819 4831
(...skipping 143 matching lines...) Expand 10 before | Expand all | Expand 10 after
4963 Token::Kind kind = iterator.CurrentTokenKind(); 4975 Token::Kind kind = iterator.CurrentTokenKind();
4964 while (index < src_pos && kind != Token::kEOS) { 4976 while (index < src_pos && kind != Token::kEOS) {
4965 iterator.Advance(); 4977 iterator.Advance();
4966 kind = iterator.CurrentTokenKind(); 4978 kind = iterator.CurrentTokenKind();
4967 index += 1; 4979 index += 1;
4968 } 4980 }
4969 return iterator.CurrentPosition(); 4981 return iterator.CurrentPosition();
4970 } 4982 }
4971 4983
4972 4984
4985 RawTokenStream* TokenStream::New() {
4986 ASSERT(Object::token_stream_class() != Class::null());
4987 TokenStream& result = TokenStream::Handle();
4988 {
4989 RawObject* raw = Object::Allocate(TokenStream::kClassId,
4990 TokenStream::InstanceSize(),
4991 Heap::kOld);
4992 NoGCScope no_gc;
4993 result ^= raw;
4994 }
4995 return result.raw();
4996 }
4997
4998
4973 RawTokenStream* TokenStream::New(intptr_t len) { 4999 RawTokenStream* TokenStream::New(intptr_t len) {
4974 ASSERT(Object::token_stream_class() != Class::null());
4975 if (len < 0 || len > kMaxElements) { 5000 if (len < 0 || len > kMaxElements) {
4976 // This should be caught before we reach here. 5001 // This should be caught before we reach here.
4977 FATAL1("Fatal error in TokenStream::New: invalid len %"Pd"\n", len); 5002 FATAL1("Fatal error in TokenStream::New: invalid len %"Pd"\n", len);
4978 } 5003 }
4979 TokenStream& result = TokenStream::Handle(); 5004 uint8_t* data = reinterpret_cast<uint8_t*>(::malloc(len));
4980 { 5005 ASSERT(data != NULL);
4981 RawObject* raw = Object::Allocate(TokenStream::kClassId, 5006 const ExternalUint8Array& stream = ExternalUint8Array::Handle(
4982 TokenStream::InstanceSize(len), 5007 ExternalUint8Array::New(data, len, data, DataFinalizer, Heap::kOld));
4983 Heap::kOld); 5008 const TokenStream& result = TokenStream::Handle(TokenStream::New());
4984 NoGCScope no_gc; 5009 result.SetStream(stream);
4985 result ^= raw;
4986 result.SetLength(len);
4987 }
4988 return result.raw(); 5010 return result.raw();
4989 } 5011 }
4990 5012
4991 5013
4992 // Helper class for creation of compressed token stream data. 5014 // Helper class for creation of compressed token stream data.
4993 class CompressedTokenStreamData : public ValueObject { 5015 class CompressedTokenStreamData : public ValueObject {
4994 public: 5016 public:
4995 CompressedTokenStreamData() : 5017 CompressedTokenStreamData() :
4996 buffer_(NULL), 5018 buffer_(NULL),
4997 stream_(&buffer_, Reallocate), 5019 stream_(&buffer_, Reallocate),
4998 token_objects_(GrowableObjectArray::Handle( 5020 token_objects_(GrowableObjectArray::Handle(
4999 GrowableObjectArray::New(kInitialTokenCount, Heap::kOld))), 5021 GrowableObjectArray::New(kInitialTokenCount, Heap::kOld))),
5000 token_obj_(Object::Handle()), 5022 token_obj_(Object::Handle()),
5001 literal_token_(LiteralToken::Handle()), 5023 literal_token_(LiteralToken::Handle()),
5002 literal_str_(String::Handle()) { 5024 literal_str_(String::Handle()) {
5003 const String& empty_literal = String::Handle(); 5025 const String& empty_literal = String::Handle();
5004 token_objects_.Add(empty_literal); 5026 token_objects_.Add(empty_literal);
5005 } 5027 }
5006 ~CompressedTokenStreamData() { 5028 ~CompressedTokenStreamData() {
5007 free(buffer_);
5008 } 5029 }
5009 5030
5010 // Add an IDENT token into the stream and the token objects array. 5031 // Add an IDENT token into the stream and the token objects array.
5011 void AddIdentToken(String* ident) { 5032 void AddIdentToken(String* ident) {
5012 if (ident != NULL) { 5033 if (ident != NULL) {
5013 // If the IDENT token is already in the tokens object array use the 5034 // If the IDENT token is already in the tokens object array use the
5014 // same index instead of duplicating it. 5035 // same index instead of duplicating it.
5015 intptr_t index = FindIdentIndex(ident); 5036 intptr_t index = FindIdentIndex(ident);
5016 if (index == -1) { 5037 if (index == -1) {
5017 WriteIndex(token_objects_.Length()); 5038 WriteIndex(token_objects_.Length());
(...skipping 129 matching lines...) Expand 10 before | Expand all | Expand 10 after
5147 ASSERT(token.kind < Token::kNumTokens); 5168 ASSERT(token.kind < Token::kNumTokens);
5148 data.AddSimpleToken(token.kind); 5169 data.AddSimpleToken(token.kind);
5149 } 5170 }
5150 } 5171 }
5151 if (FLAG_compiler_stats) { 5172 if (FLAG_compiler_stats) {
5152 CompilerStats::num_tokens_total += len; 5173 CompilerStats::num_tokens_total += len;
5153 } 5174 }
5154 data.AddSimpleToken(Token::kEOS); // End of stream. 5175 data.AddSimpleToken(Token::kEOS); // End of stream.
5155 5176
5156 // Create and setup the token stream object. 5177 // Create and setup the token stream object.
5157 const TokenStream& result = TokenStream::Handle(New(data.Length())); 5178 const ExternalUint8Array& stream = ExternalUint8Array::Handle(
5179 ExternalUint8Array::New(data.GetStream(),
5180 data.Length(),
5181 data.GetStream(),
5182 DataFinalizer,
5183 Heap::kOld));
5184 const TokenStream& result = TokenStream::Handle(New());
5158 result.SetPrivateKey(private_key); 5185 result.SetPrivateKey(private_key);
5159 { 5186 {
5160 NoGCScope no_gc; 5187 NoGCScope no_gc;
5161 memmove(result.EntryAddr(0), data.GetStream(), data.Length()); 5188 result.SetStream(stream);
5162 const Array& tokens = Array::Handle(Array::MakeArray(data.TokenObjects())); 5189 const Array& tokens = Array::Handle(Array::MakeArray(data.TokenObjects()));
5163 result.SetTokenObjects(tokens); 5190 result.SetTokenObjects(tokens);
5164 } 5191 }
5165 return result.raw(); 5192 return result.raw();
5166 } 5193 }
5167 5194
5168 5195
5169 const char* TokenStream::ToCString() const { 5196 const char* TokenStream::ToCString() const {
5170 return "TokenStream"; 5197 return "TokenStream";
5171 } 5198 }
5172 5199
5173 5200
5174 TokenStream::Iterator::Iterator(const TokenStream& tokens, intptr_t token_pos) 5201 TokenStream::Iterator::Iterator(const TokenStream& tokens, intptr_t token_pos)
5175 : tokens_(tokens), 5202 : tokens_(tokens),
5203 data_(ExternalUint8Array::Handle(tokens.GetStream())),
5204 stream_(data_.ByteAddr(0), data_.Length()),
5176 token_objects_(Array::Handle(tokens.TokenObjects())), 5205 token_objects_(Array::Handle(tokens.TokenObjects())),
5177 obj_(Object::Handle()), 5206 obj_(Object::Handle()),
5178 cur_token_pos_(token_pos), 5207 cur_token_pos_(token_pos),
5179 stream_token_pos_(token_pos),
5180 cur_token_kind_(Token::kILLEGAL), 5208 cur_token_kind_(Token::kILLEGAL),
5181 cur_token_obj_index_(-1) { 5209 cur_token_obj_index_(-1) {
5182 SetCurrentPosition(token_pos); 5210 SetCurrentPosition(token_pos);
5183 } 5211 }
5184 5212
5185 5213
5186 bool TokenStream::Iterator::IsValid() const { 5214 bool TokenStream::Iterator::IsValid() const {
5187 return !tokens_.IsNull(); 5215 return !tokens_.IsNull();
5188 } 5216 }
5189 5217
5190 5218
5191 Token::Kind TokenStream::Iterator::LookaheadTokenKind(intptr_t num_tokens) { 5219 Token::Kind TokenStream::Iterator::LookaheadTokenKind(intptr_t num_tokens) {
5192 intptr_t saved_position = stream_token_pos_; 5220 intptr_t saved_position = stream_.Position();
5193 Token::Kind kind = Token::kILLEGAL; 5221 Token::Kind kind = Token::kILLEGAL;
5194 intptr_t value = -1; 5222 intptr_t value = -1;
5195 intptr_t count = 0; 5223 intptr_t count = 0;
5196 while (count < num_tokens && value != Token::kEOS) { 5224 while (count < num_tokens && value != Token::kEOS) {
5197 value = ReadToken(); 5225 value = ReadToken();
5198 count += 1; 5226 count += 1;
5199 } 5227 }
5200 if (value < Token::kNumTokens) { 5228 if (value < Token::kNumTokens) {
5201 kind = static_cast<Token::Kind>(value); 5229 kind = static_cast<Token::Kind>(value);
5202 } else { 5230 } else {
5203 value = value - Token::kNumTokens; 5231 value = value - Token::kNumTokens;
5204 obj_ = token_objects_.At(value); 5232 obj_ = token_objects_.At(value);
5205 if (obj_.IsLiteralToken()) { 5233 if (obj_.IsLiteralToken()) {
5206 const LiteralToken& literal_token = LiteralToken::Cast(obj_); 5234 const LiteralToken& literal_token = LiteralToken::Cast(obj_);
5207 kind = literal_token.kind(); 5235 kind = literal_token.kind();
5208 } else { 5236 } else {
5209 ASSERT(obj_.IsString()); // Must be an identifier. 5237 ASSERT(obj_.IsString()); // Must be an identifier.
5210 kind = Token::kIDENT; 5238 kind = Token::kIDENT;
5211 } 5239 }
5212 } 5240 }
5213 stream_token_pos_ = saved_position; 5241 stream_.SetPosition(saved_position);
5214 return kind; 5242 return kind;
5215 } 5243 }
5216 5244
5217 5245
5218 intptr_t TokenStream::Iterator::CurrentPosition() const { 5246 intptr_t TokenStream::Iterator::CurrentPosition() const {
5219 return cur_token_pos_; 5247 return cur_token_pos_;
5220 } 5248 }
5221 5249
5222 5250
5223 void TokenStream::Iterator::SetCurrentPosition(intptr_t value) { 5251 void TokenStream::Iterator::SetCurrentPosition(intptr_t value) {
5224 stream_token_pos_ = value; 5252 stream_.SetPosition(value);
5225 Advance(); 5253 Advance();
5226 } 5254 }
5227 5255
5228 5256
5229 void TokenStream::Iterator::Advance() { 5257 void TokenStream::Iterator::Advance() {
5230 cur_token_pos_ = stream_token_pos_; 5258 cur_token_pos_ = stream_.Position();
5231 intptr_t value = ReadToken(); 5259 intptr_t value = ReadToken();
5232 if (value < Token::kNumTokens) { 5260 if (value < Token::kNumTokens) {
5233 cur_token_kind_ = static_cast<Token::Kind>(value); 5261 cur_token_kind_ = static_cast<Token::Kind>(value);
5234 cur_token_obj_index_ = -1; 5262 cur_token_obj_index_ = -1;
5235 return; 5263 return;
5236 } 5264 }
5237 cur_token_obj_index_ = value - Token::kNumTokens; 5265 cur_token_obj_index_ = value - Token::kNumTokens;
5238 obj_ = token_objects_.At(cur_token_obj_index_); 5266 obj_ = token_objects_.At(cur_token_obj_index_);
5239 if (obj_.IsLiteralToken()) { 5267 if (obj_.IsLiteralToken()) {
5240 const LiteralToken& literal_token = LiteralToken::Cast(obj_); 5268 const LiteralToken& literal_token = LiteralToken::Cast(obj_);
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
5281 } 5309 }
5282 return Symbols::New(Token::Str(kind)); 5310 return Symbols::New(Token::Str(kind));
5283 } else { 5311 } else {
5284 ASSERT(obj.IsLiteralToken()); // Must be a literal token. 5312 ASSERT(obj.IsLiteralToken()); // Must be a literal token.
5285 const LiteralToken& literal_token = LiteralToken::Cast(obj); 5313 const LiteralToken& literal_token = LiteralToken::Cast(obj);
5286 return literal_token.literal(); 5314 return literal_token.literal();
5287 } 5315 }
5288 } 5316 }
5289 5317
5290 5318
5291 intptr_t TokenStream::Iterator::ReadToken() {
5292 uint8_t b = ReadByte();
5293 if (b > kMaxUnsignedDataPerByte) {
5294 return static_cast<intptr_t>(b) - kEndUnsignedByteMarker;
5295 }
5296 intptr_t value = 0;
5297 uint8_t s = 0;
5298 do {
5299 value |= static_cast<intptr_t>(b) << s;
5300 s += kDataBitsPerByte;
5301 b = ReadByte();
5302 } while (b <= kMaxUnsignedDataPerByte);
5303 value |= ((static_cast<intptr_t>(b) - kEndUnsignedByteMarker) << s);
5304 ASSERT((value >= 0) && (value <= kIntptrMax));
5305 return value;
5306 }
5307
5308
5309 uint8_t TokenStream::Iterator::ReadByte() {
5310 ASSERT(stream_token_pos_ < tokens_.Length());
5311 return *(tokens_.EntryAddr(stream_token_pos_++));
5312 }
5313
5314
5315 bool Script::HasSource() const { 5319 bool Script::HasSource() const {
5316 return raw_ptr()->source_ != String::null(); 5320 return raw_ptr()->source_ != String::null();
5317 } 5321 }
5318 5322
5319 5323
5320 RawString* Script::Source() const { 5324 RawString* Script::Source() const {
5321 String& source = String::Handle(raw_ptr()->source_); 5325 String& source = String::Handle(raw_ptr()->source_);
5322 if (source.IsNull()) { 5326 if (source.IsNull()) {
5323 const TokenStream& token_stream = TokenStream::Handle(tokens()); 5327 const TokenStream& token_stream = TokenStream::Handle(tokens());
5324 return token_stream.GenerateSource(); 5328 return token_stream.GenerateSource();
(...skipping 5873 matching lines...) Expand 10 before | Expand all | Expand 10 after
11198 } 11202 }
11199 return result.raw(); 11203 return result.raw();
11200 } 11204 }
11201 11205
11202 11206
11203 const char* WeakProperty::ToCString() const { 11207 const char* WeakProperty::ToCString() const {
11204 return "_WeakProperty"; 11208 return "_WeakProperty";
11205 } 11209 }
11206 11210
11207 } // namespace dart 11211 } // namespace dart
OLDNEW
« no previous file with comments | « vm/object.h ('k') | vm/raw_object.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698