Implement tracking and optimizations of packed arrays

src/objects-inl.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 110 matching lines @@
 
 #define BOOL_ACCESSORS(holder, field, name, offset)        \
   bool holder::name() {                                    \
     return BooleanBit::get(field(), offset);               \
   }                                                        \
   void holder::set_##name(bool value) {                    \
     set_##field(BooleanBit::set(field(), offset, value));  \
   }
 
 
-bool IsMoreGeneralElementsKindTransition(ElementsKind from_kind,
-                                         ElementsKind to_kind) {
-  if (to_kind == FAST_ELEMENTS) {
-    return from_kind == FAST_SMI_ONLY_ELEMENTS ||
-        from_kind == FAST_DOUBLE_ELEMENTS;
-  } else {
-    return to_kind == FAST_DOUBLE_ELEMENTS &&
-        from_kind == FAST_SMI_ONLY_ELEMENTS;
-  }
-}
-
-
 bool Object::IsFixedArrayBase() {
   return IsFixedArray() || IsFixedDoubleArray();
 }
 
 
 bool Object::IsInstanceOf(FunctionTemplateInfo* expected) {
   // There is a constraint on the object; check.
   if (!this->IsJSObject()) return false;
   // Fetch the constructor function of the object.
   Object* cons_obj = JSObject::cast(this)->map()->constructor();
@@ skipping 1084 matching lines @@
   }
   return true;
 }
 
 
 FixedArrayBase* JSObject::elements() {
   Object* array = READ_FIELD(this, kElementsOffset);
   return static_cast<FixedArrayBase*>(array);
 }
 
-void JSObject::ValidateSmiOnlyElements() {
+
+void JSObject::ValidateElements() {
 #if DEBUG
-  if (map()->elements_kind() == FAST_SMI_ONLY_ELEMENTS) {
-    Heap* heap = GetHeap();
-    // Don't use elements, since integrity checks will fail if there
-    // are filler pointers in the array.
-    FixedArray* fixed_array =
-        reinterpret_cast<FixedArray*>(READ_FIELD(this, kElementsOffset));
-    Map* map = fixed_array->map();
-    // Arrays that have been shifted in place can't be verified.
-    if (map != heap->raw_unchecked_one_pointer_filler_map() &&
-        map != heap->raw_unchecked_two_pointer_filler_map() &&
-        map != heap->free_space_map()) {
-      for (int i = 0; i < fixed_array->length(); i++) {
-        Object* current = fixed_array->get(i);
-        ASSERT(current->IsSmi() || current->IsTheHole());
-      }
-    }
+  if (FLAG_enable_slow_asserts) {
+    ElementsAccessor* accessor = GetElementsAccessor();
+    accessor->Validate(this);
   }
 #endif
 }
 
 
 MaybeObject* JSObject::EnsureCanContainHeapObjectElements() {
-#if DEBUG
-  ValidateSmiOnlyElements();
-#endif
-  if ((map()->elements_kind() != FAST_ELEMENTS)) {
-    return TransitionElementsKind(FAST_ELEMENTS);
+  ValidateElements();
+  ElementsKind elements_kind = map()->elements_kind();
+  if (!IsFastObjectElementsKind(elements_kind)) {
+    if (IsFastHoleyElementsKind(elements_kind)) {
+      return TransitionElementsKind(FAST_HOLEY_ELEMENTS);
+    } else {
+      return TransitionElementsKind(FAST_ELEMENTS);
+    }
   }
   return this;
 }
 
 
 MaybeObject* JSObject::EnsureCanContainElements(Object** objects,
                                                 uint32_t count,
                                                 EnsureElementsMode mode) {
   ElementsKind current_kind = map()->elements_kind();
   ElementsKind target_kind = current_kind;
   ASSERT(mode != ALLOW_COPIED_DOUBLE_ELEMENTS);
-  if (current_kind == FAST_ELEMENTS) return this;
-
+  bool is_holey = IsFastHoleyElementsKind(current_kind);
+  if (current_kind == FAST_HOLEY_ELEMENTS) return this;
   Heap* heap = GetHeap();
   Object* the_hole = heap->the_hole_value();
   Object* heap_number_map = heap->heap_number_map();
   for (uint32_t i = 0; i < count; ++i) {
     Object* current = *objects++;
-    if (!current->IsSmi() && current != the_hole) {
+    if (current == the_hole) {
+      is_holey = true;
+      target_kind = GetHoleyElementsKind(target_kind);
+    } else if (!current->IsSmi()) {
       if (mode == ALLOW_CONVERTED_DOUBLE_ELEMENTS &&
-          HeapObject::cast(current)->map() == heap_number_map) {
-        target_kind = FAST_DOUBLE_ELEMENTS;
+          HeapObject::cast(current)->map() == heap_number_map &&
+          IsFastSmiElementsKind(target_kind)) {
+        if (is_holey) {
+          target_kind = FAST_HOLEY_DOUBLE_ELEMENTS;
+        } else {
+          target_kind = FAST_DOUBLE_ELEMENTS;
+        }
       } else {
-        target_kind = FAST_ELEMENTS;
-        break;
+        if (!current->IsNumber()) {
+          if (is_holey) {
+            target_kind = FAST_HOLEY_ELEMENTS;
+            break;
+          } else {
+            target_kind = FAST_ELEMENTS;
+          }
+        }
       }
     }
   }
 
   if (target_kind != current_kind) {
     return TransitionElementsKind(target_kind);
   }
   return this;
 }
 
 
 MaybeObject* JSObject::EnsureCanContainElements(FixedArrayBase* elements,
+                                                uint32_t length,
                                                 EnsureElementsMode mode) {
   if (elements->map() != GetHeap()->fixed_double_array_map()) {
     ASSERT(elements->map() == GetHeap()->fixed_array_map() ||
            elements->map() == GetHeap()->fixed_cow_array_map());
     if (mode == ALLOW_COPIED_DOUBLE_ELEMENTS) {
       mode = DONT_ALLOW_DOUBLE_ELEMENTS;
     }
     Object** objects = FixedArray::cast(elements)->GetFirstElementAddress();
-    return EnsureCanContainElements(objects, elements->length(), mode);
+    return EnsureCanContainElements(objects, length, mode);
   }
 
   ASSERT(mode == ALLOW_COPIED_DOUBLE_ELEMENTS);
-  if (GetElementsKind() == FAST_SMI_ONLY_ELEMENTS) {
+  if (GetElementsKind() == FAST_HOLEY_SMI_ELEMENTS) {
+    return TransitionElementsKind(FAST_HOLEY_DOUBLE_ELEMENTS);
+  } else if (GetElementsKind() == FAST_SMI_ELEMENTS) {
+    FixedDoubleArray* double_array = FixedDoubleArray::cast(elements);
+    for (uint32_t i = 0; i < length; ++i) {
+      if (double_array->is_the_hole(i)) {
+        return TransitionElementsKind(FAST_HOLEY_DOUBLE_ELEMENTS);
+      }
+    }
     return TransitionElementsKind(FAST_DOUBLE_ELEMENTS);
   }
 
   return this;
 }
 
 
 MaybeObject* JSObject::GetElementsTransitionMap(Isolate* isolate,
                                                 ElementsKind to_kind) {
   Map* current_map = map();
   ElementsKind from_kind = current_map->elements_kind();
-
   if (from_kind == to_kind) return current_map;
 
   Context* global_context = isolate->context()->global_context();
-  if (current_map == global_context->smi_js_array_map()) {
-    if (to_kind == FAST_ELEMENTS) {
-      return global_context->object_js_array_map();
-    } else {
-      if (to_kind == FAST_DOUBLE_ELEMENTS) {
-        return global_context->double_js_array_map();
-      } else {
-        ASSERT(to_kind == DICTIONARY_ELEMENTS);
+  Object* maybe_array_maps = global_context->js_array_maps();
+  if (maybe_array_maps->IsFixedArray()) {
+    FixedArray* array_maps = FixedArray::cast(maybe_array_maps);
+    if (array_maps->get(from_kind) == current_map) {
+      Object* maybe_transitioned_map = array_maps->get(to_kind);
+      if (maybe_transitioned_map->IsMap()) {
+        return Map::cast(maybe_transitioned_map);
       }
     }
   }
+
   return GetElementsTransitionMapSlow(to_kind);
 }
 
 
 void JSObject::set_map_and_elements(Map* new_map,
                                     FixedArrayBase* value,
                                     WriteBarrierMode mode) {
   ASSERT(value->HasValidElements());
-#ifdef DEBUG
-  ValidateSmiOnlyElements();
-#endif
   if (new_map != NULL) {
     if (mode == UPDATE_WRITE_BARRIER) {
       set_map(new_map);
     } else {
       ASSERT(mode == SKIP_WRITE_BARRIER);
       set_map_no_write_barrier(new_map);
     }
   }
-  ASSERT((map()->has_fast_elements() ||
-          map()->has_fast_smi_only_elements() ||
+  ASSERT((map()->has_fast_smi_or_object_elements() ||
           (value == GetHeap()->empty_fixed_array())) ==
          (value->map() == GetHeap()->fixed_array_map() ||
           value->map() == GetHeap()->fixed_cow_array_map()));
   ASSERT((value == GetHeap()->empty_fixed_array()) ||
          (map()->has_fast_double_elements() == value->IsFixedDoubleArray()));
   WRITE_FIELD(this, kElementsOffset, value);
   CONDITIONAL_WRITE_BARRIER(GetHeap(), this, kElementsOffset, value, mode);
 }
 
 
 void JSObject::set_elements(FixedArrayBase* value, WriteBarrierMode mode) {
   set_map_and_elements(NULL, value, mode);
 }
 
 
 void JSObject::initialize_properties() {
   ASSERT(!GetHeap()->InNewSpace(GetHeap()->empty_fixed_array()));
   WRITE_FIELD(this, kPropertiesOffset, GetHeap()->empty_fixed_array());
 }
 
 
 void JSObject::initialize_elements() {
-  ASSERT(map()->has_fast_elements() ||
-         map()->has_fast_smi_only_elements() ||
+  ASSERT(map()->has_fast_smi_or_object_elements() ||
          map()->has_fast_double_elements());
   ASSERT(!GetHeap()->InNewSpace(GetHeap()->empty_fixed_array()));
   WRITE_FIELD(this, kElementsOffset, GetHeap()->empty_fixed_array());
 }
 
 
 MaybeObject* JSObject::ResetElements() {
   Object* obj;
-  ElementsKind elements_kind = FLAG_smi_only_arrays
-      ? FAST_SMI_ONLY_ELEMENTS
-      : FAST_ELEMENTS;
+  ElementsKind elements_kind = GetInitialFastElementsKind();
+  if (!FLAG_smi_only_arrays) {
+    elements_kind = FastSmiToObjectElementsKind(elements_kind);
+  }
   MaybeObject* maybe_obj = GetElementsTransitionMap(GetIsolate(),
                                                     elements_kind);
   if (!maybe_obj->ToObject(&obj)) return maybe_obj;
   set_map(Map::cast(obj));
   initialize_elements();
   return this;
 }
 
 
 ACCESSORS(Oddball, to_string, String, kToStringOffset)
@@ skipping 251 matching lines @@
   return reinterpret_cast<FixedArrayBase*>(object);
 }
 
 
 Object* FixedArray::get(int index) {
   ASSERT(index >= 0 && index < this->length());
   return READ_FIELD(this, kHeaderSize + index * kPointerSize);
 }
 
 
+bool FixedArray::is_the_hole(int index) {
+  return get(index) == GetHeap()->the_hole_value();
+}
+
+
 void FixedArray::set(int index, Smi* value) {
   ASSERT(map() != HEAP->fixed_cow_array_map());
   ASSERT(index >= 0 && index < this->length());
   ASSERT(reinterpret_cast<Object*>(value)->IsSmi());
   int offset = kHeaderSize + index * kPointerSize;
   WRITE_FIELD(this, offset, value);
 }
 
 
 void FixedArray::set(int index, Object* value) {
@@ skipping 1161 matching lines @@
 }
 
 
 bool Map::has_non_instance_prototype() {
   return ((1 << kHasNonInstancePrototype) & bit_field()) != 0;
 }
 
 
 void Map::set_function_with_prototype(bool value) {
   if (value) {
-    set_bit_field2(bit_field2() | (1 << kFunctionWithPrototype));
+    set_bit_field3(bit_field3() | (1 << kFunctionWithPrototype));
   } else {
-    set_bit_field2(bit_field2() & ~(1 << kFunctionWithPrototype));
+    set_bit_field3(bit_field3() & ~(1 << kFunctionWithPrototype));
   }
 }
 
 
 bool Map::function_with_prototype() {
-  return ((1 << kFunctionWithPrototype) & bit_field2()) != 0;
+  return ((1 << kFunctionWithPrototype) & bit_field3()) != 0;
 }
 
 
 void Map::set_is_access_check_needed(bool access_check_needed) {
   if (access_check_needed) {
     set_bit_field(bit_field() | (1 << kIsAccessCheckNeeded));
   } else {
     set_bit_field(bit_field() & ~(1 << kIsAccessCheckNeeded));
   }
 }
@@ skipping 1122 matching lines @@
 }
 
 
 MaybeObject* JSFunction::set_initial_map_and_cache_transitions(
     Map* initial_map) {
   Context* global_context = context()->global_context();
   Object* array_function =
       global_context->get(Context::ARRAY_FUNCTION_INDEX);
   if (array_function->IsJSFunction() &&
       this == JSFunction::cast(array_function)) {
-    ASSERT(initial_map->elements_kind() == FAST_SMI_ONLY_ELEMENTS);
+    // Replace all of the cached initial array maps in the global context with
+    // the appropriate transitioned elements kind maps.
+    Heap* heap = GetHeap();
+    MaybeObject* maybe_maps =
+        heap->AllocateFixedArrayWithHoles(kElementsKindCount);
+    FixedArray* maps;
+    if (!maybe_maps->To(&maps)) return maybe_maps;
 
-    MaybeObject* maybe_map = initial_map->CopyDropTransitions();
-    Map* new_double_map = NULL;
-    if (!maybe_map->To<Map>(&new_double_map)) return maybe_map;
-    new_double_map->set_elements_kind(FAST_DOUBLE_ELEMENTS);
-    maybe_map = initial_map->AddElementsTransition(FAST_DOUBLE_ELEMENTS,
-                                                   new_double_map);
-    if (maybe_map->IsFailure()) return maybe_map;
-
-    maybe_map = new_double_map->CopyDropTransitions();
-    Map* new_object_map = NULL;
-    if (!maybe_map->To<Map>(&new_object_map)) return maybe_map;
-    new_object_map->set_elements_kind(FAST_ELEMENTS);
-    maybe_map = new_double_map->AddElementsTransition(FAST_ELEMENTS,
-                                                      new_object_map);
-    if (maybe_map->IsFailure()) return maybe_map;
-
-    global_context->set_smi_js_array_map(initial_map);
-    global_context->set_double_js_array_map(new_double_map);
-    global_context->set_object_js_array_map(new_object_map);
+    Map* current_map = initial_map;
+    ElementsKind kind = current_map->elements_kind();
+    ASSERT(kind == GetInitialFastElementsKind());
+    maps->set(kind, current_map);
+    for (int i = GetSequenceIndexFromFastElementsKind(kind) + 1;
+         i < kFastElementsKindCount; ++i) {
+      ElementsKind transitioned_kind = GetFastElementsKindFromSequenceIndex(i);
+      MaybeObject* maybe_new_map = current_map->CopyDropTransitions();
+      Map* new_map = NULL;
+      if (!maybe_new_map->To<Map>(&new_map)) return maybe_new_map;
+      new_map->set_elements_kind(transitioned_kind);
+      maybe_new_map = current_map->AddElementsTransition(transitioned_kind,
+                                                         new_map);
+      if (maybe_new_map->IsFailure()) return maybe_new_map;
+      maps->set(transitioned_kind, new_map);
+      current_map = new_map;
+    }
+    global_context->set_js_array_maps(maps);
   }
   set_initial_map(initial_map);
   return this;
 }
 
 
 bool JSFunction::has_initial_map() {
   return prototype_or_initial_map()->IsMap();
 }
 
@@ skipping 315 matching lines @@
   }
 }
 
 
 ElementsKind JSObject::GetElementsKind() {
   ElementsKind kind = map()->elements_kind();
 #if DEBUG
   FixedArrayBase* fixed_array =
       reinterpret_cast<FixedArrayBase*>(READ_FIELD(this, kElementsOffset));
   Map* map = fixed_array->map();
-    ASSERT(((kind == FAST_ELEMENTS || kind == FAST_SMI_ONLY_ELEMENTS) &&
-            (map == GetHeap()->fixed_array_map() ||
-             map == GetHeap()->fixed_cow_array_map())) ||
-           (kind == FAST_DOUBLE_ELEMENTS &&
-            (fixed_array->IsFixedDoubleArray() ||
-            fixed_array == GetHeap()->empty_fixed_array())) ||
-           (kind == DICTIONARY_ELEMENTS &&
+  ASSERT((IsFastSmiOrObjectElementsKind(kind) &&
+          (map == GetHeap()->fixed_array_map() ||
+           map == GetHeap()->fixed_cow_array_map())) ||
+         (IsFastDoubleElementsKind(kind) &&
+          (fixed_array->IsFixedDoubleArray() ||
+           fixed_array == GetHeap()->empty_fixed_array())) ||
+         (kind == DICTIONARY_ELEMENTS &&
             fixed_array->IsFixedArray() &&
-            fixed_array->IsDictionary()) ||
-           (kind > DICTIONARY_ELEMENTS));
-    ASSERT((kind != NON_STRICT_ARGUMENTS_ELEMENTS) ||
-           (elements()->IsFixedArray() && elements()->length() >= 2));
+          fixed_array->IsDictionary()) ||
+         (kind > DICTIONARY_ELEMENTS));
+  ASSERT((kind != NON_STRICT_ARGUMENTS_ELEMENTS) ||
+         (elements()->IsFixedArray() && elements()->length() >= 2));
 #endif
   return kind;
 }
 
 
 ElementsAccessor* JSObject::GetElementsAccessor() {
   return ElementsAccessor::ForKind(GetElementsKind());
 }
 
 
-bool JSObject::HasFastElements() {
-  return GetElementsKind() == FAST_ELEMENTS;
+bool JSObject::HasFastObjectElements() {
+  return IsFastObjectElementsKind(GetElementsKind());
 }
 
 
-bool JSObject::HasFastSmiOnlyElements() {
-  return GetElementsKind() == FAST_SMI_ONLY_ELEMENTS;
+bool JSObject::HasFastSmiElements() {
+  return IsFastSmiElementsKind(GetElementsKind());
 }
 
 
-bool JSObject::HasFastTypeElements() {
-  ElementsKind elements_kind = GetElementsKind();
-  return elements_kind == FAST_SMI_ONLY_ELEMENTS ||
-      elements_kind == FAST_ELEMENTS;
+bool JSObject::HasFastSmiOrObjectElements() {
+  return IsFastSmiOrObjectElementsKind(GetElementsKind());
 }
 
 
 bool JSObject::HasFastDoubleElements() {
-  return GetElementsKind() == FAST_DOUBLE_ELEMENTS;
+  return IsFastDoubleElementsKind(GetElementsKind());
 }
 
 
+bool JSObject::HasFastHoleyElements() {
+  return IsFastHoleyElementsKind(GetElementsKind());
+}
+
+
 bool JSObject::HasDictionaryElements() {
   return GetElementsKind() == DICTIONARY_ELEMENTS;
 }
 
 
 bool JSObject::HasNonStrictArgumentsElements() {
   return GetElementsKind() == NON_STRICT_ARGUMENTS_ELEMENTS;
 }
 
 
@@ skipping 33 matching lines @@
   return map()->has_named_interceptor();
 }
 
 
 bool JSObject::HasIndexedInterceptor() {
   return map()->has_indexed_interceptor();
 }
 
 
 MaybeObject* JSObject::EnsureWritableFastElements() {
-  ASSERT(HasFastTypeElements());
+  ASSERT(HasFastSmiOrObjectElements());
   FixedArray* elems = FixedArray::cast(elements());
   Isolate* isolate = GetIsolate();
   if (elems->map() != isolate->heap()->fixed_cow_array_map()) return elems;
   Object* writable_elems;
   { MaybeObject* maybe_writable_elems = isolate->heap()->CopyFixedArrayWithMap(
       elems, isolate->heap()->fixed_array_map());
     if (!maybe_writable_elems->ToObject(&writable_elems)) {
       return maybe_writable_elems;
     }
   }
@@ skipping 337 matching lines @@
   // No write barrier is needed since empty_fixed_array is not in new space.
   // Please note this function is used during marking:
   //  - MarkCompactCollector::MarkUnmarkedObject
   //  - IncrementalMarking::Step
   ASSERT(!heap->InNewSpace(heap->raw_unchecked_empty_fixed_array()));
   WRITE_FIELD(this, kCodeCacheOffset, heap->raw_unchecked_empty_fixed_array());
 }
 
 
 void JSArray::EnsureSize(int required_size) {
-  ASSERT(HasFastTypeElements());
+  ASSERT(HasFastSmiOrObjectElements());
   FixedArray* elts = FixedArray::cast(elements());
   const int kArraySizeThatFitsComfortablyInNewSpace = 128;
   if (elts->length() < required_size) {
     // Doubling in size would be overkill, but leave some slack to avoid
     // constantly growing.
     Expand(required_size + (required_size >> 3));
     // It's a performance benefit to keep a frequently used array in new-space.
   } else if (!GetHeap()->new_space()->Contains(elts) &&
              required_size < kArraySizeThatFitsComfortablyInNewSpace) {
     // Expand will allocate a new backing store in new space even if the size
@@ skipping 11 matching lines @@
 
 bool JSArray::AllowsSetElementsLength() {
   bool result = elements()->IsFixedArray() || elements()->IsFixedDoubleArray();
   ASSERT(result == !HasExternalArrayElements());
   return result;
 }
 
 
 MaybeObject* JSArray::SetContent(FixedArrayBase* storage) {
   MaybeObject* maybe_result = EnsureCanContainElements(
-      storage, ALLOW_COPIED_DOUBLE_ELEMENTS);
+      storage, storage->length(), ALLOW_COPIED_DOUBLE_ELEMENTS);
   if (maybe_result->IsFailure()) return maybe_result;
   ASSERT((storage->map() == GetHeap()->fixed_double_array_map() &&
-          GetElementsKind() == FAST_DOUBLE_ELEMENTS) ||
+          IsFastDoubleElementsKind(GetElementsKind())) ||
          ((storage->map() != GetHeap()->fixed_double_array_map()) &&
-          ((GetElementsKind() == FAST_ELEMENTS) ||
-           (GetElementsKind() == FAST_SMI_ONLY_ELEMENTS &&
+          (IsFastObjectElementsKind(GetElementsKind()) ||
+           (IsFastSmiElementsKind(GetElementsKind()) &&
             FixedArray::cast(storage)->ContainsOnlySmisOrHoles()))));
   set_elements(storage);
   set_length(Smi::FromInt(storage->length()));
   return this;
 }
 
 
 MaybeObject* FixedArray::Copy() {
   if (length() == 0) return this;
   return GetHeap()->CopyFixedArray(this);
@@ skipping 156 matching lines @@
 #undef WRITE_UINT32_FIELD
 #undef READ_SHORT_FIELD
 #undef WRITE_SHORT_FIELD
 #undef READ_BYTE_FIELD
 #undef WRITE_BYTE_FIELD
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_OBJECTS_INL_H_