Lattice-based representation inference, powered by left/right specific type feedback for BinaryOps …

src/x64/code-stubs-x64.cc

Index: src/x64/code-stubs-x64.cc
diff --git a/src/x64/code-stubs-x64.cc b/src/x64/code-stubs-x64.cc
index a8e52e99d306d0cedb44ab9f4ef74eedd10d85c0..be6ae99dcb4900164083e66ebcee2088dbab9a29 100644
--- a/src/x64/code-stubs-x64.cc
+++ b/src/x64/code-stubs-x64.cc
@@ -637,6 +637,10 @@ void ToBooleanStub::GenerateTypeTransition(MacroAssembler* masm) {
 
 class FloatingPointHelper : public AllStatic {
  public:
+  enum ConvertUndefined {
+    CONVERT_UNDEFINED_TO_ZERO,
+    BAILOUT_ON_UNDEFINED
+  };
   // Load the operands from rdx and rax into xmm0 and xmm1, as doubles.
   // If the operands are not both numbers, jump to not_numbers.
   // Leaves rdx and rax unchanged.  SmiOperands assumes both are smis.
@@ -672,7 +676,8 @@ class FloatingPointHelper : public AllStatic {
                             Register scratch2,
                             Register scratch3,
                             Label* on_success,
-                            Label* on_not_smis);
+                            Label* on_not_smis,
+                            ConvertUndefined convert_undefined);
 };
 
 
@@ -997,16 +1002,15 @@ void UnaryOpStub::PrintName(StringStream* stream) {
 }
 
 
+void BinaryOpStub::Initialize() {}
+
+
 void BinaryOpStub::GenerateTypeTransition(MacroAssembler* masm) {
   __ pop(rcx);  // Save return address.
   __ push(rdx);
   __ push(rax);
   // Left and right arguments are now on top.
-  // Push this stub's key. Although the operation and the type info are
-  // encoded into the key, the encoding is opaque, so push them too.
   __ Push(Smi::FromInt(MinorKey()));
-  __ Push(Smi::FromInt(op_));
-  __ Push(Smi::FromInt(operands_type_));
 
   __ push(rcx);  // Push return address.
 
@@ -1015,69 +1019,16 @@ void BinaryOpStub::GenerateTypeTransition(MacroAssembler* masm) {
   __ TailCallExternalReference(
       ExternalReference(IC_Utility(IC::kBinaryOp_Patch),
                         masm->isolate()),
-      5,
+      3,
       1);
 }
 
 
-void BinaryOpStub::Generate(MacroAssembler* masm) {
-  // Explicitly allow generation of nested stubs. It is safe here because
-  // generation code does not use any raw pointers.
-  AllowStubCallsScope allow_stub_calls(masm, true);
-
-  switch (operands_type_) {
-    case BinaryOpIC::UNINITIALIZED:
-      GenerateTypeTransition(masm);
-      break;
-    case BinaryOpIC::SMI:
-      GenerateSmiStub(masm);
-      break;
-    case BinaryOpIC::INT32:
-      UNREACHABLE();
-      // The int32 case is identical to the Smi case.  We avoid creating this
-      // ic state on x64.
-      break;
-    case BinaryOpIC::HEAP_NUMBER:
-      GenerateHeapNumberStub(masm);
-      break;
-    case BinaryOpIC::ODDBALL:
-      GenerateOddballStub(masm);
-      break;
-    case BinaryOpIC::BOTH_STRING:
-      GenerateBothStringStub(masm);
-      break;
-    case BinaryOpIC::STRING:
-      GenerateStringStub(masm);
-      break;
-    case BinaryOpIC::GENERIC:
-      GenerateGeneric(masm);
-      break;
-    default:
-      UNREACHABLE();
-  }
-}
-
-
-void BinaryOpStub::PrintName(StringStream* stream) {
-  const char* op_name = Token::Name(op_);
-  const char* overwrite_name;
-  switch (mode_) {
-    case NO_OVERWRITE: overwrite_name = "Alloc"; break;
-    case OVERWRITE_RIGHT: overwrite_name = "OverwriteRight"; break;
-    case OVERWRITE_LEFT: overwrite_name = "OverwriteLeft"; break;
-    default: overwrite_name = "UnknownOverwrite"; break;
-  }
-  stream->Add("BinaryOpStub_%s_%s_%s",
-              op_name,
-              overwrite_name,
-              BinaryOpIC::GetName(operands_type_));
-}
-
-
-void BinaryOpStub::GenerateSmiCode(
+static void BinaryOpStub_GenerateSmiCode(
     MacroAssembler* masm,
     Label* slow,
-    SmiCodeGenerateHeapNumberResults allow_heapnumber_results) {
+    BinaryOpStub::SmiCodeGenerateHeapNumberResults allow_heapnumber_results,
+    Token::Value op) {
 
   // Arguments to BinaryOpStub are in rdx and rax.
   const Register left = rdx;
@@ -1086,9 +1037,9 @@ void BinaryOpStub::GenerateSmiCode(
   // We only generate heapnumber answers for overflowing calculations
   // for the four basic arithmetic operations and logical right shift by 0.
   bool generate_inline_heapnumber_results =
-      (allow_heapnumber_results == ALLOW_HEAPNUMBER_RESULTS) &&
-      (op_ == Token::ADD || op_ == Token::SUB ||
-       op_ == Token::MUL || op_ == Token::DIV || op_ == Token::SHR);
+      (allow_heapnumber_results == BinaryOpStub::ALLOW_HEAPNUMBER_RESULTS) &&
+      (op == Token::ADD || op == Token::SUB ||
+       op == Token::MUL || op == Token::DIV || op == Token::SHR);
 
   // Smi check of both operands.  If op is BIT_OR, the check is delayed
   // until after the OR operation.
@@ -1096,7 +1047,7 @@ void BinaryOpStub::GenerateSmiCode(
   Label use_fp_on_smis;
   Label fail;
 
-  if (op_ != Token::BIT_OR) {
+  if (op != Token::BIT_OR) {
     Comment smi_check_comment(masm, "-- Smi check arguments");
     __ JumpIfNotBothSmi(left, right, &not_smis);
   }
@@ -1105,7 +1056,7 @@ void BinaryOpStub::GenerateSmiCode(
   __ bind(&smi_values);
   // Perform the operation.
   Comment perform_smi(masm, "-- Perform smi operation");
-  switch (op_) {
+  switch (op) {
     case Token::ADD:
       ASSERT(right.is(rax));
       __ SmiAdd(right, right, left, &use_fp_on_smis);  // ADD is commutative.
@@ -1177,7 +1128,7 @@ void BinaryOpStub::GenerateSmiCode(
     //    operations on known smis (e.g., if the result of the operation
     //    overflowed the smi range).
     __ bind(&use_fp_on_smis);
-    if (op_ == Token::DIV || op_ == Token::MOD) {
+    if (op == Token::DIV || op == Token::MOD) {
       // Restore left and right to rdx and rax.
       __ movq(rdx, rcx);
       __ movq(rax, rbx);
@@ -1186,12 +1137,12 @@ void BinaryOpStub::GenerateSmiCode(
     if (generate_inline_heapnumber_results) {
       __ AllocateHeapNumber(rcx, rbx, slow);
       Comment perform_float(masm, "-- Perform float operation on smis");
-      if (op_ == Token::SHR) {
+      if (op == Token::SHR) {
         __ SmiToInteger32(left, left);
         __ cvtqsi2sd(xmm0, left);
       } else {
         FloatingPointHelper::LoadSSE2SmiOperands(masm);
-        switch (op_) {
+        switch (op) {
         case Token::ADD: __ addsd(xmm0, xmm1); break;
         case Token::SUB: __ subsd(xmm0, xmm1); break;
         case Token::MUL: __ mulsd(xmm0, xmm1); break;
@@ -1214,31 +1165,50 @@ void BinaryOpStub::GenerateSmiCode(
   //    values that could be smi.
   __ bind(&not_smis);
   Comment done_comment(masm, "-- Enter non-smi code");
+  FloatingPointHelper::ConvertUndefined convert_undefined =
+      FloatingPointHelper::BAILOUT_ON_UNDEFINED;
+  // This list must be in sync with BinaryOpPatch() behavior in ic.cc.
+  if (op == Token::BIT_AND ||
+      op == Token::BIT_OR ||
+      op == Token::BIT_XOR ||
+      op == Token::SAR ||
+      op == Token::SHL ||
+      op == Token::SHR) {
+    convert_undefined = FloatingPointHelper::CONVERT_UNDEFINED_TO_ZERO;
+  }
   FloatingPointHelper::NumbersToSmis(masm, left, right, rbx, rdi, rcx,
-                                     &smi_values, &fail);
+                                     &smi_values, &fail, convert_undefined);
   __ jmp(&smi_values);
   __ bind(&fail);
 }
 
 
-void BinaryOpStub::GenerateFloatingPointCode(MacroAssembler* masm,
-                                             Label* allocation_failure,
-                                             Label* non_numeric_failure) {
-  switch (op_) {
+static void BinaryOpStub_GenerateHeapResultAllocation(MacroAssembler* masm,
+                                                      Label* alloc_failure,
+                                                      OverwriteMode mode);
+
+
+static void BinaryOpStub_GenerateFloatingPointCode(MacroAssembler* masm,
+                                                   Label* allocation_failure,
+                                                   Label* non_numeric_failure,
+                                                   Token::Value op,
+                                                   OverwriteMode mode) {
+  switch (op) {
     case Token::ADD:
     case Token::SUB:
     case Token::MUL:
     case Token::DIV: {
       FloatingPointHelper::LoadSSE2UnknownOperands(masm, non_numeric_failure);
 
-      switch (op_) {
+      switch (op) {
         case Token::ADD: __ addsd(xmm0, xmm1); break;
         case Token::SUB: __ subsd(xmm0, xmm1); break;
         case Token::MUL: __ mulsd(xmm0, xmm1); break;
         case Token::DIV: __ divsd(xmm0, xmm1); break;
         default: UNREACHABLE();
       }
-      GenerateHeapResultAllocation(masm, allocation_failure);
+      BinaryOpStub_GenerateHeapResultAllocation(
+          masm, allocation_failure, mode);
       __ movsd(FieldOperand(rax, HeapNumber::kValueOffset), xmm0);
       __ ret(0);
       break;
@@ -1259,7 +1229,7 @@ void BinaryOpStub::GenerateFloatingPointCode(MacroAssembler* masm,
       __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
       FloatingPointHelper::LoadAsIntegers(masm, non_numeric_failure,
                                           heap_number_map);
-      switch (op_) {
+      switch (op) {
         case Token::BIT_OR:  __ orl(rax, rcx); break;
         case Token::BIT_AND: __ andl(rax, rcx); break;
         case Token::BIT_XOR: __ xorl(rax, rcx); break;
@@ -1283,7 +1253,7 @@ void BinaryOpStub::GenerateFloatingPointCode(MacroAssembler* masm,
       // Logical shift right can produce an unsigned int32 that is not
       // an int32, and so is not in the smi range.  Allocate a heap number
       // in that case.
-      if (op_ == Token::SHR) {
+      if (op == Token::SHR) {
         __ bind(&non_smi_shr_result);
         Label allocation_failed;
         __ movl(rbx, rax);  // rbx holds result value (uint32 value as int64).
@@ -1320,12 +1290,12 @@ void BinaryOpStub::GenerateFloatingPointCode(MacroAssembler* masm,
   // No fall-through from this generated code.
   if (FLAG_debug_code) {
     __ Abort("Unexpected fall-through in "
-             "BinaryStub::GenerateFloatingPointCode.");
+             "BinaryStub_GenerateFloatingPointCode.");
   }
 }
 
 
-void BinaryOpStub::GenerateStringAddCode(MacroAssembler* masm) {
+void BinaryOpStub::GenerateAddStrings(MacroAssembler* masm) {
   ASSERT(op_ == Token::ADD);
   Label left_not_string, call_runtime;
 
@@ -1356,58 +1326,17 @@ void BinaryOpStub::GenerateStringAddCode(MacroAssembler* masm) {
 }
 
 
-void BinaryOpStub::GenerateCallRuntimeCode(MacroAssembler* masm) {
-  GenerateRegisterArgsPush(masm);
-  switch (op_) {
-    case Token::ADD:
-      __ InvokeBuiltin(Builtins::ADD, JUMP_FUNCTION);
-      break;
-    case Token::SUB:
-      __ InvokeBuiltin(Builtins::SUB, JUMP_FUNCTION);
-      break;
-    case Token::MUL:
-      __ InvokeBuiltin(Builtins::MUL, JUMP_FUNCTION);
-      break;
-    case Token::DIV:
-      __ InvokeBuiltin(Builtins::DIV, JUMP_FUNCTION);
-      break;
-    case Token::MOD:
-      __ InvokeBuiltin(Builtins::MOD, JUMP_FUNCTION);
-      break;
-    case Token::BIT_OR:
-      __ InvokeBuiltin(Builtins::BIT_OR, JUMP_FUNCTION);
-      break;
-    case Token::BIT_AND:
-      __ InvokeBuiltin(Builtins::BIT_AND, JUMP_FUNCTION);
-      break;
-    case Token::BIT_XOR:
-      __ InvokeBuiltin(Builtins::BIT_XOR, JUMP_FUNCTION);
-      break;
-    case Token::SAR:
-      __ InvokeBuiltin(Builtins::SAR, JUMP_FUNCTION);
-      break;
-    case Token::SHL:
-      __ InvokeBuiltin(Builtins::SHL, JUMP_FUNCTION);
-      break;
-    case Token::SHR:
-      __ InvokeBuiltin(Builtins::SHR, JUMP_FUNCTION);
-      break;
-    default:
-      UNREACHABLE();
-  }
-}
-
-
 void BinaryOpStub::GenerateSmiStub(MacroAssembler* masm) {
   Label call_runtime;
   if (result_type_ == BinaryOpIC::UNINITIALIZED ||
       result_type_ == BinaryOpIC::SMI) {
     // Only allow smi results.
-    GenerateSmiCode(masm, NULL, NO_HEAPNUMBER_RESULTS);
+    BinaryOpStub_GenerateSmiCode(masm, NULL, NO_HEAPNUMBER_RESULTS, op_);
   } else {
     // Allow heap number result and don't make a transition if a heap number
     // cannot be allocated.
-    GenerateSmiCode(masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS);
+    BinaryOpStub_GenerateSmiCode(
+        masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS, op_);
   }
 
   // Code falls through if the result is not returned as either a smi or heap
@@ -1416,24 +1345,22 @@ void BinaryOpStub::GenerateSmiStub(MacroAssembler* masm) {
 
   if (call_runtime.is_linked()) {
     __ bind(&call_runtime);
-    GenerateCallRuntimeCode(masm);
+    GenerateRegisterArgsPush(masm);
+    GenerateCallRuntime(masm);
   }
 }
 
 
-void BinaryOpStub::GenerateStringStub(MacroAssembler* masm) {
-  ASSERT(operands_type_ == BinaryOpIC::STRING);
-  ASSERT(op_ == Token::ADD);
-  GenerateStringAddCode(masm);
-  // Try to add arguments as strings, otherwise, transition to the generic
-  // BinaryOpIC type.
-  GenerateTypeTransition(masm);
+void BinaryOpStub::GenerateInt32Stub(MacroAssembler* masm) {
+  // The int32 case is identical to the Smi case.  We avoid creating this
+  // ic state on x64.
+  UNREACHABLE();
 }
 
 
 void BinaryOpStub::GenerateBothStringStub(MacroAssembler* masm) {
   Label call_runtime;
-  ASSERT(operands_type_ == BinaryOpIC::BOTH_STRING);
+  ASSERT(left_type_ == BinaryOpIC::STRING && right_type_ == BinaryOpIC::STRING);
   ASSERT(op_ == Token::ADD);
   // If both arguments are strings, call the string add stub.
   // Otherwise, do a transition.
@@ -1467,7 +1394,7 @@ void BinaryOpStub::GenerateOddballStub(MacroAssembler* masm) {
   if (op_ == Token::ADD) {
     // Handle string addition here, because it is the only operation
     // that does not do a ToNumber conversion on the operands.
-    GenerateStringAddCode(masm);
+    GenerateAddStrings(masm);
   }
 
   // Convert oddball arguments to numbers.
@@ -1494,39 +1421,79 @@ void BinaryOpStub::GenerateOddballStub(MacroAssembler* masm) {
 }
 
 
+static void BinaryOpStub_CheckSmiInput(MacroAssembler* masm,
+                                       Register input,
+                                       Label* fail) {
+  Label ok;
+  __ JumpIfSmi(input, &ok, Label::kNear);
+  Register heap_number_map = r8;
+  Register scratch1 = r9;
+  Register scratch2 = r10;
+  // HeapNumbers containing 32bit integer values are also allowed.
+  __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
+  __ cmpq(FieldOperand(input, HeapObject::kMapOffset), heap_number_map);
+  __ j(not_equal, fail);
+  __ movsd(xmm0, FieldOperand(input, HeapNumber::kValueOffset));
+  // Convert, convert back, and compare the two doubles' bits.
+  __ cvttsd2siq(scratch2, xmm0);
+  __ cvtlsi2sd(xmm1, scratch2);
+  __ movq(scratch1, xmm0);
+  __ movq(scratch2, xmm1);
+  __ cmpq(scratch1, scratch2);
+  __ j(not_equal, fail);
+  __ bind(&ok);
+}
+
+
 void BinaryOpStub::GenerateHeapNumberStub(MacroAssembler* masm) {
   Label gc_required, not_number;
-  GenerateFloatingPointCode(masm, &gc_required, &not_number);
+
+  // It could be that only SMIs have been seen at either the left
+  // or the right operand. For precise type feedback, patch the IC
+  // again if this changes.
+  if (left_type_ == BinaryOpIC::SMI) {
+    BinaryOpStub_CheckSmiInput(masm, rdx, &not_number);
+  }
+  if (right_type_ == BinaryOpIC::SMI) {
+    BinaryOpStub_CheckSmiInput(masm, rax, &not_number);
+  }
+
+  BinaryOpStub_GenerateFloatingPointCode(
+      masm, &gc_required, &not_number, op_, mode_);
 
   __ bind(&not_number);
   GenerateTypeTransition(masm);
 
   __ bind(&gc_required);
-  GenerateCallRuntimeCode(masm);
+  GenerateRegisterArgsPush(masm);
+  GenerateCallRuntime(masm);
 }
 
 
 void BinaryOpStub::GenerateGeneric(MacroAssembler* masm) {
   Label call_runtime, call_string_add_or_runtime;
 
-  GenerateSmiCode(masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS);
+  BinaryOpStub_GenerateSmiCode(
+      masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS, op_);
 
-  GenerateFloatingPointCode(masm, &call_runtime, &call_string_add_or_runtime);
+  BinaryOpStub_GenerateFloatingPointCode(
+      masm, &call_runtime, &call_string_add_or_runtime, op_, mode_);
 
   __ bind(&call_string_add_or_runtime);
   if (op_ == Token::ADD) {
-    GenerateStringAddCode(masm);
+    GenerateAddStrings(masm);
   }
 
   __ bind(&call_runtime);
-  GenerateCallRuntimeCode(masm);
+  GenerateRegisterArgsPush(masm);
+  GenerateCallRuntime(masm);
 }
 
 
-void BinaryOpStub::GenerateHeapResultAllocation(MacroAssembler* masm,
-                                                Label* alloc_failure) {
+static void BinaryOpStub_GenerateHeapResultAllocation(MacroAssembler* masm,
+                                                      Label* alloc_failure,
+                                                      OverwriteMode mode) {
   Label skip_allocation;
-  OverwriteMode mode = mode_;
   switch (mode) {
     case OVERWRITE_LEFT: {
       // If the argument in rdx is already an object, we skip the
@@ -2022,17 +1989,21 @@ void FloatingPointHelper::NumbersToSmis(MacroAssembler* masm,
                                         Register scratch2,
                                         Register scratch3,
                                         Label* on_success,
-                                        Label* on_not_smis)   {
+                                        Label* on_not_smis,
+                                        ConvertUndefined convert_undefined) {
   Register heap_number_map = scratch3;
   Register smi_result = scratch1;
-  Label done;
+  Label done, maybe_undefined_first, maybe_undefined_second, first_done;
 
   __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
 
   Label first_smi;
   __ JumpIfSmi(first, &first_smi, Label::kNear);
   __ cmpq(FieldOperand(first, HeapObject::kMapOffset), heap_number_map);
-  __ j(not_equal, on_not_smis);
+  __ j(not_equal,
+       (convert_undefined == CONVERT_UNDEFINED_TO_ZERO)
+           ? &maybe_undefined_first
+           : on_not_smis);
   // Convert HeapNumber to smi if possible.
   __ movsd(xmm0, FieldOperand(first, HeapNumber::kValueOffset));
   __ movq(scratch2, xmm0);
@@ -2045,11 +2016,15 @@ void FloatingPointHelper::NumbersToSmis(MacroAssembler* masm,
   __ j(not_equal, on_not_smis);
   __ Integer32ToSmi(first, smi_result);
 
+  __ bind(&first_done);
   __ JumpIfSmi(second, (on_success != NULL) ? on_success : &done);
   __ bind(&first_smi);
   __ AssertNotSmi(second);
   __ cmpq(FieldOperand(second, HeapObject::kMapOffset), heap_number_map);
-  __ j(not_equal, on_not_smis);
+  __ j(not_equal,
+       (convert_undefined == CONVERT_UNDEFINED_TO_ZERO)
+           ? &maybe_undefined_second
+           : on_not_smis);
   // Convert second to smi, if possible.
   __ movsd(xmm0, FieldOperand(second, HeapNumber::kValueOffset));
   __ movq(scratch2, xmm0);
@@ -2062,8 +2037,25 @@ void FloatingPointHelper::NumbersToSmis(MacroAssembler* masm,
   if (on_success != NULL) {
     __ jmp(on_success);
   } else {
-    __ bind(&done);
+    __ jmp(&done);
+  }
+
+  __ bind(&maybe_undefined_first);
+  __ CompareRoot(first, Heap::kUndefinedValueRootIndex);
+  __ j(not_equal, on_not_smis);
+  __ xor_(first, first);
+  __ jmp(&first_done);
+
+  __ bind(&maybe_undefined_second);
+  __ CompareRoot(second, Heap::kUndefinedValueRootIndex);
+  __ j(not_equal, on_not_smis);
+  __ xor_(second, second);
+  if (on_success != NULL) {
+    __ jmp(on_success);
   }
+  // Else: fall through.
+
+  __ bind(&done);
 }
 
 
@@ -3378,30 +3370,59 @@ static int NegativeComparisonResult(Condition cc) {
 }
 
 
-void CompareStub::Generate(MacroAssembler* masm) {
-  ASSERT(lhs_.is(no_reg) && rhs_.is(no_reg));
+static void CheckInputType(MacroAssembler* masm,
+                           Register input,
+                           CompareIC::State expected,
+                           Label* fail) {
+  Label ok;
+  if (expected == CompareIC::SMI) {
+    __ JumpIfNotSmi(input, fail);
+  } else if (expected == CompareIC::HEAP_NUMBER) {
+    __ JumpIfSmi(input, &ok);
+    __ CompareMap(input, masm->isolate()->factory()->heap_number_map(), NULL);
+    __ j(not_equal, fail);
+  }
+  // We could be strict about symbol/string here, but as long as
+  // hydrogen doesn't care, the stub doesn't have to care either.
+  __ bind(&ok);
+}
+
+
+static void BranchIfNonSymbol(MacroAssembler* masm,
+                              Label* label,
+                              Register object,
+                              Register scratch) {
+  __ JumpIfSmi(object, label);
+  __ movq(scratch, FieldOperand(object, HeapObject::kMapOffset));
+  __ movzxbq(scratch,
+             FieldOperand(scratch, Map::kInstanceTypeOffset));
+  // Ensure that no non-strings have the symbol bit set.
+  STATIC_ASSERT(LAST_TYPE < kNotStringTag + kIsSymbolMask);
+  STATIC_ASSERT(kSymbolTag != 0);
+  __ testb(scratch, Immediate(kIsSymbolMask));
+  __ j(zero, label);
+}
+
 
+void ICCompareStub::GenerateGeneric(MacroAssembler* masm) {
   Label check_unequal_objects, done;
+  Condition cc = GetCondition();
   Factory* factory = masm->isolate()->factory();
 
-  // Compare two smis if required.
-  if (include_smi_compare_) {
-    Label non_smi, smi_done;
-    __ JumpIfNotBothSmi(rax, rdx, &non_smi);
-    __ subq(rdx, rax);
-    __ j(no_overflow, &smi_done);
-    __ not_(rdx);  // Correct sign in case of overflow. rdx cannot be 0 here.
-    __ bind(&smi_done);
-    __ movq(rax, rdx);
-    __ ret(0);
-    __ bind(&non_smi);
-  } else if (FLAG_debug_code) {
-    Label ok;
-    __ JumpIfNotSmi(rdx, &ok);
-    __ JumpIfNotSmi(rax, &ok);
-    __ Abort("CompareStub: smi operands");
-    __ bind(&ok);
-  }
+  Label miss;
+  CheckInputType(masm, rdx, left_, &miss);
+  CheckInputType(masm, rax, right_, &miss);
+
+  // Compare two smis.
+  Label non_smi, smi_done;
+  __ JumpIfNotBothSmi(rax, rdx, &non_smi);
+  __ subq(rdx, rax);
+  __ j(no_overflow, &smi_done);
+  __ not_(rdx);  // Correct sign in case of overflow. rdx cannot be 0 here.
+  __ bind(&smi_done);
+  __ movq(rax, rdx);
+  __ ret(0);
+  __ bind(&non_smi);
 
   // The compare stub returns a positive, negative, or zero 64-bit integer
   // value in rax, corresponding to result of comparing the two inputs.
@@ -3414,66 +3435,58 @@ void CompareStub::Generate(MacroAssembler* masm) {
     __ cmpq(rax, rdx);
     __ j(not_equal, &not_identical, Label::kNear);
 
-    if (cc_ != equal) {
+    if (cc != equal) {
       // Check for undefined.  undefined OP undefined is false even though
       // undefined == undefined.
       Label check_for_nan;
       __ CompareRoot(rdx, Heap::kUndefinedValueRootIndex);
       __ j(not_equal, &check_for_nan, Label::kNear);
-      __ Set(rax, NegativeComparisonResult(cc_));
+      __ Set(rax, NegativeComparisonResult(cc));
       __ ret(0);
       __ bind(&check_for_nan);
     }
 
     // Test for NaN. Sadly, we can't just compare to FACTORY->nan_value(),
     // so we do the second best thing - test it ourselves.
-    // Note: if cc_ != equal, never_nan_nan_ is not used.
-    // We cannot set rax to EQUAL until just before return because
-    // rax must be unchanged on jump to not_identical.
-    if (never_nan_nan_ && (cc_ == equal)) {
-      __ Set(rax, EQUAL);
-      __ ret(0);
-    } else {
-      Label heap_number;
-      // If it's not a heap number, then return equal for (in)equality operator.
-      __ Cmp(FieldOperand(rdx, HeapObject::kMapOffset),
-             factory->heap_number_map());
-      __ j(equal, &heap_number, Label::kNear);
-      if (cc_ != equal) {
-        // Call runtime on identical objects.  Otherwise return equal.
-        __ CmpObjectType(rax, FIRST_SPEC_OBJECT_TYPE, rcx);
-        __ j(above_equal, &not_identical, Label::kNear);
-      }
-      __ Set(rax, EQUAL);
-      __ ret(0);
+    Label heap_number;
+    // If it's not a heap number, then return equal for (in)equality operator.
+    __ Cmp(FieldOperand(rdx, HeapObject::kMapOffset),
+           factory->heap_number_map());
+    __ j(equal, &heap_number, Label::kNear);
+    if (cc != equal) {
+      // Call runtime on identical objects.  Otherwise return equal.
+      __ CmpObjectType(rax, FIRST_SPEC_OBJECT_TYPE, rcx);
+      __ j(above_equal, &not_identical, Label::kNear);
+    }
+    __ Set(rax, EQUAL);
+    __ ret(0);
 
-      __ bind(&heap_number);
-      // It is a heap number, so return  equal if it's not NaN.
-      // For NaN, return 1 for every condition except greater and
-      // greater-equal.  Return -1 for them, so the comparison yields
-      // false for all conditions except not-equal.
-      __ Set(rax, EQUAL);
-      __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
-      __ ucomisd(xmm0, xmm0);
-      __ setcc(parity_even, rax);
-      // rax is 0 for equal non-NaN heapnumbers, 1 for NaNs.
-      if (cc_ == greater_equal || cc_ == greater) {
-        __ neg(rax);
-      }
-      __ ret(0);
+    __ bind(&heap_number);
+    // It is a heap number, so return  equal if it's not NaN.
+    // For NaN, return 1 for every condition except greater and
+    // greater-equal.  Return -1 for them, so the comparison yields
+    // false for all conditions except not-equal.
+    __ Set(rax, EQUAL);
+    __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
+    __ ucomisd(xmm0, xmm0);
+    __ setcc(parity_even, rax);
+    // rax is 0 for equal non-NaN heapnumbers, 1 for NaNs.
+    if (cc == greater_equal || cc == greater) {
+      __ neg(rax);
     }
+    __ ret(0);
 
     __ bind(&not_identical);
   }
 
-  if (cc_ == equal) {  // Both strict and non-strict.
+  if (cc == equal) {  // Both strict and non-strict.
     Label slow;  // Fallthrough label.
 
     // If we're doing a strict equality comparison, we don't have to do
     // type conversion, so we generate code to do fast comparison for objects
     // and oddballs. Non-smi numbers and strings still go through the usual
     // slow-case code.
-    if (strict_) {
+    if (strict()) {
       // If either is a Smi (we know that not both are), then they can only
       // be equal if the other is a HeapNumber. If so, use the slow case.
       {
@@ -3525,40 +3538,38 @@ void CompareStub::Generate(MacroAssembler* masm) {
   }
 
   // Generate the number comparison code.
-  if (include_number_compare_) {
-    Label non_number_comparison;
-    Label unordered;
-    FloatingPointHelper::LoadSSE2UnknownOperands(masm, &non_number_comparison);
-    __ xorl(rax, rax);
-    __ xorl(rcx, rcx);
-    __ ucomisd(xmm0, xmm1);
-
-    // Don't base result on EFLAGS when a NaN is involved.
-    __ j(parity_even, &unordered, Label::kNear);
-    // Return a result of -1, 0, or 1, based on EFLAGS.
-    __ setcc(above, rax);
-    __ setcc(below, rcx);
-    __ subq(rax, rcx);
-    __ ret(0);
+  Label non_number_comparison;
+  Label unordered;
+  FloatingPointHelper::LoadSSE2UnknownOperands(masm, &non_number_comparison);
+  __ xorl(rax, rax);
+  __ xorl(rcx, rcx);
+  __ ucomisd(xmm0, xmm1);
 
-    // If one of the numbers was NaN, then the result is always false.
-    // The cc is never not-equal.
-    __ bind(&unordered);
-    ASSERT(cc_ != not_equal);
-    if (cc_ == less || cc_ == less_equal) {
-      __ Set(rax, 1);
-    } else {
-      __ Set(rax, -1);
-    }
-    __ ret(0);
+  // Don't base result on EFLAGS when a NaN is involved.
+  __ j(parity_even, &unordered, Label::kNear);
+  // Return a result of -1, 0, or 1, based on EFLAGS.
+  __ setcc(above, rax);
+  __ setcc(below, rcx);
+  __ subq(rax, rcx);
+  __ ret(0);
 
-    // The number comparison code did not provide a valid result.
-    __ bind(&non_number_comparison);
+  // If one of the numbers was NaN, then the result is always false.
+  // The cc is never not-equal.
+  __ bind(&unordered);
+  ASSERT(cc != not_equal);
+  if (cc == less || cc == less_equal) {
+    __ Set(rax, 1);
+  } else {
+    __ Set(rax, -1);
   }
+  __ ret(0);
+
+  // The number comparison code did not provide a valid result.
+  __ bind(&non_number_comparison);
 
   // Fast negative check for symbol-to-symbol equality.
   Label check_for_strings;
-  if (cc_ == equal) {
+  if (cc == equal) {
     BranchIfNonSymbol(masm, &check_for_strings, rax, kScratchRegister);
     BranchIfNonSymbol(masm, &check_for_strings, rdx, kScratchRegister);
 
@@ -3574,7 +3585,7 @@ void CompareStub::Generate(MacroAssembler* masm) {
       rdx, rax, rcx, rbx, &check_unequal_objects);
 
   // Inline comparison of ASCII strings.
-  if (cc_ == equal) {
+  if (cc == equal) {
     StringCompareStub::GenerateFlatAsciiStringEquals(masm,
                                                      rdx,
                                                      rax,
@@ -3595,7 +3606,7 @@ void CompareStub::Generate(MacroAssembler* masm) {
 #endif
 
   __ bind(&check_unequal_objects);
-  if (cc_ == equal && !strict_) {
+  if (cc == equal && !strict()) {
     // Not strict equality.  Objects are unequal if
     // they are both JSObjects and not undetectable,
     // and their pointers are different.
@@ -3635,11 +3646,11 @@ void CompareStub::Generate(MacroAssembler* masm) {
 
   // Figure out which native to call and setup the arguments.
   Builtins::JavaScript builtin;
-  if (cc_ == equal) {
-    builtin = strict_ ? Builtins::STRICT_EQUALS : Builtins::EQUALS;
+  if (cc == equal) {
+    builtin = strict() ? Builtins::STRICT_EQUALS : Builtins::EQUALS;
   } else {
     builtin = Builtins::COMPARE;
-    __ Push(Smi::FromInt(NegativeComparisonResult(cc_)));
+    __ Push(Smi::FromInt(NegativeComparisonResult(cc)));
   }
 
   // Restore return address on the stack.
@@ -3648,22 +3659,9 @@ void CompareStub::Generate(MacroAssembler* masm) {
   // Call the native; it returns -1 (less), 0 (equal), or 1 (greater)
   // tagged as a small integer.
   __ InvokeBuiltin(builtin, JUMP_FUNCTION);
-}
-
 
-void CompareStub::BranchIfNonSymbol(MacroAssembler* masm,
-                                    Label* label,
-                                    Register object,
-                                    Register scratch) {
-  __ JumpIfSmi(object, label);
-  __ movq(scratch, FieldOperand(object, HeapObject::kMapOffset));
-  __ movzxbq(scratch,
-             FieldOperand(scratch, Map::kInstanceTypeOffset));
-  // Ensure that no non-strings have the symbol bit set.
-  STATIC_ASSERT(LAST_TYPE < kNotStringTag + kIsSymbolMask);
-  STATIC_ASSERT(kSymbolTag != 0);
-  __ testb(scratch, Immediate(kIsSymbolMask));
-  __ j(zero, label);
+  __ bind(&miss);
+  GenerateMiss(masm);
 }
 
 
@@ -4421,44 +4419,6 @@ Register InstanceofStub::left() { return no_reg; }
 Register InstanceofStub::right() { return no_reg; }
 
 
-int CompareStub::MinorKey() {
-  // Encode the three parameters in a unique 16 bit value. To avoid duplicate
-  // stubs the never NaN NaN condition is only taken into account if the
-  // condition is equals.
-  ASSERT(static_cast<unsigned>(cc_) < (1 << 12));
-  ASSERT(lhs_.is(no_reg) && rhs_.is(no_reg));
-  return ConditionField::encode(static_cast<unsigned>(cc_))
-         | RegisterField::encode(false)    // lhs_ and rhs_ are not used
-         | StrictField::encode(strict_)
-         | NeverNanNanField::encode(cc_ == equal ? never_nan_nan_ : false)
-         | IncludeNumberCompareField::encode(include_number_compare_)
-         | IncludeSmiCompareField::encode(include_smi_compare_);
-}
-
-
-// Unfortunately you have to run without snapshots to see most of these
-// names in the profile since most compare stubs end up in the snapshot.
-void CompareStub::PrintName(StringStream* stream) {
-  ASSERT(lhs_.is(no_reg) && rhs_.is(no_reg));
-  const char* cc_name;
-  switch (cc_) {
-    case less: cc_name = "LT"; break;
-    case greater: cc_name = "GT"; break;
-    case less_equal: cc_name = "LE"; break;
-    case greater_equal: cc_name = "GE"; break;
-    case equal: cc_name = "EQ"; break;
-    case not_equal: cc_name = "NE"; break;
-    default: cc_name = "UnknownCondition"; break;
-  }
-  bool is_equality = cc_ == equal || cc_ == not_equal;
-  stream->Add("CompareStub_%s", cc_name);
-  if (strict_ && is_equality) stream->Add("_STRICT");
-  if (never_nan_nan_ && is_equality) stream->Add("_NO_NAN");
-  if (!include_number_compare_) stream->Add("_NO_NUMBER");
-  if (!include_smi_compare_) stream->Add("_NO_SMI");
-}
-
-
 // -------------------------------------------------------------------------
 // StringCharCodeAtGenerator
 
@@ -5568,7 +5528,7 @@ void StringCompareStub::Generate(MacroAssembler* masm) {
 
 
 void ICCompareStub::GenerateSmis(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::SMIS);
+  ASSERT(state_ == CompareIC::SMI);
   Label miss;
   __ JumpIfNotBothSmi(rdx, rax, &miss, Label::kNear);
 
@@ -5592,23 +5552,41 @@ void ICCompareStub::GenerateSmis(MacroAssembler* masm) {
 
 
 void ICCompareStub::GenerateHeapNumbers(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::HEAP_NUMBERS);
+  ASSERT(state_ == CompareIC::HEAP_NUMBER);
 
   Label generic_stub;
   Label unordered, maybe_undefined1, maybe_undefined2;
   Label miss;
-  Condition either_smi = masm->CheckEitherSmi(rax, rdx);
-  __ j(either_smi, &generic_stub, Label::kNear);
 
-  __ CmpObjectType(rax, HEAP_NUMBER_TYPE, rcx);
+  if (left_ == CompareIC::SMI) {
+    __ JumpIfNotSmi(rdx, &miss);
+  }
+  if (right_ == CompareIC::SMI) {
+    __ JumpIfNotSmi(rax, &miss);
+  }
+
+  // Load left and right operand.
+  Label done, left, left_smi, right_smi;
+  __ JumpIfSmi(rax, &right_smi, Label::kNear);
+  __ CompareMap(rax, masm->isolate()->factory()->heap_number_map(), NULL);
   __ j(not_equal, &maybe_undefined1, Label::kNear);
-  __ CmpObjectType(rdx, HEAP_NUMBER_TYPE, rcx);
+  __ movsd(xmm1, FieldOperand(rax, HeapNumber::kValueOffset));
+  __ jmp(&left, Label::kNear);
+  __ bind(&right_smi);
+  __ SmiToInteger32(rcx, rax);  // Can't clobber rax yet.
+  __ cvtlsi2sd(xmm1, rcx);
+
+  __ bind(&left);
+  __ JumpIfSmi(rdx, &left_smi, Label::kNear);
+  __ CompareMap(rdx, masm->isolate()->factory()->heap_number_map(), NULL);
   __ j(not_equal, &maybe_undefined2, Label::kNear);
-
-  // Load left and right operand
   __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
-  __ movsd(xmm1, FieldOperand(rax, HeapNumber::kValueOffset));
+  __ jmp(&done);
+  __ bind(&left_smi);
+  __ SmiToInteger32(rcx, rdx);  // Can't clobber rdx yet.
+  __ cvtlsi2sd(xmm0, rcx);
 
+  __ bind(&done);
   // Compare operands
   __ ucomisd(xmm0, xmm1);
 
@@ -5624,14 +5602,16 @@ void ICCompareStub::GenerateHeapNumbers(MacroAssembler* masm) {
   __ ret(0);
 
   __ bind(&unordered);
-  CompareStub stub(GetCondition(), strict(), NO_COMPARE_FLAGS);
   __ bind(&generic_stub);
+  ICCompareStub stub(op_, CompareIC::GENERIC, CompareIC::GENERIC,
+                     CompareIC::GENERIC);
   __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET);
 
   __ bind(&maybe_undefined1);
   if (Token::IsOrderedRelationalCompareOp(op_)) {
     __ Cmp(rax, masm->isolate()->factory()->undefined_value());
     __ j(not_equal, &miss);
+    __ JumpIfSmi(rdx, &unordered);
     __ CmpObjectType(rdx, HEAP_NUMBER_TYPE, rcx);
     __ j(not_equal, &maybe_undefined2, Label::kNear);
     __ jmp(&unordered);
@@ -5649,7 +5629,7 @@ void ICCompareStub::GenerateHeapNumbers(MacroAssembler* masm) {
 
 
 void ICCompareStub::GenerateSymbols(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::SYMBOLS);
+  ASSERT(state_ == CompareIC::SYMBOL);
   ASSERT(GetCondition() == equal);
 
   // Registers containing left and right operands respectively.
@@ -5692,7 +5672,7 @@ void ICCompareStub::GenerateSymbols(MacroAssembler* masm) {
 
 
 void ICCompareStub::GenerateStrings(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::STRINGS);
+  ASSERT(state_ == CompareIC::STRING);
   Label miss;
 
   bool equality = Token::IsEqualityOp(op_);
@@ -5778,7 +5758,7 @@ void ICCompareStub::GenerateStrings(MacroAssembler* masm) {
 
 
 void ICCompareStub::GenerateObjects(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::OBJECTS);
+  ASSERT(state_ == CompareIC::OBJECT);
   Label miss;
   Condition either_smi = masm->CheckEitherSmi(rdx, rax);
   __ j(either_smi, &miss, Label::kNear);