Fuse compare with branch at graph building time.

runtime/vm/intermediate_language_ia32.cc

Index: runtime/vm/intermediate_language_ia32.cc
===================================================================
--- runtime/vm/intermediate_language_ia32.cc	(revision 9728)
+++ runtime/vm/intermediate_language_ia32.cc	(working copy)
@@ -203,9 +203,7 @@
     locs->set_in(0, Location::RequiresRegister());
     locs->set_in(1, Location::RequiresRegister());
     locs->set_temp(0, Location::RequiresRegister());
-    if (!is_fused_with_branch()) {
-      locs->set_out(Location::RequiresRegister());
-    }
+    locs->set_out(Location::RequiresRegister());
     return locs;
   }
   if (HasICData() && (ic_data()->NumberOfChecks() > 0)) {
@@ -216,9 +214,7 @@
     locs->set_in(0, Location::RequiresRegister());
     locs->set_in(1, Location::RequiresRegister());
     locs->set_temp(0, Location::RequiresRegister());
-    if (!is_fused_with_branch()) {
-      locs->set_out(Location::RegisterLocation(EAX));
-    }
+    locs->set_out(Location::RegisterLocation(EAX));
     return locs;
   }
   const intptr_t kNumTemps = 0;
@@ -227,9 +223,7 @@
                                               LocationSummary::kCall);
   locs->set_in(0, Location::RequiresRegister());
   locs->set_in(1, Location::RequiresRegister());
-  if (!is_fused_with_branch()) {
-    locs->set_out(Location::RegisterLocation(EAX));
-  }
+  locs->set_out(Location::RegisterLocation(EAX));
   return locs;
 }
 
@@ -252,18 +246,14 @@
   __ testl(temp, Immediate(kSmiTagMask));
   __ j(NOT_ZERO, deopt);
   __ cmpl(left, right);
-  if (comp->is_fused_with_branch()) {
-    comp->fused_with_branch()->EmitBranchOnCondition(compiler, EQUAL);
-  } else {
-    Register result = comp->locs()->out().reg();
-    Label load_true, done;
-    __ j(EQUAL, &load_true, Assembler::kNearJump);
-    __ LoadObject(result, compiler->bool_false());
-    __ jmp(&done, Assembler::kNearJump);
-    __ Bind(&load_true);
-    __ LoadObject(result, compiler->bool_true());
-    __ Bind(&done);
-  }
+  Register result = comp->locs()->out().reg();
+  Label load_true, done;
+  __ j(EQUAL, &load_true, Assembler::kNearJump);
+  __ LoadObject(result, compiler->bool_false());
+  __ jmp(&done, Assembler::kNearJump);
+  __ Bind(&load_true);
+  __ LoadObject(result, compiler->bool_true());
+  __ Bind(&done);
 }
 
 
@@ -287,13 +277,8 @@
   __ j(NOT_EQUAL, deopt);
   __ movsd(XMM0, FieldAddress(left, Double::value_offset()));
   __ movsd(XMM1, FieldAddress(right, Double::value_offset()));
-  if (comp->is_fused_with_branch()) {
-    compiler->EmitDoubleCompareBranch(
-        EQUAL, XMM0, XMM1, comp->fused_with_branch());
-  } else {
-    compiler->EmitDoubleCompareBool(
-        EQUAL, XMM0, XMM1, comp->locs()->out().reg());
-  }
+  compiler->EmitDoubleCompareBool(
+      EQUAL, XMM0, XMM1, comp->locs()->out().reg());
 }
 
 
@@ -315,29 +300,29 @@
                                  kNumberOfArguments,
                                  kNoArgumentNames,
                                  kNumArgumentsChecked);
-  ASSERT(comp->is_fused_with_branch() || (comp->locs()->out().reg() == EAX));
-
-  if (comp->is_fused_with_branch()) {
-    __ CompareObject(EAX, compiler->bool_true());
-    comp->fused_with_branch()->EmitBranchOnCondition(compiler, EQUAL);
-  }
+  ASSERT(comp->locs()->out().reg() == EAX);
 }
 
 
 static void EmitEqualityAsPolymorphicCall(FlowGraphCompiler* compiler,
-                                          EqualityCompareComp* comp,
-                                          Register left,
-                                          Register right) {
-  ASSERT(comp->HasICData());
-  const ICData& ic_data = ICData::Handle(comp->ic_data()->AsUnaryClassChecks());
+                                          const ICData& orig_ic_data,
+                                          const LocationSummary& locs,
+                                          BranchInstr* branch,
+                                          Token::Kind kind,
+                                          intptr_t cid,
+                                          intptr_t token_pos,
+                                          intptr_t try_index) {
+  const ICData& ic_data = ICData::Handle(orig_ic_data.AsUnaryClassChecks());
   ASSERT(ic_data.NumberOfChecks() > 0);
   ASSERT(ic_data.num_args_tested() == 1);
-  Label* deopt = compiler->AddDeoptStub(comp->cid(),
-                                        comp->token_pos(),
-                                        comp->try_index(),
+  Label* deopt = compiler->AddDeoptStub(cid,
+                                        token_pos,
+                                        try_index,
                                         kDeoptEquality);
+  Register left = locs.in(0).reg();
+  Register right = locs.in(1).reg();
   __ testl(left, Immediate(kSmiTagMask));
-  Register temp = comp->locs()->temp(0).reg();
+  Register temp = locs.temp(0).reg();
   if (ic_data.GetReceiverClassIdAt(0) == kSmi) {
     Label done, load_class_id;
     __ j(NOT_ZERO, &load_class_id, Assembler::kNearJump);
@@ -350,6 +335,7 @@
     __ j(ZERO, deopt);  // Smi deopts.
     __ LoadClassId(temp, left);
   }
+  Condition cond = (kind == Token::kEQ) ? EQUAL : NOT_EQUAL;
   Label done;
   for (intptr_t i = 0; i < ic_data.NumberOfChecks(); i++) {
     ASSERT((ic_data.GetReceiverClassIdAt(i) != kSmi) || (i == 0));
@@ -362,13 +348,13 @@
       // Object.== is same as ===.
       __ Drop(2);
       __ cmpl(left, right);
-      if (comp->is_fused_with_branch()) {
-        comp->fused_with_branch()->EmitBranchOnCondition(compiler, EQUAL);
+      if (branch != NULL) {
+        branch->EmitBranchOnCondition(compiler, cond);
       } else {
         // This case should be rare.
-        Register result = comp->locs()->out().reg();
+        Register result = locs.out().reg();
         Label load_true;
-        __ j(EQUAL, &load_true, Assembler::kNearJump);
+        __ j(cond, &load_true, Assembler::kNearJump);
         __ LoadObject(result, compiler->bool_false());
         __ jmp(&done);
         __ Bind(&load_true);
@@ -377,17 +363,15 @@
     } else {
       const int kNumberOfArguments = 2;
       const Array& kNoArgumentNames = Array::Handle();
-      compiler->GenerateStaticCall(comp->cid(),
-                                   comp->token_pos(),
-                                   comp->try_index(),
+      compiler->GenerateStaticCall(cid,
+                                   token_pos,
+                                   try_index,
                                    target,
                                    kNumberOfArguments,
                                    kNoArgumentNames);
-      ASSERT(comp->is_fused_with_branch() ||
-            (comp->locs()->out().reg() == EAX));
-      if (comp->is_fused_with_branch()) {
+      if (branch != NULL) {
         __ CompareObject(EAX, compiler->bool_true());
-        comp->fused_with_branch()->EmitBranchOnCondition(compiler, EQUAL);
+        branch->EmitBranchOnCondition(compiler, cond);
       }
     }
     __ jmp(&done);
@@ -403,9 +387,16 @@
 // If type feedback was provided (lists of <class-id, target>), do a
 // type by type check (either === or static call to the operator.
 static void EmitGenericEqualityCompare(FlowGraphCompiler* compiler,
-                                       EqualityCompareComp* comp) {
-  Register left = comp->locs()->in(0).reg();
-  Register right = comp->locs()->in(1).reg();
+                                       const LocationSummary& locs,
+                                       Token::Kind kind,
+                                       BranchInstr* branch,
+                                       const ICData& ic_data,
+                                       intptr_t cid,
+                                       intptr_t token_pos,
+                                       intptr_t try_index) {
+  ASSERT(!ic_data.IsNull() && (ic_data.NumberOfChecks() > 0));
+  Register left = locs.in(0).reg();
+  Register right = locs.in(1).reg();
   const Immediate raw_null =
       Immediate(reinterpret_cast<intptr_t>(Object::null()));
   Label done, non_null_compare;
@@ -413,27 +404,24 @@
   __ j(NOT_EQUAL, &non_null_compare, Assembler::kNearJump);
   // Comparison with NULL is "===".
   __ cmpl(left, right);
-  if (comp->is_fused_with_branch()) {
-    comp->fused_with_branch()->EmitBranchOnCondition(compiler, EQUAL);
+  Condition cond = (kind == Token::kEQ) ? EQUAL : NOT_EQUAL;
+  if (branch != NULL) {
+    branch->EmitBranchOnCondition(compiler, cond);
   } else {
-    Register result = comp->locs()->out().reg();
+    Register result = locs.out().reg();
     Label load_true;
-    __ j(EQUAL, &load_true, Assembler::kNearJump);
+    __ j(cond, &load_true, Assembler::kNearJump);
     __ LoadObject(result, compiler->bool_false());
     __ jmp(&done);
     __ Bind(&load_true);
     __ LoadObject(result, compiler->bool_true());
   }
   __ jmp(&done);
-
   __ Bind(&non_null_compare);  // Receiver is not null.
   __ pushl(left);
   __ pushl(right);
-  if (comp->HasICData() && (comp->ic_data()->NumberOfChecks() > 0)) {
-    EmitEqualityAsPolymorphicCall(compiler, comp, left, right);
-  } else {
-    EmitEqualityAsInstanceCall(compiler, comp);
-  }
+  EmitEqualityAsPolymorphicCall(compiler, ic_data, locs, branch, kind,
+                                cid, token_pos, try_index);
   __ Bind(&done);
 }
 
@@ -447,7 +435,16 @@
     EmitDoubleEqualityCompare(compiler, this);
     return;
   }
-  EmitGenericEqualityCompare(compiler, this);
+  if (HasICData() && (ic_data()->NumberOfChecks() > 0)) {
+    EmitGenericEqualityCompare(compiler, *locs(), Token::kEQ, NULL,
+                               *ic_data(), cid(), token_pos(), try_index());
+  } else {
+    Register left = locs()->in(0).reg();
+    Register right = locs()->in(1).reg();
+    __ pushl(left);
+    __ pushl(right);
+    EmitEqualityAsInstanceCall(compiler, this);
+  }
 }
 
 
@@ -460,13 +457,10 @@
                                                    LocationSummary::kNoCall);
     summary->set_in(0, Location::RequiresRegister());
     summary->set_in(1, Location::RequiresRegister());
-    if (!is_fused_with_branch()) {
-      summary->set_out(Location::RequiresRegister());
-    }
+    summary->set_out(Location::RequiresRegister());
     summary->set_temp(0, Location::RequiresRegister());
     return summary;
   }
-  ASSERT(!is_fused_with_branch());
   ASSERT(operands_class_id() == kObject);
   return MakeCallSummary();
 }
@@ -487,14 +481,19 @@
 }
 
 
-static void EmitSmiRelationalOp(FlowGraphCompiler* compiler,
-                                RelationalOpComp* comp) {
-  Register left = comp->locs()->in(0).reg();
-  Register right = comp->locs()->in(1).reg();
-  Register temp = comp->locs()->temp(0).reg();
-  Label* deopt = compiler->AddDeoptStub(comp->cid(),
-                                        comp->token_pos(),
-                                        comp->try_index(),
+static void EmitSmiComparisonOp(FlowGraphCompiler* compiler,
+                                const LocationSummary& locs,
+                                Token::Kind kind,
+                                BranchInstr* branch,
+                                intptr_t cid,
+                                intptr_t token_pos,
+                                intptr_t try_index) {
+  Register left = locs.in(0).reg();
+  Register right = locs.in(1).reg();
+  Register temp = locs.temp(0).reg();
+  Label* deopt = compiler->AddDeoptStub(cid,
+                                        token_pos,
+                                        try_index,
                                         kDeoptSmiCompareSmi,
                                         left,
                                         right);
@@ -503,13 +502,13 @@
   __ testl(temp, Immediate(kSmiTagMask));
   __ j(NOT_ZERO, deopt);
 
-  Condition true_condition = TokenKindToSmiCondition(comp->kind());
+  Condition true_condition = TokenKindToSmiCondition(kind);
   __ cmpl(left, right);
 
-  if (comp->is_fused_with_branch()) {
-    comp->fused_with_branch()->EmitBranchOnCondition(compiler, true_condition);
+  if (branch != NULL) {
+    branch->EmitBranchOnCondition(compiler, true_condition);
   } else {
-    Register result = comp->locs()->out().reg();
+    Register result = locs.out().reg();
     Label done, is_true;
     __ j(true_condition, &is_true);
     __ LoadObject(result, compiler->bool_false());
@@ -524,6 +523,7 @@
 static Condition TokenKindToDoubleCondition(Token::Kind kind) {
   switch (kind) {
     case Token::kEQ: return EQUAL;
+    case Token::kNE: return NOT_EQUAL;
     case Token::kLT: return BELOW;
     case Token::kGT: return ABOVE;
     case Token::kLTE: return BELOW_EQUAL;
@@ -535,39 +535,46 @@
 }
 
 
-static void EmitDoubleRelationalOp(FlowGraphCompiler* compiler,
-                                   RelationalOpComp* comp) {
-  Register left = comp->locs()->in(0).reg();
-  Register right = comp->locs()->in(1).reg();
+static void EmitDoubleComparisonOp(FlowGraphCompiler* compiler,
+                                   const LocationSummary& locs,
+                                   Token::Kind kind,
+                                   BranchInstr* branch,
+                                   intptr_t cid,
+                                   intptr_t token_pos,
+                                   intptr_t try_index) {
+  Register left = locs.in(0).reg();
+  Register right = locs.in(1).reg();
   // TODO(srdjan): temp is only needed if a conversion Smi->Double occurs.
-  Register temp = comp->locs()->temp(0).reg();
-  Label* deopt = compiler->AddDeoptStub(comp->cid(),
-                                        comp->token_pos(),
-                                        comp->try_index(),
+  Register temp = locs.temp(0).reg();
+  Label* deopt = compiler->AddDeoptStub(cid,
+                                        token_pos,
+                                        try_index,
                                         kDeoptDoubleComparison,
                                         left,
                                         right);
   compiler->LoadDoubleOrSmiToXmm(XMM0, left, temp, deopt);
   compiler->LoadDoubleOrSmiToXmm(XMM1, right, temp, deopt);
 
-  Condition true_condition = TokenKindToDoubleCondition(comp->kind());
-  if (comp->is_fused_with_branch()) {
+  Condition true_condition = TokenKindToDoubleCondition(kind);
+  if (branch != NULL) {
     compiler->EmitDoubleCompareBranch(
-        true_condition, XMM0, XMM1, comp->fused_with_branch());
+        true_condition, XMM0, XMM1, branch);
   } else {
     compiler->EmitDoubleCompareBool(
-        true_condition, XMM0, XMM1, comp->locs()->out().reg());
+        true_condition, XMM0, XMM1, locs.out().reg());
   }
 }
 
 
 void RelationalOpComp::EmitNativeCode(FlowGraphCompiler* compiler) {
   if (operands_class_id() == kSmi) {
-    EmitSmiRelationalOp(compiler, this);
+    EmitSmiComparisonOp(compiler, *locs(), kind(), NULL,
+                        cid(), token_pos(), try_index());
     return;
   }
   if (operands_class_id() == kDouble) {
-    EmitDoubleRelationalOp(compiler, this);
+    EmitDoubleComparisonOp(compiler, *locs(), kind(), NULL,
+                           cid(), token_pos(), try_index());
     return;
   }
   if (HasICData() && (ic_data()->NumberOfChecks() > 0)) {
@@ -2028,6 +2035,114 @@
   __ Bind(&done);
 }
 
+
+// TODO(srdjan): Move to shared.
+static bool ICDataWithBothClassIds(const ICData& ic_data, intptr_t class_id) {
+  if (ic_data.num_args_tested() != 2) return false;
+  if (ic_data.NumberOfChecks() != 1) return false;
+  Function& target = Function::Handle();
+  GrowableArray<intptr_t> class_ids;
+  ic_data.GetCheckAt(0, &class_ids, &target);
+  return (class_ids[0] == class_id) && (class_ids[1] == class_id);
+}
+
+
+LocationSummary* BranchInstr::MakeLocationSummary() const {
+  if ((kind() == Token::kEQ_STRICT) || (kind() == Token::kNE_STRICT)) {
+    const int kNumInputs = 2;
+    const int kNumTemps = 0;
+    LocationSummary* locs = new LocationSummary(kNumInputs,
+                                                kNumTemps,
+                                                LocationSummary::kNoCall);
+    locs->set_in(0, Location::RequiresRegister());
+    locs->set_in(1, Location::RequiresRegister());
+    return locs;
+  }
+  if (HasICData() && (ic_data()->NumberOfChecks() > 0)) {
+    if (ICDataWithBothClassIds(*ic_data(), kSmi) ||
+        ICDataWithBothClassIds(*ic_data(), kDouble)) {
+      const intptr_t kNumInputs = 2;
+      const intptr_t kNumTemps = 1;
+      LocationSummary* summary = new LocationSummary(kNumInputs,
+                                                     kNumTemps,
+                                                     LocationSummary::kNoCall);
+      summary->set_in(0, Location::RequiresRegister());
+      summary->set_in(1, Location::RequiresRegister());
+      summary->set_temp(0, Location::RequiresRegister());
+      return summary;
+    }
+    if ((kind() == Token::kEQ) || (kind() == Token::kNE)) {
+      const intptr_t kNumInputs = 2;
+      const intptr_t kNumTemps = 1;
+      LocationSummary* locs = new LocationSummary(kNumInputs,
+                                                  kNumTemps,
+                                                  LocationSummary::kCall);
+      locs->set_in(0, Location::RequiresRegister());
+      locs->set_in(1, Location::RequiresRegister());
+      locs->set_temp(0, Location::RequiresRegister());
+      return locs;
+    }
+    // Otherwise polymorphic dispatch.
+  }
+  // Call.
+  LocationSummary* result = new LocationSummary(0, 0, LocationSummary::kCall);
+  result->set_out(Location::RegisterLocation(EAX));
+  return result;
+}
+
+
+void BranchInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  if ((kind() == Token::kEQ_STRICT) || (kind() == Token::kNE_STRICT)) {
+    Register left = locs()->in(0).reg();
+    Register right = locs()->in(1).reg();
+    __ cmpl(left, right);
+    Condition cond = (kind() == Token::kEQ_STRICT) ? EQUAL : NOT_EQUAL;
+    EmitBranchOnCondition(compiler, cond);
+    return;
+  }
+  // Relational or equality.
+  if (HasICData() && (ic_data()->NumberOfChecks() > 0)) {
+    if (ICDataWithBothClassIds(*ic_data(), kSmi)) {
+      EmitSmiComparisonOp(compiler, *locs(), kind(), this,
+                          cid(), token_pos(), try_index());
+      return;
+    }
+    if (ICDataWithBothClassIds(*ic_data(), kDouble)) {
+      EmitDoubleComparisonOp(compiler, *locs(), kind(), this,
+                             cid(), token_pos(), try_index());
+      return;
+    }
+    // TODO(srdjan): Add Smi/Double, Double/Smi comparisons.
+    if ((kind() == Token::kEQ) || (kind() == Token::kNE)) {
+      EmitGenericEqualityCompare(compiler, *locs(), kind(), this, *ic_data(),
+                                 cid(), token_pos(), try_index());
+      return;
+    }
+    // Otherwise polymorphic dispatch?
+  }
+  // Not equal is always split into '==' and negate,
+  Condition branch_condition = (kind() == Token::kNE) ? NOT_EQUAL : EQUAL;
+  Token::Kind call_kind = (kind() == Token::kNE) ? Token::kEQ : kind();
+  const String& function_name =
+      String::ZoneHandle(String::NewSymbol(Token::Str(call_kind)));
+  compiler->AddCurrentDescriptor(PcDescriptors::kDeopt,
+                                 cid(),
+                                 token_pos(),
+                                 try_index());
+  const intptr_t kNumArguments = 2;
+  const intptr_t kNumArgsChecked = 2;  // Type-feedback.
+  compiler->GenerateInstanceCall(cid(),
+                                 token_pos(),
+                                 try_index(),
+                                 function_name,
+                                 kNumArguments,
+                                 Array::ZoneHandle(),  // No optional arguments.
+                                 kNumArgsChecked);
+  ASSERT(locs()->out().reg() == EAX);
+  __ CompareObject(locs()->out().reg(), compiler->bool_true());
+  EmitBranchOnCondition(compiler, branch_condition);
+}
+
 }  // namespace dart
 
 #undef __