[wasm] simd scalar lowering F32x4Add and I32x4Add

src/compiler/simd-scalar-lowering.cc

+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/compiler/simd-scalar-lowering.h"
+#include "src/compiler/diamond.h"
+#include "src/compiler/linkage.h"
+#include "src/compiler/node-matchers.h"
+#include "src/compiler/node-properties.h"
+
+#include "src/compiler/node.h"
+#include "src/wasm/wasm-module.h"
+
+namespace v8 {
+namespace internal {
+namespace compiler {
+
+SimdScalarLowering::SimdScalarLowering(
+    Graph* graph, MachineOperatorBuilder* machine,
+    CommonOperatorBuilder* common, Zone* zone,
+    Signature<MachineRepresentation>* signature)
+    : zone_(zone),
+      graph_(graph),
+      machine_(machine),
+      common_(common),
+      state_(graph, 3),
+      stack_(zone),
+      replacements_(nullptr),
+      signature_(signature),
+      placeholder_(
+          graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
+      parameter_count_after_lowering_(-1) {
+  DCHECK_NOT_NULL(graph);
+  DCHECK_NOT_NULL(graph->end());
+  replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
+  memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
+}
+
+void SimdScalarLowering::LowerGraph() {
+  stack_.push_back({graph()->end(), 0});
+  state_.Set(graph()->end(), State::kOnStack);
+  replacements_[graph()->end()->id()].type = SimdType::kInt32;
+
+  while (!stack_.empty()) {
+    NodeState& top = stack_.back();
+    if (top.input_index == top.node->InputCount()) {
+      // All inputs of top have already been lowered, now lower top.
+      stack_.pop_back();
+      state_.Set(top.node, State::kVisited);
+      LowerNode(top.node);
+    } else {
+      // Push the next input onto the stack.
+      Node* input = top.node->InputAt(top.input_index++);
+      if (state_.Get(input) == State::kUnvisited) {
+        SetLoweredType(input, top.node);
+        if (input->opcode() == IrOpcode::kPhi) {
+          // To break cycles with phi nodes we push phis on a separate stack so
+          // that they are processed after all other nodes.
+          PreparePhiReplacement(input);
+          stack_.push_front({input, 0});
+        } else {
+          stack_.push_back({input, 0});
+        }
+        state_.Set(input, State::kOnStack);
+      }
+    }
+  }
+}
+
+#define FOREACH_INT32X4_OPCODE(V) \
+  V(Int32x4Add)                   \
+  V(Int32x4ExtractLane)           \
+  V(CreateInt32x4)
+
+#define FOREACH_FLOAT32X4_OPCODE(V) \
+  V(Float32x4Add)                   \
+  V(Float32x4ExtractLane)           \
+  V(CreateFloat32x4)
+
+void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
+  switch (node->opcode()) {
+#define CASE_STMT(name) case IrOpcode::k##name:
+    FOREACH_INT32X4_OPCODE(CASE_STMT)
+    case IrOpcode::kReturn:
+    case IrOpcode::kParameter:
+    case IrOpcode::kCall: {
+      replacements_[node->id()].type = SimdType::kInt32;
+      break;
+    }
+      FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
+        replacements_[node->id()].type = SimdType::kFloat32;
+        break;
+      }
+#undef CASE_STMT
+    default:
+      replacements_[node->id()].type = replacements_[output->id()].type;
+  }
+}
+
+static int GetParameterIndexAfterLowering(
+    Signature<MachineRepresentation>* signature, int old_index) {
+  // In function calls, the simd128 types are passed as 4 Int32 types. The
+  // parameters are typecast to the types as needed for various operations.
+  int result = old_index;
+  for (int i = 0; i < old_index; i++) {
+    if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
+      result += 3;
+    }
+  }
+  return result;
+}
+
+int SimdScalarLowering::GetParameterCountAfterLowering() {
+  if (parameter_count_after_lowering_ == -1) {
+    // GetParameterIndexAfterLowering(parameter_count) returns the parameter
+    // count after lowering.
+    parameter_count_after_lowering_ = GetParameterIndexAfterLowering(
+        signature(), static_cast<int>(signature()->parameter_count()));
+  }
+  return parameter_count_after_lowering_;
+}
+
+static int GetReturnCountAfterLowering(
+    Signature<MachineRepresentation>* signature) {
+  int result = static_cast<int>(signature->return_count());
+  for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
+    if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
+      result += 3;
+    }
+  }
+  return result;
+}
+
+void SimdScalarLowering::LowerNode(Node* node) {
+  SimdType rep_type = ReplacementType(node);
+  switch (node->opcode()) {
+    case IrOpcode::kStart: {
+      int parameter_count = GetParameterCountAfterLowering();
+      // Only exchange the node if the parameter count actually changed.
+      if (parameter_count != signature()->parameter_count()) {
+        int delta =
+            parameter_count - static_cast<int>(signature()->parameter_count());
+        int new_output_count = node->op()->ValueOutputCount() + delta;
+        NodeProperties::ChangeOp(node, common()->Start(new_output_count));
+      }
+      break;
+    }
+    case IrOpcode::kParameter: {
+      DCHECK(node->InputCount() == 1);
+      // Only exchange the node if the parameter count actually changed. We do
+      // not even have to do the default lowering because the the start node,
+      // the only input of a parameter node, only changes if the parameter count
+      // changes.
+      if (GetParameterCountAfterLowering() != signature()->parameter_count()) {
+        int old_index = ParameterIndexOf(node->op());
+        int new_index = GetParameterIndexAfterLowering(signature(), old_index);
+        if (old_index == new_index) {
+          NodeProperties::ChangeOp(node, common()->Parameter(new_index));
+
+          Node* new_node[kMaxLanes];
+          for (int i = 0; i < kMaxLanes; i++) {
+            new_node[i] = nullptr;
+          }
+          new_node[0] = node;
+          if (signature()->GetParam(old_index) ==
+              MachineRepresentation::kSimd128) {
+            for (int i = 1; i < kMaxLanes; i++) {
+              new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
+                                             graph()->start());
+            }
+          }
+          ReplaceNode(node, new_node);
+        }
+      }
+      break;
+    }
+    case IrOpcode::kReturn: {
+      DefaultLowering(node);
+      int new_return_count = GetReturnCountAfterLowering(signature());
+      if (signature()->return_count() != new_return_count) {
+        NodeProperties::ChangeOp(node, common()->Return(new_return_count));
+      }
+      break;
+    }
+    case IrOpcode::kCall: {
+      // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor.
+      CallDescriptor* descriptor =
+          const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
+      if (DefaultLowering(node) ||
+          (descriptor->ReturnCount() == 1 &&
+           descriptor->GetReturnType(0) == MachineType::Simd128())) {
+        // We have to adjust the call descriptor.
+        const Operator* op =
+            common()->Call(wasm::ModuleEnv::GetI32WasmCallDescriptorForSimd(
+                zone(), descriptor));
+        NodeProperties::ChangeOp(node, op);
+      }
+      if (descriptor->ReturnCount() == 1 &&
+          descriptor->GetReturnType(0) == MachineType::Simd128()) {
+        // We access the additional return values through projections.
+        Node* rep_node[kMaxLanes];
+        for (int i = 0; i < kMaxLanes; i++) {
+          rep_node[i] =
+              graph()->NewNode(common()->Projection(i), node, graph()->start());
+        }
+        ReplaceNode(node, rep_node);
+      }
+      break;
+    }
+    case IrOpcode::kPhi: {
+      MachineRepresentation rep = PhiRepresentationOf(node->op());
+      if (rep == MachineRepresentation::kSimd128) {
+        // The replacement nodes have already been created, we only have to
+        // replace placeholder nodes.
+        Node** rep_node = GetReplacements(node);
+        for (int i = 0; i < node->op()->ValueInputCount(); i++) {
+          Node** rep_input =
+              GetReplacementsWithType(node->InputAt(i), rep_type);
+          for (int j = 0; j < kMaxLanes; j++) {
+            rep_node[j]->ReplaceInput(i, rep_input[j]);
+          }
+        }
+      } else {
+        DefaultLowering(node);
+      }
+      break;
+    }
+
+    case IrOpcode::kInt32x4Add: {
+      DCHECK(node->InputCount() == 2);
+      Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
+      Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
+      Node* rep_node[kMaxLanes];
+      for (int i = 0; i < kMaxLanes; i++) {
+        rep_node[i] =
+            graph()->NewNode(machine()->Int32Add(), rep_left[i], rep_right[i]);
+      }
+      ReplaceNode(node, rep_node);
+      break;
+    }
+
+    case IrOpcode::kCreateInt32x4: {
+      Node* rep_node[kMaxLanes];
+      for (int i = 0; i < kMaxLanes; i++) {
+        DCHECK(!HasReplacement(1, node->InputAt(i)));
+        rep_node[i] = node->InputAt(i);
+      }
+      ReplaceNode(node, rep_node);
+      break;
+    }
+
+    case IrOpcode::kInt32x4ExtractLane: {
+      Node* laneNode = node->InputAt(1);
+      DCHECK_EQ(laneNode->opcode(), IrOpcode::kInt32Constant);
+      int32_t lane = OpParameter<int32_t>(laneNode);
+      Node* rep_node[kMaxLanes] = {
+          GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr,
+          nullptr, nullptr};
+      ReplaceNode(node, rep_node);
+      break;
+    }
+
+    case IrOpcode::kFloat32x4Add: {
+      DCHECK(node->InputCount() == 2);
+      Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
+      Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
+      Node* rep_node[kMaxLanes];
+      for (int i = 0; i < kMaxLanes; i++) {
+        rep_node[i] = graph()->NewNode(machine()->Float32Add(), rep_left[i],
+                                       rep_right[i]);
+      }
+      ReplaceNode(node, rep_node);
+      break;
+    }
+
+    case IrOpcode::kCreateFloat32x4: {
+      Node* rep_node[kMaxLanes];
+      for (int i = 0; i < kMaxLanes; i++) {
+        DCHECK(!HasReplacement(1, node->InputAt(i)));
+        rep_node[i] = node->InputAt(i);
+      }
+      ReplaceNode(node, rep_node);
+      break;
+    }
+
+    case IrOpcode::kFloat32x4ExtractLane: {
+      Node* laneNode = node->InputAt(1);
+      DCHECK_EQ(laneNode->opcode(), IrOpcode::kInt32Constant);
+      int32_t lane = OpParameter<int32_t>(laneNode);
+      Node* rep_node[kMaxLanes] = {
+          GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr,
+          nullptr, nullptr};
+      ReplaceNode(node, rep_node);
+      break;
+    }
+
+    default: { DefaultLowering(node); }
+  }
+}
+
+bool SimdScalarLowering::DefaultLowering(Node* node) {
+  bool something_changed = false;
+  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
+    Node* input = node->InputAt(i);
+    if (HasReplacement(0, input)) {
+      something_changed = true;
+      node->ReplaceInput(i, GetReplacements(input)[0]);
+    }
+    if (HasReplacement(1, input)) {
+      something_changed = true;
+      for (int j = 1; j < kMaxLanes; j++) {
+        node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
+      }
+    }
+  }
+  return something_changed;
+}
+
+void SimdScalarLowering::ReplaceNode(Node* old, Node** new_node) {
+  // if new_low == nullptr, then also new_high == nullptr.
+  DCHECK(new_node[0] != nullptr ||
+         (new_node[1] == nullptr && new_node[2] == nullptr &&
+          new_node[3] == nullptr));
+  for (int i = 0; i < kMaxLanes; i++) {
+    replacements_[old->id()].node[i] = new_node[i];
+  }
+}
+
+bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
+  return replacements_[node->id()].node[index] != nullptr;
+}
+
+SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
+  return replacements_[node->id()].type;
+}
+
+Node** SimdScalarLowering::GetReplacements(Node* node) {
+  Node** result = replacements_[node->id()].node;
+  DCHECK(result);
+  return result;
+}
+
+Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
+  Node** replacements = GetReplacements(node);
+  if (ReplacementType(node) == type) {
+    return GetReplacements(node);
+  }
+  Node** result = zone()->NewArray<Node*>(kMaxLanes);
+  if (ReplacementType(node) == SimdType::kInt32 && type == SimdType::kFloat32) {
+    for (int i = 0; i < kMaxLanes; i++) {
+      if (replacements[i] != nullptr) {
+        result[i] = graph()->NewNode(machine()->BitcastInt32ToFloat32(),
+                                     replacements[i]);
+      } else {
+        result[i] = nullptr;
+      }
+    }
+  } else {
+    for (int i = 0; i < kMaxLanes; i++) {
+      if (replacements[i] != nullptr) {
+        result[i] = graph()->NewNode(machine()->BitcastFloat32ToInt32(),
+                                     replacements[i]);
+      } else {
+        result[i] = nullptr;
+      }
+    }
+  }
+  return result;
+}
+
+void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
+  MachineRepresentation rep = PhiRepresentationOf(phi->op());
+  if (rep == MachineRepresentation::kSimd128) {
+    // We have to create the replacements for a phi node before we actually
+    // lower the phi to break potential cycles in the graph. The replacements of
+    // input nodes do not exist yet, so we use a placeholder node to pass the
+    // graph verifier.
+    int value_count = phi->op()->ValueInputCount();
+    SimdType type = ReplacementType(phi);
+    Node** inputs_rep[kMaxLanes];
+    for (int i = 0; i < kMaxLanes; i++) {
+      inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
+      inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
+    }
+    for (int i = 0; i < value_count; i++) {
+      for (int j = 0; j < kMaxLanes; j++) {
+        inputs_rep[j][i] = placeholder_;
+      }
+    }
+    Node* rep_nodes[kMaxLanes];
+    for (int i = 0; i < kMaxLanes; i++) {
+      if (type == SimdType::kInt32) {
+        rep_nodes[i] = graph()->NewNode(
+            common()->Phi(MachineRepresentation::kWord32, value_count),
+            value_count + 1, inputs_rep[i], false);
+      } else if (type == SimdType::kFloat32) {
+        rep_nodes[i] = graph()->NewNode(
+            common()->Phi(MachineRepresentation::kFloat32, value_count),
+            value_count + 1, inputs_rep[i], false);
+      } else {
+        UNREACHABLE();
+      }
+    }
+    ReplaceNode(phi, rep_nodes);
+  }
+}
+}  // namespace compiler
+}  // namespace internal
+}  // namespace v8