OLD | NEW |
---|---|
1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
2 // | 2 // |
3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
4 // | 4 // |
5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
7 // | 7 // |
8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
9 /// | 9 /// |
10 /// \file | 10 /// \file |
(...skipping 779 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
790 if (Var->mustHaveReg()) { | 790 if (Var->mustHaveReg()) { |
791 llvm_unreachable("Infinite-weight Variable has no register assigned"); | 791 llvm_unreachable("Infinite-weight Variable has no register assigned"); |
792 } | 792 } |
793 int32_t Offset = Var->getStackOffset(); | 793 int32_t Offset = Var->getStackOffset(); |
794 int32_t BaseRegNum = Var->getBaseRegNum(); | 794 int32_t BaseRegNum = Var->getBaseRegNum(); |
795 if (Var->getBaseRegNum() == Variable::NoRegister) { | 795 if (Var->getBaseRegNum() == Variable::NoRegister) { |
796 BaseRegNum = getFrameOrStackReg(); | 796 BaseRegNum = getFrameOrStackReg(); |
797 if (!hasFramePointer()) | 797 if (!hasFramePointer()) |
798 Offset += getStackAdjustment(); | 798 Offset += getStackAdjustment(); |
799 } | 799 } |
800 static constexpr AssemblerFixup *Fixup = nullptr; | |
800 return typename Traits::Address( | 801 return typename Traits::Address( |
801 Traits::RegisterSet::getEncodedGPR(BaseRegNum), Offset); | 802 Traits::RegisterSet::getEncodedGPR(BaseRegNum), Offset, Fixup); |
802 } | 803 } |
803 | 804 |
804 /// Helper function for addProlog(). | 805 /// Helper function for addProlog(). |
805 /// | 806 /// |
806 /// This assumes Arg is an argument passed on the stack. This sets the frame | 807 /// This assumes Arg is an argument passed on the stack. This sets the frame |
807 /// offset for Arg and updates InArgsSizeBytes according to Arg's width. For an | 808 /// offset for Arg and updates InArgsSizeBytes according to Arg's width. For an |
808 /// I64 arg that has been split into Lo and Hi components, it calls itself | 809 /// I64 arg that has been split into Lo and Hi components, it calls itself |
809 /// recursively on the components, taking care to handle Lo first because of the | 810 /// recursively on the components, taking care to handle Lo first because of the |
810 /// little-endian architecture. Lastly, this function generates an instruction | 811 /// little-endian architecture. Lastly, this function generates an instruction |
811 /// to copy Arg into its assigned register if applicable. | 812 /// to copy Arg into its assigned register if applicable. |
(...skipping 2032 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2844 // Optimize comparisons with zero. | 2845 // Optimize comparisons with zero. |
2845 if (isZero(Src1)) { | 2846 if (isZero(Src1)) { |
2846 Constant *SignMask = Ctx->getConstantInt32(0x80000000); | 2847 Constant *SignMask = Ctx->getConstantInt32(0x80000000); |
2847 Variable *Temp = nullptr; | 2848 Variable *Temp = nullptr; |
2848 switch (Condition) { | 2849 switch (Condition) { |
2849 default: | 2850 default: |
2850 llvm_unreachable("unexpected condition"); | 2851 llvm_unreachable("unexpected condition"); |
2851 break; | 2852 break; |
2852 case InstIcmp::Eq: | 2853 case InstIcmp::Eq: |
2853 case InstIcmp::Ule: | 2854 case InstIcmp::Ule: |
2854 _mov(Temp, Src0LoRM); | 2855 // Mov Src0HiRM first, because it was legalized most recently, and will |
2855 _or(Temp, Src0HiRM); | 2856 // sometimes avoid a move before the OR. |
2857 _mov(Temp, Src0HiRM); | |
2858 _or(Temp, Src0LoRM); | |
2856 Context.insert(InstFakeUse::create(Func, Temp)); | 2859 Context.insert(InstFakeUse::create(Func, Temp)); |
2857 setccOrBr(Traits::Cond::Br_e, Dest, Br); | 2860 setccOrBr(Traits::Cond::Br_e, Dest, Br); |
2858 return; | 2861 return; |
2859 case InstIcmp::Ne: | 2862 case InstIcmp::Ne: |
2860 case InstIcmp::Ugt: | 2863 case InstIcmp::Ugt: |
2861 _mov(Temp, Src0LoRM); | 2864 // Mov Src0HiRM first, because it was legalized most recently, and will |
2862 _or(Temp, Src0HiRM); | 2865 // sometimes avoid a move before the OR. |
2866 _mov(Temp, Src0HiRM); | |
2867 _or(Temp, Src0LoRM); | |
2863 Context.insert(InstFakeUse::create(Func, Temp)); | 2868 Context.insert(InstFakeUse::create(Func, Temp)); |
2864 setccOrBr(Traits::Cond::Br_ne, Dest, Br); | 2869 setccOrBr(Traits::Cond::Br_ne, Dest, Br); |
2865 return; | 2870 return; |
2866 case InstIcmp::Uge: | 2871 case InstIcmp::Uge: |
2867 movOrBr(true, Dest, Br); | 2872 movOrBr(true, Dest, Br); |
2868 return; | 2873 return; |
2869 case InstIcmp::Ult: | 2874 case InstIcmp::Ult: |
2870 movOrBr(false, Dest, Br); | 2875 movOrBr(false, Dest, Br); |
2871 return; | 2876 return; |
2872 case InstIcmp::Sgt: | 2877 case InstIcmp::Sgt: |
(...skipping 1214 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
4087 const SizeT BundleSize = | 4092 const SizeT BundleSize = |
4088 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); | 4093 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); |
4089 _and(Target, Ctx->getConstantInt32(~(BundleSize - 1))); | 4094 _and(Target, Ctx->getConstantInt32(~(BundleSize - 1))); |
4090 } | 4095 } |
4091 _jmp(Target); | 4096 _jmp(Target); |
4092 if (NeedSandboxing) | 4097 if (NeedSandboxing) |
4093 _bundle_unlock(); | 4098 _bundle_unlock(); |
4094 } | 4099 } |
4095 | 4100 |
4096 inline bool isAdd(const Inst *Inst) { | 4101 inline bool isAdd(const Inst *Inst) { |
4097 if (const InstArithmetic *Arith = | 4102 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { |
4098 llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { | |
4099 return (Arith->getOp() == InstArithmetic::Add); | 4103 return (Arith->getOp() == InstArithmetic::Add); |
4100 } | 4104 } |
4101 return false; | 4105 return false; |
4102 } | 4106 } |
4103 | 4107 |
4104 inline void dumpAddressOpt(const Cfg *Func, const Variable *Base, | 4108 inline void dumpAddressOpt(const Cfg *Func, |
4109 const ConstantRelocatable *Relocatable, | |
4110 int32_t Offset, const Variable *Base, | |
4105 const Variable *Index, uint16_t Shift, | 4111 const Variable *Index, uint16_t Shift, |
4106 int32_t Offset, const Inst *Reason) { | 4112 const Inst *Reason) { |
4107 if (!BuildDefs::dump()) | 4113 if (!BuildDefs::dump()) |
4108 return; | 4114 return; |
4109 if (!Func->isVerbose(IceV_AddrOpt)) | 4115 if (!Func->isVerbose(IceV_AddrOpt)) |
4110 return; | 4116 return; |
4111 OstreamLocker L(Func->getContext()); | 4117 OstreamLocker L(Func->getContext()); |
4112 Ostream &Str = Func->getContext()->getStrDump(); | 4118 Ostream &Str = Func->getContext()->getStrDump(); |
4113 Str << "Instruction: "; | 4119 Str << "Instruction: "; |
4114 Reason->dumpDecorated(Func); | 4120 Reason->dumpDecorated(Func); |
4115 Str << " results in Base="; | 4121 Str << " results in Base="; |
4116 if (Base) | 4122 if (Base) |
4117 Base->dump(Func); | 4123 Base->dump(Func); |
4118 else | 4124 else |
4119 Str << "<null>"; | 4125 Str << "<null>"; |
4120 Str << ", Index="; | 4126 Str << ", Index="; |
4121 if (Index) | 4127 if (Index) |
4122 Index->dump(Func); | 4128 Index->dump(Func); |
4123 else | 4129 else |
4124 Str << "<null>"; | 4130 Str << "<null>"; |
4125 Str << ", Shift=" << Shift << ", Offset=" << Offset << "\n"; | 4131 Str << ", Shift=" << Shift << ", Offset=" << Offset |
4132 << ", Relocatable=" << Relocatable << "\n"; | |
4126 } | 4133 } |
4127 | 4134 |
4128 inline bool matchTransitiveAssign(const VariablesMetadata *VMetadata, | 4135 inline bool matchAssign(const VariablesMetadata *VMetadata, Variable *&Var, |
4129 Variable *&Var, const Inst *&Reason) { | 4136 ConstantRelocatable *&Relocatable, int32_t &Offset, |
4137 const Inst *&Reason) { | |
4130 // Var originates from Var=SrcVar ==> set Var:=SrcVar | 4138 // Var originates from Var=SrcVar ==> set Var:=SrcVar |
4131 if (Var == nullptr) | 4139 if (Var == nullptr) |
4132 return false; | 4140 return false; |
4133 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { | 4141 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { |
4134 assert(!VMetadata->isMultiDef(Var)); | 4142 assert(!VMetadata->isMultiDef(Var)); |
4135 if (llvm::isa<InstAssign>(VarAssign)) { | 4143 if (llvm::isa<InstAssign>(VarAssign)) { |
4136 Operand *SrcOp = VarAssign->getSrc(0); | 4144 Operand *SrcOp = VarAssign->getSrc(0); |
4137 assert(SrcOp); | 4145 assert(SrcOp); |
4138 if (Variable *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { | 4146 if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { |
4139 if (!VMetadata->isMultiDef(SrcVar) && | 4147 if (!VMetadata->isMultiDef(SrcVar) && |
4140 // TODO: ensure SrcVar stays single-BB | 4148 // TODO: ensure SrcVar stays single-BB |
4141 true) { | 4149 true) { |
4142 Var = SrcVar; | 4150 Var = SrcVar; |
4143 Reason = VarAssign; | 4151 Reason = VarAssign; |
4144 return true; | 4152 return true; |
4145 } | 4153 } |
4154 } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { | |
4155 int32_t MoreOffset = Const->getValue(); | |
4156 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) | |
4157 return false; | |
4158 Var = nullptr; | |
4159 Offset += MoreOffset; | |
4160 Reason = VarAssign; | |
4161 return true; | |
4162 } else if (auto *AddReloc = llvm::dyn_cast<ConstantRelocatable>(SrcOp)) { | |
4163 if (Relocatable == nullptr) { | |
4164 Var = nullptr; | |
4165 Relocatable = AddReloc; | |
4166 Reason = VarAssign; | |
4167 return true; | |
4168 } | |
4146 } | 4169 } |
4147 } | 4170 } |
4148 } | 4171 } |
4149 return false; | 4172 return false; |
4150 } | 4173 } |
4151 | 4174 |
4152 inline bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, | 4175 inline bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, |
4153 Variable *&Base, Variable *&Index, | 4176 Variable *&Base, Variable *&Index, |
4154 uint16_t &Shift, const Inst *&Reason) { | 4177 uint16_t &Shift, const Inst *&Reason) { |
4155 // Index==nullptr && Base is Base=Var1+Var2 ==> | 4178 // Index==nullptr && Base is Base=Var1+Var2 ==> |
4156 // set Base=Var1, Index=Var2, Shift=0 | 4179 // set Base=Var1, Index=Var2, Shift=0 |
4157 if (Base == nullptr) | 4180 if (Base == nullptr) |
4158 return false; | 4181 return false; |
4159 if (Index != nullptr) | 4182 if (Index != nullptr) |
4160 return false; | 4183 return false; |
4161 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); | 4184 auto *BaseInst = VMetadata->getSingleDefinition(Base); |
4162 if (BaseInst == nullptr) | 4185 if (BaseInst == nullptr) |
4163 return false; | 4186 return false; |
4164 assert(!VMetadata->isMultiDef(Base)); | 4187 assert(!VMetadata->isMultiDef(Base)); |
4165 if (BaseInst->getSrcSize() < 2) | 4188 if (BaseInst->getSrcSize() < 2) |
4166 return false; | 4189 return false; |
4167 if (Variable *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { | 4190 if (auto *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { |
4168 if (VMetadata->isMultiDef(Var1)) | 4191 if (VMetadata->isMultiDef(Var1)) |
4169 return false; | 4192 return false; |
4170 if (Variable *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { | 4193 if (auto *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { |
4171 if (VMetadata->isMultiDef(Var2)) | 4194 if (VMetadata->isMultiDef(Var2)) |
4172 return false; | 4195 return false; |
4173 if (isAdd(BaseInst) && | 4196 if (isAdd(BaseInst) && |
4174 // TODO: ensure Var1 and Var2 stay single-BB | 4197 // TODO: ensure Var1 and Var2 stay single-BB |
4175 true) { | 4198 true) { |
4176 Base = Var1; | 4199 Base = Var1; |
4177 Index = Var2; | 4200 Index = Var2; |
4178 Shift = 0; // should already have been 0 | 4201 Shift = 0; // should already have been 0 |
4179 Reason = BaseInst; | 4202 Reason = BaseInst; |
4180 return true; | 4203 return true; |
4181 } | 4204 } |
4182 } | 4205 } |
4183 } | 4206 } |
4184 return false; | 4207 return false; |
4185 } | 4208 } |
4186 | 4209 |
4187 inline bool matchShiftedIndex(const VariablesMetadata *VMetadata, | 4210 inline bool matchShiftedIndex(const VariablesMetadata *VMetadata, |
4188 Variable *&Index, uint16_t &Shift, | 4211 Variable *&Index, uint16_t &Shift, |
4189 const Inst *&Reason) { | 4212 const Inst *&Reason) { |
4190 // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> | 4213 // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> |
4191 // Index=Var, Shift+=log2(Const) | 4214 // Index=Var, Shift+=log2(Const) |
4192 if (Index == nullptr) | 4215 if (Index == nullptr) |
4193 return false; | 4216 return false; |
4194 const Inst *IndexInst = VMetadata->getSingleDefinition(Index); | 4217 auto *IndexInst = VMetadata->getSingleDefinition(Index); |
4195 if (IndexInst == nullptr) | 4218 if (IndexInst == nullptr) |
4196 return false; | 4219 return false; |
4197 assert(!VMetadata->isMultiDef(Index)); | 4220 assert(!VMetadata->isMultiDef(Index)); |
4198 if (IndexInst->getSrcSize() < 2) | 4221 if (IndexInst->getSrcSize() < 2) |
4199 return false; | 4222 return false; |
4200 if (const InstArithmetic *ArithInst = | 4223 if (auto *ArithInst = llvm::dyn_cast<InstArithmetic>(IndexInst)) { |
4201 llvm::dyn_cast<InstArithmetic>(IndexInst)) { | 4224 if (auto *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { |
4202 if (Variable *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 4225 if (auto *Const = |
4203 if (ConstantInteger32 *Const = | |
4204 llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { | 4226 llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { |
4205 if (ArithInst->getOp() == InstArithmetic::Mul && | 4227 if (VMetadata->isMultiDef(Var) || Const->getType() != IceType_i32) |
4206 !VMetadata->isMultiDef(Var) && Const->getType() == IceType_i32) { | 4228 return false; |
4207 uint64_t Mult = Const->getValue(); | 4229 switch (ArithInst->getOp()) { |
4230 default: | |
4231 return false; | |
4232 case InstArithmetic::Mul: { | |
4233 uint32_t Mult = Const->getValue(); | |
4208 uint32_t LogMult; | 4234 uint32_t LogMult; |
4209 switch (Mult) { | 4235 switch (Mult) { |
4210 case 1: | 4236 case 1: |
4211 LogMult = 0; | 4237 LogMult = 0; |
4212 break; | 4238 break; |
4213 case 2: | 4239 case 2: |
4214 LogMult = 1; | 4240 LogMult = 1; |
4215 break; | 4241 break; |
4216 case 4: | 4242 case 4: |
4217 LogMult = 2; | 4243 LogMult = 2; |
4218 break; | 4244 break; |
4219 case 8: | 4245 case 8: |
4220 LogMult = 3; | 4246 LogMult = 3; |
4221 break; | 4247 break; |
4222 default: | 4248 default: |
4223 return false; | 4249 return false; |
4224 } | 4250 } |
4225 if (Shift + LogMult <= 3) { | 4251 if (Shift + LogMult <= 3) { |
4226 Index = Var; | 4252 Index = Var; |
4227 Shift += LogMult; | 4253 Shift += LogMult; |
4228 Reason = IndexInst; | 4254 Reason = IndexInst; |
4229 return true; | 4255 return true; |
4230 } | 4256 } |
4231 } | 4257 } |
4258 case InstArithmetic::Shl: { | |
4259 uint32_t ShiftAmount = Const->getValue(); | |
4260 switch (ShiftAmount) { | |
4261 case 0: | |
4262 case 1: | |
4263 case 2: | |
4264 case 3: | |
4265 break; | |
4266 default: | |
4267 return false; | |
4268 } | |
4269 if (Shift + ShiftAmount <= 3) { | |
4270 Index = Var; | |
4271 Shift += ShiftAmount; | |
4272 Reason = IndexInst; | |
4273 return true; | |
4274 } | |
4275 } | |
4276 } | |
4232 } | 4277 } |
4233 } | 4278 } |
4234 } | 4279 } |
4235 return false; | 4280 return false; |
4236 } | 4281 } |
4237 | 4282 |
4238 inline bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable *&Base, | 4283 inline bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable *&Base, |
4239 int32_t &Offset, const Inst *&Reason) { | 4284 ConstantRelocatable *&Relocatable, int32_t &Offset, |
4285 const Inst *&Reason) { | |
4240 // Base is Base=Var+Const || Base is Base=Const+Var ==> | 4286 // Base is Base=Var+Const || Base is Base=Const+Var ==> |
4241 // set Base=Var, Offset+=Const | 4287 // set Base=Var, Offset+=Const |
4242 // Base is Base=Var-Const ==> | 4288 // Base is Base=Var-Const ==> |
4243 // set Base=Var, Offset-=Const | 4289 // set Base=Var, Offset-=Const |
4244 if (Base == nullptr) | 4290 if (Base == nullptr) { |
4245 return false; | 4291 return false; |
4292 } | |
4246 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); | 4293 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); |
4247 if (BaseInst == nullptr) | 4294 if (BaseInst == nullptr) { |
4248 return false; | 4295 return false; |
4296 } | |
4249 assert(!VMetadata->isMultiDef(Base)); | 4297 assert(!VMetadata->isMultiDef(Base)); |
4250 if (const InstArithmetic *ArithInst = | 4298 if (auto *ArithInst = llvm::dyn_cast<const InstArithmetic>(BaseInst)) { |
4251 llvm::dyn_cast<const InstArithmetic>(BaseInst)) { | |
4252 if (ArithInst->getOp() != InstArithmetic::Add && | 4299 if (ArithInst->getOp() != InstArithmetic::Add && |
4253 ArithInst->getOp() != InstArithmetic::Sub) | 4300 ArithInst->getOp() != InstArithmetic::Sub) |
4254 return false; | 4301 return false; |
4255 bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; | 4302 bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; |
4256 Variable *Var = nullptr; | 4303 Operand *Src0 = ArithInst->getSrc(0); |
4257 ConstantInteger32 *Const = nullptr; | 4304 Operand *Src1 = ArithInst->getSrc(1); |
4258 if (Variable *VariableOperand = | 4305 auto *Var0 = llvm::dyn_cast<Variable>(Src0); |
4259 llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 4306 auto *Var1 = llvm::dyn_cast<Variable>(Src1); |
4260 Var = VariableOperand; | 4307 auto *Const0 = llvm::dyn_cast<ConstantInteger32>(Src0); |
4261 Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1)); | 4308 auto *Const1 = llvm::dyn_cast<ConstantInteger32>(Src1); |
4262 } else if (IsAdd) { | 4309 auto *Reloc0 = llvm::dyn_cast<ConstantRelocatable>(Src0); |
4263 Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(0)); | 4310 auto *Reloc1 = llvm::dyn_cast<ConstantRelocatable>(Src1); |
4264 Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(1)); | 4311 Variable *NewBase = nullptr; |
4312 int32_t NewOffset = Offset; | |
4313 ConstantRelocatable *NewRelocatable = Relocatable; | |
4314 if (Var0 && Var1) | |
4315 // TODO(sehr): merge base/index splitting into here. | |
4316 return false; | |
4317 else if (!IsAdd && Var1) | |
Jim Stichnoth
2015/10/27 22:19:52
Don't use "else" after "if (...) return;".
(here a
sehr
2015/10/27 23:24:05
Done.
| |
4318 return false; | |
4319 else if (Var0) | |
4320 NewBase = Var0; | |
4321 else if (Var1) | |
4322 NewBase = Var1; | |
4323 // Don't know how to add/subtract two relocatables. | |
4324 if ((Relocatable && (Reloc0 || Reloc1)) || (Reloc0 && Reloc1)) | |
4325 return false; | |
4326 // Don't know how to subtract a relocatable. | |
4327 if (!IsAdd && Reloc1) | |
4328 return false; | |
4329 // Incorporate ConstantRelocatables. | |
4330 if (Reloc0) | |
4331 NewRelocatable = Reloc0; | |
4332 else if (Reloc1) | |
4333 NewRelocatable = Reloc1; | |
4334 // Compute the updated constant offset. | |
4335 if (Const0) { | |
4336 int32_t MoreOffset = IsAdd ? Const0->getValue() : -Const0->getValue(); | |
4337 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | |
4338 return false; | |
4339 NewOffset += MoreOffset; | |
4265 } | 4340 } |
4266 if (Var == nullptr || Const == nullptr || VMetadata->isMultiDef(Var)) | 4341 if (Const1) { |
4267 return false; | 4342 int32_t MoreOffset = IsAdd ? Const1->getValue() : -Const1->getValue(); |
4268 int32_t MoreOffset = IsAdd ? Const->getValue() : -Const->getValue(); | 4343 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
4269 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) | 4344 return false; |
4270 return false; | 4345 NewOffset += MoreOffset; |
4271 Base = Var; | 4346 } |
4272 Offset += MoreOffset; | 4347 // Update the computed address parameters once we are sure optimization |
4348 // is valid. | |
4349 Base = NewBase; | |
4350 Offset = NewOffset; | |
4351 Relocatable = NewRelocatable; | |
4273 Reason = BaseInst; | 4352 Reason = BaseInst; |
4274 return true; | 4353 return true; |
4275 } | 4354 } |
4276 return false; | 4355 return false; |
4277 } | 4356 } |
4278 | 4357 |
4279 inline void computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *&Base, | 4358 // Builds information for a canonical address expresion: |
4280 Variable *&Index, uint16_t &Shift, | 4359 // <Relocatable + Offset>(Base, Index, Shift) |
4281 int32_t &Offset) { | 4360 // On entry: |
4361 // Relocatable == null, | |
4362 // Offset == 0, | |
4363 // Base is a Variable, | |
4364 // Index == nullptr, | |
4365 // Shift == 0 | |
4366 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, | |
4367 ConstantRelocatable *&Relocatable, | |
4368 int32_t &Offset, Variable *&Base, | |
4369 Variable *&Index, uint16_t &Shift) { | |
4370 bool AddressWasOptimized = false; | |
4282 Func->resetCurrentNode(); | 4371 Func->resetCurrentNode(); |
4283 if (Func->isVerbose(IceV_AddrOpt)) { | 4372 if (Func->isVerbose(IceV_AddrOpt)) { |
4284 OstreamLocker L(Func->getContext()); | 4373 OstreamLocker L(Func->getContext()); |
4285 Ostream &Str = Func->getContext()->getStrDump(); | 4374 Ostream &Str = Func->getContext()->getStrDump(); |
4286 Str << "\nStarting computeAddressOpt for instruction:\n "; | 4375 Str << "\nStarting computeAddressOpt for instruction:\n "; |
4287 Instr->dumpDecorated(Func); | 4376 Instr->dumpDecorated(Func); |
4288 } | 4377 } |
4289 (void)Offset; // TODO: pattern-match for non-zero offsets. | |
4290 if (Base == nullptr) | 4378 if (Base == nullptr) |
4291 return; | 4379 return AddressWasOptimized; |
4292 // If the Base has more than one use or is live across multiple blocks, then | 4380 // If the Base has more than one use or is live across multiple blocks, then |
4293 // don't go further. Alternatively (?), never consider a transformation that | 4381 // don't go further. Alternatively (?), never consider a transformation that |
4294 // would change a variable that is currently *not* live across basic block | 4382 // would change a variable that is currently *not* live across basic block |
4295 // boundaries into one that *is*. | 4383 // boundaries into one that *is*. |
4296 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) | 4384 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) |
4297 return; | 4385 return AddressWasOptimized; |
4298 | 4386 |
4299 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); | 4387 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); |
4300 const VariablesMetadata *VMetadata = Func->getVMetadata(); | 4388 const VariablesMetadata *VMetadata = Func->getVMetadata(); |
4301 bool Continue = true; | 4389 const Inst *Reason = nullptr; |
4302 while (Continue) { | 4390 do { |
4303 const Inst *Reason = nullptr; | 4391 if (Reason) { |
4304 if (matchTransitiveAssign(VMetadata, Base, Reason) || | 4392 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); |
4305 matchTransitiveAssign(VMetadata, Index, Reason) || | 4393 AddressWasOptimized = true; |
4306 (!MockBounds && | 4394 Reason = nullptr; |
4307 matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) || | |
4308 (!MockBounds && matchShiftedIndex(VMetadata, Index, Shift, Reason)) || | |
4309 matchOffsetBase(VMetadata, Base, Offset, Reason)) { | |
4310 dumpAddressOpt(Func, Base, Index, Shift, Offset, Reason); | |
4311 } else { | |
4312 Continue = false; | |
4313 } | 4395 } |
4396 // Update Base and Index to follow through assignments to definitions. | |
4397 if (matchAssign(VMetadata, Base, Relocatable, Offset, Reason)) { | |
4398 // Assignments of Base from a Relocatable or ConstantInt32 can result | |
4399 // in Base becoming nullptr. To avoid code duplication in this loop we | |
4400 // prefer that Base be non-nullptr if possible. | |
4401 if ((Base == nullptr) && (Index != nullptr) && (Shift == 0)) | |
4402 std::swap(Base, Index); | |
4403 continue; | |
4404 } | |
4405 if (matchAssign(VMetadata, Index, Relocatable, Offset, Reason)) | |
4406 continue; | |
4314 | 4407 |
4315 // Index is Index=Var<<Const && Const+Shift<=3 ==> | 4408 if (!MockBounds) { |
4316 // Index=Var, Shift+=Const | 4409 // Transition from: |
4317 | 4410 // <Relocatable + Offset>(Base) to |
4318 // Index is Index=Const*Var && log2(Const)+Shift<=3 ==> | 4411 // <Relocatable + Offset>(Base, Index) |
4319 // Index=Var, Shift+=log2(Const) | 4412 if (matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) |
4320 | 4413 continue; |
4321 // Index && Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> | 4414 // Recognize multiply/shift and update Shift amount. |
4322 // swap(Index,Base) | 4415 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> |
4323 // Similar for Base=Const*Var and Base=Var<<Const | 4416 // Index=Var, Shift+=Const |
4324 | 4417 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> |
4418 // Index=Var, Shift+=log2(Const) | |
4419 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) | |
4420 continue; | |
4421 // If Shift is zero, the choice of Base and Index was purely arbitrary. | |
4422 // Recognize multiply/shift and set Shift amount. | |
4423 // Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> | |
4424 // swap(Index,Base) | |
4425 // Similar for Base=Const*Var and Base=Var<<Const | |
4426 if ((Shift == 0) && matchShiftedIndex(VMetadata, Base, Shift, Reason)) { | |
4427 std::swap(Base, Index); | |
4428 continue; | |
4429 } | |
4430 } | |
4431 // Update Offset to reflect additions/subtractions with constants and | |
4432 // relocatables. | |
4433 // TODO: consider overflow issues with respect to Offset. | |
4434 // TODO: handle symbolic constants. | |
4435 if (matchOffsetBase(VMetadata, Base, Relocatable, Offset, Reason)) | |
4436 continue; | |
4437 // TODO(sehr, stichnot): Handle updates of Index with Shift != 0. | |
4325 // Index is Index=Var+Const ==> | 4438 // Index is Index=Var+Const ==> |
4326 // set Index=Var, Offset+=(Const<<Shift) | 4439 // set Index=Var, Offset+=(Const<<Shift) |
4327 | |
4328 // Index is Index=Const+Var ==> | 4440 // Index is Index=Const+Var ==> |
4329 // set Index=Var, Offset+=(Const<<Shift) | 4441 // set Index=Var, Offset+=(Const<<Shift) |
4330 | |
4331 // Index is Index=Var-Const ==> | 4442 // Index is Index=Var-Const ==> |
4332 // set Index=Var, Offset-=(Const<<Shift) | 4443 // set Index=Var, Offset-=(Const<<Shift) |
4333 | 4444 break; |
4334 // TODO: consider overflow issues with respect to Offset. | 4445 } while (Reason); |
4335 // TODO: handle symbolic constants. | 4446 return AddressWasOptimized; |
4336 } | |
4337 } | 4447 } |
4338 | 4448 |
4339 /// Add a mock bounds check on the memory address before using it as a load or | 4449 /// Add a mock bounds check on the memory address before using it as a load or |
4340 /// store operand. The basic idea is that given a memory operand [reg], we | 4450 /// store operand. The basic idea is that given a memory operand [reg], we |
4341 /// would first add bounds-check code something like: | 4451 /// would first add bounds-check code something like: |
4342 /// | 4452 /// |
4343 /// cmp reg, <lb> | 4453 /// cmp reg, <lb> |
4344 /// jl out_of_line_error | 4454 /// jl out_of_line_error |
4345 /// cmp reg, <ub> | 4455 /// cmp reg, <ub> |
4346 /// jg out_of_line_error | 4456 /// jg out_of_line_error |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
4408 doMockBoundsCheck(Src0); | 4518 doMockBoundsCheck(Src0); |
4409 InstAssign *Assign = InstAssign::create(Func, DestLoad, Src0); | 4519 InstAssign *Assign = InstAssign::create(Func, DestLoad, Src0); |
4410 lowerAssign(Assign); | 4520 lowerAssign(Assign); |
4411 } | 4521 } |
4412 | 4522 |
4413 template <class Machine> void TargetX86Base<Machine>::doAddressOptLoad() { | 4523 template <class Machine> void TargetX86Base<Machine>::doAddressOptLoad() { |
4414 Inst *Inst = Context.getCur(); | 4524 Inst *Inst = Context.getCur(); |
4415 Variable *Dest = Inst->getDest(); | 4525 Variable *Dest = Inst->getDest(); |
4416 Operand *Addr = Inst->getSrc(0); | 4526 Operand *Addr = Inst->getSrc(0); |
4417 Variable *Index = nullptr; | 4527 Variable *Index = nullptr; |
4528 ConstantRelocatable *Relocatable = nullptr; | |
4418 uint16_t Shift = 0; | 4529 uint16_t Shift = 0; |
4419 int32_t Offset = 0; // TODO: make Constant | 4530 int32_t Offset = 0; |
4420 // Vanilla ICE load instructions should not use the segment registers, and | 4531 // Vanilla ICE load instructions should not use the segment registers, and |
4421 // computeAddressOpt only works at the level of Variables and Constants, not | 4532 // computeAddressOpt only works at the level of Variables and Constants, not |
4422 // other Traits::X86OperandMem, so there should be no mention of segment | 4533 // other Traits::X86OperandMem, so there should be no mention of segment |
4423 // registers there either. | 4534 // registers there either. |
4424 const typename Traits::X86OperandMem::SegmentRegisters SegmentReg = | 4535 const typename Traits::X86OperandMem::SegmentRegisters SegmentReg = |
4425 Traits::X86OperandMem::DefaultSegment; | 4536 Traits::X86OperandMem::DefaultSegment; |
4426 Variable *Base = llvm::dyn_cast<Variable>(Addr); | 4537 auto *Base = llvm::dyn_cast<Variable>(Addr); |
4427 computeAddressOpt(Func, Inst, Base, Index, Shift, Offset); | 4538 if (computeAddressOpt(Func, Inst, Relocatable, Offset, Base, Index, Shift)) { |
4428 if (Base && Addr != Base) { | |
4429 Inst->setDeleted(); | 4539 Inst->setDeleted(); |
4430 Constant *OffsetOp = Ctx->getConstantInt32(Offset); | 4540 Constant *OffsetOp = nullptr; |
4541 if (Relocatable == nullptr) { | |
4542 OffsetOp = Ctx->getConstantInt32(Offset); | |
4543 } else { | |
4544 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | |
4545 Relocatable->getName(), | |
4546 Relocatable->getSuppressMangling()); | |
4547 } | |
4431 Addr = Traits::X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, | 4548 Addr = Traits::X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, |
4432 Index, Shift, SegmentReg); | 4549 Index, Shift, SegmentReg); |
4433 Context.insert(InstLoad::create(Func, Dest, Addr)); | 4550 Context.insert(InstLoad::create(Func, Dest, Addr)); |
4434 } | 4551 } |
4435 } | 4552 } |
4436 | 4553 |
4437 template <class Machine> | 4554 template <class Machine> |
4438 void TargetX86Base<Machine>::randomlyInsertNop(float Probability, | 4555 void TargetX86Base<Machine>::randomlyInsertNop(float Probability, |
4439 RandomNumberGenerator &RNG) { | 4556 RandomNumberGenerator &RNG) { |
4440 RandomNumberGeneratorWrapper RNGW(RNG); | 4557 RandomNumberGeneratorWrapper RNGW(RNG); |
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
4616 Value = legalize(Value, Legal_Reg | Legal_Imm); | 4733 Value = legalize(Value, Legal_Reg | Legal_Imm); |
4617 _store(Value, NewAddr); | 4734 _store(Value, NewAddr); |
4618 } | 4735 } |
4619 } | 4736 } |
4620 | 4737 |
4621 template <class Machine> void TargetX86Base<Machine>::doAddressOptStore() { | 4738 template <class Machine> void TargetX86Base<Machine>::doAddressOptStore() { |
4622 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); | 4739 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); |
4623 Operand *Data = Inst->getData(); | 4740 Operand *Data = Inst->getData(); |
4624 Operand *Addr = Inst->getAddr(); | 4741 Operand *Addr = Inst->getAddr(); |
4625 Variable *Index = nullptr; | 4742 Variable *Index = nullptr; |
4743 ConstantRelocatable *Relocatable = nullptr; | |
4626 uint16_t Shift = 0; | 4744 uint16_t Shift = 0; |
4627 int32_t Offset = 0; // TODO: make Constant | 4745 int32_t Offset = 0; |
4628 Variable *Base = llvm::dyn_cast<Variable>(Addr); | 4746 auto *Base = llvm::dyn_cast<Variable>(Addr); |
4629 // Vanilla ICE store instructions should not use the segment registers, and | 4747 // Vanilla ICE store instructions should not use the segment registers, and |
4630 // computeAddressOpt only works at the level of Variables and Constants, not | 4748 // computeAddressOpt only works at the level of Variables and Constants, not |
4631 // other Traits::X86OperandMem, so there should be no mention of segment | 4749 // other Traits::X86OperandMem, so there should be no mention of segment |
4632 // registers there either. | 4750 // registers there either. |
4633 const typename Traits::X86OperandMem::SegmentRegisters SegmentReg = | 4751 const typename Traits::X86OperandMem::SegmentRegisters SegmentReg = |
4634 Traits::X86OperandMem::DefaultSegment; | 4752 Traits::X86OperandMem::DefaultSegment; |
4635 computeAddressOpt(Func, Inst, Base, Index, Shift, Offset); | 4753 if (computeAddressOpt(Func, Inst, Relocatable, Offset, Base, Index, Shift)) { |
4636 if (Base && Addr != Base) { | |
4637 Inst->setDeleted(); | 4754 Inst->setDeleted(); |
4638 Constant *OffsetOp = Ctx->getConstantInt32(Offset); | 4755 Constant *OffsetOp = nullptr; |
4756 if (Relocatable == nullptr) { | |
4757 OffsetOp = Ctx->getConstantInt32(Offset); | |
4758 } else { | |
4759 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | |
4760 Relocatable->getName(), | |
4761 Relocatable->getSuppressMangling()); | |
4762 } | |
4639 Addr = Traits::X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, | 4763 Addr = Traits::X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, |
4640 Index, Shift, SegmentReg); | 4764 Index, Shift, SegmentReg); |
4641 InstStore *NewStore = InstStore::create(Func, Data, Addr); | 4765 InstStore *NewStore = InstStore::create(Func, Data, Addr); |
4642 if (Inst->getDest()) | 4766 if (Inst->getDest()) |
4643 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 4767 NewStore->setRmwBeacon(Inst->getRmwBeacon()); |
4644 Context.insert(NewStore); | 4768 Context.insert(NewStore); |
4645 } | 4769 } |
4646 } | 4770 } |
4647 | 4771 |
4648 template <class Machine> | 4772 template <class Machine> |
(...skipping 992 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
5641 } | 5765 } |
5642 // the offset is not eligible for blinding or pooling, return the original | 5766 // the offset is not eligible for blinding or pooling, return the original |
5643 // mem operand | 5767 // mem operand |
5644 return MemOperand; | 5768 return MemOperand; |
5645 } | 5769 } |
5646 | 5770 |
5647 } // end of namespace X86Internal | 5771 } // end of namespace X86Internal |
5648 } // end of namespace Ice | 5772 } // end of namespace Ice |
5649 | 5773 |
5650 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 5774 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
OLD | NEW |