LLVM API Documentation
00001 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// 00002 // 00003 // The LLVM Compiler Infrastructure 00004 // 00005 // This file is distributed under the University of Illinois Open Source 00006 // License. See LICENSE.TXT for details. 00007 // 00008 //===----------------------------------------------------------------------===// 00009 // 00010 // This file defines the interfaces that X86 uses to lower LLVM code into a 00011 // selection DAG. 00012 // 00013 //===----------------------------------------------------------------------===// 00014 00015 #include "X86.h" 00016 #include "X86InstrBuilder.h" 00017 #include "X86ISelLowering.h" 00018 #include "X86TargetMachine.h" 00019 #include "llvm/CallingConv.h" 00020 #include "llvm/Constants.h" 00021 #include "llvm/DerivedTypes.h" 00022 #include "llvm/GlobalAlias.h" 00023 #include "llvm/GlobalVariable.h" 00024 #include "llvm/Function.h" 00025 #include "llvm/Instructions.h" 00026 #include "llvm/Intrinsics.h" 00027 #include "llvm/LLVMContext.h" 00028 #include "llvm/ADT/BitVector.h" 00029 #include "llvm/ADT/VectorExtras.h" 00030 #include "llvm/CodeGen/MachineFrameInfo.h" 00031 #include "llvm/CodeGen/MachineFunction.h" 00032 #include "llvm/CodeGen/MachineInstrBuilder.h" 00033 #include "llvm/CodeGen/MachineModuleInfo.h" 00034 #include "llvm/CodeGen/MachineRegisterInfo.h" 00035 #include "llvm/CodeGen/PseudoSourceValue.h" 00036 #include "llvm/Support/MathExtras.h" 00037 #include "llvm/Support/Debug.h" 00038 #include "llvm/Support/ErrorHandling.h" 00039 #include "llvm/Target/TargetLoweringObjectFile.h" 00040 #include "llvm/Target/TargetOptions.h" 00041 #include "llvm/ADT/SmallSet.h" 00042 #include "llvm/ADT/StringExtras.h" 00043 #include "llvm/Support/CommandLine.h" 00044 #include "llvm/Support/raw_ostream.h" 00045 using namespace llvm; 00046 00047 static cl::opt<bool> 00048 DisableMMX("disable-mmx", cl::Hidden, cl::desc("Disable use of MMX")); 00049 00050 // Forward declarations. 00051 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 00052 SDValue V2); 00053 00054 static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) { 00055 switch (TM.getSubtarget<X86Subtarget>().TargetType) { 00056 default: llvm_unreachable("unknown subtarget type"); 00057 case X86Subtarget::isDarwin: 00058 return new TargetLoweringObjectFileMachO(); 00059 case X86Subtarget::isELF: 00060 return new TargetLoweringObjectFileELF(); 00061 case X86Subtarget::isMingw: 00062 case X86Subtarget::isCygwin: 00063 case X86Subtarget::isWindows: 00064 return new TargetLoweringObjectFileCOFF(); 00065 } 00066 00067 } 00068 00069 X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) 00070 : TargetLowering(TM, createTLOF(TM)) { 00071 Subtarget = &TM.getSubtarget<X86Subtarget>(); 00072 X86ScalarSSEf64 = Subtarget->hasSSE2(); 00073 X86ScalarSSEf32 = Subtarget->hasSSE1(); 00074 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; 00075 00076 RegInfo = TM.getRegisterInfo(); 00077 TD = getTargetData(); 00078 00079 // Set up the TargetLowering object. 00080 00081 // X86 is weird, it always uses i8 for shift amounts and setcc results. 00082 setShiftAmountType(MVT::i8); 00083 setBooleanContents(ZeroOrOneBooleanContent); 00084 setSchedulingPreference(SchedulingForRegPressure); 00085 setStackPointerRegisterToSaveRestore(X86StackPtr); 00086 00087 if (Subtarget->isTargetDarwin()) { 00088 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. 00089 setUseUnderscoreSetJmp(false); 00090 setUseUnderscoreLongJmp(false); 00091 } else if (Subtarget->isTargetMingw()) { 00092 // MS runtime is weird: it exports _setjmp, but longjmp! 00093 setUseUnderscoreSetJmp(true); 00094 setUseUnderscoreLongJmp(false); 00095 } else { 00096 setUseUnderscoreSetJmp(true); 00097 setUseUnderscoreLongJmp(true); 00098 } 00099 00100 // Set up the register classes. 00101 addRegisterClass(MVT::i8, X86::GR8RegisterClass); 00102 addRegisterClass(MVT::i16, X86::GR16RegisterClass); 00103 addRegisterClass(MVT::i32, X86::GR32RegisterClass); 00104 if (Subtarget->is64Bit()) 00105 addRegisterClass(MVT::i64, X86::GR64RegisterClass); 00106 00107 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 00108 00109 // We don't accept any truncstore of integer registers. 00110 setTruncStoreAction(MVT::i64, MVT::i32, Expand); 00111 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 00112 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); 00113 setTruncStoreAction(MVT::i32, MVT::i16, Expand); 00114 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); 00115 setTruncStoreAction(MVT::i16, MVT::i8, Expand); 00116 00117 // SETOEQ and SETUNE require checking two conditions. 00118 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); 00119 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); 00120 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); 00121 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); 00122 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); 00123 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); 00124 00125 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this 00126 // operation. 00127 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); 00128 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); 00129 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); 00130 00131 if (Subtarget->is64Bit()) { 00132 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); 00133 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); 00134 } else if (!UseSoftFloat) { 00135 if (X86ScalarSSEf64) { 00136 // We have an impenetrably clever algorithm for ui64->double only. 00137 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); 00138 } 00139 // We have an algorithm for SSE2, and we turn this into a 64-bit 00140 // FILD for other targets. 00141 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); 00142 } 00143 00144 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have 00145 // this operation. 00146 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); 00147 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); 00148 00149 if (!UseSoftFloat) { 00150 // SSE has no i16 to fp conversion, only i32 00151 if (X86ScalarSSEf32) { 00152 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 00153 // f32 and f64 cases are Legal, f80 case is not 00154 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 00155 } else { 00156 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); 00157 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); 00158 } 00159 } else { 00160 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); 00161 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); 00162 } 00163 00164 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 00165 // are Legal, f80 is custom lowered. 00166 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); 00167 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); 00168 00169 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have 00170 // this operation. 00171 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); 00172 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); 00173 00174 if (X86ScalarSSEf32) { 00175 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); 00176 // f32 and f64 cases are Legal, f80 case is not 00177 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 00178 } else { 00179 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); 00180 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); 00181 } 00182 00183 // Handle FP_TO_UINT by promoting the destination to a larger signed 00184 // conversion. 00185 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); 00186 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); 00187 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); 00188 00189 if (Subtarget->is64Bit()) { 00190 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); 00191 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); 00192 } else if (!UseSoftFloat) { 00193 if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) 00194 // Expand FP_TO_UINT into a select. 00195 // FIXME: We would like to use a Custom expander here eventually to do 00196 // the optimal thing for SSE vs. the default expansion in the legalizer. 00197 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); 00198 else 00199 // With SSE3 we can use fisttpll to convert to a signed i64; without 00200 // SSE, we're stuck with a fistpll. 00201 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); 00202 } 00203 00204 // TODO: when we have SSE, these could be more efficient, by using movd/movq. 00205 if (!X86ScalarSSEf64) { 00206 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); 00207 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); 00208 } 00209 00210 // Scalar integer divide and remainder are lowered to use operations that 00211 // produce two results, to match the available instructions. This exposes 00212 // the two-result form to trivial CSE, which is able to combine x/y and x%y 00213 // into a single instruction. 00214 // 00215 // Scalar integer multiply-high is also lowered to use two-result 00216 // operations, to match the available instructions. However, plain multiply 00217 // (low) operations are left as Legal, as there are single-result 00218 // instructions for this in x86. Using the two-result multiply instructions 00219 // when both high and low results are needed must be arranged by dagcombine. 00220 setOperationAction(ISD::MULHS , MVT::i8 , Expand); 00221 setOperationAction(ISD::MULHU , MVT::i8 , Expand); 00222 setOperationAction(ISD::SDIV , MVT::i8 , Expand); 00223 setOperationAction(ISD::UDIV , MVT::i8 , Expand); 00224 setOperationAction(ISD::SREM , MVT::i8 , Expand); 00225 setOperationAction(ISD::UREM , MVT::i8 , Expand); 00226 setOperationAction(ISD::MULHS , MVT::i16 , Expand); 00227 setOperationAction(ISD::MULHU , MVT::i16 , Expand); 00228 setOperationAction(ISD::SDIV , MVT::i16 , Expand); 00229 setOperationAction(ISD::UDIV , MVT::i16 , Expand); 00230 setOperationAction(ISD::SREM , MVT::i16 , Expand); 00231 setOperationAction(ISD::UREM , MVT::i16 , Expand); 00232 setOperationAction(ISD::MULHS , MVT::i32 , Expand); 00233 setOperationAction(ISD::MULHU , MVT::i32 , Expand); 00234 setOperationAction(ISD::SDIV , MVT::i32 , Expand); 00235 setOperationAction(ISD::UDIV , MVT::i32 , Expand); 00236 setOperationAction(ISD::SREM , MVT::i32 , Expand); 00237 setOperationAction(ISD::UREM , MVT::i32 , Expand); 00238 setOperationAction(ISD::MULHS , MVT::i64 , Expand); 00239 setOperationAction(ISD::MULHU , MVT::i64 , Expand); 00240 setOperationAction(ISD::SDIV , MVT::i64 , Expand); 00241 setOperationAction(ISD::UDIV , MVT::i64 , Expand); 00242 setOperationAction(ISD::SREM , MVT::i64 , Expand); 00243 setOperationAction(ISD::UREM , MVT::i64 , Expand); 00244 00245 setOperationAction(ISD::BR_JT , MVT::Other, Expand); 00246 setOperationAction(ISD::BRCOND , MVT::Other, Custom); 00247 setOperationAction(ISD::BR_CC , MVT::Other, Expand); 00248 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); 00249 if (Subtarget->is64Bit()) 00250 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 00251 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); 00252 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 00253 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); 00254 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); 00255 setOperationAction(ISD::FREM , MVT::f32 , Expand); 00256 setOperationAction(ISD::FREM , MVT::f64 , Expand); 00257 setOperationAction(ISD::FREM , MVT::f80 , Expand); 00258 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); 00259 00260 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); 00261 setOperationAction(ISD::CTTZ , MVT::i8 , Custom); 00262 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); 00263 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); 00264 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); 00265 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); 00266 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); 00267 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); 00268 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); 00269 if (Subtarget->is64Bit()) { 00270 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); 00271 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); 00272 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); 00273 } 00274 00275 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); 00276 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); 00277 00278 // These should be promoted to a larger select which is supported. 00279 setOperationAction(ISD::SELECT , MVT::i1 , Promote); 00280 setOperationAction(ISD::SELECT , MVT::i8 , Promote); 00281 // X86 wants to expand cmov itself. 00282 setOperationAction(ISD::SELECT , MVT::i16 , Custom); 00283 setOperationAction(ISD::SELECT , MVT::i32 , Custom); 00284 setOperationAction(ISD::SELECT , MVT::f32 , Custom); 00285 setOperationAction(ISD::SELECT , MVT::f64 , Custom); 00286 setOperationAction(ISD::SELECT , MVT::f80 , Custom); 00287 setOperationAction(ISD::SETCC , MVT::i8 , Custom); 00288 setOperationAction(ISD::SETCC , MVT::i16 , Custom); 00289 setOperationAction(ISD::SETCC , MVT::i32 , Custom); 00290 setOperationAction(ISD::SETCC , MVT::f32 , Custom); 00291 setOperationAction(ISD::SETCC , MVT::f64 , Custom); 00292 setOperationAction(ISD::SETCC , MVT::f80 , Custom); 00293 if (Subtarget->is64Bit()) { 00294 setOperationAction(ISD::SELECT , MVT::i64 , Custom); 00295 setOperationAction(ISD::SETCC , MVT::i64 , Custom); 00296 } 00297 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); 00298 00299 // Darwin ABI issue. 00300 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); 00301 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); 00302 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); 00303 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); 00304 if (Subtarget->is64Bit()) 00305 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 00306 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); 00307 if (Subtarget->is64Bit()) { 00308 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); 00309 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); 00310 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); 00311 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); 00312 } 00313 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) 00314 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); 00315 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); 00316 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); 00317 if (Subtarget->is64Bit()) { 00318 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); 00319 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); 00320 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); 00321 } 00322 00323 if (Subtarget->hasSSE1()) 00324 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); 00325 00326 if (!Subtarget->hasSSE2()) 00327 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); 00328 00329 // Expand certain atomics 00330 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, Custom); 00331 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, Custom); 00332 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 00333 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); 00334 00335 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i8, Custom); 00336 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i16, Custom); 00337 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 00338 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); 00339 00340 if (!Subtarget->is64Bit()) { 00341 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom); 00342 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); 00343 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); 00344 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, Custom); 00345 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom); 00346 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom); 00347 setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom); 00348 } 00349 00350 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. 00351 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); 00352 // FIXME - use subtarget debug flags 00353 if (!Subtarget->isTargetDarwin() && 00354 !Subtarget->isTargetELF() && 00355 !Subtarget->isTargetCygMing()) { 00356 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); 00357 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); 00358 } 00359 00360 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); 00361 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); 00362 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); 00363 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); 00364 if (Subtarget->is64Bit()) { 00365 setExceptionPointerRegister(X86::RAX); 00366 setExceptionSelectorRegister(X86::RDX); 00367 } else { 00368 setExceptionPointerRegister(X86::EAX); 00369 setExceptionSelectorRegister(X86::EDX); 00370 } 00371 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); 00372 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); 00373 00374 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); 00375 00376 setOperationAction(ISD::TRAP, MVT::Other, Legal); 00377 00378 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 00379 setOperationAction(ISD::VASTART , MVT::Other, Custom); 00380 setOperationAction(ISD::VAEND , MVT::Other, Expand); 00381 if (Subtarget->is64Bit()) { 00382 setOperationAction(ISD::VAARG , MVT::Other, Custom); 00383 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 00384 } else { 00385 setOperationAction(ISD::VAARG , MVT::Other, Expand); 00386 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 00387 } 00388 00389 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 00390 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 00391 if (Subtarget->is64Bit()) 00392 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); 00393 if (Subtarget->isTargetCygMing()) 00394 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); 00395 else 00396 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); 00397 00398 if (!UseSoftFloat && X86ScalarSSEf64) { 00399 // f32 and f64 use SSE. 00400 // Set up the FP register classes. 00401 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 00402 addRegisterClass(MVT::f64, X86::FR64RegisterClass); 00403 00404 // Use ANDPD to simulate FABS. 00405 setOperationAction(ISD::FABS , MVT::f64, Custom); 00406 setOperationAction(ISD::FABS , MVT::f32, Custom); 00407 00408 // Use XORP to simulate FNEG. 00409 setOperationAction(ISD::FNEG , MVT::f64, Custom); 00410 setOperationAction(ISD::FNEG , MVT::f32, Custom); 00411 00412 // Use ANDPD and ORPD to simulate FCOPYSIGN. 00413 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); 00414 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 00415 00416 // We don't support sin/cos/fmod 00417 setOperationAction(ISD::FSIN , MVT::f64, Expand); 00418 setOperationAction(ISD::FCOS , MVT::f64, Expand); 00419 setOperationAction(ISD::FSIN , MVT::f32, Expand); 00420 setOperationAction(ISD::FCOS , MVT::f32, Expand); 00421 00422 // Expand FP immediates into loads from the stack, except for the special 00423 // cases we handle. 00424 addLegalFPImmediate(APFloat(+0.0)); // xorpd 00425 addLegalFPImmediate(APFloat(+0.0f)); // xorps 00426 } else if (!UseSoftFloat && X86ScalarSSEf32) { 00427 // Use SSE for f32, x87 for f64. 00428 // Set up the FP register classes. 00429 addRegisterClass(MVT::f32, X86::FR32RegisterClass); 00430 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 00431 00432 // Use ANDPS to simulate FABS. 00433 setOperationAction(ISD::FABS , MVT::f32, Custom); 00434 00435 // Use XORP to simulate FNEG. 00436 setOperationAction(ISD::FNEG , MVT::f32, Custom); 00437 00438 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 00439 00440 // Use ANDPS and ORPS to simulate FCOPYSIGN. 00441 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 00442 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); 00443 00444 // We don't support sin/cos/fmod 00445 setOperationAction(ISD::FSIN , MVT::f32, Expand); 00446 setOperationAction(ISD::FCOS , MVT::f32, Expand); 00447 00448 // Special cases we handle for FP constants. 00449 addLegalFPImmediate(APFloat(+0.0f)); // xorps 00450 addLegalFPImmediate(APFloat(+0.0)); // FLD0 00451 addLegalFPImmediate(APFloat(+1.0)); // FLD1 00452 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 00453 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 00454 00455 if (!UnsafeFPMath) { 00456 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 00457 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 00458 } 00459 } else if (!UseSoftFloat) { 00460 // f32 and f64 in x87. 00461 // Set up the FP register classes. 00462 addRegisterClass(MVT::f64, X86::RFP64RegisterClass); 00463 addRegisterClass(MVT::f32, X86::RFP32RegisterClass); 00464 00465 setOperationAction(ISD::UNDEF, MVT::f64, Expand); 00466 setOperationAction(ISD::UNDEF, MVT::f32, Expand); 00467 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 00468 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 00469 00470 if (!UnsafeFPMath) { 00471 setOperationAction(ISD::FSIN , MVT::f64 , Expand); 00472 setOperationAction(ISD::FCOS , MVT::f64 , Expand); 00473 } 00474 addLegalFPImmediate(APFloat(+0.0)); // FLD0 00475 addLegalFPImmediate(APFloat(+1.0)); // FLD1 00476 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS 00477 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS 00478 addLegalFPImmediate(APFloat(+0.0f)); // FLD0 00479 addLegalFPImmediate(APFloat(+1.0f)); // FLD1 00480 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS 00481 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS 00482 } 00483 00484 // Long double always uses X87. 00485 if (!UseSoftFloat) { 00486 addRegisterClass(MVT::f80, X86::RFP80RegisterClass); 00487 setOperationAction(ISD::UNDEF, MVT::f80, Expand); 00488 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); 00489 { 00490 bool ignored; 00491 APFloat TmpFlt(+0.0); 00492 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, 00493 &ignored); 00494 addLegalFPImmediate(TmpFlt); // FLD0 00495 TmpFlt.changeSign(); 00496 addLegalFPImmediate(TmpFlt); // FLD0/FCHS 00497 APFloat TmpFlt2(+1.0); 00498 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, 00499 &ignored); 00500 addLegalFPImmediate(TmpFlt2); // FLD1 00501 TmpFlt2.changeSign(); 00502 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS 00503 } 00504 00505 if (!UnsafeFPMath) { 00506 setOperationAction(ISD::FSIN , MVT::f80 , Expand); 00507 setOperationAction(ISD::FCOS , MVT::f80 , Expand); 00508 } 00509 } 00510 00511 // Always use a library call for pow. 00512 setOperationAction(ISD::FPOW , MVT::f32 , Expand); 00513 setOperationAction(ISD::FPOW , MVT::f64 , Expand); 00514 setOperationAction(ISD::FPOW , MVT::f80 , Expand); 00515 00516 setOperationAction(ISD::FLOG, MVT::f80, Expand); 00517 setOperationAction(ISD::FLOG2, MVT::f80, Expand); 00518 setOperationAction(ISD::FLOG10, MVT::f80, Expand); 00519 setOperationAction(ISD::FEXP, MVT::f80, Expand); 00520 setOperationAction(ISD::FEXP2, MVT::f80, Expand); 00521 00522 // First set operation action for all vector types to either promote 00523 // (for widening) or expand (for scalarization). Then we will selectively 00524 // turn on ones that can be effectively codegen'd. 00525 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 00526 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { 00527 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); 00528 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); 00529 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); 00530 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); 00531 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); 00532 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); 00533 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); 00534 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); 00535 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); 00536 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); 00537 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); 00538 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); 00539 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); 00540 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); 00541 setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand); 00542 setOperationAction(ISD::EXTRACT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand); 00543 setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand); 00544 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); 00545 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); 00546 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); 00547 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); 00548 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); 00549 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); 00550 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); 00551 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 00552 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); 00553 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); 00554 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); 00555 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); 00556 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); 00557 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); 00558 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); 00559 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); 00560 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); 00561 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); 00562 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); 00563 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); 00564 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); 00565 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); 00566 setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand); 00567 setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand); 00568 setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand); 00569 setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand); 00570 setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand); 00571 setOperationAction(ISD::FP_TO_UINT, (MVT::SimpleValueType)VT, Expand); 00572 setOperationAction(ISD::FP_TO_SINT, (MVT::SimpleValueType)VT, Expand); 00573 setOperationAction(ISD::UINT_TO_FP, (MVT::SimpleValueType)VT, Expand); 00574 setOperationAction(ISD::SINT_TO_FP, (MVT::SimpleValueType)VT, Expand); 00575 } 00576 00577 // FIXME: In order to prevent SSE instructions being expanded to MMX ones 00578 // with -msoft-float, disable use of MMX as well. 00579 if (!UseSoftFloat && !DisableMMX && Subtarget->hasMMX()) { 00580 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); 00581 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); 00582 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); 00583 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); 00584 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); 00585 00586 setOperationAction(ISD::ADD, MVT::v8i8, Legal); 00587 setOperationAction(ISD::ADD, MVT::v4i16, Legal); 00588 setOperationAction(ISD::ADD, MVT::v2i32, Legal); 00589 setOperationAction(ISD::ADD, MVT::v1i64, Legal); 00590 00591 setOperationAction(ISD::SUB, MVT::v8i8, Legal); 00592 setOperationAction(ISD::SUB, MVT::v4i16, Legal); 00593 setOperationAction(ISD::SUB, MVT::v2i32, Legal); 00594 setOperationAction(ISD::SUB, MVT::v1i64, Legal); 00595 00596 setOperationAction(ISD::MULHS, MVT::v4i16, Legal); 00597 setOperationAction(ISD::MUL, MVT::v4i16, Legal); 00598 00599 setOperationAction(ISD::AND, MVT::v8i8, Promote); 00600 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); 00601 setOperationAction(ISD::AND, MVT::v4i16, Promote); 00602 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); 00603 setOperationAction(ISD::AND, MVT::v2i32, Promote); 00604 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); 00605 setOperationAction(ISD::AND, MVT::v1i64, Legal); 00606 00607 setOperationAction(ISD::OR, MVT::v8i8, Promote); 00608 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); 00609 setOperationAction(ISD::OR, MVT::v4i16, Promote); 00610 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); 00611 setOperationAction(ISD::OR, MVT::v2i32, Promote); 00612 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); 00613 setOperationAction(ISD::OR, MVT::v1i64, Legal); 00614 00615 setOperationAction(ISD::XOR, MVT::v8i8, Promote); 00616 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); 00617 setOperationAction(ISD::XOR, MVT::v4i16, Promote); 00618 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); 00619 setOperationAction(ISD::XOR, MVT::v2i32, Promote); 00620 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); 00621 setOperationAction(ISD::XOR, MVT::v1i64, Legal); 00622 00623 setOperationAction(ISD::LOAD, MVT::v8i8, Promote); 00624 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); 00625 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); 00626 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); 00627 setOperationAction(ISD::LOAD, MVT::v2i32, Promote); 00628 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); 00629 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 00630 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); 00631 setOperationAction(ISD::LOAD, MVT::v1i64, Legal); 00632 00633 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); 00634 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); 00635 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); 00636 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); 00637 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); 00638 00639 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); 00640 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); 00641 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); 00642 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); 00643 00644 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom); 00645 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); 00646 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); 00647 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); 00648 00649 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); 00650 00651 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); 00652 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Expand); 00653 setOperationAction(ISD::SELECT, MVT::v8i8, Promote); 00654 setOperationAction(ISD::SELECT, MVT::v4i16, Promote); 00655 setOperationAction(ISD::SELECT, MVT::v2i32, Promote); 00656 setOperationAction(ISD::SELECT, MVT::v1i64, Custom); 00657 setOperationAction(ISD::VSETCC, MVT::v8i8, Custom); 00658 setOperationAction(ISD::VSETCC, MVT::v4i16, Custom); 00659 setOperationAction(ISD::VSETCC, MVT::v2i32, Custom); 00660 } 00661 00662 if (!UseSoftFloat && Subtarget->hasSSE1()) { 00663 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); 00664 00665 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 00666 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 00667 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 00668 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 00669 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 00670 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); 00671 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); 00672 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 00673 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); 00674 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 00675 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); 00676 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); 00677 } 00678 00679 if (!UseSoftFloat && Subtarget->hasSSE2()) { 00680 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); 00681 00682 // FIXME: Unfortunately -soft-float and -no-implicit-float means XMM 00683 // registers cannot be used even for integer operations. 00684 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); 00685 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); 00686 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); 00687 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); 00688 00689 setOperationAction(ISD::ADD, MVT::v16i8, Legal); 00690 setOperationAction(ISD::ADD, MVT::v8i16, Legal); 00691 setOperationAction(ISD::ADD, MVT::v4i32, Legal); 00692 setOperationAction(ISD::ADD, MVT::v2i64, Legal); 00693 setOperationAction(ISD::MUL, MVT::v2i64, Custom); 00694 setOperationAction(ISD::SUB, MVT::v16i8, Legal); 00695 setOperationAction(ISD::SUB, MVT::v8i16, Legal); 00696 setOperationAction(ISD::SUB, MVT::v4i32, Legal); 00697 setOperationAction(ISD::SUB, MVT::v2i64, Legal); 00698 setOperationAction(ISD::MUL, MVT::v8i16, Legal); 00699 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 00700 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 00701 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 00702 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 00703 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 00704 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); 00705 00706 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); 00707 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); 00708 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); 00709 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); 00710 00711 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); 00712 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); 00713 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 00714 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 00715 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 00716 00717 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 00718 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { 00719 EVT VT = (MVT::SimpleValueType)i; 00720 // Do not attempt to custom lower non-power-of-2 vectors 00721 if (!isPowerOf2_32(VT.getVectorNumElements())) 00722 continue; 00723 // Do not attempt to custom lower non-128-bit vectors 00724 if (!VT.is128BitVector()) 00725 continue; 00726 setOperationAction(ISD::BUILD_VECTOR, 00727 VT.getSimpleVT().SimpleTy, Custom); 00728 setOperationAction(ISD::VECTOR_SHUFFLE, 00729 VT.getSimpleVT().SimpleTy, Custom); 00730 setOperationAction(ISD::EXTRACT_VECTOR_ELT, 00731 VT.getSimpleVT().SimpleTy, Custom); 00732 } 00733 00734 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 00735 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 00736 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); 00737 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); 00738 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 00739 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 00740 00741 if (Subtarget->is64Bit()) { 00742 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); 00743 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); 00744 } 00745 00746 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. 00747 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; i++) { 00748 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i; 00749 EVT VT = SVT; 00750 00751 // Do not attempt to promote non-128-bit vectors 00752 if (!VT.is128BitVector()) { 00753 continue; 00754 } 00755 setOperationAction(ISD::AND, SVT, Promote); 00756 AddPromotedToType (ISD::AND, SVT, MVT::v2i64); 00757 setOperationAction(ISD::OR, SVT, Promote); 00758 AddPromotedToType (ISD::OR, SVT, MVT::v2i64); 00759 setOperationAction(ISD::XOR, SVT, Promote); 00760 AddPromotedToType (ISD::XOR, SVT, MVT::v2i64); 00761 setOperationAction(ISD::LOAD, SVT, Promote); 00762 AddPromotedToType (ISD::LOAD, SVT, MVT::v2i64); 00763 setOperationAction(ISD::SELECT, SVT, Promote); 00764 AddPromotedToType (ISD::SELECT, SVT, MVT::v2i64); 00765 } 00766 00767 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 00768 00769 // Custom lower v2i64 and v2f64 selects. 00770 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 00771 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); 00772 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); 00773 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); 00774 00775 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 00776 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 00777 if (!DisableMMX && Subtarget->hasMMX()) { 00778 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom); 00779 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); 00780 } 00781 } 00782 00783 if (Subtarget->hasSSE41()) { 00784 // FIXME: Do we need to handle scalar-to-vector here? 00785 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 00786 00787 // i8 and i16 vectors are custom , because the source register and source 00788 // source memory operand types are not the same width. f32 vectors are 00789 // custom since the immediate controlling the insert encodes additional 00790 // information. 00791 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 00792 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 00793 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 00794 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 00795 00796 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); 00797 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); 00798 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); 00799 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 00800 00801 if (Subtarget->is64Bit()) { 00802 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); 00803 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 00804 } 00805 } 00806 00807 if (Subtarget->hasSSE42()) { 00808 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); 00809 } 00810 00811 if (!UseSoftFloat && Subtarget->hasAVX()) { 00812 addRegisterClass(MVT::v8f32, X86::VR256RegisterClass); 00813 addRegisterClass(MVT::v4f64, X86::VR256RegisterClass); 00814 addRegisterClass(MVT::v8i32, X86::VR256RegisterClass); 00815 addRegisterClass(MVT::v4i64, X86::VR256RegisterClass); 00816 00817 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); 00818 setOperationAction(ISD::LOAD, MVT::v8i32, Legal); 00819 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); 00820 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); 00821 setOperationAction(ISD::FADD, MVT::v8f32, Legal); 00822 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); 00823 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); 00824 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); 00825 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); 00826 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); 00827 //setOperationAction(ISD::BUILD_VECTOR, MVT::v8f32, Custom); 00828 //setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Custom); 00829 //setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8f32, Custom); 00830 //setOperationAction(ISD::SELECT, MVT::v8f32, Custom); 00831 //setOperationAction(ISD::VSETCC, MVT::v8f32, Custom); 00832 00833 // Operations to consider commented out -v16i16 v32i8 00834 //setOperationAction(ISD::ADD, MVT::v16i16, Legal); 00835 setOperationAction(ISD::ADD, MVT::v8i32, Custom); 00836 setOperationAction(ISD::ADD, MVT::v4i64, Custom); 00837 //setOperationAction(ISD::SUB, MVT::v32i8, Legal); 00838 //setOperationAction(ISD::SUB, MVT::v16i16, Legal); 00839 setOperationAction(ISD::SUB, MVT::v8i32, Custom); 00840 setOperationAction(ISD::SUB, MVT::v4i64, Custom); 00841 //setOperationAction(ISD::MUL, MVT::v16i16, Legal); 00842 setOperationAction(ISD::FADD, MVT::v4f64, Legal); 00843 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); 00844 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); 00845 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); 00846 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); 00847 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); 00848 00849 setOperationAction(ISD::VSETCC, MVT::v4f64, Custom); 00850 // setOperationAction(ISD::VSETCC, MVT::v32i8, Custom); 00851 // setOperationAction(ISD::VSETCC, MVT::v16i16, Custom); 00852 setOperationAction(ISD::VSETCC, MVT::v8i32, Custom); 00853 00854 // setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32i8, Custom); 00855 // setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i16, Custom); 00856 // setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i16, Custom); 00857 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i32, Custom); 00858 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f32, Custom); 00859 00860 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); 00861 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i64, Custom); 00862 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f64, Custom); 00863 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i64, Custom); 00864 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f64, Custom); 00865 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f64, Custom); 00866 00867 #if 0 00868 // Not sure we want to do this since there are no 256-bit integer 00869 // operations in AVX 00870 00871 // Custom lower build_vector, vector_shuffle, and extract_vector_elt. 00872 // This includes 256-bit vectors 00873 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v4i64; ++i) { 00874 EVT VT = (MVT::SimpleValueType)i; 00875 00876 // Do not attempt to custom lower non-power-of-2 vectors 00877 if (!isPowerOf2_32(VT.getVectorNumElements())) 00878 continue; 00879 00880 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 00881 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 00882 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); 00883 } 00884 00885 if (Subtarget->is64Bit()) { 00886 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i64, Custom); 00887 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i64, Custom); 00888 } 00889 #endif 00890 00891 #if 0 00892 // Not sure we want to do this since there are no 256-bit integer 00893 // operations in AVX 00894 00895 // Promote v32i8, v16i16, v8i32 load, select, and, or, xor to v4i64. 00896 // Including 256-bit vectors 00897 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v4i64; i++) { 00898 EVT VT = (MVT::SimpleValueType)i; 00899 00900 if (!VT.is256BitVector()) { 00901 continue; 00902 } 00903 setOperationAction(ISD::AND, VT, Promote); 00904 AddPromotedToType (ISD::AND, VT, MVT::v4i64); 00905 setOperationAction(ISD::OR, VT, Promote); 00906 AddPromotedToType (ISD::OR, VT, MVT::v4i64); 00907 setOperationAction(ISD::XOR, VT, Promote); 00908 AddPromotedToType (ISD::XOR, VT, MVT::v4i64); 00909 setOperationAction(ISD::LOAD, VT, Promote); 00910 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64); 00911 setOperationAction(ISD::SELECT, VT, Promote); 00912 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64); 00913 } 00914 00915 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 00916 #endif 00917 } 00918 00919 // We want to custom lower some of our intrinsics. 00920 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 00921 00922 // Add/Sub/Mul with overflow operations are custom lowered. 00923 setOperationAction(ISD::SADDO, MVT::i32, Custom); 00924 setOperationAction(ISD::SADDO, MVT::i64, Custom); 00925 setOperationAction(ISD::UADDO, MVT::i32, Custom); 00926 setOperationAction(ISD::UADDO, MVT::i64, Custom); 00927 setOperationAction(ISD::SSUBO, MVT::i32, Custom); 00928 setOperationAction(ISD::SSUBO, MVT::i64, Custom); 00929 setOperationAction(ISD::USUBO, MVT::i32, Custom); 00930 setOperationAction(ISD::USUBO, MVT::i64, Custom); 00931 setOperationAction(ISD::SMULO, MVT::i32, Custom); 00932 setOperationAction(ISD::SMULO, MVT::i64, Custom); 00933 00934 if (!Subtarget->is64Bit()) { 00935 // These libcalls are not available in 32-bit. 00936 setLibcallName(RTLIB::SHL_I128, 0); 00937 setLibcallName(RTLIB::SRL_I128, 0); 00938 setLibcallName(RTLIB::SRA_I128, 0); 00939 } 00940 00941 // We have target-specific dag combine patterns for the following nodes: 00942 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 00943 setTargetDAGCombine(ISD::BUILD_VECTOR); 00944 setTargetDAGCombine(ISD::SELECT); 00945 setTargetDAGCombine(ISD::SHL); 00946 setTargetDAGCombine(ISD::SRA); 00947 setTargetDAGCombine(ISD::SRL); 00948 setTargetDAGCombine(ISD::STORE); 00949 setTargetDAGCombine(ISD::MEMBARRIER); 00950 if (Subtarget->is64Bit()) 00951 setTargetDAGCombine(ISD::MUL); 00952 00953 computeRegisterProperties(); 00954 00955 // FIXME: These should be based on subtarget info. Plus, the values should 00956 // be smaller when we are in optimizing for size mode. 00957 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores 00958 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores 00959 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores 00960 setPrefLoopAlignment(16); 00961 benefitFromCodePlacementOpt = true; 00962 } 00963 00964 00965 MVT::SimpleValueType X86TargetLowering::getSetCCResultType(EVT VT) const { 00966 return MVT::i8; 00967 } 00968 00969 00970 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine 00971 /// the desired ByVal argument alignment. 00972 static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { 00973 if (MaxAlign == 16) 00974 return; 00975 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { 00976 if (VTy->getBitWidth() == 128) 00977 MaxAlign = 16; 00978 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 00979 unsigned EltAlign = 0; 00980 getMaxByValAlign(ATy->getElementType(), EltAlign); 00981 if (EltAlign > MaxAlign) 00982 MaxAlign = EltAlign; 00983 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { 00984 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 00985 unsigned EltAlign = 0; 00986 getMaxByValAlign(STy->getElementType(i), EltAlign); 00987 if (EltAlign > MaxAlign) 00988 MaxAlign = EltAlign; 00989 if (MaxAlign == 16) 00990 break; 00991 } 00992 } 00993 return; 00994 } 00995 00996 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 00997 /// function arguments in the caller parameter area. For X86, aggregates 00998 /// that contain SSE vectors are placed at 16-byte boundaries while the rest 00999 /// are at 4-byte boundaries. 01000 unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { 01001 if (Subtarget->is64Bit()) { 01002 // Max of 8 and alignment of type. 01003 unsigned TyAlign = TD->getABITypeAlignment(Ty); 01004 if (TyAlign > 8) 01005 return TyAlign; 01006 return 8; 01007 } 01008 01009 unsigned Align = 4; 01010 if (Subtarget->hasSSE1()) 01011 getMaxByValAlign(Ty, Align); 01012 return Align; 01013 } 01014 01015 /// getOptimalMemOpType - Returns the target specific optimal type for load 01016 /// and store operations as a result of memset, memcpy, and memmove 01017 /// lowering. It returns MVT::iAny if SelectionDAG should be responsible for 01018 /// determining it. 01019 EVT 01020 X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, 01021 bool isSrcConst, bool isSrcStr, 01022 SelectionDAG &DAG) const { 01023 // FIXME: This turns off use of xmm stores for memset/memcpy on targets like 01024 // linux. This is because the stack realignment code can't handle certain 01025 // cases like PR2962. This should be removed when PR2962 is fixed. 01026 const Function *F = DAG.getMachineFunction().getFunction(); 01027 bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat); 01028 if (!NoImplicitFloatOps && Subtarget->getStackAlignment() >= 16) { 01029 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) 01030 return MVT::v4i32; 01031 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) 01032 return MVT::v4f32; 01033 } 01034 if (Subtarget->is64Bit() && Size >= 8) 01035 return MVT::i64; 01036 return MVT::i32; 01037 } 01038 01039 /// getPICJumpTableRelocaBase - Returns relocation base for the given PIC 01040 /// jumptable. 01041 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, 01042 SelectionDAG &DAG) const { 01043 if (usesGlobalOffsetTable()) 01044 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy()); 01045 if (!Subtarget->is64Bit()) 01046 // This doesn't have DebugLoc associated with it, but is not really the 01047 // same as a Register. 01048 return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc::getUnknownLoc(), 01049 getPointerTy()); 01050 return Table; 01051 } 01052 01053 /// getFunctionAlignment - Return the Log2 alignment of this function. 01054 unsigned X86TargetLowering::getFunctionAlignment(const Function *F) const { 01055 return F->hasFnAttr(Attribute::OptimizeForSize) ? 0 : 4; 01056 } 01057 01058 //===----------------------------------------------------------------------===// 01059 // Return Value Calling Convention Implementation 01060 //===----------------------------------------------------------------------===// 01061 01062 #include "X86GenCallingConv.inc" 01063 01064 SDValue 01065 X86TargetLowering::LowerReturn(SDValue Chain, 01066 unsigned CallConv, bool isVarArg, 01067 const SmallVectorImpl<ISD::OutputArg> &Outs, 01068 DebugLoc dl, SelectionDAG &DAG) { 01069 01070 SmallVector<CCValAssign, 16> RVLocs; 01071 CCState CCInfo(CallConv, isVarArg, getTargetMachine(), 01072 RVLocs, *DAG.getContext()); 01073 CCInfo.AnalyzeReturn(Outs, RetCC_X86); 01074 01075 // If this is the first return lowered for this function, add the regs to the 01076 // liveout set for the function. 01077 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { 01078 for (unsigned i = 0; i != RVLocs.size(); ++i) 01079 if (RVLocs[i].isRegLoc()) 01080 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 01081 } 01082 01083 SDValue Flag; 01084 01085 SmallVector<SDValue, 6> RetOps; 01086 RetOps.push_back(Chain); // Operand #0 = Chain (updated below) 01087 // Operand #1 = Bytes To Pop 01088 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); 01089 01090 // Copy the result values into the output registers. 01091 for (unsigned i = 0; i != RVLocs.size(); ++i) { 01092 CCValAssign &VA = RVLocs[i]; 01093 assert(VA.isRegLoc() && "Can only return in registers!"); 01094 SDValue ValToCopy = Outs[i].Val; 01095 01096 // Returns in ST0/ST1 are handled specially: these are pushed as operands to 01097 // the RET instruction and handled by the FP Stackifier. 01098 if (VA.getLocReg() == X86::ST0 || 01099 VA.getLocReg() == X86::ST1) { 01100 // If this is a copy from an xmm register to ST(0), use an FPExtend to 01101 // change the value to the FP stack register class. 01102 if (isScalarFPTypeInSSEReg(VA.getValVT())) 01103 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); 01104 RetOps.push_back(ValToCopy); 01105 // Don't emit a copytoreg. 01106 continue; 01107 } 01108 01109 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64 01110 // which is returned in RAX / RDX. 01111 if (Subtarget->is64Bit()) { 01112 EVT ValVT = ValToCopy.getValueType(); 01113 if (ValVT.isVector() && ValVT.getSizeInBits() == 64) { 01114 ValToCopy = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, ValToCopy); 01115 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) 01116 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, ValToCopy); 01117 } 01118 } 01119 01120 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag); 01121 Flag = Chain.getValue(1); 01122 } 01123 01124 // The x86-64 ABI for returning structs by value requires that we copy 01125 // the sret argument into %rax for the return. We saved the argument into 01126 // a virtual register in the entry block, so now we copy the value out 01127 // and into %rax. 01128 if (Subtarget->is64Bit() && 01129 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { 01130 MachineFunction &MF = DAG.getMachineFunction(); 01131 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 01132 unsigned Reg = FuncInfo->getSRetReturnReg(); 01133 if (!Reg) { 01134 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 01135 FuncInfo->setSRetReturnReg(Reg); 01136 } 01137 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy()); 01138 01139 Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Val, Flag); 01140 Flag = Chain.getValue(1); 01141 } 01142 01143 RetOps[0] = Chain; // Update chain. 01144 01145 // Add the flag if we have it. 01146 if (Flag.getNode()) 01147 RetOps.push_back(Flag); 01148 01149 return DAG.getNode(X86ISD::RET_FLAG, dl, 01150 MVT::Other, &RetOps[0], RetOps.size()); 01151 } 01152 01153 /// LowerCallResult - Lower the result values of a call into the 01154 /// appropriate copies out of appropriate physical registers. 01155 /// 01156 SDValue 01157 X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag, 01158 unsigned CallConv, bool isVarArg, 01159 const SmallVectorImpl<ISD::InputArg> &Ins, 01160 DebugLoc dl, SelectionDAG &DAG, 01161 SmallVectorImpl<SDValue> &InVals) { 01162 01163 // Assign locations to each value returned by this call. 01164 SmallVector<CCValAssign, 16> RVLocs; 01165 bool Is64Bit = Subtarget->is64Bit(); 01166 CCState CCInfo(CallConv, isVarArg, getTargetMachine(), 01167 RVLocs, *DAG.getContext()); 01168 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 01169 01170 // Copy all of the result registers out of their specified physreg. 01171 for (unsigned i = 0; i != RVLocs.size(); ++i) { 01172 CCValAssign &VA = RVLocs[i]; 01173 EVT CopyVT = VA.getValVT(); 01174 01175 // If this is x86-64, and we disabled SSE, we can't return FP values 01176 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) && 01177 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) { 01178 llvm_report_error("SSE register return with SSE disabled"); 01179 } 01180 01181 // If this is a call to a function that returns an fp value on the floating 01182 // point stack, but where we prefer to use the value in xmm registers, copy 01183 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. 01184 if ((VA.getLocReg() == X86::ST0 || 01185 VA.getLocReg() == X86::ST1) && 01186 isScalarFPTypeInSSEReg(VA.getValVT())) { 01187 CopyVT = MVT::f80; 01188 } 01189 01190 SDValue Val; 01191 if (Is64Bit && CopyVT.isVector() && CopyVT.getSizeInBits() == 64) { 01192 // For x86-64, MMX values are returned in XMM0 / XMM1 except for v1i64. 01193 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) { 01194 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), 01195 MVT::v2i64, InFlag).getValue(1); 01196 Val = Chain.getValue(0); 01197 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, 01198 Val, DAG.getConstant(0, MVT::i64)); 01199 } else { 01200 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), 01201 MVT::i64, InFlag).getValue(1); 01202 Val = Chain.getValue(0); 01203 } 01204 Val = DAG.getNode(ISD::BIT_CONVERT, dl, CopyVT, Val); 01205 } else { 01206 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), 01207 CopyVT, InFlag).getValue(1); 01208 Val = Chain.getValue(0); 01209 } 01210 InFlag = Chain.getValue(2); 01211 01212 if (CopyVT != VA.getValVT()) { 01213 // Round the F80 the right size, which also moves to the appropriate xmm 01214 // register. 01215 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, 01216 // This truncation won't change the value. 01217 DAG.getIntPtrConstant(1)); 01218 } 01219 01220 InVals.push_back(Val); 01221 } 01222 01223 return Chain; 01224 } 01225 01226 01227 //===----------------------------------------------------------------------===// 01228 // C & StdCall & Fast Calling Convention implementation 01229 //===----------------------------------------------------------------------===// 01230 // StdCall calling convention seems to be standard for many Windows' API 01231 // routines and around. It differs from C calling convention just a little: 01232 // callee should clean up the stack, not caller. Symbols should be also 01233 // decorated in some fancy way :) It doesn't support any vector arguments. 01234 // For info on fast calling convention see Fast Calling Convention (tail call) 01235 // implementation LowerX86_32FastCCCallTo. 01236 01237 /// CallIsStructReturn - Determines whether a call uses struct return 01238 /// semantics. 01239 static bool CallIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) { 01240 if (Outs.empty()) 01241 return false; 01242 01243 return Outs[0].Flags.isSRet(); 01244 } 01245 01246 /// ArgsAreStructReturn - Determines whether a function uses struct 01247 /// return semantics. 01248 static bool 01249 ArgsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) { 01250 if (Ins.empty()) 01251 return false; 01252 01253 return Ins[0].Flags.isSRet(); 01254 } 01255 01256 /// IsCalleePop - Determines whether the callee is required to pop its 01257 /// own arguments. Callee pop is necessary to support tail calls. 01258 bool X86TargetLowering::IsCalleePop(bool IsVarArg, unsigned CallingConv) { 01259 if (IsVarArg) 01260 return false; 01261 01262 switch (CallingConv) { 01263 default: 01264 return false; 01265 case CallingConv::X86_StdCall: 01266 return !Subtarget->is64Bit(); 01267 case CallingConv::X86_FastCall: 01268 return !Subtarget->is64Bit(); 01269 case CallingConv::Fast: 01270 return PerformTailCallOpt; 01271 } 01272 } 01273 01274 /// CCAssignFnForNode - Selects the correct CCAssignFn for a the 01275 /// given CallingConvention value. 01276 CCAssignFn *X86TargetLowering::CCAssignFnForNode(unsigned CC) const { 01277 if (Subtarget->is64Bit()) { 01278 if (Subtarget->isTargetWin64()) 01279 return CC_X86_Win64_C; 01280 else 01281 return CC_X86_64_C; 01282 } 01283 01284 if (CC == CallingConv::X86_FastCall) 01285 return CC_X86_32_FastCall; 01286 else if (CC == CallingConv::Fast) 01287 return CC_X86_32_FastCC; 01288 else 01289 return CC_X86_32_C; 01290 } 01291 01292 /// NameDecorationForCallConv - Selects the appropriate decoration to 01293 /// apply to a MachineFunction containing a given calling convention. 01294 NameDecorationStyle 01295 X86TargetLowering::NameDecorationForCallConv(unsigned CallConv) { 01296 if (CallConv == CallingConv::X86_FastCall) 01297 return FastCall; 01298 else if (CallConv == CallingConv::X86_StdCall) 01299 return StdCall; 01300 return None; 01301 } 01302 01303 01304 /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 01305 /// by "Src" to address "Dst" with size and alignment information specified by 01306 /// the specific parameter attribute. The copy will be passed as a byval 01307 /// function parameter. 01308 static SDValue 01309 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, 01310 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, 01311 DebugLoc dl) { 01312 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); 01313 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 01314 /*AlwaysInline=*/true, NULL, 0, NULL, 0); 01315 } 01316 01317 SDValue 01318 X86TargetLowering::LowerMemArgument(SDValue Chain, 01319 unsigned CallConv, 01320 const SmallVectorImpl<ISD::InputArg> &Ins, 01321 DebugLoc dl, SelectionDAG &DAG, 01322 const CCValAssign &VA, 01323 MachineFrameInfo *MFI, 01324 unsigned i) { 01325 01326 // Create the nodes corresponding to a load from this parameter slot. 01327 ISD::ArgFlagsTy Flags = Ins[i].Flags; 01328 bool AlwaysUseMutable = (CallConv==CallingConv::Fast) && PerformTailCallOpt; 01329 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); 01330 EVT ValVT; 01331 01332 // If value is passed by pointer we have address passed instead of the value 01333 // itself. 01334 if (VA.getLocInfo() == CCValAssign::Indirect) 01335 ValVT = VA.getLocVT(); 01336 else 01337 ValVT = VA.getValVT(); 01338 01339 // FIXME: For now, all byval parameter objects are marked mutable. This can be 01340 // changed with more analysis. 01341 // In case of tail call optimization mark all arguments mutable. Since they 01342 // could be overwritten by lowering of arguments in case of a tail call. 01343 int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8, 01344 VA.getLocMemOffset(), isImmutable); 01345 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 01346 if (Flags.isByVal()) 01347 return FIN; 01348 return DAG.getLoad(ValVT, dl, Chain, FIN, 01349 PseudoSourceValue::getFixedStack(FI), 0); 01350 } 01351 01352 SDValue 01353 X86TargetLowering::LowerFormalArguments(SDValue Chain, 01354 unsigned CallConv, 01355 bool isVarArg, 01356 const SmallVectorImpl<ISD::InputArg> &Ins, 01357 DebugLoc dl, 01358 SelectionDAG &DAG, 01359 SmallVectorImpl<SDValue> &InVals) { 01360 01361 MachineFunction &MF = DAG.getMachineFunction(); 01362 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 01363 01364 const Function* Fn = MF.getFunction(); 01365 if (Fn->hasExternalLinkage() && 01366 Subtarget->isTargetCygMing() && 01367 Fn->getName() == "main") 01368 FuncInfo->setForceFramePointer(true); 01369 01370 // Decorate the function name. 01371 FuncInfo->setDecorationStyle(NameDecorationForCallConv(CallConv)); 01372 01373 MachineFrameInfo *MFI = MF.getFrameInfo(); 01374 bool Is64Bit = Subtarget->is64Bit(); 01375 bool IsWin64 = Subtarget->isTargetWin64(); 01376 01377 assert(!(isVarArg && CallConv == CallingConv::Fast) && 01378 "Var args not supported with calling convention fastcc"); 01379 01380 // Assign locations to all of the incoming arguments. 01381 SmallVector<CCValAssign, 16> ArgLocs; 01382 CCState CCInfo(CallConv, isVarArg, getTargetMachine(), 01383 ArgLocs, *DAG.getContext()); 01384 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForNode(CallConv)); 01385 01386 unsigned LastVal = ~0U; 01387 SDValue ArgValue; 01388 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 01389 CCValAssign &VA = ArgLocs[i]; 01390 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later 01391 // places. 01392 assert(VA.getValNo() != LastVal && 01393 "Don't support value assigned to multiple locs yet"); 01394 LastVal = VA.getValNo(); 01395 01396 if (VA.isRegLoc()) { 01397 EVT RegVT = VA.getLocVT(); 01398 TargetRegisterClass *RC = NULL; 01399 if (RegVT == MVT::i32) 01400 RC = X86::GR32RegisterClass; 01401 else if (Is64Bit && RegVT == MVT::i64) 01402 RC = X86::GR64RegisterClass; 01403 else if (RegVT == MVT::f32) 01404 RC = X86::FR32RegisterClass; 01405 else if (RegVT == MVT::f64) 01406 RC = X86::FR64RegisterClass; 01407 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) 01408 RC = X86::VR128RegisterClass; 01409 else if (RegVT.isVector() && RegVT.getSizeInBits() == 64) 01410 RC = X86::VR64RegisterClass; 01411 else 01412 llvm_unreachable("Unknown argument type!"); 01413 01414 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 01415 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); 01416 01417 // If this is an 8 or 16-bit value, it is really passed promoted to 32 01418 // bits. Insert an assert[sz]ext to capture this, then truncate to the 01419 // right size. 01420 if (VA.getLocInfo() == CCValAssign::SExt) 01421 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, 01422 DAG.getValueType(VA.getValVT())); 01423 else if (VA.getLocInfo() == CCValAssign::ZExt) 01424 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, 01425 DAG.getValueType(VA.getValVT())); 01426 else if (VA.getLocInfo() == CCValAssign::BCvt) 01427 ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue); 01428 01429 if (VA.isExtInLoc()) { 01430 // Handle MMX values passed in XMM regs. 01431 if (RegVT.isVector()) { 01432 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, 01433 ArgValue, DAG.getConstant(0, MVT::i64)); 01434 ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue); 01435 } else 01436 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); 01437 } 01438 } else { 01439 assert(VA.isMemLoc()); 01440 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i); 01441 } 01442 01443 // If value is passed via pointer - do a load. 01444 if (VA.getLocInfo() == CCValAssign::Indirect) 01445 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, NULL, 0); 01446 01447 InVals.push_back(ArgValue); 01448 } 01449 01450 // The x86-64 ABI for returning structs by value requires that we copy 01451 // the sret argument into %rax for the return. Save the argument into 01452 // a virtual register so that we can access it from the return points. 01453 if (Is64Bit && MF.getFunction()->hasStructRetAttr()) { 01454 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 01455 unsigned Reg = FuncInfo->getSRetReturnReg(); 01456 if (!Reg) { 01457 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); 01458 FuncInfo->setSRetReturnReg(Reg); 01459 } 01460 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]); 01461 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); 01462 } 01463 01464 unsigned StackSize = CCInfo.getNextStackOffset(); 01465 // align stack specially for tail calls 01466 if (PerformTailCallOpt && CallConv == CallingConv::Fast) 01467 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); 01468 01469 // If the function takes variable number of arguments, make a frame index for 01470 // the start of the first vararg value... for expansion of llvm.va_start. 01471 if (isVarArg) { 01472 if (Is64Bit || CallConv != CallingConv::X86_FastCall) { 01473 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); 01474 } 01475 if (Is64Bit) { 01476 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; 01477 01478 // FIXME: We should really autogenerate these arrays 01479 static const unsigned GPR64ArgRegsWin64[] = { 01480 X86::RCX, X86::RDX, X86::R8, X86::R9 01481 }; 01482 static const unsigned XMMArgRegsWin64[] = { 01483 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 01484 }; 01485 static const unsigned GPR64ArgRegs64Bit[] = { 01486 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 01487 }; 01488 static const unsigned XMMArgRegs64Bit[] = { 01489 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 01490 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 01491 }; 01492 const unsigned *GPR64ArgRegs, *XMMArgRegs; 01493 01494 if (IsWin64) { 01495 TotalNumIntRegs = 4; TotalNumXMMRegs = 4; 01496 GPR64ArgRegs = GPR64ArgRegsWin64; 01497 XMMArgRegs = XMMArgRegsWin64; 01498 } else { 01499 TotalNumIntRegs = 6; TotalNumXMMRegs = 8; 01500 GPR64ArgRegs = GPR64ArgRegs64Bit; 01501 XMMArgRegs = XMMArgRegs64Bit; 01502 } 01503 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 01504 TotalNumIntRegs); 01505 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 01506 TotalNumXMMRegs); 01507 01508 bool NoImplicitFloatOps = Fn->hasFnAttr(Attribute::NoImplicitFloat); 01509 assert(!(NumXMMRegs && !Subtarget->hasSSE1()) && 01510 "SSE register cannot be used when SSE is disabled!"); 01511 assert(!(NumXMMRegs && UseSoftFloat && NoImplicitFloatOps) && 01512 "SSE register cannot be used when SSE is disabled!"); 01513 if (UseSoftFloat || NoImplicitFloatOps || !Subtarget->hasSSE1()) 01514 // Kernel mode asks for SSE to be disabled, so don't push them 01515 // on the stack. 01516 TotalNumXMMRegs = 0; 01517 01518 // For X86-64, if there are vararg parameters that are passed via 01519 // registers, then we must store them to their spots on the stack so they 01520 // may be loaded by deferencing the result of va_next. 01521 VarArgsGPOffset = NumIntRegs * 8; 01522 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; 01523 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + 01524 TotalNumXMMRegs * 16, 16); 01525 01526 // Store the integer parameter registers. 01527 SmallVector<SDValue, 8> MemOps; 01528 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 01529 unsigned Offset = VarArgsGPOffset; 01530 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { 01531 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN, 01532 DAG.getIntPtrConstant(Offset)); 01533 unsigned VReg = MF.addLiveIn(GPR64ArgRegs[NumIntRegs], 01534 X86::GR64RegisterClass); 01535 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 01536 SDValue Store = 01537 DAG.getStore(Val.getValue(1), dl, Val, FIN, 01538 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 01539 Offset); 01540 MemOps.push_back(Store); 01541 Offset += 8; 01542 } 01543 01544 if (TotalNumXMMRegs != 0 && NumXMMRegs != TotalNumXMMRegs) { 01545 // Now store the XMM (fp + vector) parameter registers. 01546 SmallVector<SDValue, 11> SaveXMMOps; 01547 SaveXMMOps.push_back(Chain); 01548 01549 unsigned AL = MF.addLiveIn(X86::AL, X86::GR8RegisterClass); 01550 SDValue ALVal = DAG.getCopyFromReg(DAG.getEntryNode(), dl, AL, MVT::i8); 01551 SaveXMMOps.push_back(ALVal); 01552 01553 SaveXMMOps.push_back(DAG.getIntPtrConstant(RegSaveFrameIndex)); 01554 SaveXMMOps.push_back(DAG.getIntPtrConstant(VarArgsFPOffset)); 01555 01556 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { 01557 unsigned VReg = MF.addLiveIn(XMMArgRegs[NumXMMRegs], 01558 X86::VR128RegisterClass); 01559 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32); 01560 SaveXMMOps.push_back(Val); 01561 } 01562 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl, 01563 MVT::Other, 01564 &SaveXMMOps[0], SaveXMMOps.size())); 01565 } 01566 01567 if (!MemOps.empty()) 01568 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 01569 &MemOps[0], MemOps.size()); 01570 } 01571 } 01572 01573 // Some CCs need callee pop. 01574 if (IsCalleePop(isVarArg, CallConv)) { 01575 BytesToPopOnReturn = StackSize; // Callee pops everything. 01576 BytesCallerReserves = 0; 01577 } else { 01578 BytesToPopOnReturn = 0; // Callee pops nothing. 01579 // If this is an sret function, the return should pop the hidden pointer. 01580 if (!Is64Bit && CallConv != CallingConv::Fast && ArgsAreStructReturn(Ins)) 01581 BytesToPopOnReturn = 4; 01582 BytesCallerReserves = StackSize; 01583 } 01584 01585 if (!Is64Bit) { 01586 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. 01587 if (CallConv == CallingConv::X86_FastCall) 01588 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. 01589 } 01590 01591 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); 01592 01593 return Chain; 01594 } 01595 01596 SDValue 01597 X86TargetLowering::LowerMemOpCallTo(SDValue Chain, 01598 SDValue StackPtr, SDValue Arg, 01599 DebugLoc dl, SelectionDAG &DAG, 01600 const CCValAssign &VA, 01601 ISD::ArgFlagsTy Flags) { 01602 const unsigned FirstStackArgOffset = (Subtarget->isTargetWin64() ? 32 : 0); 01603 unsigned LocMemOffset = FirstStackArgOffset + VA.getLocMemOffset(); 01604 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); 01605 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); 01606 if (Flags.isByVal()) { 01607 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl); 01608 } 01609 return DAG.getStore(Chain, dl, Arg, PtrOff, 01610 PseudoSourceValue::getStack(), LocMemOffset); 01611 } 01612 01613 /// EmitTailCallLoadRetAddr - Emit a load of return address if tail call 01614 /// optimization is performed and it is required. 01615 SDValue 01616 X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, 01617 SDValue &OutRetAddr, 01618 SDValue Chain, 01619 bool IsTailCall, 01620 bool Is64Bit, 01621 int FPDiff, 01622 DebugLoc dl) { 01623 if (!IsTailCall || FPDiff==0) return Chain; 01624 01625 // Adjust the Return address stack slot. 01626 EVT VT = getPointerTy(); 01627 OutRetAddr = getReturnAddressFrameIndex(DAG); 01628 01629 // Load the "old" Return address. 01630 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, NULL, 0); 01631 return SDValue(OutRetAddr.getNode(), 1); 01632 } 01633 01634 /// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call 01635 /// optimization is performed and it is required (FPDiff!=0). 01636 static SDValue 01637 EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, 01638 SDValue Chain, SDValue RetAddrFrIdx, 01639 bool Is64Bit, int FPDiff, DebugLoc dl) { 01640 // Store the return address to the appropriate stack slot. 01641 if (!FPDiff) return Chain; 01642 // Calculate the new stack slot for the return address. 01643 int SlotSize = Is64Bit ? 8 : 4; 01644 int NewReturnAddrFI = 01645 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); 01646 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 01647 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); 01648 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, 01649 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); 01650 return Chain; 01651 } 01652 01653 SDValue 01654 X86TargetLowering::LowerCall(SDValue Chain, SDValue Callee, 01655 unsigned CallConv, bool isVarArg, bool isTailCall, 01656 const SmallVectorImpl<ISD::OutputArg> &Outs, 01657 const SmallVectorImpl<ISD::InputArg> &Ins, 01658 DebugLoc dl, SelectionDAG &DAG, 01659 SmallVectorImpl<SDValue> &InVals) { 01660 01661 MachineFunction &MF = DAG.getMachineFunction(); 01662 bool Is64Bit = Subtarget->is64Bit(); 01663 bool IsStructRet = CallIsStructReturn(Outs); 01664 01665 assert((!isTailCall || 01666 (CallConv == CallingConv::Fast && PerformTailCallOpt)) && 01667 "IsEligibleForTailCallOptimization missed a case!"); 01668 assert(!(isVarArg && CallConv == CallingConv::Fast) && 01669 "Var args not supported with calling convention fastcc"); 01670 01671 // Analyze operands of the call, assigning locations to each operand. 01672 SmallVector<CCValAssign, 16> ArgLocs; 01673 CCState CCInfo(CallConv, isVarArg, getTargetMachine(), 01674 ArgLocs, *DAG.getContext()); 01675 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForNode(CallConv)); 01676 01677 // Get a count of how many bytes are to be pushed on the stack. 01678 unsigned NumBytes = CCInfo.getNextStackOffset(); 01679 if (PerformTailCallOpt && CallConv == CallingConv::Fast) 01680 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); 01681 01682 int FPDiff = 0; 01683 if (isTailCall) { 01684 // Lower arguments at fp - stackoffset + fpdiff. 01685 unsigned NumBytesCallerPushed = 01686 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); 01687 FPDiff = NumBytesCallerPushed - NumBytes; 01688 01689 // Set the delta of movement of the returnaddr stackslot. 01690 // But only set if delta is greater than previous delta. 01691 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) 01692 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); 01693 } 01694 01695 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true)); 01696 01697 SDValue RetAddrFrIdx; 01698 // Load return adress for tail calls. 01699 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall, Is64Bit, 01700 FPDiff, dl); 01701 01702 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 01703 SmallVector<SDValue, 8> MemOpChains; 01704 SDValue StackPtr; 01705 01706 // Walk the register/memloc assignments, inserting copies/loads. In the case 01707 // of tail call optimization arguments are handle later. 01708 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 01709 CCValAssign &VA = ArgLocs[i]; 01710 EVT RegVT = VA.getLocVT(); 01711 SDValue Arg = Outs[i].Val; 01712 ISD::ArgFlagsTy Flags = Outs[i].Flags; 01713 bool isByVal = Flags.isByVal(); 01714 01715 // Promote the value if needed. 01716 switch (VA.getLocInfo()) { 01717 default: llvm_unreachable("Unknown loc info!"); 01718 case CCValAssign::Full: break; 01719 case CCValAssign::SExt: 01720 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); 01721 break; 01722 case CCValAssign::ZExt: 01723 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg); 01724 break; 01725 case CCValAssign::AExt: 01726 if (RegVT.isVector() && RegVT.getSizeInBits() == 128) { 01727 // Special case: passing MMX values in XMM registers. 01728 Arg = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, Arg); 01729 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); 01730 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); 01731 } else 01732 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg); 01733 break; 01734 case CCValAssign::BCvt: 01735 Arg = DAG.getNode(ISD::BIT_CONVERT, dl, RegVT, Arg); 01736 break; 01737 case CCValAssign::Indirect: { 01738 // Store the argument. 01739 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 01740 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 01741 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot, 01742 PseudoSourceValue::getFixedStack(FI), 0); 01743 Arg = SpillSlot; 01744 break; 01745 } 01746 } 01747 01748 if (VA.isRegLoc()) { 01749 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 01750 } else { 01751 if (!isTailCall || (isTailCall && isByVal)) { 01752 assert(VA.isMemLoc()); 01753 if (StackPtr.getNode() == 0) 01754 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy()); 01755 01756 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg, 01757 dl, DAG, VA, Flags)); 01758 } 01759 } 01760 } 01761 01762 if (!MemOpChains.empty()) 01763 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 01764 &MemOpChains[0], MemOpChains.size()); 01765 01766 // Build a sequence of copy-to-reg nodes chained together with token chain 01767 // and flag operands which copy the outgoing args into registers. 01768 SDValue InFlag; 01769 // Tail call byval lowering might overwrite argument registers so in case of 01770 // tail call optimization the copies to registers are lowered later. 01771 if (!isTailCall) 01772 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 01773 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 01774 RegsToPass[i].second, InFlag); 01775 InFlag = Chain.getValue(1); 01776 } 01777 01778 01779 if (Subtarget->isPICStyleGOT()) { 01780 // ELF / PIC requires GOT in the EBX register before function calls via PLT 01781 // GOT pointer. 01782 if (!isTailCall) { 01783 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX, 01784 DAG.getNode(X86ISD::GlobalBaseReg, 01785 DebugLoc::getUnknownLoc(), 01786 getPointerTy()), 01787 InFlag); 01788 InFlag = Chain.getValue(1); 01789 } else { 01790 // If we are tail calling and generating PIC/GOT style code load the 01791 // address of the callee into ECX. The value in ecx is used as target of 01792 // the tail jump. This is done to circumvent the ebx/callee-saved problem 01793 // for tail calls on PIC/GOT architectures. Normally we would just put the 01794 // address of GOT into ebx and then call target@PLT. But for tail calls 01795 // ebx would be restored (since ebx is callee saved) before jumping to the 01796 // target@PLT. 01797 01798 // Note: The actual moving to ECX is done further down. 01799 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 01800 if (G && !G->getGlobal()->hasHiddenVisibility() && 01801 !G->getGlobal()->hasProtectedVisibility()) 01802 Callee = LowerGlobalAddress(Callee, DAG); 01803 else if (isa<ExternalSymbolSDNode>(Callee)) 01804 Callee = LowerExternalSymbol(Callee, DAG); 01805 } 01806 } 01807 01808 if (Is64Bit && isVarArg) { 01809 // From AMD64 ABI document: 01810 // For calls that may call functions that use varargs or stdargs 01811 // (prototype-less calls or calls to functions containing ellipsis (...) in 01812 // the declaration) %al is used as hidden argument to specify the number 01813 // of SSE registers used. The contents of %al do not need to match exactly 01814 // the number of registers, but must be an ubound on the number of SSE 01815 // registers used and is in the range 0 - 8 inclusive. 01816 01817 // FIXME: Verify this on Win64 01818 // Count the number of XMM registers allocated. 01819 static const unsigned XMMArgRegs[] = { 01820 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, 01821 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 01822 }; 01823 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); 01824 assert((Subtarget->hasSSE1() || !NumXMMRegs) 01825 && "SSE registers cannot be used when SSE is disabled"); 01826 01827 Chain = DAG.getCopyToReg(Chain, dl, X86::AL, 01828 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); 01829 InFlag = Chain.getValue(1); 01830 } 01831 01832 01833 // For tail calls lower the arguments to the 'real' stack slot. 01834 if (isTailCall) { 01835 // Force all the incoming stack arguments to be loaded from the stack 01836 // before any new outgoing arguments are stored to the stack, because the 01837 // outgoing stack slots may alias the incoming argument stack slots, and 01838 // the alias isn't otherwise explicit. This is slightly more conservative 01839 // than necessary, because it means that each store effectively depends 01840 // on every argument instead of just those arguments it would clobber. 01841 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain); 01842 01843 SmallVector<SDValue, 8> MemOpChains2; 01844 SDValue FIN; 01845 int FI = 0; 01846 // Do not flag preceeding copytoreg stuff together with the following stuff. 01847 InFlag = SDValue(); 01848 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 01849 CCValAssign &VA = ArgLocs[i]; 01850 if (!VA.isRegLoc()) { 01851 assert(VA.isMemLoc()); 01852 SDValue Arg = Outs[i].Val; 01853 ISD::ArgFlagsTy Flags = Outs[i].Flags; 01854 // Create frame index. 01855 int32_t Offset = VA.getLocMemOffset()+FPDiff; 01856 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; 01857 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); 01858 FIN = DAG.getFrameIndex(FI, getPointerTy()); 01859 01860 if (Flags.isByVal()) { 01861 // Copy relative to framepointer. 01862 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); 01863 if (StackPtr.getNode() == 0) 01864 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, 01865 getPointerTy()); 01866 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source); 01867 01868 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, 01869 ArgChain, 01870 Flags, DAG, dl)); 01871 } else { 01872 // Store relative to framepointer. 01873 MemOpChains2.push_back( 01874 DAG.getStore(ArgChain, dl, Arg, FIN, 01875 PseudoSourceValue::getFixedStack(FI), 0)); 01876 } 01877 } 01878 } 01879 01880 if (!MemOpChains2.empty()) 01881 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 01882 &MemOpChains2[0], MemOpChains2.size()); 01883 01884 // Copy arguments to their registers. 01885 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 01886 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 01887 RegsToPass[i].second, InFlag); 01888 InFlag = Chain.getValue(1); 01889 } 01890 InFlag =SDValue(); 01891 01892 // Store the return address to the appropriate stack slot. 01893 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, 01894 FPDiff, dl); 01895 } 01896 01897 // If the callee is a GlobalAddress node (quite common, every direct call is) 01898 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 01899 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 01900 // We should use extra load for direct calls to dllimported functions in 01901 // non-JIT mode. 01902 GlobalValue *GV = G->getGlobal(); 01903 if (!GV->hasDLLImportLinkage()) { 01904 unsigned char OpFlags = 0; 01905 01906 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to 01907 // external symbols most go through the PLT in PIC mode. If the symbol 01908 // has hidden or protected visibility, or if it is static or local, then 01909 // we don't need to use the PLT - we can directly call it. 01910 if (Subtarget->isTargetELF() && 01911 getTargetMachine().getRelocationModel() == Reloc::PIC_ && 01912 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) { 01913 OpFlags = X86II::MO_PLT; 01914 } else if (Subtarget->isPICStyleStubAny() && 01915 (GV->isDeclaration() || GV->isWeakForLinker()) && 01916 Subtarget->getDarwinVers() < 9) { 01917 // PC-relative references to external symbols should go through $stub, 01918 // unless we're building with the leopard linker or later, which 01919 // automatically synthesizes these stubs. 01920 OpFlags = X86II::MO_DARWIN_STUB; 01921 } 01922 01923 Callee = DAG.getTargetGlobalAddress(GV, getPointerTy(), 01924 G->getOffset(), OpFlags); 01925 } 01926 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 01927 unsigned char OpFlags = 0; 01928 01929 // On ELF targets, in either X86-64 or X86-32 mode, direct calls to external 01930 // symbols should go through the PLT. 01931 if (Subtarget->isTargetELF() && 01932 getTargetMachine().getRelocationModel() == Reloc::PIC_) { 01933 OpFlags = X86II::MO_PLT; 01934 } else if (Subtarget->isPICStyleStubAny() && 01935 Subtarget->getDarwinVers() < 9) { 01936 // PC-relative references to external symbols should go through $stub, 01937 // unless we're building with the leopard linker or later, which 01938 // automatically synthesizes these stubs. 01939 OpFlags = X86II::MO_DARWIN_STUB; 01940 } 01941 01942 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(), 01943 OpFlags); 01944 } else if (isTailCall) { 01945 unsigned Opc = Is64Bit ? X86::R11 : X86::EAX; 01946 01947 Chain = DAG.getCopyToReg(Chain, dl, 01948 DAG.getRegister(Opc, getPointerTy()), 01949 Callee,InFlag); 01950 Callee = DAG.getRegister(Opc, getPointerTy()); 01951 // Add register as live out. 01952 MF.getRegInfo().addLiveOut(Opc); 01953 } 01954 01955 // Returns a chain & a flag for retval copy to use. 01956 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 01957 SmallVector<SDValue, 8> Ops; 01958 01959 if (isTailCall) { 01960 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), 01961 DAG.getIntPtrConstant(0, true), InFlag); 01962 InFlag = Chain.getValue(1); 01963 } 01964 01965 Ops.push_back(Chain); 01966 Ops.push_back(Callee); 01967 01968 if (isTailCall) 01969 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); 01970 01971 // Add argument registers to the end of the list so that they are known live 01972 // into the call. 01973 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 01974 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 01975 RegsToPass[i].second.getValueType())); 01976 01977 // Add an implicit use GOT pointer in EBX. 01978 if (!isTailCall && Subtarget->isPICStyleGOT()) 01979 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); 01980 01981 // Add an implicit use of AL for x86 vararg functions. 01982 if (Is64Bit && isVarArg) 01983 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); 01984 01985 if (InFlag.getNode()) 01986 Ops.push_back(InFlag); 01987 01988 if (isTailCall) { 01989 // If this is the first return lowered for this function, add the regs 01990 // to the liveout set for the function. 01991 if (MF.getRegInfo().liveout_empty()) { 01992 SmallVector<CCValAssign, 16> RVLocs; 01993 CCState CCInfo(CallConv, isVarArg, getTargetMachine(), RVLocs, 01994 *DAG.getContext()); 01995 CCInfo.AnalyzeCallResult(Ins, RetCC_X86); 01996 for (unsigned i = 0; i != RVLocs.size(); ++i) 01997 if (RVLocs[i].isRegLoc()) 01998 MF.getRegInfo().addLiveOut(RVLocs[i].getLocReg()); 01999 } 02000 02001 assert(((Callee.getOpcode() == ISD::Register && 02002 (cast<RegisterSDNode>(Callee)->getReg() == X86::EAX || 02003 cast<RegisterSDNode>(Callee)->getReg() == X86::R9)) || 02004 Callee.getOpcode() == ISD::TargetExternalSymbol || 02005 Callee.getOpcode() == ISD::TargetGlobalAddress) && 02006 "Expecting an global address, external symbol, or register"); 02007 02008 return DAG.getNode(X86ISD::TC_RETURN, dl, 02009 NodeTys, &Ops[0], Ops.size()); 02010 } 02011 02012 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size()); 02013 InFlag = Chain.getValue(1); 02014 02015 // Create the CALLSEQ_END node. 02016 unsigned NumBytesForCalleeToPush; 02017 if (IsCalleePop(isVarArg, CallConv)) 02018 NumBytesForCalleeToPush = NumBytes; // Callee pops everything 02019 else if (!Is64Bit && CallConv != CallingConv::Fast && IsStructRet) 02020 // If this is is a call to a struct-return function, the callee 02021 // pops the hidden struct pointer, so we have to push it back. 02022 // This is common for Darwin/X86, Linux & Mingw32 targets. 02023 NumBytesForCalleeToPush = 4; 02024 else 02025 NumBytesForCalleeToPush = 0; // Callee pops nothing. 02026 02027 // Returns a flag for retval copy to use. 02028 Chain = DAG.getCALLSEQ_END(Chain, 02029 DAG.getIntPtrConstant(NumBytes, true), 02030 DAG.getIntPtrConstant(NumBytesForCalleeToPush, 02031 true), 02032 InFlag); 02033 InFlag = Chain.getValue(1); 02034 02035 // Handle result values, copying them out of physregs into vregs that we 02036 // return. 02037 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, 02038 Ins, dl, DAG, InVals); 02039 } 02040 02041 02042 //===----------------------------------------------------------------------===// 02043 // Fast Calling Convention (tail call) implementation 02044 //===----------------------------------------------------------------------===// 02045 02046 // Like std call, callee cleans arguments, convention except that ECX is 02047 // reserved for storing the tail called function address. Only 2 registers are 02048 // free for argument passing (inreg). Tail call optimization is performed 02049 // provided: 02050 // * tailcallopt is enabled 02051 // * caller/callee are fastcc 02052 // On X86_64 architecture with GOT-style position independent code only local 02053 // (within module) calls are supported at the moment. 02054 // To keep the stack aligned according to platform abi the function 02055 // GetAlignedArgumentStackSize ensures that argument delta is always multiples 02056 // of stack alignment. (Dynamic linkers need this - darwin's dyld for example) 02057 // If a tail called function callee has more arguments than the caller the 02058 // caller needs to make sure that there is room to move the RETADDR to. This is 02059 // achieved by reserving an area the size of the argument delta right after the 02060 // original REtADDR, but before the saved framepointer or the spilled registers 02061 // e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) 02062 // stack layout: 02063 // arg1 02064 // arg2 02065 // RETADDR 02066 // [ new RETADDR 02067 // move area ] 02068 // (possible EBP) 02069 // ESI 02070 // EDI 02071 // local1 .. 02072 02073 /// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned 02074 /// for a 16 byte align requirement. 02075 unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, 02076 SelectionDAG& DAG) { 02077 MachineFunction &MF = DAG.getMachineFunction(); 02078 const TargetMachine &TM = MF.getTarget(); 02079 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 02080 unsigned StackAlignment = TFI.getStackAlignment(); 02081 uint64_t AlignMask = StackAlignment - 1; 02082 int64_t Offset = StackSize; 02083 uint64_t SlotSize = TD->getPointerSize(); 02084 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { 02085 // Number smaller than 12 so just add the difference. 02086 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); 02087 } else { 02088 // Mask out lower bits, add stackalignment once plus the 12 bytes. 02089 Offset = ((~AlignMask) & Offset) + StackAlignment + 02090 (StackAlignment-SlotSize); 02091 } 02092 return Offset; 02093 } 02094 02095 /// IsEligibleForTailCallOptimization - Check whether the call is eligible 02096 /// for tail call optimization. Targets which want to do tail call 02097 /// optimization should implement this function. 02098 bool 02099 X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 02100 unsigned CalleeCC, 02101 bool isVarArg, 02102 const SmallVectorImpl<ISD::InputArg> &Ins, 02103 SelectionDAG& DAG) const { 02104 MachineFunction &MF = DAG.getMachineFunction(); 02105 unsigned CallerCC = MF.getFunction()->getCallingConv(); 02106 return CalleeCC == CallingConv::Fast && CallerCC == CalleeCC; 02107 } 02108 02109 FastISel * 02110 X86TargetLowering::createFastISel(MachineFunction &mf, 02111 MachineModuleInfo *mmo, 02112 DwarfWriter *dw, 02113 DenseMap<const Value *, unsigned> &vm, 02114 DenseMap<const BasicBlock *, 02115 MachineBasicBlock *> &bm, 02116 DenseMap<const AllocaInst *, int> &am 02117 #ifndef NDEBUG 02118 , SmallSet<Instruction*, 8> &cil 02119 #endif 02120 ) { 02121 return X86::createFastISel(mf, mmo, dw, vm, bm, am 02122 #ifndef NDEBUG 02123 , cil 02124 #endif 02125 ); 02126 } 02127 02128 02129 //===----------------------------------------------------------------------===// 02130 // Other Lowering Hooks 02131 //===----------------------------------------------------------------------===// 02132 02133 02134 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { 02135 MachineFunction &MF = DAG.getMachineFunction(); 02136 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 02137 int ReturnAddrIndex = FuncInfo->getRAIndex(); 02138 02139 if (ReturnAddrIndex == 0) { 02140 // Set up a frame object for the return address. 02141 uint64_t SlotSize = TD->getPointerSize(); 02142 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize); 02143 FuncInfo->setRAIndex(ReturnAddrIndex); 02144 } 02145 02146 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); 02147 } 02148 02149 02150 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, 02151 bool hasSymbolicDisplacement) { 02152 // Offset should fit into 32 bit immediate field. 02153 if (!isInt32(Offset)) 02154 return false; 02155 02156 // If we don't have a symbolic displacement - we don't have any extra 02157 // restrictions. 02158 if (!hasSymbolicDisplacement) 02159 return true; 02160 02161 // FIXME: Some tweaks might be needed for medium code model. 02162 if (M != CodeModel::Small && M != CodeModel::Kernel) 02163 return false; 02164 02165 // For small code model we assume that latest object is 16MB before end of 31 02166 // bits boundary. We may also accept pretty large negative constants knowing 02167 // that all objects are in the positive half of address space. 02168 if (M == CodeModel::Small && Offset < 16*1024*1024) 02169 return true; 02170 02171 // For kernel code model we know that all object resist in the negative half 02172 // of 32bits address space. We may not accept negative offsets, since they may 02173 // be just off and we may accept pretty large positive ones. 02174 if (M == CodeModel::Kernel && Offset > 0) 02175 return true; 02176 02177 return false; 02178 } 02179 02180 /// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 02181 /// specific condition code, returning the condition code and the LHS/RHS of the 02182 /// comparison to make. 02183 static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP, 02184 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) { 02185 if (!isFP) { 02186 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 02187 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { 02188 // X > -1 -> X == 0, jump !sign. 02189 RHS = DAG.getConstant(0, RHS.getValueType()); 02190 return X86::COND_NS; 02191 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { 02192 // X < 0 -> X == 0, jump on sign. 02193 return X86::COND_S; 02194 } else if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) { 02195 // X < 1 -> X <= 0 02196 RHS = DAG.getConstant(0, RHS.getValueType()); 02197 return X86::COND_LE; 02198 } 02199 } 02200 02201 switch (SetCCOpcode) { 02202 default: llvm_unreachable("Invalid integer condition!"); 02203 case ISD::SETEQ: return X86::COND_E; 02204 case ISD::SETGT: return X86::COND_G; 02205 case ISD::SETGE: return X86::COND_GE; 02206 case ISD::SETLT: return X86::COND_L; 02207 case ISD::SETLE: return X86::COND_LE; 02208 case ISD::SETNE: return X86::COND_NE; 02209 case ISD::SETULT: return X86::COND_B; 02210 case ISD::SETUGT: return X86::COND_A; 02211 case ISD::SETULE: return X86::COND_BE; 02212 case ISD::SETUGE: return X86::COND_AE; 02213 } 02214 } 02215 02216 // First determine if it is required or is profitable to flip the operands. 02217 02218 // If LHS is a foldable load, but RHS is not, flip the condition. 02219 if ((ISD::isNON_EXTLoad(LHS.getNode()) && LHS.hasOneUse()) && 02220 !(ISD::isNON_EXTLoad(RHS.getNode()) && RHS.hasOneUse())) { 02221 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode); 02222 std::swap(LHS, RHS); 02223 } 02224 02225 switch (SetCCOpcode) { 02226 default: break; 02227 case ISD::SETOLT: 02228 case ISD::SETOLE: 02229 case ISD::SETUGT: 02230 case ISD::SETUGE: 02231 std::swap(LHS, RHS); 02232 break; 02233 } 02234 02235 // On a floating point condition, the flags are set as follows: 02236 // ZF PF CF op 02237 // 0 | 0 | 0 | X > Y 02238 // 0 | 0 | 1 | X < Y 02239 // 1 | 0 | 0 | X == Y 02240 // 1 | 1 | 1 | unordered 02241 switch (SetCCOpcode) { 02242 default: llvm_unreachable("Condcode should be pre-legalized away"); 02243 case ISD::SETUEQ: 02244 case ISD::SETEQ: return X86::COND_E; 02245 case ISD::SETOLT: // flipped 02246 case ISD::SETOGT: 02247 case ISD::SETGT: return X86::COND_A; 02248 case ISD::SETOLE: // flipped 02249 case ISD::SETOGE: 02250 case ISD::SETGE: return X86::COND_AE; 02251 case ISD::SETUGT: // flipped 02252 case ISD::SETULT: 02253 case ISD::SETLT: return X86::COND_B; 02254 case ISD::SETUGE: // flipped 02255 case ISD::SETULE: 02256 case ISD::SETLE: return X86::COND_BE; 02257 case ISD::SETONE: 02258 case ISD::SETNE: return X86::COND_NE; 02259 case ISD::SETUO: return X86::COND_P; 02260 case ISD::SETO: return X86::COND_NP; 02261 } 02262 } 02263 02264 /// hasFPCMov - is there a floating point cmov for the specific X86 condition 02265 /// code. Current x86 isa includes the following FP cmov instructions: 02266 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. 02267 static bool hasFPCMov(unsigned X86CC) { 02268 switch (X86CC) { 02269 default: 02270 return false; 02271 case X86::COND_B: 02272 case X86::COND_BE: 02273 case X86::COND_E: 02274 case X86::COND_P: 02275 case X86::COND_A: 02276 case X86::COND_AE: 02277 case X86::COND_NE: 02278 case X86::COND_NP: 02279 return true; 02280 } 02281 } 02282 02283 /// isUndefOrInRange - Return true if Val is undef or if its value falls within 02284 /// the specified range (L, H]. 02285 static bool isUndefOrInRange(int Val, int Low, int Hi) { 02286 return (Val < 0) || (Val >= Low && Val < Hi); 02287 } 02288 02289 /// isUndefOrEqual - Val is either less than zero (undef) or equal to the 02290 /// specified value. 02291 static bool isUndefOrEqual(int Val, int CmpVal) { 02292 if (Val < 0 || Val == CmpVal) 02293 return true; 02294 return false; 02295 } 02296 02297 /// isPSHUFDMask - Return true if the node specifies a shuffle of elements that 02298 /// is suitable for input to PSHUFD or PSHUFW. That is, it doesn't reference 02299 /// the second operand. 02300 static bool isPSHUFDMask(const SmallVectorImpl<int> &Mask, EVT VT) { 02301 if (VT == MVT::v4f32 || VT == MVT::v4i32 || VT == MVT::v4i16) 02302 return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4); 02303 if (VT == MVT::v2f64 || VT == MVT::v2i64) 02304 return (Mask[0] < 2 && Mask[1] < 2); 02305 return false; 02306 } 02307 02308 bool X86::isPSHUFDMask(ShuffleVectorSDNode *N) { 02309 SmallVector<int, 8> M; 02310 N->getMask(M); 02311 return ::isPSHUFDMask(M, N->getValueType(0)); 02312 } 02313 02314 /// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that 02315 /// is suitable for input to PSHUFHW. 02316 static bool isPSHUFHWMask(const SmallVectorImpl<int> &Mask, EVT VT) { 02317 if (VT != MVT::v8i16) 02318 return false; 02319 02320 // Lower quadword copied in order or undef. 02321 for (int i = 0; i != 4; ++i) 02322 if (Mask[i] >= 0 && Mask[i] != i) 02323 return false; 02324 02325 // Upper quadword shuffled. 02326 for (int i = 4; i != 8; ++i) 02327 if (Mask[i] >= 0 && (Mask[i] < 4 || Mask[i] > 7)) 02328 return false; 02329 02330 return true; 02331 } 02332 02333 bool X86::isPSHUFHWMask(ShuffleVectorSDNode *N) { 02334 SmallVector<int, 8> M; 02335 N->getMask(M); 02336 return ::isPSHUFHWMask(M, N->getValueType(0)); 02337 } 02338 02339 /// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that 02340 /// is suitable for input to PSHUFLW. 02341 static bool isPSHUFLWMask(const SmallVectorImpl<int> &Mask, EVT VT) { 02342 if (VT != MVT::v8i16) 02343 return false; 02344 02345 // Upper quadword copied in order. 02346 for (int i = 4; i != 8; ++i) 02347 if (Mask[i] >= 0 && Mask[i] != i) 02348 return false; 02349 02350 // Lower quadword shuffled. 02351 for (int i = 0; i != 4; ++i) 02352 if (Mask[i] >= 4) 02353 return false; 02354 02355 return true; 02356 } 02357 02358 bool X86::isPSHUFLWMask(ShuffleVectorSDNode *N) { 02359 SmallVector<int, 8> M; 02360 N->getMask(M); 02361 return ::isPSHUFLWMask(M, N->getValueType(0)); 02362 } 02363 02364 /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand 02365 /// specifies a shuffle of elements that is suitable for input to SHUFP*. 02366 static bool isSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) { 02367 int NumElems = VT.getVectorNumElements(); 02368 if (NumElems != 2 && NumElems != 4) 02369 return false; 02370 02371 int Half = NumElems / 2; 02372 for (int i = 0; i < Half; ++i) 02373 if (!isUndefOrInRange(Mask[i], 0, NumElems)) 02374 return false; 02375 for (int i = Half; i < NumElems; ++i) 02376 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2)) 02377 return false; 02378 02379 return true; 02380 } 02381 02382 bool X86::isSHUFPMask(ShuffleVectorSDNode *N) { 02383 SmallVector<int, 8> M; 02384 N->getMask(M); 02385 return ::isSHUFPMask(M, N->getValueType(0)); 02386 } 02387 02388 /// isCommutedSHUFP - Returns true if the shuffle mask is exactly 02389 /// the reverse of what x86 shuffles want. x86 shuffles requires the lower 02390 /// half elements to come from vector 1 (which would equal the dest.) and 02391 /// the upper half to come from vector 2. 02392 static bool isCommutedSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) { 02393 int NumElems = VT.getVectorNumElements(); 02394 02395 if (NumElems != 2 && NumElems != 4) 02396 return false; 02397 02398 int Half = NumElems / 2; 02399 for (int i = 0; i < Half; ++i) 02400 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2)) 02401 return false; 02402 for (int i = Half; i < NumElems; ++i) 02403 if (!isUndefOrInRange(Mask[i], 0, NumElems)) 02404 return false; 02405 return true; 02406 } 02407 02408 static bool isCommutedSHUFP(ShuffleVectorSDNode *N) { 02409 SmallVector<int, 8> M; 02410 N->getMask(M); 02411 return isCommutedSHUFPMask(M, N->getValueType(0)); 02412 } 02413 02414 /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand 02415 /// specifies a shuffle of elements that is suitable for input to MOVHLPS. 02416 bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) { 02417 if (N->getValueType(0).getVectorNumElements() != 4) 02418 return false; 02419 02420 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 02421 return isUndefOrEqual(N->getMaskElt(0), 6) && 02422 isUndefOrEqual(N->getMaskElt(1), 7) && 02423 isUndefOrEqual(N->getMaskElt(2), 2) && 02424 isUndefOrEqual(N->getMaskElt(3), 3); 02425 } 02426 02427 /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand 02428 /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. 02429 bool X86::isMOVLPMask(ShuffleVectorSDNode *N) { 02430 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 02431 02432 if (NumElems != 2 && NumElems != 4) 02433 return false; 02434 02435 for (unsigned i = 0; i < NumElems/2; ++i) 02436 if (!isUndefOrEqual(N->getMaskElt(i), i + NumElems)) 02437 return false; 02438 02439 for (unsigned i = NumElems/2; i < NumElems; ++i) 02440 if (!isUndefOrEqual(N->getMaskElt(i), i)) 02441 return false; 02442 02443 return true; 02444 } 02445 02446 /// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand 02447 /// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} 02448 /// and MOVLHPS. 02449 bool X86::isMOVHPMask(ShuffleVectorSDNode *N) { 02450 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 02451 02452 if (NumElems != 2 && NumElems != 4) 02453 return false; 02454 02455 for (unsigned i = 0; i < NumElems/2; ++i) 02456 if (!isUndefOrEqual(N->getMaskElt(i), i)) 02457 return false; 02458 02459 for (unsigned i = 0; i < NumElems/2; ++i) 02460 if (!isUndefOrEqual(N->getMaskElt(i + NumElems/2), i + NumElems)) 02461 return false; 02462 02463 return true; 02464 } 02465 02466 /// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form 02467 /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, 02468 /// <2, 3, 2, 3> 02469 bool X86::isMOVHLPS_v_undef_Mask(ShuffleVectorSDNode *N) { 02470 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 02471 02472 if (NumElems != 4) 02473 return false; 02474 02475 return isUndefOrEqual(N->getMaskElt(0), 2) && 02476 isUndefOrEqual(N->getMaskElt(1), 3) && 02477 isUndefOrEqual(N->getMaskElt(2), 2) && 02478 isUndefOrEqual(N->getMaskElt(3), 3); 02479 } 02480 02481 /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand 02482 /// specifies a shuffle of elements that is suitable for input to UNPCKL. 02483 static bool isUNPCKLMask(const SmallVectorImpl<int> &Mask, EVT VT, 02484 bool V2IsSplat = false) { 02485 int NumElts = VT.getVectorNumElements(); 02486 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 02487 return false; 02488 02489 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) { 02490 int BitI = Mask[i]; 02491 int BitI1 = Mask[i+1]; 02492 if (!isUndefOrEqual(BitI, j)) 02493 return false; 02494 if (V2IsSplat) { 02495 if (!isUndefOrEqual(BitI1, NumElts)) 02496 return false; 02497 } else { 02498 if (!isUndefOrEqual(BitI1, j + NumElts)) 02499 return false; 02500 } 02501 } 02502 return true; 02503 } 02504 02505 bool X86::isUNPCKLMask(ShuffleVectorSDNode *N, bool V2IsSplat) { 02506 SmallVector<int, 8> M; 02507 N->getMask(M); 02508 return ::isUNPCKLMask(M, N->getValueType(0), V2IsSplat); 02509 } 02510 02511 /// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand 02512 /// specifies a shuffle of elements that is suitable for input to UNPCKH. 02513 static bool isUNPCKHMask(const SmallVectorImpl<int> &Mask, EVT VT, 02514 bool V2IsSplat = false) { 02515 int NumElts = VT.getVectorNumElements(); 02516 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) 02517 return false; 02518 02519 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) { 02520 int BitI = Mask[i]; 02521 int BitI1 = Mask[i+1]; 02522 if (!isUndefOrEqual(BitI, j + NumElts/2)) 02523 return false; 02524 if (V2IsSplat) { 02525 if (isUndefOrEqual(BitI1, NumElts)) 02526 return false; 02527 } else { 02528 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) 02529 return false; 02530 } 02531 } 02532 return true; 02533 } 02534 02535 bool X86::isUNPCKHMask(ShuffleVectorSDNode *N, bool V2IsSplat) { 02536 SmallVector<int, 8> M; 02537 N->getMask(M); 02538 return ::isUNPCKHMask(M, N->getValueType(0), V2IsSplat); 02539 } 02540 02541 /// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form 02542 /// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, 02543 /// <0, 0, 1, 1> 02544 static bool isUNPCKL_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) { 02545 int NumElems = VT.getVectorNumElements(); 02546 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 02547 return false; 02548 02549 for (int i = 0, j = 0; i != NumElems; i += 2, ++j) { 02550 int BitI = Mask[i]; 02551 int BitI1 = Mask[i+1]; 02552 if (!isUndefOrEqual(BitI, j)) 02553 return false; 02554 if (!isUndefOrEqual(BitI1, j)) 02555 return false; 02556 } 02557 return true; 02558 } 02559 02560 bool X86::isUNPCKL_v_undef_Mask(ShuffleVectorSDNode *N) { 02561 SmallVector<int, 8> M; 02562 N->getMask(M); 02563 return ::isUNPCKL_v_undef_Mask(M, N->getValueType(0)); 02564 } 02565 02566 /// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form 02567 /// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, 02568 /// <2, 2, 3, 3> 02569 static bool isUNPCKH_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) { 02570 int NumElems = VT.getVectorNumElements(); 02571 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) 02572 return false; 02573 02574 for (int i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { 02575 int BitI = Mask[i]; 02576 int BitI1 = Mask[i+1]; 02577 if (!isUndefOrEqual(BitI, j)) 02578 return false; 02579 if (!isUndefOrEqual(BitI1, j)) 02580 return false; 02581 } 02582 return true; 02583 } 02584 02585 bool X86::isUNPCKH_v_undef_Mask(ShuffleVectorSDNode *N) { 02586 SmallVector<int, 8> M; 02587 N->getMask(M); 02588 return ::isUNPCKH_v_undef_Mask(M, N->getValueType(0)); 02589 } 02590 02591 /// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand 02592 /// specifies a shuffle of elements that is suitable for input to MOVSS, 02593 /// MOVSD, and MOVD, i.e. setting the lowest element. 02594 static bool isMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT) { 02595 if (VT.getVectorElementType().getSizeInBits() < 32) 02596 return false; 02597 02598 int NumElts = VT.getVectorNumElements(); 02599 02600 if (!isUndefOrEqual(Mask[0], NumElts)) 02601 return false; 02602 02603 for (int i = 1; i < NumElts; ++i) 02604 if (!isUndefOrEqual(Mask[i], i)) 02605 return false; 02606 02607 return true; 02608 } 02609 02610 bool X86::isMOVLMask(ShuffleVectorSDNode *N) { 02611 SmallVector<int, 8> M; 02612 N->getMask(M); 02613 return ::isMOVLMask(M, N->getValueType(0)); 02614 } 02615 02616 /// isCommutedMOVL - Returns true if the shuffle mask is except the reverse 02617 /// of what x86 movss want. X86 movs requires the lowest element to be lowest 02618 /// element of vector 2 and the other elements to come from vector 1 in order. 02619 static bool isCommutedMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT, 02620 bool V2IsSplat = false, bool V2IsUndef = false) { 02621 int NumOps = VT.getVectorNumElements(); 02622 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) 02623 return false; 02624 02625 if (!isUndefOrEqual(Mask[0], 0)) 02626 return false; 02627 02628 for (int i = 1; i < NumOps; ++i) 02629 if (!(isUndefOrEqual(Mask[i], i+NumOps) || 02630 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) || 02631 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps)))) 02632 return false; 02633 02634 return true; 02635 } 02636 02637 static bool isCommutedMOVL(ShuffleVectorSDNode *N, bool V2IsSplat = false, 02638 bool V2IsUndef = false) { 02639 SmallVector<int, 8> M; 02640 N->getMask(M); 02641 return isCommutedMOVLMask(M, N->getValueType(0), V2IsSplat, V2IsUndef); 02642 } 02643 02644 /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand 02645 /// specifies a shuffle of elements that is suitable for input to MOVSHDUP. 02646 bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N) { 02647 if (N->getValueType(0).getVectorNumElements() != 4) 02648 return false; 02649 02650 // Expect 1, 1, 3, 3 02651 for (unsigned i = 0; i < 2; ++i) { 02652 int Elt = N->getMaskElt(i); 02653 if (Elt >= 0 && Elt != 1) 02654 return false; 02655 } 02656 02657 bool HasHi = false; 02658 for (unsigned i = 2; i < 4; ++i) { 02659 int Elt = N->getMaskElt(i); 02660 if (Elt >= 0 && Elt != 3) 02661 return false; 02662 if (Elt == 3) 02663 HasHi = true; 02664 } 02665 // Don't use movshdup if it can be done with a shufps. 02666 // FIXME: verify that matching u, u, 3, 3 is what we want. 02667 return HasHi; 02668 } 02669 02670 /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand 02671 /// specifies a shuffle of elements that is suitable for input to MOVSLDUP. 02672 bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N) { 02673 if (N->getValueType(0).getVectorNumElements() != 4) 02674 return false; 02675 02676 // Expect 0, 0, 2, 2 02677 for (unsigned i = 0; i < 2; ++i) 02678 if (N->getMaskElt(i) > 0) 02679 return false; 02680 02681 bool HasHi = false; 02682 for (unsigned i = 2; i < 4; ++i) { 02683 int Elt = N->getMaskElt(i); 02684 if (Elt >= 0 && Elt != 2) 02685 return false; 02686 if (Elt == 2) 02687 HasHi = true; 02688 } 02689 // Don't use movsldup if it can be done with a shufps. 02690 return HasHi; 02691 } 02692 02693 /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand 02694 /// specifies a shuffle of elements that is suitable for input to MOVDDUP. 02695 bool X86::isMOVDDUPMask(ShuffleVectorSDNode *N) { 02696 int e = N->getValueType(0).getVectorNumElements() / 2; 02697 02698 for (int i = 0; i < e; ++i) 02699 if (!isUndefOrEqual(N->getMaskElt(i), i)) 02700 return false; 02701 for (int i = 0; i < e; ++i) 02702 if (!isUndefOrEqual(N->getMaskElt(e+i), i)) 02703 return false; 02704 return true; 02705 } 02706 02707 /// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle 02708 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* 02709 /// instructions. 02710 unsigned X86::getShuffleSHUFImmediate(SDNode *N) { 02711 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 02712 int NumOperands = SVOp->getValueType(0).getVectorNumElements(); 02713 02714 unsigned Shift = (NumOperands == 4) ? 2 : 1; 02715 unsigned Mask = 0; 02716 for (int i = 0; i < NumOperands; ++i) { 02717 int Val = SVOp->getMaskElt(NumOperands-i-1); 02718 if (Val < 0) Val = 0; 02719 if (Val >= NumOperands) Val -= NumOperands; 02720 Mask |= Val; 02721 if (i != NumOperands - 1) 02722 Mask <<= Shift; 02723 } 02724 return Mask; 02725 } 02726 02727 /// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle 02728 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW 02729 /// instructions. 02730 unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { 02731 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 02732 unsigned Mask = 0; 02733 // 8 nodes, but we only care about the last 4. 02734 for (unsigned i = 7; i >= 4; --i) { 02735 int Val = SVOp->getMaskElt(i); 02736 if (Val >= 0) 02737 Mask |= (Val - 4); 02738 if (i != 4) 02739 Mask <<= 2; 02740 } 02741 return Mask; 02742 } 02743 02744 /// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle 02745 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW 02746 /// instructions. 02747 unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { 02748 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 02749 unsigned Mask = 0; 02750 // 8 nodes, but we only care about the first 4. 02751 for (int i = 3; i >= 0; --i) { 02752 int Val = SVOp->getMaskElt(i); 02753 if (Val >= 0) 02754 Mask |= Val; 02755 if (i != 0) 02756 Mask <<= 2; 02757 } 02758 return Mask; 02759 } 02760 02761 /// isZeroNode - Returns true if Elt is a constant zero or a floating point 02762 /// constant +0.0. 02763 bool X86::isZeroNode(SDValue Elt) { 02764 return ((isa<ConstantSDNode>(Elt) && 02765 cast<ConstantSDNode>(Elt)->getZExtValue() == 0) || 02766 (isa<ConstantFPSDNode>(Elt) && 02767 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); 02768 } 02769 02770 /// CommuteVectorShuffle - Swap vector_shuffle operands as well as values in 02771 /// their permute mask. 02772 static SDValue CommuteVectorShuffle(ShuffleVectorSDNode *SVOp, 02773 SelectionDAG &DAG) { 02774 EVT VT = SVOp->getValueType(0); 02775 unsigned NumElems = VT.getVectorNumElements(); 02776 SmallVector<int, 8> MaskVec; 02777 02778 for (unsigned i = 0; i != NumElems; ++i) { 02779 int idx = SVOp->getMaskElt(i); 02780 if (idx < 0) 02781 MaskVec.push_back(idx); 02782 else if (idx < (int)NumElems) 02783 MaskVec.push_back(idx + NumElems); 02784 else 02785 MaskVec.push_back(idx - NumElems); 02786 } 02787 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1), 02788 SVOp->getOperand(0), &MaskVec[0]); 02789 } 02790 02791 /// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming 02792 /// the two vector operands have swapped position. 02793 static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask, EVT VT) { 02794 unsigned NumElems = VT.getVectorNumElements(); 02795 for (unsigned i = 0; i != NumElems; ++i) { 02796 int idx = Mask[i]; 02797 if (idx < 0) 02798 continue; 02799 else if (idx < (int)NumElems) 02800 Mask[i] = idx + NumElems; 02801 else 02802 Mask[i] = idx - NumElems; 02803 } 02804 } 02805 02806 /// ShouldXformToMOVHLPS - Return true if the node should be transformed to 02807 /// match movhlps. The lower half elements should come from upper half of 02808 /// V1 (and in order), and the upper half elements should come from the upper 02809 /// half of V2 (and in order). 02810 static bool ShouldXformToMOVHLPS(ShuffleVectorSDNode *Op) { 02811 if (Op->getValueType(0).getVectorNumElements() != 4) 02812 return false; 02813 for (unsigned i = 0, e = 2; i != e; ++i) 02814 if (!isUndefOrEqual(Op->getMaskElt(i), i+2)) 02815 return false; 02816 for (unsigned i = 2; i != 4; ++i) 02817 if (!isUndefOrEqual(Op->getMaskElt(i), i+4)) 02818 return false; 02819 return true; 02820 } 02821 02822 /// isScalarLoadToVector - Returns true if the node is a scalar load that 02823 /// is promoted to a vector. It also returns the LoadSDNode by reference if 02824 /// required. 02825 static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { 02826 if (N->getOpcode() != ISD::SCALAR_TO_VECTOR) 02827 return false; 02828 N = N->getOperand(0).getNode(); 02829 if (!ISD::isNON_EXTLoad(N)) 02830 return false; 02831 if (LD) 02832 *LD = cast<LoadSDNode>(N); 02833 return true; 02834 } 02835 02836 /// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to 02837 /// match movlp{s|d}. The lower half elements should come from lower half of 02838 /// V1 (and in order), and the upper half elements should come from the upper 02839 /// half of V2 (and in order). And since V1 will become the source of the 02840 /// MOVLP, it must be either a vector load or a scalar load to vector. 02841 static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, 02842 ShuffleVectorSDNode *Op) { 02843 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) 02844 return false; 02845 // Is V2 is a vector load, don't do this transformation. We will try to use 02846 // load folding shufps op. 02847 if (ISD::isNON_EXTLoad(V2)) 02848 return false; 02849 02850 unsigned NumElems = Op->getValueType(0).getVectorNumElements(); 02851 02852 if (NumElems != 2 && NumElems != 4) 02853 return false; 02854 for (unsigned i = 0, e = NumElems/2; i != e; ++i) 02855 if (!isUndefOrEqual(Op->getMaskElt(i), i)) 02856 return false; 02857 for (unsigned i = NumElems/2; i != NumElems; ++i) 02858 if (!isUndefOrEqual(Op->getMaskElt(i), i+NumElems)) 02859 return false; 02860 return true; 02861 } 02862 02863 /// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are 02864 /// all the same. 02865 static bool isSplatVector(SDNode *N) { 02866 if (N->getOpcode() != ISD::BUILD_VECTOR) 02867 return false; 02868 02869 SDValue SplatValue = N->getOperand(0); 02870 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 02871 if (N->getOperand(i) != SplatValue) 02872 return false; 02873 return true; 02874 } 02875 02876 /// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved 02877 /// to an zero vector. 02878 /// FIXME: move to dag combiner / method on ShuffleVectorSDNode 02879 static bool isZeroShuffle(ShuffleVectorSDNode *N) { 02880 SDValue V1 = N->getOperand(0); 02881 SDValue V2 = N->getOperand(1); 02882 unsigned NumElems = N->getValueType(0).getVectorNumElements(); 02883 for (unsigned i = 0; i != NumElems; ++i) { 02884 int Idx = N->getMaskElt(i); 02885 if (Idx >= (int)NumElems) { 02886 unsigned Opc = V2.getOpcode(); 02887 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) 02888 continue; 02889 if (Opc != ISD::BUILD_VECTOR || 02890 !X86::isZeroNode(V2.getOperand(Idx-NumElems))) 02891 return false; 02892 } else if (Idx >= 0) { 02893 unsigned Opc = V1.getOpcode(); 02894 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) 02895 continue; 02896 if (Opc != ISD::BUILD_VECTOR || 02897 !X86::isZeroNode(V1.getOperand(Idx))) 02898 return false; 02899 } 02900 } 02901 return true; 02902 } 02903 02904 /// getZeroVector - Returns a vector of specified type with all zero elements. 02905 /// 02906 static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG, 02907 DebugLoc dl) { 02908 assert(VT.isVector() && "Expected a vector type"); 02909 02910 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest 02911 // type. This ensures they get CSE'd. 02912 SDValue Vec; 02913 if (VT.getSizeInBits() == 64) { // MMX 02914 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 02915 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst); 02916 } else if (HasSSE2) { // SSE2 02917 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); 02918 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 02919 } else { // SSE1 02920 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); 02921 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); 02922 } 02923 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec); 02924 } 02925 02926 /// getOnesVector - Returns a vector of specified type with all bits set. 02927 /// 02928 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) { 02929 assert(VT.isVector() && "Expected a vector type"); 02930 02931 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest 02932 // type. This ensures they get CSE'd. 02933 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); 02934 SDValue Vec; 02935 if (VT.getSizeInBits() == 64) // MMX 02936 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst); 02937 else // SSE 02938 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); 02939 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec); 02940 } 02941 02942 02943 /// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements 02944 /// that point to V2 points to its first element. 02945 static SDValue NormalizeMask(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 02946 EVT VT = SVOp->getValueType(0); 02947 unsigned NumElems = VT.getVectorNumElements(); 02948 02949 bool Changed = false; 02950 SmallVector<int, 8> MaskVec; 02951 SVOp->getMask(MaskVec); 02952 02953 for (unsigned i = 0; i != NumElems; ++i) { 02954 if (MaskVec[i] > (int)NumElems) { 02955 MaskVec[i] = NumElems; 02956 Changed = true; 02957 } 02958 } 02959 if (Changed) 02960 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(0), 02961 SVOp->getOperand(1), &MaskVec[0]); 02962 return SDValue(SVOp, 0); 02963 } 02964 02965 /// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd 02966 /// operation of specified width. 02967 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 02968 SDValue V2) { 02969 unsigned NumElems = VT.getVectorNumElements(); 02970 SmallVector<int, 8> Mask; 02971 Mask.push_back(NumElems); 02972 for (unsigned i = 1; i != NumElems; ++i) 02973 Mask.push_back(i); 02974 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 02975 } 02976 02977 /// getUnpackl - Returns a vector_shuffle node for an unpackl operation. 02978 static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 02979 SDValue V2) { 02980 unsigned NumElems = VT.getVectorNumElements(); 02981 SmallVector<int, 8> Mask; 02982 for (unsigned i = 0, e = NumElems/2; i != e; ++i) { 02983 Mask.push_back(i); 02984 Mask.push_back(i + NumElems); 02985 } 02986 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 02987 } 02988 02989 /// getUnpackhMask - Returns a vector_shuffle node for an unpackh operation. 02990 static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, 02991 SDValue V2) { 02992 unsigned NumElems = VT.getVectorNumElements(); 02993 unsigned Half = NumElems/2; 02994 SmallVector<int, 8> Mask; 02995 for (unsigned i = 0; i != Half; ++i) { 02996 Mask.push_back(i + Half); 02997 Mask.push_back(i + NumElems + Half); 02998 } 02999 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); 03000 } 03001 03002 /// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. 03003 static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG, 03004 bool HasSSE2) { 03005 if (SV->getValueType(0).getVectorNumElements() <= 4) 03006 return SDValue(SV, 0); 03007 03008 EVT PVT = MVT::v4f32; 03009 EVT VT = SV->getValueType(0); 03010 DebugLoc dl = SV->getDebugLoc(); 03011 SDValue V1 = SV->getOperand(0); 03012 int NumElems = VT.getVectorNumElements(); 03013 int EltNo = SV->getSplatIndex(); 03014 03015 // unpack elements to the correct location 03016 while (NumElems > 4) { 03017 if (EltNo < NumElems/2) { 03018 V1 = getUnpackl(DAG, dl, VT, V1, V1); 03019 } else { 03020 V1 = getUnpackh(DAG, dl, VT, V1, V1); 03021 EltNo -= NumElems/2; 03022 } 03023 NumElems >>= 1; 03024 } 03025 03026 // Perform the splat. 03027 int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo }; 03028 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, PVT, V1); 03029 V1 = DAG.getVectorShuffle(PVT, dl, V1, DAG.getUNDEF(PVT), &SplatMask[0]); 03030 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, V1); 03031 } 03032 03033 /// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified 03034 /// vector of zero or undef vector. This produces a shuffle where the low 03035 /// element of V2 is swizzled into the zero/undef vector, landing at element 03036 /// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). 03037 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, 03038 bool isZero, bool HasSSE2, 03039 SelectionDAG &DAG) { 03040 EVT VT = V2.getValueType(); 03041 SDValue V1 = isZero 03042 ? getZeroVector(VT, HasSSE2, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT); 03043 unsigned NumElems = VT.getVectorNumElements(); 03044 SmallVector<int, 16> MaskVec; 03045 for (unsigned i = 0; i != NumElems; ++i) 03046 // If this is the insertion idx, put the low elt of V2 here. 03047 MaskVec.push_back(i == Idx ? NumElems : i); 03048 return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]); 03049 } 03050 03051 /// getNumOfConsecutiveZeros - Return the number of elements in a result of 03052 /// a shuffle that is zero. 03053 static 03054 unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp, int NumElems, 03055 bool Low, SelectionDAG &DAG) { 03056 unsigned NumZeros = 0; 03057 for (int i = 0; i < NumElems; ++i) { 03058 unsigned Index = Low ? i : NumElems-i-1; 03059 int Idx = SVOp->getMaskElt(Index); 03060 if (Idx < 0) { 03061 ++NumZeros; 03062 continue; 03063 } 03064 SDValue Elt = DAG.getShuffleScalarElt(SVOp, Index); 03065 if (Elt.getNode() && X86::isZeroNode(Elt)) 03066 ++NumZeros; 03067 else 03068 break; 03069 } 03070 return NumZeros; 03071 } 03072 03073 /// isVectorShift - Returns true if the shuffle can be implemented as a 03074 /// logical left or right shift of a vector. 03075 /// FIXME: split into pslldqi, psrldqi, palignr variants. 03076 static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, 03077 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { 03078 int NumElems = SVOp->getValueType(0).getVectorNumElements(); 03079 03080 isLeft = true; 03081 unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, true, DAG); 03082 if (!NumZeros) { 03083 isLeft = false; 03084 NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, false, DAG); 03085 if (!NumZeros) 03086 return false; 03087 } 03088 bool SeenV1 = false; 03089 bool SeenV2 = false; 03090 for (int i = NumZeros; i < NumElems; ++i) { 03091 int Val = isLeft ? (i - NumZeros) : i; 03092 int Idx = SVOp->getMaskElt(isLeft ? i : (i - NumZeros)); 03093 if (Idx < 0) 03094 continue; 03095 if (Idx < NumElems) 03096 SeenV1 = true; 03097 else { 03098 Idx -= NumElems; 03099 SeenV2 = true; 03100 } 03101 if (Idx != Val) 03102 return false; 03103 } 03104 if (SeenV1 && SeenV2) 03105 return false; 03106 03107 ShVal = SeenV1 ? SVOp->getOperand(0) : SVOp->getOperand(1); 03108 ShAmt = NumZeros; 03109 return true; 03110 } 03111 03112 03113 /// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. 03114 /// 03115 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, 03116 unsigned NumNonZero, unsigned NumZero, 03117 SelectionDAG &DAG, TargetLowering &TLI) { 03118 if (NumNonZero > 8) 03119 return SDValue(); 03120 03121 DebugLoc dl = Op.getDebugLoc(); 03122 SDValue V(0, 0); 03123 bool First = true; 03124 for (unsigned i = 0; i < 16; ++i) { 03125 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; 03126 if (ThisIsNonZero && First) { 03127 if (NumZero) 03128 V = getZeroVector(MVT::v8i16, true, DAG, dl); 03129 else 03130 V = DAG.getUNDEF(MVT::v8i16); 03131 First = false; 03132 } 03133 03134 if ((i & 1) != 0) { 03135 SDValue ThisElt(0, 0), LastElt(0, 0); 03136 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; 03137 if (LastIsNonZero) { 03138 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, 03139 MVT::i16, Op.getOperand(i-1)); 03140 } 03141 if (ThisIsNonZero) { 03142 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); 03143 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, 03144 ThisElt, DAG.getConstant(8, MVT::i8)); 03145 if (LastIsNonZero) 03146 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); 03147 } else 03148 ThisElt = LastElt; 03149 03150 if (ThisElt.getNode()) 03151 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, 03152 DAG.getIntPtrConstant(i/2)); 03153 } 03154 } 03155 03156 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V); 03157 } 03158 03159 /// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. 03160 /// 03161 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, 03162 unsigned NumNonZero, unsigned NumZero, 03163 SelectionDAG &DAG, TargetLowering &TLI) { 03164 if (NumNonZero > 4) 03165 return SDValue(); 03166 03167 DebugLoc dl = Op.getDebugLoc(); 03168 SDValue V(0, 0); 03169 bool First = true; 03170 for (unsigned i = 0; i < 8; ++i) { 03171 bool isNonZero = (NonZeros & (1 << i)) != 0; 03172 if (isNonZero) { 03173 if (First) { 03174 if (NumZero) 03175 V = getZeroVector(MVT::v8i16, true, DAG, dl); 03176 else 03177 V = DAG.getUNDEF(MVT::v8i16); 03178 First = false; 03179 } 03180 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 03181 MVT::v8i16, V, Op.getOperand(i), 03182 DAG.getIntPtrConstant(i)); 03183 } 03184 } 03185 03186 return V; 03187 } 03188 03189 /// getVShift - Return a vector logical shift node. 03190 /// 03191 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, 03192 unsigned NumBits, SelectionDAG &DAG, 03193 const TargetLowering &TLI, DebugLoc dl) { 03194 bool isMMX = VT.getSizeInBits() == 64; 03195 EVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; 03196 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; 03197 SrcOp = DAG.getNode(ISD::BIT_CONVERT, dl, ShVT, SrcOp); 03198 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, 03199 DAG.getNode(Opc, dl, ShVT, SrcOp, 03200 DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); 03201 } 03202 03203 SDValue 03204 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) { 03205 DebugLoc dl = Op.getDebugLoc(); 03206 // All zero's are handled with pxor, all one's are handled with pcmpeqd. 03207 if (ISD::isBuildVectorAllZeros(Op.getNode()) 03208 || ISD::isBuildVectorAllOnes(Op.getNode())) { 03209 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to 03210 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are 03211 // eliminated on x86-32 hosts. 03212 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) 03213 return Op; 03214 03215 if (ISD::isBuildVectorAllOnes(Op.getNode())) 03216 return getOnesVector(Op.getValueType(), DAG, dl); 03217 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG, dl); 03218 } 03219 03220 EVT VT = Op.getValueType(); 03221 EVT ExtVT = VT.getVectorElementType(); 03222 unsigned EVTBits = ExtVT.getSizeInBits(); 03223 03224 unsigned NumElems = Op.getNumOperands(); 03225 unsigned NumZero = 0; 03226 unsigned NumNonZero = 0; 03227 unsigned NonZeros = 0; 03228 bool IsAllConstants = true; 03229 SmallSet<SDValue, 8> Values; 03230 for (unsigned i = 0; i < NumElems; ++i) { 03231 SDValue Elt = Op.getOperand(i); 03232 if (Elt.getOpcode() == ISD::UNDEF) 03233 continue; 03234 Values.insert(Elt); 03235 if (Elt.getOpcode() != ISD::Constant && 03236 Elt.getOpcode() != ISD::ConstantFP) 03237 IsAllConstants = false; 03238 if (X86::isZeroNode(Elt)) 03239 NumZero++; 03240 else { 03241 NonZeros |= (1 << i); 03242 NumNonZero++; 03243 } 03244 } 03245 03246 if (NumNonZero == 0) { 03247 // All undef vector. Return an UNDEF. All zero vectors were handled above. 03248 return DAG.getUNDEF(VT); 03249 } 03250 03251 // Special case for single non-zero, non-undef, element. 03252 if (NumNonZero == 1) { 03253 unsigned Idx = CountTrailingZeros_32(NonZeros); 03254 SDValue Item = Op.getOperand(Idx); 03255 03256 // If this is an insertion of an i64 value on x86-32, and if the top bits of 03257 // the value are obviously zero, truncate the value to i32 and do the 03258 // insertion that way. Only do this if the value is non-constant or if the 03259 // value is a constant being inserted into element 0. It is cheaper to do 03260 // a constant pool load than it is to do a movd + shuffle. 03261 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && 03262 (!IsAllConstants || Idx == 0)) { 03263 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { 03264 // Handle MMX and SSE both. 03265 EVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; 03266 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; 03267 03268 // Truncate the value (which may itself be a constant) to i32, and 03269 // convert it to a vector with movd (S2V+shuffle to zero extend). 03270 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); 03271 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); 03272 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 03273 Subtarget->hasSSE2(), DAG); 03274 03275 // Now we have our 32-bit value zero extended in the low element of 03276 // a vector. If Idx != 0, swizzle it into place. 03277 if (Idx != 0) { 03278 SmallVector<int, 4> Mask; 03279 Mask.push_back(Idx); 03280 for (unsigned i = 1; i != VecElts; ++i) 03281 Mask.push_back(i); 03282 Item = DAG.getVectorShuffle(VecVT, dl, Item, 03283 DAG.getUNDEF(Item.getValueType()), 03284 &Mask[0]); 03285 } 03286 return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(), Item); 03287 } 03288 } 03289 03290 // If we have a constant or non-constant insertion into the low element of 03291 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into 03292 // the rest of the elements. This will be matched as movd/movq/movss/movsd 03293 // depending on what the source datatype is. 03294 if (Idx == 0) { 03295 if (NumZero == 0) { 03296 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 03297 } else if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || 03298 (ExtVT == MVT::i64 && Subtarget->is64Bit())) { 03299 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 03300 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. 03301 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget->hasSSE2(), 03302 DAG); 03303 } else if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { 03304 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); 03305 EVT MiddleVT = VT.getSizeInBits() == 64 ? MVT::v2i32 : MVT::v4i32; 03306 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MiddleVT, Item); 03307 Item = getShuffleVectorZeroOrUndef(Item, 0, true, 03308 Subtarget->hasSSE2(), DAG); 03309 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Item); 03310 } 03311 } 03312 03313 // Is it a vector logical left shift? 03314 if (NumElems == 2 && Idx == 1 && 03315 X86::isZeroNode(Op.getOperand(0)) && 03316 !X86::isZeroNode(Op.getOperand(1))) { 03317 unsigned NumBits = VT.getSizeInBits(); 03318 return getVShift(true, VT, 03319 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 03320 VT, Op.getOperand(1)), 03321 NumBits/2, DAG, *this, dl); 03322 } 03323 03324 if (IsAllConstants) // Otherwise, it's better to do a constpool load. 03325 return SDValue(); 03326 03327 // Otherwise, if this is a vector with i32 or f32 elements, and the element 03328 // is a non-constant being inserted into an element other than the low one, 03329 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka 03330 // movd/movss) to move this into the low element, then shuffle it into 03331 // place. 03332 if (EVTBits == 32) { 03333 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); 03334 03335 // Turn it into a shuffle of zero and zero-extended scalar to vector. 03336 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, 03337 Subtarget->hasSSE2(), DAG); 03338 SmallVector<int, 8> MaskVec; 03339 for (unsigned i = 0; i < NumElems; i++) 03340 MaskVec.push_back(i == Idx ? 0 : 1); 03341 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]); 03342 } 03343 } 03344 03345 // Splat is obviously ok. Let legalizer expand it to a shuffle. 03346 if (Values.size() == 1) 03347 return SDValue(); 03348 03349 // A vector full of immediates; various special cases are already 03350 // handled, so this is best done with a single constant-pool load. 03351 if (IsAllConstants) 03352 return SDValue(); 03353 03354 // Let legalizer expand 2-wide build_vectors. 03355 if (EVTBits == 64) { 03356 if (NumNonZero == 1) { 03357 // One half is zero or undef. 03358 unsigned Idx = CountTrailingZeros_32(NonZeros); 03359 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, 03360 Op.getOperand(Idx)); 03361 return getShuffleVectorZeroOrUndef(V2, Idx, true, 03362 Subtarget->hasSSE2(), DAG); 03363 } 03364 return SDValue(); 03365 } 03366 03367 // If element VT is < 32 bits, convert it to inserts into a zero vector. 03368 if (EVTBits == 8 && NumElems == 16) { 03369 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, 03370 *this); 03371 if (V.getNode()) return V; 03372 } 03373 03374 if (EVTBits == 16 && NumElems == 8) { 03375 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, 03376 *this); 03377 if (V.getNode()) return V; 03378 } 03379 03380 // If element VT is == 32 bits, turn it into a number of shuffles. 03381 SmallVector<SDValue, 8> V; 03382 V.resize(NumElems); 03383 if (NumElems == 4 && NumZero > 0) { 03384 for (unsigned i = 0; i < 4; ++i) { 03385 bool isZero = !(NonZeros & (1 << i)); 03386 if (isZero) 03387 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl); 03388 else 03389 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); 03390 } 03391 03392 for (unsigned i = 0; i < 2; ++i) { 03393 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { 03394 default: break; 03395 case 0: 03396 V[i] = V[i*2]; // Must be a zero vector. 03397 break; 03398 case 1: 03399 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]); 03400 break; 03401 case 2: 03402 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]); 03403 break; 03404 case 3: 03405 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]); 03406 break; 03407 } 03408 } 03409 03410 SmallVector<int, 8> MaskVec; 03411 bool Reverse = (NonZeros & 0x3) == 2; 03412 for (unsigned i = 0; i < 2; ++i) 03413 MaskVec.push_back(Reverse ? 1-i : i); 03414 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; 03415 for (unsigned i = 0; i < 2; ++i) 03416 MaskVec.push_back(Reverse ? 1-i+NumElems : i+NumElems); 03417 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]); 03418 } 03419 03420 if (Values.size() > 2) { 03421 // If we have SSE 4.1, Expand into a number of inserts unless the number of 03422 // values to be inserted is equal to the number of elements, in which case 03423 // use the unpack code below in the hopes of matching the consecutive elts 03424 // load merge pattern for shuffles. 03425 // FIXME: We could probably just check that here directly. 03426 if (Values.size() < NumElems && VT.getSizeInBits() == 128 && 03427 getSubtarget()->hasSSE41()) { 03428 V[0] = DAG.getUNDEF(VT); 03429 for (unsigned i = 0; i < NumElems; ++i) 03430 if (Op.getOperand(i).getOpcode() != ISD::UNDEF) 03431 V[0] = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V[0], 03432 Op.getOperand(i), DAG.getIntPtrConstant(i)); 03433 return V[0]; 03434 } 03435 // Expand into a number of unpckl*. 03436 // e.g. for v4f32 03437 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> 03438 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> 03439 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> 03440 for (unsigned i = 0; i < NumElems; ++i) 03441 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); 03442 NumElems >>= 1; 03443 while (NumElems != 0) { 03444 for (unsigned i = 0; i < NumElems; ++i) 03445 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + NumElems]); 03446 NumElems >>= 1; 03447 } 03448 return V[0]; 03449 } 03450 03451 return SDValue(); 03452 } 03453 03454 // v8i16 shuffles - Prefer shuffles in the following order: 03455 // 1. [all] pshuflw, pshufhw, optional move 03456 // 2. [ssse3] 1 x pshufb 03457 // 3. [ssse3] 2 x pshufb + 1 x por 03458 // 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw) 03459 static 03460 SDValue LowerVECTOR_SHUFFLEv8i16(ShuffleVectorSDNode *SVOp, 03461 SelectionDAG &DAG, X86TargetLowering &TLI) { 03462 SDValue V1 = SVOp->getOperand(0); 03463 SDValue V2 = SVOp->getOperand(1); 03464 DebugLoc dl = SVOp->getDebugLoc(); 03465 SmallVector<int, 8> MaskVals; 03466 03467 // Determine if more than 1 of the words in each of the low and high quadwords 03468 // of the result come from the same quadword of one of the two inputs. Undef 03469 // mask values count as coming from any quadword, for better codegen. 03470 SmallVector<unsigned, 4> LoQuad(4); 03471 SmallVector<unsigned, 4> HiQuad(4); 03472 BitVector InputQuads(4); 03473 for (unsigned i = 0; i < 8; ++i) { 03474 SmallVectorImpl<unsigned> &Quad = i < 4 ? LoQuad : HiQuad; 03475 int EltIdx = SVOp->getMaskElt(i); 03476 MaskVals.push_back(EltIdx); 03477 if (EltIdx < 0) { 03478 ++Quad[0]; 03479 ++Quad[1]; 03480 ++Quad[2]; 03481 ++Quad[3]; 03482 continue; 03483 } 03484 ++Quad[EltIdx / 4]; 03485 InputQuads.set(EltIdx / 4); 03486 } 03487 03488 int BestLoQuad = -1; 03489 unsigned MaxQuad = 1; 03490 for (unsigned i = 0; i < 4; ++i) { 03491 if (LoQuad[i] > MaxQuad) { 03492 BestLoQuad = i; 03493 MaxQuad = LoQuad[i]; 03494 } 03495 } 03496 03497 int BestHiQuad = -1; 03498 MaxQuad = 1; 03499 for (unsigned i = 0; i < 4; ++i) { 03500 if (HiQuad[i] > MaxQuad) { 03501 BestHiQuad = i; 03502 MaxQuad = HiQuad[i]; 03503 } 03504 } 03505 03506 // For SSSE3, If all 8 words of the result come from only 1 quadword of each 03507 // of the two input vectors, shuffle them into one input vector so only a 03508 // single pshufb instruction is necessary. If There are more than 2 input 03509 // quads, disable the next transformation since it does not help SSSE3. 03510 bool V1Used = InputQuads[0] || InputQuads[1]; 03511 bool V2Used = InputQuads[2] || InputQuads[3]; 03512 if (TLI.getSubtarget()->hasSSSE3()) { 03513 if (InputQuads.count() == 2 && V1Used && V2Used) { 03514 BestLoQuad = InputQuads.find_first(); 03515 BestHiQuad = InputQuads.find_next(BestLoQuad); 03516 } 03517 if (InputQuads.count() > 2) { 03518 BestLoQuad = -1; 03519 BestHiQuad = -1; 03520 } 03521 } 03522 03523 // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update 03524 // the shuffle mask. If a quad is scored as -1, that means that it contains 03525 // words from all 4 input quadwords. 03526 SDValue NewV; 03527 if (BestLoQuad >= 0 || BestHiQuad >= 0) { 03528 SmallVector<int, 8> MaskV; 03529 MaskV.push_back(BestLoQuad < 0 ? 0 : BestLoQuad); 03530 MaskV.push_back(BestHiQuad < 0 ? 1 : BestHiQuad); 03531 NewV = DAG.getVectorShuffle(MVT::v2i64, dl, 03532 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, V1), 03533 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, V2), &MaskV[0]); 03534 NewV = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, NewV); 03535 03536 // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the 03537 // source words for the shuffle, to aid later transformations. 03538 bool AllWordsInNewV = true; 03539 bool InOrder[2] = { true, true }; 03540 for (unsigned i = 0; i != 8; ++i) { 03541 int idx = MaskVals[i]; 03542 if (idx != (int)i) 03543 InOrder[i/4] = false; 03544 if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad) 03545 continue; 03546 AllWordsInNewV = false; 03547 break; 03548 } 03549 03550 bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV; 03551 if (AllWordsInNewV) { 03552 for (int i = 0; i != 8; ++i) { 03553 int idx = MaskVals[i]; 03554 if (idx < 0) 03555 continue; 03556 idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4; 03557 if ((idx != i) && idx < 4) 03558 pshufhw = false; 03559 if ((idx != i) && idx > 3) 03560 pshuflw = false; 03561 } 03562 V1 = NewV; 03563 V2Used = false; 03564 BestLoQuad = 0; 03565 BestHiQuad = 1; 03566 } 03567 03568 // If we've eliminated the use of V2, and the new mask is a pshuflw or 03569 // pshufhw, that's as cheap as it gets. Return the new shuffle. 03570 if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) { 03571 return DAG.getVectorShuffle(MVT::v8i16, dl, NewV, 03572 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]); 03573 } 03574 } 03575 03576 // If we have SSSE3, and all words of the result are from 1 input vector, 03577 // case 2 is generated, otherwise case 3 is generated. If no SSSE3 03578 // is present, fall back to case 4. 03579 if (TLI.getSubtarget()->hasSSSE3()) { 03580 SmallVector<SDValue,16> pshufbMask; 03581 03582 // If we have elements from both input vectors, set the high bit of the 03583 // shuffle mask element to zero out elements that come from V2 in the V1 03584 // mask, and elements that come from V1 in the V2 mask, so that the two 03585 // results can be OR'd together. 03586 bool TwoInputs = V1Used && V2Used; 03587 for (unsigned i = 0; i != 8; ++i) { 03588 int EltIdx = MaskVals[i] * 2; 03589 if (TwoInputs && (EltIdx >= 16)) { 03590 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 03591 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 03592 continue; 03593 } 03594 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); 03595 pshufbMask.push_back(DAG.getConstant(EltIdx+1, MVT::i8)); 03596 } 03597 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V1); 03598 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, 03599 DAG.getNode(ISD::BUILD_VECTOR, dl, 03600 MVT::v16i8, &pshufbMask[0], 16)); 03601 if (!TwoInputs) 03602 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1); 03603 03604 // Calculate the shuffle mask for the second input, shuffle it, and 03605 // OR it with the first shuffled input. 03606 pshufbMask.clear(); 03607 for (unsigned i = 0; i != 8; ++i) { 03608 int EltIdx = MaskVals[i] * 2; 03609 if (EltIdx < 16) { 03610 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 03611 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 03612 continue; 03613 } 03614 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); 03615 pshufbMask.push_back(DAG.getConstant(EltIdx - 15, MVT::i8)); 03616 } 03617 V2 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V2); 03618 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, 03619 DAG.getNode(ISD::BUILD_VECTOR, dl, 03620 MVT::v16i8, &pshufbMask[0], 16)); 03621 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); 03622 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1); 03623 } 03624 03625 // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order, 03626 // and update MaskVals with new element order. 03627 BitVector InOrder(8); 03628 if (BestLoQuad >= 0) { 03629 SmallVector<int, 8> MaskV; 03630 for (int i = 0; i != 4; ++i) { 03631 int idx = MaskVals[i]; 03632 if (idx < 0) { 03633 MaskV.push_back(-1); 03634 InOrder.set(i); 03635 } else if ((idx / 4) == BestLoQuad) { 03636 MaskV.push_back(idx & 3); 03637 InOrder.set(i); 03638 } else { 03639 MaskV.push_back(-1); 03640 } 03641 } 03642 for (unsigned i = 4; i != 8; ++i) 03643 MaskV.push_back(i); 03644 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), 03645 &MaskV[0]); 03646 } 03647 03648 // If BestHi >= 0, generate a pshufhw to put the high elements in order, 03649 // and update MaskVals with the new element order. 03650 if (BestHiQuad >= 0) { 03651 SmallVector<int, 8> MaskV; 03652 for (unsigned i = 0; i != 4; ++i) 03653 MaskV.push_back(i); 03654 for (unsigned i = 4; i != 8; ++i) { 03655 int idx = MaskVals[i]; 03656 if (idx < 0) { 03657 MaskV.push_back(-1); 03658 InOrder.set(i); 03659 } else if ((idx / 4) == BestHiQuad) { 03660 MaskV.push_back((idx & 3) + 4); 03661 InOrder.set(i); 03662 } else { 03663 MaskV.push_back(-1); 03664 } 03665 } 03666 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), 03667 &MaskV[0]); 03668 } 03669 03670 // In case BestHi & BestLo were both -1, which means each quadword has a word 03671 // from each of the four input quadwords, calculate the InOrder bitvector now 03672 // before falling through to the insert/extract cleanup. 03673 if (BestLoQuad == -1 && BestHiQuad == -1) { 03674 NewV = V1; 03675 for (int i = 0; i != 8; ++i) 03676 if (MaskVals[i] < 0 || MaskVals[i] == i) 03677 InOrder.set(i); 03678 } 03679 03680 // The other elements are put in the right place using pextrw and pinsrw. 03681 for (unsigned i = 0; i != 8; ++i) { 03682 if (InOrder[i]) 03683 continue; 03684 int EltIdx = MaskVals[i]; 03685 if (EltIdx < 0) 03686 continue; 03687 SDValue ExtOp = (EltIdx < 8) 03688 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1, 03689 DAG.getIntPtrConstant(EltIdx)) 03690 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2, 03691 DAG.getIntPtrConstant(EltIdx - 8)); 03692 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp, 03693 DAG.getIntPtrConstant(i)); 03694 } 03695 return NewV; 03696 } 03697 03698 // v16i8 shuffles - Prefer shuffles in the following order: 03699 // 1. [ssse3] 1 x pshufb 03700 // 2. [ssse3] 2 x pshufb + 1 x por 03701 // 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw 03702 static 03703 SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp, 03704 SelectionDAG &DAG, X86TargetLowering &TLI) { 03705 SDValue V1 = SVOp->getOperand(0); 03706 SDValue V2 = SVOp->getOperand(1); 03707 DebugLoc dl = SVOp->getDebugLoc(); 03708 SmallVector<int, 16> MaskVals; 03709 SVOp->getMask(MaskVals); 03710 03711 // If we have SSSE3, case 1 is generated when all result bytes come from 03712 // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is 03713 // present, fall back to case 3. 03714 // FIXME: kill V2Only once shuffles are canonizalized by getNode. 03715 bool V1Only = true; 03716 bool V2Only = true; 03717 for (unsigned i = 0; i < 16; ++i) { 03718 int EltIdx = MaskVals[i]; 03719 if (EltIdx < 0) 03720 continue; 03721 if (EltIdx < 16) 03722 V2Only = false; 03723 else 03724 V1Only = false; 03725 } 03726 03727 // If SSSE3, use 1 pshufb instruction per vector with elements in the result. 03728 if (TLI.getSubtarget()->hasSSSE3()) { 03729 SmallVector<SDValue,16> pshufbMask; 03730 03731 // If all result elements are from one input vector, then only translate 03732 // undef mask values to 0x80 (zero out result) in the pshufb mask. 03733 // 03734 // Otherwise, we have elements from both input vectors, and must zero out 03735 // elements that come from V2 in the first mask, and V1 in the second mask 03736 // so that we can OR them together. 03737 bool TwoInputs = !(V1Only || V2Only); 03738 for (unsigned i = 0; i != 16; ++i) { 03739 int EltIdx = MaskVals[i]; 03740 if (EltIdx < 0 || (TwoInputs && EltIdx >= 16)) { 03741 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 03742 continue; 03743 } 03744 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); 03745 } 03746 // If all the elements are from V2, assign it to V1 and return after 03747 // building the first pshufb. 03748 if (V2Only) 03749 V1 = V2; 03750 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, 03751 DAG.getNode(ISD::BUILD_VECTOR, dl, 03752 MVT::v16i8, &pshufbMask[0], 16)); 03753 if (!TwoInputs) 03754 return V1; 03755 03756 // Calculate the shuffle mask for the second input, shuffle it, and 03757 // OR it with the first shuffled input. 03758 pshufbMask.clear(); 03759 for (unsigned i = 0; i != 16; ++i) { 03760 int EltIdx = MaskVals[i]; 03761 if (EltIdx < 16) { 03762 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); 03763 continue; 03764 } 03765 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); 03766 } 03767 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, 03768 DAG.getNode(ISD::BUILD_VECTOR, dl, 03769 MVT::v16i8, &pshufbMask[0], 16)); 03770 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); 03771 } 03772 03773 // No SSSE3 - Calculate in place words and then fix all out of place words 03774 // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from 03775 // the 16 different words that comprise the two doublequadword input vectors. 03776 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1); 03777 V2 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V2); 03778 SDValue NewV = V2Only ? V2 : V1; 03779 for (int i = 0; i != 8; ++i) { 03780 int Elt0 = MaskVals[i*2]; 03781 int Elt1 = MaskVals[i*2+1]; 03782 03783 // This word of the result is all undef, skip it. 03784 if (Elt0 < 0 && Elt1 < 0) 03785 continue; 03786 03787 // This word of the result is already in the correct place, skip it. 03788 if (V1Only && (Elt0 == i*2) && (Elt1 == i*2+1)) 03789 continue; 03790 if (V2Only && (Elt0 == i*2+16) && (Elt1 == i*2+17)) 03791 continue; 03792 03793 SDValue Elt0Src = Elt0 < 16 ? V1 : V2; 03794 SDValue Elt1Src = Elt1 < 16 ? V1 : V2; 03795 SDValue InsElt; 03796 03797 // If Elt0 and Elt1 are defined, are consecutive, and can be load 03798 // using a single extract together, load it and store it. 03799 if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) { 03800 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, 03801 DAG.getIntPtrConstant(Elt1 / 2)); 03802 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, 03803 DAG.getIntPtrConstant(i)); 03804 continue; 03805 } 03806 03807 // If Elt1 is defined, extract it from the appropriate source. If the 03808 // source byte is not also odd, shift the extracted word left 8 bits 03809 // otherwise clear the bottom 8 bits if we need to do an or. 03810 if (Elt1 >= 0) { 03811 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, 03812 DAG.getIntPtrConstant(Elt1 / 2)); 03813 if ((Elt1 & 1) == 0) 03814 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt, 03815 DAG.getConstant(8, TLI.getShiftAmountTy())); 03816 else if (Elt0 >= 0) 03817 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt, 03818 DAG.getConstant(0xFF00, MVT::i16)); 03819 } 03820 // If Elt0 is defined, extract it from the appropriate source. If the 03821 // source byte is not also even, shift the extracted word right 8 bits. If 03822 // Elt1 was also defined, OR the extracted values together before 03823 // inserting them in the result. 03824 if (Elt0 >= 0) { 03825 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, 03826 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2)); 03827 if ((Elt0 & 1) != 0) 03828 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0, 03829 DAG.getConstant(8, TLI.getShiftAmountTy())); 03830 else if (Elt1 >= 0) 03831 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0, 03832 DAG.getConstant(0x00FF, MVT::i16)); 03833 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0) 03834 : InsElt0; 03835 } 03836 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, 03837 DAG.getIntPtrConstant(i)); 03838 } 03839 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, NewV); 03840 } 03841 03842 /// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide 03843 /// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be 03844 /// done when every pair / quad of shuffle mask elements point to elements in 03845 /// the right sequence. e.g. 03846 /// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> 03847 static 03848 SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp, 03849 SelectionDAG &DAG, 03850 TargetLowering &TLI, DebugLoc dl) { 03851 EVT VT = SVOp->getValueType(0); 03852 SDValue V1 = SVOp->getOperand(0); 03853 SDValue V2 = SVOp->getOperand(1); 03854 unsigned NumElems = VT.getVectorNumElements(); 03855 unsigned NewWidth = (NumElems == 4) ? 2 : 4; 03856 EVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); 03857 EVT MaskEltVT = MaskVT.getVectorElementType(); 03858 EVT NewVT = MaskVT; 03859 switch (VT.getSimpleVT().SimpleTy) { 03860 default: assert(false && "Unexpected!"); 03861 case MVT::v4f32: NewVT = MVT::v2f64; break; 03862 case MVT::v4i32: NewVT = MVT::v2i64; break; 03863 case MVT::v8i16: NewVT = MVT::v4i32; break; 03864 case MVT::v16i8: NewVT = MVT::v4i32; break; 03865 } 03866 03867 if (NewWidth == 2) { 03868 if (VT.isInteger()) 03869 NewVT = MVT::v2i64; 03870 else 03871 NewVT = MVT::v2f64; 03872 } 03873 int Scale = NumElems / NewWidth; 03874 SmallVector<int, 8> MaskVec; 03875 for (unsigned i = 0; i < NumElems; i += Scale) { 03876 int StartIdx = -1; 03877 for (int j = 0; j < Scale; ++j) { 03878 int EltIdx = SVOp->getMaskElt(i+j); 03879 if (EltIdx < 0) 03880 continue; 03881 if (StartIdx == -1) 03882 StartIdx = EltIdx - (EltIdx % Scale); 03883 if (EltIdx != StartIdx + j) 03884 return SDValue(); 03885 } 03886 if (StartIdx == -1) 03887 MaskVec.push_back(-1); 03888 else 03889 MaskVec.push_back(StartIdx / Scale); 03890 } 03891 03892 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, V1); 03893 V2 = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, V2); 03894 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]); 03895 } 03896 03897 /// getVZextMovL - Return a zero-extending vector move low node. 03898 /// 03899 static SDValue getVZextMovL(EVT VT, EVT OpVT, 03900 SDValue SrcOp, SelectionDAG &DAG, 03901 const X86Subtarget *Subtarget, DebugLoc dl) { 03902 if (VT == MVT::v2f64 || VT == MVT::v4f32) { 03903 LoadSDNode *LD = NULL; 03904 if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) 03905 LD = dyn_cast<LoadSDNode>(SrcOp); 03906 if (!LD) { 03907 // movssrr and movsdrr do not clear top bits. Try to use movd, movq 03908 // instead. 03909 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; 03910 if ((ExtVT.SimpleTy != MVT::i64 || Subtarget->is64Bit()) && 03911 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && 03912 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && 03913 SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) { 03914 // PR2108 03915 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; 03916 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, 03917 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, 03918 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 03919 OpVT, 03920 SrcOp.getOperand(0) 03921 .getOperand(0)))); 03922 } 03923 } 03924 } 03925 03926 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, 03927 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, 03928 DAG.getNode(ISD::BIT_CONVERT, dl, 03929 OpVT, SrcOp))); 03930 } 03931 03932 /// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of 03933 /// shuffles. 03934 static SDValue 03935 LowerVECTOR_SHUFFLE_4wide(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { 03936 SDValue V1 = SVOp->getOperand(0); 03937 SDValue V2 = SVOp->getOperand(1); 03938 DebugLoc dl = SVOp->getDebugLoc(); 03939 EVT VT = SVOp->getValueType(0); 03940 03941 SmallVector<std::pair<int, int>, 8> Locs; 03942 Locs.resize(4); 03943 SmallVector<int, 8> Mask1(4U, -1); 03944 SmallVector<int, 8> PermMask; 03945 SVOp->getMask(PermMask); 03946 03947 unsigned NumHi = 0; 03948 unsigned NumLo = 0; 03949 for (unsigned i = 0; i != 4; ++i) { 03950 int Idx = PermMask[i]; 03951 if (Idx < 0) { 03952 Locs[i] = std::make_pair(-1, -1); 03953 } else { 03954 assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!"); 03955 if (Idx < 4) { 03956 Locs[i] = std::make_pair(0, NumLo); 03957 Mask1[NumLo] = Idx; 03958 NumLo++; 03959 } else { 03960 Locs[i] = std::make_pair(1, NumHi); 03961 if (2+NumHi < 4) 03962 Mask1[2+NumHi] = Idx; 03963 NumHi++; 03964 } 03965 } 03966 } 03967 03968 if (NumLo <= 2 && NumHi <= 2) { 03969 // If no more than two elements come from either vector. This can be 03970 // implemented with two shuffles. First shuffle gather the elements. 03971 // The second shuffle, which takes the first shuffle as both of its 03972 // vector operands, put the elements into the right order. 03973 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 03974 03975 SmallVector<int, 8> Mask2(4U, -1); 03976 03977 for (unsigned i = 0; i != 4; ++i) { 03978 if (Locs[i].first == -1) 03979 continue; 03980 else { 03981 unsigned Idx = (i < 2) ? 0 : 4; 03982 Idx += Locs[i].first * 2 + Locs[i].second; 03983 Mask2[i] = Idx; 03984 } 03985 } 03986 03987 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]); 03988 } else if (NumLo == 3 || NumHi == 3) { 03989 // Otherwise, we must have three elements from one vector, call it X, and 03990 // one element from the other, call it Y. First, use a shufps to build an 03991 // intermediate vector with the one element from Y and the element from X 03992 // that will be in the same half in the final destination (the indexes don't 03993 // matter). Then, use a shufps to build the final vector, taking the half 03994 // containing the element from Y from the intermediate, and the other half 03995 // from X. 03996 if (NumHi == 3) { 03997 // Normalize it so the 3 elements come from V1. 03998 CommuteVectorShuffleMask(PermMask, VT); 03999 std::swap(V1, V2); 04000 } 04001 04002 // Find the element from V2. 04003 unsigned HiIndex; 04004 for (HiIndex = 0; HiIndex < 3; ++HiIndex) { 04005 int Val = PermMask[HiIndex]; 04006 if (Val < 0) 04007 continue; 04008 if (Val >= 4) 04009 break; 04010 } 04011 04012 Mask1[0] = PermMask[HiIndex]; 04013 Mask1[1] = -1; 04014 Mask1[2] = PermMask[HiIndex^1]; 04015 Mask1[3] = -1; 04016 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 04017 04018 if (HiIndex >= 2) { 04019 Mask1[0] = PermMask[0]; 04020 Mask1[1] = PermMask[1]; 04021 Mask1[2] = HiIndex & 1 ? 6 : 4; 04022 Mask1[3] = HiIndex & 1 ? 4 : 6; 04023 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); 04024 } else { 04025 Mask1[0] = HiIndex & 1 ? 2 : 0; 04026 Mask1[1] = HiIndex & 1 ? 0 : 2; 04027 Mask1[2] = PermMask[2]; 04028 Mask1[3] = PermMask[3]; 04029 if (Mask1[2] >= 0) 04030 Mask1[2] += 4; 04031 if (Mask1[3] >= 0) 04032 Mask1[3] += 4; 04033 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]); 04034 } 04035 } 04036 04037 // Break it into (shuffle shuffle_hi, shuffle_lo). 04038 Locs.clear(); 04039 SmallVector<int,8> LoMask(4U, -1); 04040 SmallVector<int,8> HiMask(4U, -1); 04041 04042 SmallVector<int,8> *MaskPtr = &LoMask; 04043 unsigned MaskIdx = 0; 04044 unsigned LoIdx = 0; 04045 unsigned HiIdx = 2; 04046 for (unsigned i = 0; i != 4; ++i) { 04047 if (i == 2) { 04048 MaskPtr = &HiMask; 04049 MaskIdx = 1; 04050 LoIdx = 0; 04051 HiIdx = 2; 04052 } 04053 int Idx = PermMask[i]; 04054 if (Idx < 0) { 04055 Locs[i] = std::make_pair(-1, -1); 04056 } else if (Idx < 4) { 04057 Locs[i] = std::make_pair(MaskIdx, LoIdx); 04058 (*MaskPtr)[LoIdx] = Idx; 04059 LoIdx++; 04060 } else { 04061 Locs[i] = std::make_pair(MaskIdx, HiIdx); 04062 (*MaskPtr)[HiIdx] = Idx; 04063 HiIdx++; 04064 } 04065 } 04066 04067 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]); 04068 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]); 04069 SmallVector<int, 8> MaskOps; 04070 for (unsigned i = 0; i != 4; ++i) { 04071 if (Locs[i].first == -1) { 04072 MaskOps.push_back(-1); 04073 } else { 04074 unsigned Idx = Locs[i].first * 4 + Locs[i].second; 04075 MaskOps.push_back(Idx); 04076 } 04077 } 04078 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]); 04079 } 04080 04081 SDValue 04082 X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { 04083 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 04084 SDValue V1 = Op.getOperand(0); 04085 SDValue V2 = Op.getOperand(1); 04086 EVT VT = Op.getValueType(); 04087 DebugLoc dl = Op.getDebugLoc(); 04088 unsigned NumElems = VT.getVectorNumElements(); 04089 bool isMMX = VT.getSizeInBits() == 64; 04090 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; 04091 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; 04092 bool V1IsSplat = false; 04093 bool V2IsSplat = false; 04094 04095 if (isZeroShuffle(SVOp)) 04096 return getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl); 04097 04098 // Promote splats to v4f32. 04099 if (SVOp->isSplat()) { 04100 if (isMMX || NumElems < 4) 04101 return Op; 04102 return PromoteSplat(SVOp, DAG, Subtarget->hasSSE2()); 04103 } 04104 04105 // If the shuffle can be profitably rewritten as a narrower shuffle, then 04106 // do it! 04107 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 04108 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl); 04109 if (NewOp.getNode()) 04110 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, 04111 LowerVECTOR_SHUFFLE(NewOp, DAG)); 04112 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { 04113 // FIXME: Figure out a cleaner way to do this. 04114 // Try to make use of movq to zero out the top part. 04115 if (ISD::isBuildVectorAllZeros(V2.getNode())) { 04116 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl); 04117 if (NewOp.getNode()) { 04118 if (isCommutedMOVL(cast<ShuffleVectorSDNode>(NewOp), true, false)) 04119 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(0), 04120 DAG, Subtarget, dl); 04121 } 04122 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { 04123 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl); 04124 if (NewOp.getNode() && X86::isMOVLMask(cast<ShuffleVectorSDNode>(NewOp))) 04125 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), 04126 DAG, Subtarget, dl); 04127 } 04128 } 04129 04130 if (X86::isPSHUFDMask(SVOp)) 04131 return Op; 04132 04133 // Check if this can be converted into a logical shift. 04134 bool isLeft = false; 04135 unsigned ShAmt = 0; 04136 SDValue ShVal; 04137 bool isShift = getSubtarget()->hasSSE2() && 04138 isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt); 04139 if (isShift && ShVal.hasOneUse()) { 04140 // If the shifted value has multiple uses, it may be cheaper to use 04141 // v_set0 + movlhps or movhlps, etc. 04142 EVT EVT = VT.getVectorElementType(); 04143 ShAmt *= EVT.getSizeInBits(); 04144 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); 04145 } 04146 04147 if (X86::isMOVLMask(SVOp)) { 04148 if (V1IsUndef) 04149 return V2; 04150 if (ISD::isBuildVectorAllZeros(V1.getNode())) 04151 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl); 04152 if (!isMMX) 04153 return Op; 04154 } 04155 04156 // FIXME: fold these into legal mask. 04157 if (!isMMX && (X86::isMOVSHDUPMask(SVOp) || 04158 X86::isMOVSLDUPMask(SVOp) || 04159 X86::isMOVHLPSMask(SVOp) || 04160 X86::isMOVHPMask(SVOp) || 04161 X86::isMOVLPMask(SVOp))) 04162 return Op; 04163 04164 if (ShouldXformToMOVHLPS(SVOp) || 04165 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), SVOp)) 04166 return CommuteVectorShuffle(SVOp, DAG); 04167 04168 if (isShift) { 04169 // No better options. Use a vshl / vsrl. 04170 EVT EVT = VT.getVectorElementType(); 04171 ShAmt *= EVT.getSizeInBits(); 04172 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); 04173 } 04174 04175 bool Commuted = false; 04176 // FIXME: This should also accept a bitcast of a splat? Be careful, not 04177 // 1,1,1,1 -> v8i16 though. 04178 V1IsSplat = isSplatVector(V1.getNode()); 04179 V2IsSplat = isSplatVector(V2.getNode()); 04180 04181 // Canonicalize the splat or undef, if present, to be on the RHS. 04182 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { 04183 Op = CommuteVectorShuffle(SVOp, DAG); 04184 SVOp = cast<ShuffleVectorSDNode>(Op); 04185 V1 = SVOp->getOperand(0); 04186 V2 = SVOp->getOperand(1); 04187 std::swap(V1IsSplat, V2IsSplat); 04188 std::swap(V1IsUndef, V2IsUndef); 04189 Commuted = true; 04190 } 04191 04192 if (isCommutedMOVL(SVOp, V2IsSplat, V2IsUndef)) { 04193 // Shuffling low element of v1 into undef, just return v1. 04194 if (V2IsUndef) 04195 return V1; 04196 // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which 04197 // the instruction selector will not match, so get a canonical MOVL with 04198 // swapped operands to undo the commute. 04199 return getMOVL(DAG, dl, VT, V2, V1); 04200 } 04201 04202 if (X86::isUNPCKL_v_undef_Mask(SVOp) || 04203 X86::isUNPCKH_v_undef_Mask(SVOp) || 04204 X86::isUNPCKLMask(SVOp) || 04205 X86::isUNPCKHMask(SVOp)) 04206 return Op; 04207 04208 if (V2IsSplat) { 04209 // Normalize mask so all entries that point to V2 points to its first 04210 // element then try to match unpck{h|l} again. If match, return a 04211 // new vector_shuffle with the corrected mask. 04212 SDValue NewMask = NormalizeMask(SVOp, DAG); 04213 ShuffleVectorSDNode *NSVOp = cast<ShuffleVectorSDNode>(NewMask); 04214 if (NSVOp != SVOp) { 04215 if (X86::isUNPCKLMask(NSVOp, true)) { 04216 return NewMask; 04217 } else if (X86::isUNPCKHMask(NSVOp, true)) { 04218 return NewMask; 04219 } 04220 } 04221 } 04222 04223 if (Commuted) { 04224 // Commute is back and try unpck* again. 04225 // FIXME: this seems wrong. 04226 SDValue NewOp = CommuteVectorShuffle(SVOp, DAG); 04227 ShuffleVectorSDNode *NewSVOp = cast<ShuffleVectorSDNode>(NewOp); 04228 if (X86::isUNPCKL_v_undef_Mask(NewSVOp) || 04229 X86::isUNPCKH_v_undef_Mask(NewSVOp) || 04230 X86::isUNPCKLMask(NewSVOp) || 04231 X86::isUNPCKHMask(NewSVOp)) 04232 return NewOp; 04233 } 04234 04235 // FIXME: for mmx, bitcast v2i32 to v4i16 for shuffle. 04236 04237 // Normalize the node to match x86 shuffle ops if needed 04238 if (!isMMX && V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(SVOp)) 04239 return CommuteVectorShuffle(SVOp, DAG); 04240 04241 // Check for legal shuffle and return? 04242 SmallVector<int, 16> PermMask; 04243 SVOp->getMask(PermMask); 04244 if (isShuffleMaskLegal(PermMask, VT)) 04245 return Op; 04246 04247 // Handle v8i16 specifically since SSE can do byte extraction and insertion. 04248 if (VT == MVT::v8i16) { 04249 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(SVOp, DAG, *this); 04250 if (NewOp.getNode()) 04251 return NewOp; 04252 } 04253 04254 if (VT == MVT::v16i8) { 04255 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, DAG, *this); 04256 if (NewOp.getNode()) 04257 return NewOp; 04258 } 04259 04260 // Handle all 4 wide cases with a number of shuffles except for MMX. 04261 if (NumElems == 4 && !isMMX) 04262 return LowerVECTOR_SHUFFLE_4wide(SVOp, DAG); 04263 04264 return SDValue(); 04265 } 04266 04267 SDValue 04268 X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, 04269 SelectionDAG &DAG) { 04270 EVT VT = Op.getValueType(); 04271 DebugLoc dl = Op.getDebugLoc(); 04272 if (VT.getSizeInBits() == 8) { 04273 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, 04274 Op.getOperand(0), Op.getOperand(1)); 04275 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, 04276 DAG.getValueType(VT)); 04277 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 04278 } else if (VT.getSizeInBits() == 16) { 04279 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 04280 // If Idx is 0, it's cheaper to do a move instead of a pextrw. 04281 if (Idx == 0) 04282 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, 04283 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 04284 DAG.getNode(ISD::BIT_CONVERT, dl, 04285 MVT::v4i32, 04286 Op.getOperand(0)), 04287 Op.getOperand(1))); 04288 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, 04289 Op.getOperand(0), Op.getOperand(1)); 04290 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, 04291 DAG.getValueType(VT)); 04292 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 04293 } else if (VT == MVT::f32) { 04294 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy 04295 // the result back to FR32 register. It's only worth matching if the 04296 // result has a single use which is a store or a bitcast to i32. And in 04297 // the case of a store, it's not worth it if the index is a constant 0, 04298 // because a MOVSSmr can be used instead, which is smaller and faster. 04299 if (!Op.hasOneUse()) 04300 return SDValue(); 04301 SDNode *User = *Op.getNode()->use_begin(); 04302 if ((User->getOpcode() != ISD::STORE || 04303 (isa<ConstantSDNode>(Op.getOperand(1)) && 04304 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) && 04305 (User->getOpcode() != ISD::BIT_CONVERT || 04306 User->getValueType(0) != MVT::i32)) 04307 return SDValue(); 04308 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 04309 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4i32, 04310 Op.getOperand(0)), 04311 Op.getOperand(1)); 04312 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, Extract); 04313 } else if (VT == MVT::i32) { 04314 // ExtractPS works with constant index. 04315 if (isa<ConstantSDNode>(Op.getOperand(1))) 04316 return Op; 04317 } 04318 return SDValue(); 04319 } 04320 04321 04322 SDValue 04323 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 04324 if (!isa<ConstantSDNode>(Op.getOperand(1))) 04325 return SDValue(); 04326 04327 if (Subtarget->hasSSE41()) { 04328 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); 04329 if (Res.getNode()) 04330 return Res; 04331 } 04332 04333 EVT VT = Op.getValueType(); 04334 DebugLoc dl = Op.getDebugLoc(); 04335 // TODO: handle v16i8. 04336 if (VT.getSizeInBits() == 16) { 04337 SDValue Vec = Op.getOperand(0); 04338 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 04339 if (Idx == 0) 04340 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, 04341 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, 04342 DAG.getNode(ISD::BIT_CONVERT, dl, 04343 MVT::v4i32, Vec), 04344 Op.getOperand(1))); 04345 // Transform it so it match pextrw which produces a 32-bit result. 04346 EVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy+1); 04347 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EVT, 04348 Op.getOperand(0), Op.getOperand(1)); 04349 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EVT, Extract, 04350 DAG.getValueType(VT)); 04351 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); 04352 } else if (VT.getSizeInBits() == 32) { 04353 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 04354 if (Idx == 0) 04355 return Op; 04356 04357 // SHUFPS the element to the lowest double word, then movss. 04358 int Mask[4] = { Idx, -1, -1, -1 }; 04359 EVT VVT = Op.getOperand(0).getValueType(); 04360 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), 04361 DAG.getUNDEF(VVT), Mask); 04362 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, 04363 DAG.getIntPtrConstant(0)); 04364 } else if (VT.getSizeInBits() == 64) { 04365 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b 04366 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught 04367 // to match extract_elt for f64. 04368 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 04369 if (Idx == 0) 04370 return Op; 04371 04372 // UNPCKHPD the element to the lowest double word, then movsd. 04373 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored 04374 // to a f64mem, the whole operation is folded into a single MOVHPDmr. 04375 int Mask[2] = { 1, -1 }; 04376 EVT VVT = Op.getOperand(0).getValueType(); 04377 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), 04378 DAG.getUNDEF(VVT), Mask); 04379 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, 04380 DAG.getIntPtrConstant(0)); 04381 } 04382 04383 return SDValue(); 04384 } 04385 04386 SDValue 04387 X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG){ 04388 EVT VT = Op.getValueType(); 04389 EVT EVT = VT.getVectorElementType(); 04390 DebugLoc dl = Op.getDebugLoc(); 04391 04392 SDValue N0 = Op.getOperand(0); 04393 SDValue N1 = Op.getOperand(1); 04394 SDValue N2 = Op.getOperand(2); 04395 04396 if ((EVT.getSizeInBits() == 8 || EVT.getSizeInBits() == 16) && 04397 isa<ConstantSDNode>(N2)) { 04398 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB 04399 : X86ISD::PINSRW; 04400 // Transform it so it match pinsr{b,w} which expects a GR32 as its second 04401 // argument. 04402 if (N1.getValueType() != MVT::i32) 04403 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); 04404 if (N2.getValueType() != MVT::i32) 04405 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); 04406 return DAG.getNode(Opc, dl, VT, N0, N1, N2); 04407 } else if (EVT == MVT::f32 && isa<ConstantSDNode>(N2)) { 04408 // Bits [7:6] of the constant are the source select. This will always be 04409 // zero here. The DAG Combiner may combine an extract_elt index into these 04410 // bits. For example (insert (extract, 3), 2) could be matched by putting 04411 // the '3' into bits [7:6] of X86ISD::INSERTPS. 04412 // Bits [5:4] of the constant are the destination select. This is the 04413 // value of the incoming immediate. 04414 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may 04415 // combine either bitwise AND or insert of float 0.0 to set these bits. 04416 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4); 04417 // Create this as a scalar to vector.. 04418 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); 04419 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2); 04420 } else if (EVT == MVT::i32 && isa<ConstantSDNode>(N2)) { 04421 // PINSR* works with constant index. 04422 return Op; 04423 } 04424 return SDValue(); 04425 } 04426 04427 SDValue 04428 X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { 04429 EVT VT = Op.getValueType(); 04430 EVT EVT = VT.getVectorElementType(); 04431 04432 if (Subtarget->hasSSE41()) 04433 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); 04434 04435 if (EVT == MVT::i8) 04436 return SDValue(); 04437 04438 DebugLoc dl = Op.getDebugLoc(); 04439 SDValue N0 = Op.getOperand(0); 04440 SDValue N1 = Op.getOperand(1); 04441 SDValue N2 = Op.getOperand(2); 04442 04443 if (EVT.getSizeInBits() == 16 && isa<ConstantSDNode>(N2)) { 04444 // Transform it so it match pinsrw which expects a 16-bit value in a GR32 04445 // as its second argument. 04446 if (N1.getValueType() != MVT::i32) 04447 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); 04448 if (N2.getValueType() != MVT::i32) 04449 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); 04450 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2); 04451 } 04452 return SDValue(); 04453 } 04454 04455 SDValue 04456 X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) { 04457 DebugLoc dl = Op.getDebugLoc(); 04458 if (Op.getValueType() == MVT::v2f32) 04459 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f32, 04460 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i32, 04461 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, 04462 Op.getOperand(0)))); 04463 04464 if (Op.getValueType() == MVT::v1i64 && Op.getOperand(0).getValueType() == MVT::i64) 04465 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); 04466 04467 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); 04468 EVT VT = MVT::v2i32; 04469 switch (Op.getValueType().getSimpleVT().SimpleTy) { 04470 default: break; 04471 case MVT::v16i8: 04472 case MVT::v8i16: 04473 VT = MVT::v4i32; 04474 break; 04475 } 04476 return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(), 04477 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, AnyExt)); 04478 } 04479 04480 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as 04481 // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is 04482 // one of the above mentioned nodes. It has to be wrapped because otherwise 04483 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only 04484 // be used to form addressing mode. These wrapped nodes will be selected 04485 // into MOV32ri. 04486 SDValue 04487 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) { 04488 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 04489 04490 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 04491 // global base reg. 04492 unsigned char OpFlag = 0; 04493 unsigned WrapperKind = X86ISD::Wrapper; 04494 CodeModel::Model M = getTargetMachine().getCodeModel(); 04495 04496 if (Subtarget->isPICStyleRIPRel() && 04497 (M == CodeModel::Small || M == CodeModel::Kernel)) 04498 WrapperKind = X86ISD::WrapperRIP; 04499 else if (Subtarget->isPICStyleGOT()) 04500 OpFlag = X86II::MO_GOTOFF; 04501 else if (Subtarget->isPICStyleStubPIC()) 04502 OpFlag = X86II::MO_PIC_BASE_OFFSET; 04503 04504 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(), 04505 CP->getAlignment(), 04506 CP->getOffset(), OpFlag); 04507 DebugLoc DL = CP->getDebugLoc(); 04508 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 04509 // With PIC, the address is actually $g + Offset. 04510 if (OpFlag) { 04511 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 04512 DAG.getNode(X86ISD::GlobalBaseReg, 04513 DebugLoc::getUnknownLoc(), getPointerTy()), 04514 Result); 04515 } 04516 04517 return Result; 04518 } 04519 04520 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) { 04521 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 04522 04523 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 04524 // global base reg. 04525 unsigned char OpFlag = 0; 04526 unsigned WrapperKind = X86ISD::Wrapper; 04527 CodeModel::Model M = getTargetMachine().getCodeModel(); 04528 04529 if (Subtarget->isPICStyleRIPRel() && 04530 (M == CodeModel::Small || M == CodeModel::Kernel)) 04531 WrapperKind = X86ISD::WrapperRIP; 04532 else if (Subtarget->isPICStyleGOT()) 04533 OpFlag = X86II::MO_GOTOFF; 04534 else if (Subtarget->isPICStyleStubPIC()) 04535 OpFlag = X86II::MO_PIC_BASE_OFFSET; 04536 04537 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(), 04538 OpFlag); 04539 DebugLoc DL = JT->getDebugLoc(); 04540 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 04541 04542 // With PIC, the address is actually $g + Offset. 04543 if (OpFlag) { 04544 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 04545 DAG.getNode(X86ISD::GlobalBaseReg, 04546 DebugLoc::getUnknownLoc(), getPointerTy()), 04547 Result); 04548 } 04549 04550 return Result; 04551 } 04552 04553 SDValue 04554 X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) { 04555 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); 04556 04557 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the 04558 // global base reg. 04559 unsigned char OpFlag = 0; 04560 unsigned WrapperKind = X86ISD::Wrapper; 04561 CodeModel::Model M = getTargetMachine().getCodeModel(); 04562 04563 if (Subtarget->isPICStyleRIPRel() && 04564 (M == CodeModel::Small || M == CodeModel::Kernel)) 04565 WrapperKind = X86ISD::WrapperRIP; 04566 else if (Subtarget->isPICStyleGOT()) 04567 OpFlag = X86II::MO_GOTOFF; 04568 else if (Subtarget->isPICStyleStubPIC()) 04569 OpFlag = X86II::MO_PIC_BASE_OFFSET; 04570 04571 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag); 04572 04573 DebugLoc DL = Op.getDebugLoc(); 04574 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); 04575 04576 04577 // With PIC, the address is actually $g + Offset. 04578 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && 04579 !Subtarget->is64Bit()) { 04580 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), 04581 DAG.getNode(X86ISD::GlobalBaseReg, 04582 DebugLoc::getUnknownLoc(), 04583 getPointerTy()), 04584 Result); 04585 } 04586 04587 return Result; 04588 } 04589 04590 SDValue 04591 X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, DebugLoc dl, 04592 int64_t Offset, 04593 SelectionDAG &DAG) const { 04594 // Create the TargetGlobalAddress node, folding in the constant 04595 // offset if it is legal. 04596 unsigned char OpFlags = 04597 Subtarget->ClassifyGlobalReference(GV, getTargetMachine()); 04598 CodeModel::Model M = getTargetMachine().getCodeModel(); 04599 SDValue Result; 04600 if (OpFlags == X86II::MO_NO_FLAG && 04601 X86::isOffsetSuitableForCodeModel(Offset, M)) { 04602 // A direct static reference to a global. 04603 Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), Offset); 04604 Offset = 0; 04605 } else { 04606 Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), 0, OpFlags); 04607 } 04608 04609 if (Subtarget->isPICStyleRIPRel() && 04610 (M == CodeModel::Small || M == CodeModel::Kernel)) 04611 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); 04612 else 04613 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); 04614 04615 // With PIC, the address is actually $g + Offset. 04616 if (isGlobalRelativeToPICBase(OpFlags)) { 04617 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), 04618 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), 04619 Result); 04620 } 04621 04622 // For globals that require a load from a stub to get the address, emit the 04623 // load. 04624 if (isGlobalStubReference(OpFlags)) 04625 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result, 04626 PseudoSourceValue::getGOT(), 0); 04627 04628 // If there was a non-zero offset that we didn't fold, create an explicit 04629 // addition for it. 04630 if (Offset != 0) 04631 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result, 04632 DAG.getConstant(Offset, getPointerTy())); 04633 04634 return Result; 04635 } 04636 04637 SDValue 04638 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) { 04639 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); 04640 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset(); 04641 return LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG); 04642 } 04643 04644 static SDValue 04645 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, 04646 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, 04647 unsigned char OperandFlags) { 04648 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 04649 DebugLoc dl = GA->getDebugLoc(); 04650 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), 04651 GA->getValueType(0), 04652 GA->getOffset(), 04653 OperandFlags); 04654 if (InFlag) { 04655 SDValue Ops[] = { Chain, TGA, *InFlag }; 04656 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3); 04657 } else { 04658 SDValue Ops[] = { Chain, TGA }; 04659 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2); 04660 } 04661 SDValue Flag = Chain.getValue(1); 04662 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag); 04663 } 04664 04665 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit 04666 static SDValue 04667 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, 04668 const EVT PtrVT) { 04669 SDValue InFlag; 04670 DebugLoc dl = GA->getDebugLoc(); // ? function entry point might be better 04671 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX, 04672 DAG.getNode(X86ISD::GlobalBaseReg, 04673 DebugLoc::getUnknownLoc(), 04674 PtrVT), InFlag); 04675 InFlag = Chain.getValue(1); 04676 04677 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD); 04678 } 04679 04680 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit 04681 static SDValue 04682 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, 04683 const EVT PtrVT) { 04684 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT, 04685 X86::RAX, X86II::MO_TLSGD); 04686 } 04687 04688 // Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or 04689 // "local exec" model. 04690 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, 04691 const EVT PtrVT, TLSModel::Model model, 04692 bool is64Bit) { 04693 DebugLoc dl = GA->getDebugLoc(); 04694 // Get the Thread Pointer 04695 SDValue Base = DAG.getNode(X86ISD::SegmentBaseAddress, 04696 DebugLoc::getUnknownLoc(), PtrVT, 04697 DAG.getRegister(is64Bit? X86::FS : X86::GS, 04698 MVT::i32)); 04699 04700 SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Base, 04701 NULL, 0); 04702 04703 unsigned char OperandFlags = 0; 04704 // Most TLS accesses are not RIP relative, even on x86-64. One exception is 04705 // initialexec. 04706 unsigned WrapperKind = X86ISD::Wrapper; 04707 if (model == TLSModel::LocalExec) { 04708 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF; 04709 } else if (is64Bit) { 04710 assert(model == TLSModel::InitialExec); 04711 OperandFlags = X86II::MO_GOTTPOFF; 04712 WrapperKind = X86ISD::WrapperRIP; 04713 } else { 04714 assert(model == TLSModel::InitialExec); 04715 OperandFlags = X86II::MO_INDNTPOFF; 04716 } 04717 04718 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial 04719 // exec) 04720 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), 04721 GA->getOffset(), OperandFlags); 04722 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); 04723 04724 if (model == TLSModel::InitialExec) 04725 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset, 04726 PseudoSourceValue::getGOT(), 0); 04727 04728 // The address of the thread local variable is the add of the thread 04729 // pointer with the offset of the variable. 04730 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); 04731 } 04732 04733 SDValue 04734 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) { 04735 // TODO: implement the "local dynamic" model 04736 // TODO: implement the "initial exec"model for pic executables 04737 assert(Subtarget->isTargetELF() && 04738 "TLS not implemented for non-ELF targets"); 04739 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 04740 const GlobalValue *GV = GA->getGlobal(); 04741 04742 // If GV is an alias then use the aliasee for determining 04743 // thread-localness. 04744 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 04745 GV = GA->resolveAliasedGlobal(false); 04746 04747 TLSModel::Model model = getTLSModel(GV, 04748 getTargetMachine().getRelocationModel()); 04749 04750 switch (model) { 04751 case TLSModel::GeneralDynamic: 04752 case TLSModel::LocalDynamic: // not implemented 04753 if (Subtarget->is64Bit()) 04754 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); 04755 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); 04756 04757 case TLSModel::InitialExec: 04758 case TLSModel::LocalExec: 04759 return LowerToTLSExecModel(GA, DAG, getPointerTy(), model, 04760 Subtarget->is64Bit()); 04761 } 04762 04763 llvm_unreachable("Unreachable"); 04764 return SDValue(); 04765 } 04766 04767 04768 /// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and 04769 /// take a 2 x i32 value to shift plus a shift amount. 04770 SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) { 04771 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 04772 EVT VT = Op.getValueType(); 04773 unsigned VTBits = VT.getSizeInBits(); 04774 DebugLoc dl = Op.getDebugLoc(); 04775 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; 04776 SDValue ShOpLo = Op.getOperand(0); 04777 SDValue ShOpHi = Op.getOperand(1); 04778 SDValue ShAmt = Op.getOperand(2); 04779 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, 04780 DAG.getConstant(VTBits - 1, MVT::i8)) 04781 : DAG.getConstant(0, VT); 04782 04783 SDValue Tmp2, Tmp3; 04784 if (Op.getOpcode() == ISD::SHL_PARTS) { 04785 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt); 04786 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 04787 } else { 04788 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt); 04789 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt); 04790 } 04791 04792 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, 04793 DAG.getConstant(VTBits, MVT::i8)); 04794 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, VT, 04795 AndNode, DAG.getConstant(0, MVT::i8)); 04796 04797 SDValue Hi, Lo; 04798 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); 04799 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; 04800 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; 04801 04802 if (Op.getOpcode() == ISD::SHL_PARTS) { 04803 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); 04804 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); 04805 } else { 04806 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); 04807 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); 04808 } 04809 04810 SDValue Ops[2] = { Lo, Hi }; 04811 return DAG.getMergeValues(Ops, 2, dl); 04812 } 04813 04814 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 04815 EVT SrcVT = Op.getOperand(0).getValueType(); 04816 04817 if (SrcVT.isVector()) { 04818 if (SrcVT == MVT::v2i32 && Op.getValueType() == MVT::v2f64) { 04819 return Op; 04820 } 04821 return SDValue(); 04822 } 04823 04824 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && 04825 "Unknown SINT_TO_FP to lower!"); 04826 04827 // These are really Legal; return the operand so the caller accepts it as 04828 // Legal. 04829 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) 04830 return Op; 04831 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) && 04832 Subtarget->is64Bit()) { 04833 return Op; 04834 } 04835 04836 DebugLoc dl = Op.getDebugLoc(); 04837 unsigned Size = SrcVT.getSizeInBits()/8; 04838 MachineFunction &MF = DAG.getMachineFunction(); 04839 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); 04840 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 04841 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 04842 StackSlot, 04843 PseudoSourceValue::getFixedStack(SSFI), 0); 04844 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG); 04845 } 04846 04847 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, 04848 SDValue StackSlot, 04849 SelectionDAG &DAG) { 04850 // Build the FILD 04851 DebugLoc dl = Op.getDebugLoc(); 04852 SDVTList Tys; 04853 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); 04854 if (useSSE) 04855 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); 04856 else 04857 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); 04858 SmallVector<SDValue, 8> Ops; 04859 Ops.push_back(Chain); 04860 Ops.push_back(StackSlot); 04861 Ops.push_back(DAG.getValueType(SrcVT)); 04862 SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, dl, 04863 Tys, &Ops[0], Ops.size()); 04864 04865 if (useSSE) { 04866 Chain = Result.getValue(1); 04867 SDValue InFlag = Result.getValue(2); 04868 04869 // FIXME: Currently the FST is flagged to the FILD_FLAG. This 04870 // shouldn't be necessary except that RFP cannot be live across 04871 // multiple blocks. When stackifier is fixed, they can be uncoupled. 04872 MachineFunction &MF = DAG.getMachineFunction(); 04873 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); 04874 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 04875 Tys = DAG.getVTList(MVT::Other); 04876 SmallVector<SDValue, 8> Ops; 04877 Ops.push_back(Chain); 04878 Ops.push_back(Result); 04879 Ops.push_back(StackSlot); 04880 Ops.push_back(DAG.getValueType(Op.getValueType())); 04881 Ops.push_back(InFlag); 04882 Chain = DAG.getNode(X86ISD::FST, dl, Tys, &Ops[0], Ops.size()); 04883 Result = DAG.getLoad(Op.getValueType(), dl, Chain, StackSlot, 04884 PseudoSourceValue::getFixedStack(SSFI), 0); 04885 } 04886 04887 return Result; 04888 } 04889 04890 // LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion. 04891 SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG) { 04892 // This algorithm is not obvious. Here it is in C code, more or less: 04893 /* 04894 double uint64_to_double( uint32_t hi, uint32_t lo ) { 04895 static const __m128i exp = { 0x4330000045300000ULL, 0 }; 04896 static const __m128d bias = { 0x1.0p84, 0x1.0p52 }; 04897 04898 // Copy ints to xmm registers. 04899 __m128i xh = _mm_cvtsi32_si128( hi ); 04900 __m128i xl = _mm_cvtsi32_si128( lo ); 04901 04902 // Combine into low half of a single xmm register. 04903 __m128i x = _mm_unpacklo_epi32( xh, xl ); 04904 __m128d d; 04905 double sd; 04906 04907 // Merge in appropriate exponents to give the integer bits the right 04908 // magnitude. 04909 x = _mm_unpacklo_epi32( x, exp ); 04910 04911 // Subtract away the biases to deal with the IEEE-754 double precision 04912 // implicit 1. 04913 d = _mm_sub_pd( (__m128d) x, bias ); 04914 04915 // All conversions up to here are exact. The correctly rounded result is 04916 // calculated using the current rounding mode using the following 04917 // horizontal add. 04918 d = _mm_add_sd( d, _mm_unpackhi_pd( d, d ) ); 04919 _mm_store_sd( &sd, d ); // Because we are returning doubles in XMM, this 04920 // store doesn't really need to be here (except 04921 // maybe to zero the other double) 04922 return sd; 04923 } 04924 */ 04925 04926 DebugLoc dl = Op.getDebugLoc(); 04927 LLVMContext *Context = DAG.getContext(); 04928 04929 // Build some magic constants. 04930 std::vector<Constant*> CV0; 04931 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x45300000))); 04932 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x43300000))); 04933 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0))); 04934 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0))); 04935 Constant *C0 = ConstantVector::get(CV0); 04936 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16); 04937 04938 std::vector<Constant*> CV1; 04939 CV1.push_back( 04940 ConstantFP::get(*Context, APFloat(APInt(64, 0x4530000000000000ULL)))); 04941 CV1.push_back( 04942 ConstantFP::get(*Context, APFloat(APInt(64, 0x4330000000000000ULL)))); 04943 Constant *C1 = ConstantVector::get(CV1); 04944 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16); 04945 04946 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 04947 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 04948 Op.getOperand(0), 04949 DAG.getIntPtrConstant(1))); 04950 SDValue XR2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 04951 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 04952 Op.getOperand(0), 04953 DAG.getIntPtrConstant(0))); 04954 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, XR1, XR2); 04955 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, 04956 PseudoSourceValue::getConstantPool(), 0, 04957 false, 16); 04958 SDValue Unpck2 = getUnpackl(DAG, dl, MVT::v4i32, Unpck1, CLod0); 04959 SDValue XR2F = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Unpck2); 04960 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, 04961 PseudoSourceValue::getConstantPool(), 0, 04962 false, 16); 04963 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); 04964 04965 // Add the halves; easiest way is to swap them into another reg first. 04966 int ShufMask[2] = { 1, -1 }; 04967 SDValue Shuf = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, 04968 DAG.getUNDEF(MVT::v2f64), ShufMask); 04969 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuf, Sub); 04970 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Add, 04971 DAG.getIntPtrConstant(0)); 04972 } 04973 04974 // LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion. 04975 SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG) { 04976 DebugLoc dl = Op.getDebugLoc(); 04977 // FP constant to bias correct the final result. 04978 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), 04979 MVT::f64); 04980 04981 // Load the 32-bit value into an XMM register. 04982 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, 04983 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 04984 Op.getOperand(0), 04985 DAG.getIntPtrConstant(0))); 04986 04987 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 04988 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Load), 04989 DAG.getIntPtrConstant(0)); 04990 04991 // Or the load with the bias. 04992 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, 04993 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, 04994 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 04995 MVT::v2f64, Load)), 04996 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, 04997 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 04998 MVT::v2f64, Bias))); 04999 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, 05000 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Or), 05001 DAG.getIntPtrConstant(0)); 05002 05003 // Subtract the bias. 05004 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); 05005 05006 // Handle final rounding. 05007 EVT DestVT = Op.getValueType(); 05008 05009 if (DestVT.bitsLT(MVT::f64)) { 05010 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 05011 DAG.getIntPtrConstant(0)); 05012 } else if (DestVT.bitsGT(MVT::f64)) { 05013 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 05014 } 05015 05016 // Handle final rounding. 05017 return Sub; 05018 } 05019 05020 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) { 05021 SDValue N0 = Op.getOperand(0); 05022 DebugLoc dl = Op.getDebugLoc(); 05023 05024 // Now not UINT_TO_FP is legal (it's marked custom), dag combiner won't 05025 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform 05026 // the optimization here. 05027 if (DAG.SignBitIsZero(N0)) 05028 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0); 05029 05030 EVT SrcVT = N0.getValueType(); 05031 if (SrcVT == MVT::i64) { 05032 // We only handle SSE2 f64 target here; caller can expand the rest. 05033 if (Op.getValueType() != MVT::f64 || !X86ScalarSSEf64) 05034 return SDValue(); 05035 05036 return LowerUINT_TO_FP_i64(Op, DAG); 05037 } else if (SrcVT == MVT::i32 && X86ScalarSSEf64) { 05038 return LowerUINT_TO_FP_i32(Op, DAG); 05039 } 05040 05041 assert(SrcVT == MVT::i32 && "Unknown UINT_TO_FP to lower!"); 05042 05043 // Make a 64-bit buffer, and use it to build an FILD. 05044 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); 05045 SDValue WordOff = DAG.getConstant(4, getPointerTy()); 05046 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl, 05047 getPointerTy(), StackSlot, WordOff); 05048 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), 05049 StackSlot, NULL, 0); 05050 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32), 05051 OffsetSlot, NULL, 0); 05052 return BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); 05053 } 05054 05055 std::pair<SDValue,SDValue> X86TargetLowering:: 05056 FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned) { 05057 DebugLoc dl = Op.getDebugLoc(); 05058 05059 EVT DstTy = Op.getValueType(); 05060 05061 if (!IsSigned) { 05062 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); 05063 DstTy = MVT::i64; 05064 } 05065 05066 assert(DstTy.getSimpleVT() <= MVT::i64 && 05067 DstTy.getSimpleVT() >= MVT::i16 && 05068 "Unknown FP_TO_SINT to lower!"); 05069 05070 // These are really Legal. 05071 if (DstTy == MVT::i32 && 05072 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 05073 return std::make_pair(SDValue(), SDValue()); 05074 if (Subtarget->is64Bit() && 05075 DstTy == MVT::i64 && 05076 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) 05077 return std::make_pair(SDValue(), SDValue()); 05078 05079 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary 05080 // stack slot. 05081 MachineFunction &MF = DAG.getMachineFunction(); 05082 unsigned MemSize = DstTy.getSizeInBits()/8; 05083 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 05084 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 05085 05086 unsigned Opc; 05087 switch (DstTy.getSimpleVT().SimpleTy) { 05088 default: llvm_unreachable("Invalid FP_TO_SINT to lower!"); 05089 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; 05090 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; 05091 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; 05092 } 05093 05094 SDValue Chain = DAG.getEntryNode(); 05095 SDValue Value = Op.getOperand(0); 05096 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { 05097 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); 05098 Chain = DAG.getStore(Chain, dl, Value, StackSlot, 05099 PseudoSourceValue::getFixedStack(SSFI), 0); 05100 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); 05101 SDValue Ops[] = { 05102 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) 05103 }; 05104 Value = DAG.getNode(X86ISD::FLD, dl, Tys, Ops, 3); 05105 Chain = Value.getValue(1); 05106 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); 05107 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 05108 } 05109 05110 // Build the FP_TO_INT*_IN_MEM 05111 SDValue Ops[] = { Chain, Value, StackSlot }; 05112 SDValue FIST = DAG.getNode(Opc, dl, MVT::Other, Ops, 3); 05113 05114 return std::make_pair(FIST, StackSlot); 05115 } 05116 05117 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) { 05118 if (Op.getValueType().isVector()) { 05119 if (Op.getValueType() == MVT::v2i32 && 05120 Op.getOperand(0).getValueType() == MVT::v2f64) { 05121 return Op; 05122 } 05123 return SDValue(); 05124 } 05125 05126 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, true); 05127 SDValue FIST = Vals.first, StackSlot = Vals.second; 05128 // If FP_TO_INTHelper failed, the node is actually supposed to be Legal. 05129 if (FIST.getNode() == 0) return Op; 05130 05131 // Load the result. 05132 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), 05133 FIST, StackSlot, NULL, 0); 05134 } 05135 05136 SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) { 05137 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, false); 05138 SDValue FIST = Vals.first, StackSlot = Vals.second; 05139 assert(FIST.getNode() && "Unexpected failure"); 05140 05141 // Load the result. 05142 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), 05143 FIST, StackSlot, NULL, 0); 05144 } 05145 05146 SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) { 05147 LLVMContext *Context = DAG.getContext(); 05148 DebugLoc dl = Op.getDebugLoc(); 05149 EVT VT = Op.getValueType(); 05150 EVT EltVT = VT; 05151 if (VT.isVector()) 05152 EltVT = VT.getVectorElementType(); 05153 std::vector<Constant*> CV; 05154 if (EltVT == MVT::f64) { 05155 Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63)))); 05156 CV.push_back(C); 05157 CV.push_back(C); 05158 } else { 05159 Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31)))); 05160 CV.push_back(C); 05161 CV.push_back(C); 05162 CV.push_back(C); 05163 CV.push_back(C); 05164 } 05165 Constant *C = ConstantVector::get(CV); 05166 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 05167 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 05168 PseudoSourceValue::getConstantPool(), 0, 05169 false, 16); 05170 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask); 05171 } 05172 05173 SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) { 05174 LLVMContext *Context = DAG.getContext(); 05175 DebugLoc dl = Op.getDebugLoc(); 05176 EVT VT = Op.getValueType(); 05177 EVT EltVT = VT; 05178 unsigned EltNum = 1; 05179 if (VT.isVector()) { 05180 EltVT = VT.getVectorElementType(); 05181 EltNum = VT.getVectorNumElements(); 05182 } 05183 std::vector<Constant*> CV; 05184 if (EltVT == MVT::f64) { 05185 Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63))); 05186 CV.push_back(C); 05187 CV.push_back(C); 05188 } else { 05189 Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31))); 05190 CV.push_back(C); 05191 CV.push_back(C); 05192 CV.push_back(C); 05193 CV.push_back(C); 05194 } 05195 Constant *C = ConstantVector::get(CV); 05196 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 05197 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 05198 PseudoSourceValue::getConstantPool(), 0, 05199 false, 16); 05200 if (VT.isVector()) { 05201 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, 05202 DAG.getNode(ISD::XOR, dl, MVT::v2i64, 05203 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, 05204 Op.getOperand(0)), 05205 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, Mask))); 05206 } else { 05207 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask); 05208 } 05209 } 05210 05211 SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { 05212 LLVMContext *Context = DAG.getContext(); 05213 SDValue Op0 = Op.getOperand(0); 05214 SDValue Op1 = Op.getOperand(1); 05215 DebugLoc dl = Op.getDebugLoc(); 05216 EVT VT = Op.getValueType(); 05217 EVT SrcVT = Op1.getValueType(); 05218 05219 // If second operand is smaller, extend it first. 05220 if (SrcVT.bitsLT(VT)) { 05221 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1); 05222 SrcVT = VT; 05223 } 05224 // And if it is bigger, shrink it first. 05225 if (SrcVT.bitsGT(VT)) { 05226 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1)); 05227 SrcVT = VT; 05228 } 05229 05230 // At this point the operands and the result should have the same 05231 // type, and that won't be f80 since that is not custom lowered. 05232 05233 // First get the sign bit of second operand. 05234 std::vector<Constant*> CV; 05235 if (SrcVT == MVT::f64) { 05236 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63)))); 05237 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0)))); 05238 } else { 05239 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31)))); 05240 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 05241 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 05242 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 05243 } 05244 Constant *C = ConstantVector::get(CV); 05245 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 05246 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx, 05247 PseudoSourceValue::getConstantPool(), 0, 05248 false, 16); 05249 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1); 05250 05251 // Shift sign bit right or left if the two operands have different types. 05252 if (SrcVT.bitsGT(VT)) { 05253 // Op0 is MVT::f32, Op1 is MVT::f64. 05254 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit); 05255 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit, 05256 DAG.getConstant(32, MVT::i32)); 05257 SignBit = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4f32, SignBit); 05258 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit, 05259 DAG.getIntPtrConstant(0)); 05260 } 05261 05262 // Clear first operand sign bit. 05263 CV.clear(); 05264 if (VT == MVT::f64) { 05265 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))))); 05266 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0)))); 05267 } else { 05268 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))))); 05269 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 05270 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 05271 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0)))); 05272 } 05273 C = ConstantVector::get(CV); 05274 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); 05275 SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 05276 PseudoSourceValue::getConstantPool(), 0, 05277 false, 16); 05278 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2); 05279 05280 // Or the value with the sign bit. 05281 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit); 05282 } 05283 05284 /// Emit nodes that will be selected as "test Op0,Op0", or something 05285 /// equivalent. 05286 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, 05287 SelectionDAG &DAG) { 05288 DebugLoc dl = Op.getDebugLoc(); 05289 05290 // CF and OF aren't always set the way we want. Determine which 05291 // of these we need. 05292 bool NeedCF = false; 05293 bool NeedOF = false; 05294 switch (X86CC) { 05295 case X86::COND_A: case X86::COND_AE: 05296 case X86::COND_B: case X86::COND_BE: 05297 NeedCF = true; 05298 break; 05299 case X86::COND_G: case X86::COND_GE: 05300 case X86::COND_L: case X86::COND_LE: 05301 case X86::COND_O: case X86::COND_NO: 05302 NeedOF = true; 05303 break; 05304 default: break; 05305 } 05306 05307 // See if we can use the EFLAGS value from the operand instead of 05308 // doing a separate TEST. TEST always sets OF and CF to 0, so unless 05309 // we prove that the arithmetic won't overflow, we can't use OF or CF. 05310 if (Op.getResNo() == 0 && !NeedOF && !NeedCF) { 05311 unsigned Opcode = 0; 05312 unsigned NumOperands = 0; 05313 switch (Op.getNode()->getOpcode()) { 05314 case ISD::ADD: 05315 // Due to an isel shortcoming, be conservative if this add is likely to 05316 // be selected as part of a load-modify-store instruction. When the root 05317 // node in a match is a store, isel doesn't know how to remap non-chain 05318 // non-flag uses of other nodes in the match, such as the ADD in this 05319 // case. This leads to the ADD being left around and reselected, with 05320 // the result being two adds in the output. 05321 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 05322 UE = Op.getNode()->use_end(); UI != UE; ++UI) 05323 if (UI->getOpcode() == ISD::STORE) 05324 goto default_case; 05325 if (ConstantSDNode *C = 05326 dyn_cast<ConstantSDNode>(Op.getNode()->getOperand(1))) { 05327 // An add of one will be selected as an INC. 05328 if (C->getAPIntValue() == 1) { 05329 Opcode = X86ISD::INC; 05330 NumOperands = 1; 05331 break; 05332 } 05333 // An add of negative one (subtract of one) will be selected as a DEC. 05334 if (C->getAPIntValue().isAllOnesValue()) { 05335 Opcode = X86ISD::DEC; 05336 NumOperands = 1; 05337 break; 05338 } 05339 } 05340 // Otherwise use a regular EFLAGS-setting add. 05341 Opcode = X86ISD::ADD; 05342 NumOperands = 2; 05343 break; 05344 case ISD::SUB: 05345 // Due to the ISEL shortcoming noted above, be conservative if this sub is 05346 // likely to be selected as part of a load-modify-store instruction. 05347 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 05348 UE = Op.getNode()->use_end(); UI != UE; ++UI) 05349 if (UI->getOpcode() == ISD::STORE) 05350 goto default_case; 05351 // Otherwise use a regular EFLAGS-setting sub. 05352 Opcode = X86ISD::SUB; 05353 NumOperands = 2; 05354 break; 05355 case X86ISD::ADD: 05356 case X86ISD::SUB: 05357 case X86ISD::INC: 05358 case X86ISD::DEC: 05359 return SDValue(Op.getNode(), 1); 05360 default: 05361 default_case: 05362 break; 05363 } 05364 if (Opcode != 0) { 05365 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); 05366 SmallVector<SDValue, 4> Ops; 05367 for (unsigned i = 0; i != NumOperands; ++i) 05368 Ops.push_back(Op.getOperand(i)); 05369 SDValue New = DAG.getNode(Opcode, dl, VTs, &Ops[0], NumOperands); 05370 DAG.ReplaceAllUsesWith(Op, New); 05371 return SDValue(New.getNode(), 1); 05372 } 05373 } 05374 05375 // Otherwise just emit a CMP with 0, which is the TEST pattern. 05376 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, 05377 DAG.getConstant(0, Op.getValueType())); 05378 } 05379 05380 /// Emit nodes that will be selected as "cmp Op0,Op1", or something 05381 /// equivalent. 05382 SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, 05383 SelectionDAG &DAG) { 05384 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) 05385 if (C->getAPIntValue() == 0) 05386 return EmitTest(Op0, X86CC, DAG); 05387 05388 DebugLoc dl = Op0.getDebugLoc(); 05389 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); 05390 } 05391 05392 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) { 05393 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); 05394 SDValue Op0 = Op.getOperand(0); 05395 SDValue Op1 = Op.getOperand(1); 05396 DebugLoc dl = Op.getDebugLoc(); 05397 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 05398 05399 // Lower (X & (1 << N)) == 0 to BT(X, N). 05400 // Lower ((X >>u N) & 1) != 0 to BT(X, N). 05401 // Lower ((X >>s N) & 1) != 0 to BT(X, N). 05402 if (Op0.getOpcode() == ISD::AND && 05403 Op0.hasOneUse() && 05404 Op1.getOpcode() == ISD::Constant && 05405 cast<ConstantSDNode>(Op1)->getZExtValue() == 0 && 05406 (CC == ISD::SETEQ || CC == ISD::SETNE)) { 05407 SDValue LHS, RHS; 05408 if (Op0.getOperand(1).getOpcode() == ISD::SHL) { 05409 if (ConstantSDNode *Op010C = 05410 dyn_cast<ConstantSDNode>(Op0.getOperand(1).getOperand(0))) 05411 if (Op010C->getZExtValue() == 1) { 05412 LHS = Op0.getOperand(0); 05413 RHS = Op0.getOperand(1).getOperand(1); 05414 } 05415 } else if (Op0.getOperand(0).getOpcode() == ISD::SHL) { 05416 if (ConstantSDNode *Op000C = 05417 dyn_cast<ConstantSDNode>(Op0.getOperand(0).getOperand(0))) 05418 if (Op000C->getZExtValue() == 1) { 05419 LHS = Op0.getOperand(1); 05420 RHS = Op0.getOperand(0).getOperand(1); 05421 } 05422 } else if (Op0.getOperand(1).getOpcode() == ISD::Constant) { 05423 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op0.getOperand(1)); 05424 SDValue AndLHS = Op0.getOperand(0); 05425 if (AndRHS->getZExtValue() == 1 && AndLHS.getOpcode() == ISD::SRL) { 05426 LHS = AndLHS.getOperand(0); 05427 RHS = AndLHS.getOperand(1); 05428 } 05429 } 05430 05431 if (LHS.getNode()) { 05432 // If LHS is i8, promote it to i16 with any_extend. There is no i8 BT 05433 // instruction. Since the shift amount is in-range-or-undefined, we know 05434 // that doing a bittest on the i16 value is ok. We extend to i32 because 05435 // the encoding for the i16 version is larger than the i32 version. 05436 if (LHS.getValueType() == MVT::i8) 05437 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); 05438 05439 // If the operand types disagree, extend the shift amount to match. Since 05440 // BT ignores high bits (like shifts) we can use anyextend. 05441 if (LHS.getValueType() != RHS.getValueType()) 05442 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS); 05443 05444 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); 05445 unsigned Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B; 05446 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 05447 DAG.getConstant(Cond, MVT::i8), BT); 05448 } 05449 } 05450 05451 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 05452 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG); 05453 05454 SDValue Cond = EmitCmp(Op0, Op1, X86CC, DAG); 05455 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 05456 DAG.getConstant(X86CC, MVT::i8), Cond); 05457 } 05458 05459 SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) { 05460 SDValue Cond; 05461 SDValue Op0 = Op.getOperand(0); 05462 SDValue Op1 = Op.getOperand(1); 05463 SDValue CC = Op.getOperand(2); 05464 EVT VT = Op.getValueType(); 05465 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); 05466 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); 05467 DebugLoc dl = Op.getDebugLoc(); 05468 05469 if (isFP) { 05470 unsigned SSECC = 8; 05471 EVT VT0 = Op0.getValueType(); 05472 assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64); 05473 unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD; 05474 bool Swap = false; 05475 05476 switch (SetCCOpcode) { 05477 default: break; 05478 case ISD::SETOEQ: 05479 case ISD::SETEQ: SSECC = 0; break; 05480 case ISD::SETOGT: 05481 case ISD::SETGT: Swap = true; // Fallthrough 05482 case ISD::SETLT: 05483 case ISD::SETOLT: SSECC = 1; break; 05484 case ISD::SETOGE: 05485 case ISD::SETGE: Swap = true; // Fallthrough 05486 case ISD::SETLE: 05487 case ISD::SETOLE: SSECC = 2; break; 05488 case ISD::SETUO: SSECC = 3; break; 05489 case ISD::SETUNE: 05490 case ISD::SETNE: SSECC = 4; break; 05491 case ISD::SETULE: Swap = true; 05492 case ISD::SETUGE: SSECC = 5; break; 05493 case ISD::SETULT: Swap = true; 05494 case ISD::SETUGT: SSECC = 6; break; 05495 case ISD::SETO: SSECC = 7; break; 05496 } 05497 if (Swap) 05498 std::swap(Op0, Op1); 05499 05500 // In the two special cases we can't handle, emit two comparisons. 05501 if (SSECC == 8) { 05502 if (SetCCOpcode == ISD::SETUEQ) { 05503 SDValue UNORD, EQ; 05504 UNORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); 05505 EQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); 05506 return DAG.getNode(ISD::OR, dl, VT, UNORD, EQ); 05507 } 05508 else if (SetCCOpcode == ISD::SETONE) { 05509 SDValue ORD, NEQ; 05510 ORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); 05511 NEQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); 05512 return DAG.getNode(ISD::AND, dl, VT, ORD, NEQ); 05513 } 05514 llvm_unreachable("Illegal FP comparison"); 05515 } 05516 // Handle all other FP comparisons here. 05517 return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); 05518 } 05519 05520 // We are handling one of the integer comparisons here. Since SSE only has 05521 // GT and EQ comparisons for integer, swapping operands and multiple 05522 // operations may be required for some comparisons. 05523 unsigned Opc = 0, EQOpc = 0, GTOpc = 0; 05524 bool Swap = false, Invert = false, FlipSigns = false; 05525 05526 switch (VT.getSimpleVT().SimpleTy) { 05527 default: break; 05528 case MVT::v8i8: 05529 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; 05530 case MVT::v4i16: 05531 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; 05532 case MVT::v2i32: 05533 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; 05534 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; 05535 } 05536 05537 switch (SetCCOpcode) { 05538 default: break; 05539 case ISD::SETNE: Invert = true; 05540 case ISD::SETEQ: Opc = EQOpc; break; 05541 case ISD::SETLT: Swap = true; 05542 case ISD::SETGT: Opc = GTOpc; break; 05543 case ISD::SETGE: Swap = true; 05544 case ISD::SETLE: Opc = GTOpc; Invert = true; break; 05545 case ISD::SETULT: Swap = true; 05546 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; 05547 case ISD::SETUGE: Swap = true; 05548 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; 05549 } 05550 if (Swap) 05551 std::swap(Op0, Op1); 05552 05553 // Since SSE has no unsigned integer comparisons, we need to flip the sign 05554 // bits of the inputs before performing those operations. 05555 if (FlipSigns) { 05556 EVT EltVT = VT.getVectorElementType(); 05557 SDValue SignBit = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), 05558 EltVT); 05559 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); 05560 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0], 05561 SignBits.size()); 05562 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec); 05563 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec); 05564 } 05565 05566 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); 05567 05568 // If the logical-not of the result is required, perform that now. 05569 if (Invert) 05570 Result = DAG.getNOT(dl, Result, VT); 05571 05572 return Result; 05573 } 05574 05575 // isX86LogicalCmp - Return true if opcode is a X86 logical comparison. 05576 static bool isX86LogicalCmp(SDValue Op) { 05577 unsigned Opc = Op.getNode()->getOpcode(); 05578 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) 05579 return true; 05580 if (Op.getResNo() == 1 && 05581 (Opc == X86ISD::ADD || 05582 Opc == X86ISD::SUB || 05583 Opc == X86ISD::SMUL || 05584 Opc == X86ISD::UMUL || 05585 Opc == X86ISD::INC || 05586 Opc == X86ISD::DEC)) 05587 return true; 05588 05589 return false; 05590 } 05591 05592 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) { 05593 bool addTest = true; 05594 SDValue Cond = Op.getOperand(0); 05595 DebugLoc dl = Op.getDebugLoc(); 05596 SDValue CC; 05597 05598 if (Cond.getOpcode() == ISD::SETCC) 05599 Cond = LowerSETCC(Cond, DAG); 05600 05601 // If condition flag is set by a X86ISD::CMP, then use it as the condition 05602 // setting operand in place of the X86ISD::SETCC. 05603 if (Cond.getOpcode() == X86ISD::SETCC) { 05604 CC = Cond.getOperand(0); 05605 05606 SDValue Cmp = Cond.getOperand(1); 05607 unsigned Opc = Cmp.getOpcode(); 05608 EVT VT = Op.getValueType(); 05609 05610 bool IllegalFPCMov = false; 05611 if (VT.isFloatingPoint() && !VT.isVector() && 05612 !isScalarFPTypeInSSEReg(VT)) // FPStack? 05613 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue()); 05614 05615 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) || 05616 Opc == X86ISD::BT) { // FIXME 05617 Cond = Cmp; 05618 addTest = false; 05619 } 05620 } 05621 05622 if (addTest) { 05623 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 05624 Cond = EmitTest(Cond, X86::COND_NE, DAG); 05625 } 05626 05627 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Flag); 05628 SmallVector<SDValue, 4> Ops; 05629 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if 05630 // condition is true. 05631 Ops.push_back(Op.getOperand(2)); 05632 Ops.push_back(Op.getOperand(1)); 05633 Ops.push_back(CC); 05634 Ops.push_back(Cond); 05635 return DAG.getNode(X86ISD::CMOV, dl, VTs, &Ops[0], Ops.size()); 05636 } 05637 05638 // isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or 05639 // ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart 05640 // from the AND / OR. 05641 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) { 05642 Opc = Op.getOpcode(); 05643 if (Opc != ISD::OR && Opc != ISD::AND) 05644 return false; 05645 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC && 05646 Op.getOperand(0).hasOneUse() && 05647 Op.getOperand(1).getOpcode() == X86ISD::SETCC && 05648 Op.getOperand(1).hasOneUse()); 05649 } 05650 05651 // isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and 05652 // 1 and that the SETCC node has a single use. 05653 static bool isXor1OfSetCC(SDValue Op) { 05654 if (Op.getOpcode() != ISD::XOR) 05655 return false; 05656 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 05657 if (N1C && N1C->getAPIntValue() == 1) { 05658 return Op.getOperand(0).getOpcode() == X86ISD::SETCC && 05659 Op.getOperand(0).hasOneUse(); 05660 } 05661 return false; 05662 } 05663 05664 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) { 05665 bool addTest = true; 05666 SDValue Chain = Op.getOperand(0); 05667 SDValue Cond = Op.getOperand(1); 05668 SDValue Dest = Op.getOperand(2); 05669 DebugLoc dl = Op.getDebugLoc(); 05670 SDValue CC; 05671 05672 if (Cond.getOpcode() == ISD::SETCC) 05673 Cond = LowerSETCC(Cond, DAG); 05674 #if 0 05675 // FIXME: LowerXALUO doesn't handle these!! 05676 else if (Cond.getOpcode() == X86ISD::ADD || 05677 Cond.getOpcode() == X86ISD::SUB || 05678 Cond.getOpcode() == X86ISD::SMUL || 05679 Cond.getOpcode() == X86ISD::UMUL) 05680 Cond = LowerXALUO(Cond, DAG); 05681 #endif 05682 05683 // If condition flag is set by a X86ISD::CMP, then use it as the condition 05684 // setting operand in place of the X86ISD::SETCC. 05685 if (Cond.getOpcode() == X86ISD::SETCC) { 05686 CC = Cond.getOperand(0); 05687 05688 SDValue Cmp = Cond.getOperand(1); 05689 unsigned Opc = Cmp.getOpcode(); 05690 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp?? 05691 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) { 05692 Cond = Cmp; 05693 addTest = false; 05694 } else { 05695 switch (cast<ConstantSDNode>(CC)->getZExtValue()) { 05696 default: break; 05697 case X86::COND_O: 05698 case X86::COND_B: 05699 // These can only come from an arithmetic instruction with overflow, 05700 // e.g. SADDO, UADDO. 05701 Cond = Cond.getNode()->getOperand(1); 05702 addTest = false; 05703 break; 05704 } 05705 } 05706 } else { 05707 unsigned CondOpc; 05708 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) { 05709 SDValue Cmp = Cond.getOperand(0).getOperand(1); 05710 if (CondOpc == ISD::OR) { 05711 // Also, recognize the pattern generated by an FCMP_UNE. We can emit 05712 // two branches instead of an explicit OR instruction with a 05713 // separate test. 05714 if (Cmp == Cond.getOperand(1).getOperand(1) && 05715 isX86LogicalCmp(Cmp)) { 05716 CC = Cond.getOperand(0).getOperand(0); 05717 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 05718 Chain, Dest, CC, Cmp); 05719 CC = Cond.getOperand(1).getOperand(0); 05720 Cond = Cmp; 05721 addTest = false; 05722 } 05723 } else { // ISD::AND 05724 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit 05725 // two branches instead of an explicit AND instruction with a 05726 // separate test. However, we only do this if this block doesn't 05727 // have a fall-through edge, because this requires an explicit 05728 // jmp when the condition is false. 05729 if (Cmp == Cond.getOperand(1).getOperand(1) && 05730 isX86LogicalCmp(Cmp) && 05731 Op.getNode()->hasOneUse()) { 05732 X86::CondCode CCode = 05733 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); 05734 CCode = X86::GetOppositeBranchCondition(CCode); 05735 CC = DAG.getConstant(CCode, MVT::i8); 05736 SDValue User = SDValue(*Op.getNode()->use_begin(), 0); 05737 // Look for an unconditional branch following this conditional branch. 05738 // We need this because we need to reverse the successors in order 05739 // to implement FCMP_OEQ. 05740 if (User.getOpcode() == ISD::BR) { 05741 SDValue FalseBB = User.getOperand(1); 05742 SDValue NewBR = 05743 DAG.UpdateNodeOperands(User, User.getOperand(0), Dest); 05744 assert(NewBR == User); 05745 Dest = FalseBB; 05746 05747 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 05748 Chain, Dest, CC, Cmp); 05749 X86::CondCode CCode = 05750 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0); 05751 CCode = X86::GetOppositeBranchCondition(CCode); 05752 CC = DAG.getConstant(CCode, MVT::i8); 05753 Cond = Cmp; 05754 addTest = false; 05755 } 05756 } 05757 } 05758 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) { 05759 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition. 05760 // It should be transformed during dag combiner except when the condition 05761 // is set by a arithmetics with overflow node. 05762 X86::CondCode CCode = 05763 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); 05764 CCode = X86::GetOppositeBranchCondition(CCode); 05765 CC = DAG.getConstant(CCode, MVT::i8); 05766 Cond = Cond.getOperand(0).getOperand(1); 05767 addTest = false; 05768 } 05769 } 05770 05771 if (addTest) { 05772 CC = DAG.getConstant(X86::COND_NE, MVT::i8); 05773 Cond = EmitTest(Cond, X86::COND_NE, DAG); 05774 } 05775 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), 05776 Chain, Dest, CC, Cond); 05777 } 05778 05779 05780 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. 05781 // Calls to _alloca is needed to probe the stack when allocating more than 4k 05782 // bytes in one go. Touching the stack at 4K increments is necessary to ensure 05783 // that the guard pages used by the OS virtual memory manager are allocated in 05784 // correct sequence. 05785 SDValue 05786 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 05787 SelectionDAG &DAG) { 05788 assert(Subtarget->isTargetCygMing() && 05789 "This should be used only on Cygwin/Mingw targets"); 05790 DebugLoc dl = Op.getDebugLoc(); 05791 05792 // Get the inputs. 05793 SDValue Chain = Op.getOperand(0); 05794 SDValue Size = Op.getOperand(1); 05795 // FIXME: Ensure alignment here 05796 05797 SDValue Flag; 05798 05799 EVT IntPtr = getPointerTy(); 05800 EVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; 05801 05802 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 05803 05804 Chain = DAG.getCopyToReg(Chain, dl, X86::EAX, Size, Flag); 05805 Flag = Chain.getValue(1); 05806 05807 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); 05808 SDValue Ops[] = { Chain, 05809 DAG.getTargetExternalSymbol("_alloca", IntPtr), 05810 DAG.getRegister(X86::EAX, IntPtr), 05811 DAG.getRegister(X86StackPtr, SPTy), 05812 Flag }; 05813 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops, 5); 05814 Flag = Chain.getValue(1); 05815 05816 Chain = DAG.getCALLSEQ_END(Chain, 05817 DAG.getIntPtrConstant(0, true), 05818 DAG.getIntPtrConstant(0, true), 05819 Flag); 05820 05821 Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1); 05822 05823 SDValue Ops1[2] = { Chain.getValue(0), Chain }; 05824 return DAG.getMergeValues(Ops1, 2, dl); 05825 } 05826 05827 SDValue 05828 X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl, 05829 SDValue Chain, 05830 SDValue Dst, SDValue Src, 05831 SDValue Size, unsigned Align, 05832 const Value *DstSV, 05833 uint64_t DstSVOff) { 05834 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 05835 05836 // If not DWORD aligned or size is more than the threshold, call the library. 05837 // The libc version is likely to be faster for these cases. It can use the 05838 // address value and run time information about the CPU. 05839 if ((Align & 3) != 0 || 05840 !ConstantSize || 05841 ConstantSize->getZExtValue() > 05842 getSubtarget()->getMaxInlineSizeThreshold()) { 05843 SDValue InFlag(0, 0); 05844 05845 // Check to see if there is a specialized entry-point for memory zeroing. 05846 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); 05847 05848 if (const char *bzeroEntry = V && 05849 V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { 05850 EVT IntPtr = getPointerTy(); 05851 const Type *IntPtrTy = TD->getIntPtrType(*DAG.getContext()); 05852 TargetLowering::ArgListTy Args; 05853 TargetLowering::ArgListEntry Entry; 05854 Entry.Node = Dst; 05855 Entry.Ty = IntPtrTy; 05856 Args.push_back(Entry); 05857 Entry.Node = Size; 05858 Args.push_back(Entry); 05859 std::pair<SDValue,SDValue> CallResult = 05860 LowerCallTo(Chain, Type::getVoidTy(*DAG.getContext()), 05861 false, false, false, false, 05862 0, CallingConv::C, false, /*isReturnValueUsed=*/false, 05863 DAG.getExternalSymbol(bzeroEntry, IntPtr), Args, DAG, dl); 05864 return CallResult.second; 05865 } 05866 05867 // Otherwise have the target-independent code call memset. 05868 return SDValue(); 05869 } 05870 05871 uint64_t SizeVal = ConstantSize->getZExtValue(); 05872 SDValue InFlag(0, 0); 05873 EVT AVT; 05874 SDValue Count; 05875 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); 05876 unsigned BytesLeft = 0; 05877 bool TwoRepStos = false; 05878 if (ValC) { 05879 unsigned ValReg; 05880 uint64_t Val = ValC->getZExtValue() & 255; 05881 05882 // If the value is a constant, then we can potentially use larger sets. 05883 switch (Align & 3) { 05884 case 2: // WORD aligned 05885 AVT = MVT::i16; 05886 ValReg = X86::AX; 05887 Val = (Val << 8) | Val; 05888 break; 05889 case 0: // DWORD aligned 05890 AVT = MVT::i32; 05891 ValReg = X86::EAX; 05892 Val = (Val << 8) | Val; 05893 Val = (Val << 16) | Val; 05894 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned 05895 AVT = MVT::i64; 05896 ValReg = X86::RAX; 05897 Val = (Val << 32) | Val; 05898 } 05899 break; 05900 default: // Byte aligned 05901 AVT = MVT::i8; 05902 ValReg = X86::AL; 05903 Count = DAG.getIntPtrConstant(SizeVal); 05904 break; 05905 } 05906 05907 if (AVT.bitsGT(MVT::i8)) { 05908 unsigned UBytes = AVT.getSizeInBits() / 8; 05909 Count = DAG.getIntPtrConstant(SizeVal / UBytes); 05910 BytesLeft = SizeVal % UBytes; 05911 } 05912 05913 Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT), 05914 InFlag); 05915 InFlag = Chain.getValue(1); 05916 } else { 05917 AVT = MVT::i8; 05918 Count = DAG.getIntPtrConstant(SizeVal); 05919 Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag); 05920 InFlag = Chain.getValue(1); 05921 } 05922 05923 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX : 05924 X86::ECX, 05925 Count, InFlag); 05926 InFlag = Chain.getValue(1); 05927 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI : 05928 X86::EDI, 05929 Dst, InFlag); 05930 InFlag = Chain.getValue(1); 05931 05932 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 05933 SmallVector<SDValue, 8> Ops; 05934 Ops.push_back(Chain); 05935 Ops.push_back(DAG.getValueType(AVT)); 05936 Ops.push_back(InFlag); 05937 Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, &Ops[0], Ops.size()); 05938 05939 if (TwoRepStos) { 05940 InFlag = Chain.getValue(1); 05941 Count = Size; 05942 EVT CVT = Count.getValueType(); 05943 SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count, 05944 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); 05945 Chain = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX : 05946 X86::ECX, 05947 Left, InFlag); 05948 InFlag = Chain.getValue(1); 05949 Tys = DAG.getVTList(MVT::Other, MVT::Flag); 05950 Ops.clear(); 05951 Ops.push_back(Chain); 05952 Ops.push_back(DAG.getValueType(MVT::i8)); 05953 Ops.push_back(InFlag); 05954 Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, &Ops[0], Ops.size()); 05955 } else if (BytesLeft) { 05956 // Handle the last 1 - 7 bytes. 05957 unsigned Offset = SizeVal - BytesLeft; 05958 EVT AddrVT = Dst.getValueType(); 05959 EVT SizeVT = Size.getValueType(); 05960 05961 Chain = DAG.getMemset(Chain, dl, 05962 DAG.getNode(ISD::ADD, dl, AddrVT, Dst, 05963 DAG.getConstant(Offset, AddrVT)), 05964 Src, 05965 DAG.getConstant(BytesLeft, SizeVT), 05966 Align, DstSV, DstSVOff + Offset); 05967 } 05968 05969 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. 05970 return Chain; 05971 } 05972 05973 SDValue 05974 X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl, 05975 SDValue Chain, SDValue Dst, SDValue Src, 05976 SDValue Size, unsigned Align, 05977 bool AlwaysInline, 05978 const Value *DstSV, uint64_t DstSVOff, 05979 const Value *SrcSV, uint64_t SrcSVOff) { 05980 // This requires the copy size to be a constant, preferrably 05981 // within a subtarget-specific limit. 05982 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 05983 if (!ConstantSize) 05984 return SDValue(); 05985 uint64_t SizeVal = ConstantSize->getZExtValue(); 05986 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) 05987 return SDValue(); 05988 05989 /// If not DWORD aligned, call the library. 05990 if ((Align & 3) != 0) 05991 return SDValue(); 05992 05993 // DWORD aligned 05994 EVT AVT = MVT::i32; 05995 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) // QWORD aligned 05996 AVT = MVT::i64; 05997 05998 unsigned UBytes = AVT.getSizeInBits() / 8; 05999 unsigned CountVal = SizeVal / UBytes; 06000 SDValue Count = DAG.getIntPtrConstant(CountVal); 06001 unsigned BytesLeft = SizeVal % UBytes; 06002 06003 SDValue InFlag(0, 0); 06004 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX : 06005 X86::ECX, 06006 Count, InFlag); 06007 InFlag = Chain.getValue(1); 06008 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI : 06009 X86::EDI, 06010 Dst, InFlag); 06011 InFlag = Chain.getValue(1); 06012 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RSI : 06013 X86::ESI, 06014 Src, InFlag); 06015 InFlag = Chain.getValue(1); 06016 06017 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 06018 SmallVector<SDValue, 8> Ops; 06019 Ops.push_back(Chain); 06020 Ops.push_back(DAG.getValueType(AVT)); 06021 Ops.push_back(InFlag); 06022 SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, &Ops[0], Ops.size()); 06023 06024 SmallVector<SDValue, 4> Results; 06025 Results.push_back(RepMovs); 06026 if (BytesLeft) { 06027 // Handle the last 1 - 7 bytes. 06028 unsigned Offset = SizeVal - BytesLeft; 06029 EVT DstVT = Dst.getValueType(); 06030 EVT SrcVT = Src.getValueType(); 06031 EVT SizeVT = Size.getValueType(); 06032 Results.push_back(DAG.getMemcpy(Chain, dl, 06033 DAG.getNode(ISD::ADD, dl, DstVT, Dst, 06034 DAG.getConstant(Offset, DstVT)), 06035 DAG.getNode(ISD::ADD, dl, SrcVT, Src, 06036 DAG.getConstant(Offset, SrcVT)), 06037 DAG.getConstant(BytesLeft, SizeVT), 06038 Align, AlwaysInline, 06039 DstSV, DstSVOff + Offset, 06040 SrcSV, SrcSVOff + Offset)); 06041 } 06042 06043 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 06044 &Results[0], Results.size()); 06045 } 06046 06047 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) { 06048 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 06049 DebugLoc dl = Op.getDebugLoc(); 06050 06051 if (!Subtarget->is64Bit()) { 06052 // vastart just stores the address of the VarArgsFrameIndex slot into the 06053 // memory location argument. 06054 SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 06055 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), SV, 0); 06056 } 06057 06058 // __va_list_tag: 06059 // gp_offset (0 - 6 * 8) 06060 // fp_offset (48 - 48 + 8 * 16) 06061 // overflow_arg_area (point to parameters coming in memory). 06062 // reg_save_area 06063 SmallVector<SDValue, 8> MemOps; 06064 SDValue FIN = Op.getOperand(1); 06065 // Store gp_offset 06066 SDValue Store = DAG.getStore(Op.getOperand(0), dl, 06067 DAG.getConstant(VarArgsGPOffset, MVT::i32), 06068 FIN, SV, 0); 06069 MemOps.push_back(Store); 06070 06071 // Store fp_offset 06072 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), 06073 FIN, DAG.getIntPtrConstant(4)); 06074 Store = DAG.getStore(Op.getOperand(0), dl, 06075 DAG.getConstant(VarArgsFPOffset, MVT::i32), 06076 FIN, SV, 0); 06077 MemOps.push_back(Store); 06078 06079 // Store ptr to overflow_arg_area 06080 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), 06081 FIN, DAG.getIntPtrConstant(4)); 06082 SDValue OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); 06083 Store = DAG.getStore(Op.getOperand(0), dl, OVFIN, FIN, SV, 0); 06084 MemOps.push_back(Store); 06085 06086 // Store ptr to reg_save_area. 06087 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), 06088 FIN, DAG.getIntPtrConstant(8)); 06089 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); 06090 Store = DAG.getStore(Op.getOperand(0), dl, RSFIN, FIN, SV, 0); 06091 MemOps.push_back(Store); 06092 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 06093 &MemOps[0], MemOps.size()); 06094 } 06095 06096 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) { 06097 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 06098 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); 06099 SDValue Chain = Op.getOperand(0); 06100 SDValue SrcPtr = Op.getOperand(1); 06101 SDValue SrcSV = Op.getOperand(2); 06102 06103 llvm_report_error("VAArgInst is not yet implemented for x86-64!"); 06104 return SDValue(); 06105 } 06106 06107 SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) { 06108 // X86-64 va_list is a struct { i32, i32, i8*, i8* }. 06109 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); 06110 SDValue Chain = Op.getOperand(0); 06111 SDValue DstPtr = Op.getOperand(1); 06112 SDValue SrcPtr = Op.getOperand(2); 06113 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 06114 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 06115 DebugLoc dl = Op.getDebugLoc(); 06116 06117 return DAG.getMemcpy(Chain, dl, DstPtr, SrcPtr, 06118 DAG.getIntPtrConstant(24), 8, false, 06119 DstSV, 0, SrcSV, 0); 06120 } 06121 06122 SDValue 06123 X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { 06124 DebugLoc dl = Op.getDebugLoc(); 06125 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 06126 switch (IntNo) { 06127 default: return SDValue(); // Don't custom lower most intrinsics. 06128 // Comparison intrinsics. 06129 case Intrinsic::x86_sse_comieq_ss: 06130 case Intrinsic::x86_sse_comilt_ss: 06131 case Intrinsic::x86_sse_comile_ss: 06132 case Intrinsic::x86_sse_comigt_ss: 06133 case Intrinsic::x86_sse_comige_ss: 06134 case Intrinsic::x86_sse_comineq_ss: 06135 case Intrinsic::x86_sse_ucomieq_ss: 06136 case Intrinsic::x86_sse_ucomilt_ss: 06137 case Intrinsic::x86_sse_ucomile_ss: 06138 case Intrinsic::x86_sse_ucomigt_ss: 06139 case Intrinsic::x86_sse_ucomige_ss: 06140 case Intrinsic::x86_sse_ucomineq_ss: 06141 case Intrinsic::x86_sse2_comieq_sd: 06142 case Intrinsic::x86_sse2_comilt_sd: 06143 case Intrinsic::x86_sse2_comile_sd: 06144 case Intrinsic::x86_sse2_comigt_sd: 06145 case Intrinsic::x86_sse2_comige_sd: 06146 case Intrinsic::x86_sse2_comineq_sd: 06147 case Intrinsic::x86_sse2_ucomieq_sd: 06148 case Intrinsic::x86_sse2_ucomilt_sd: 06149 case Intrinsic::x86_sse2_ucomile_sd: 06150 case Intrinsic::x86_sse2_ucomigt_sd: 06151 case Intrinsic::x86_sse2_ucomige_sd: 06152 case Intrinsic::x86_sse2_ucomineq_sd: { 06153 unsigned Opc = 0; 06154 ISD::CondCode CC = ISD::SETCC_INVALID; 06155 switch (IntNo) { 06156 default: break; 06157 case Intrinsic::x86_sse_comieq_ss: 06158 case Intrinsic::x86_sse2_comieq_sd: 06159 Opc = X86ISD::COMI; 06160 CC = ISD::SETEQ; 06161 break; 06162 case Intrinsic::x86_sse_comilt_ss: 06163 case Intrinsic::x86_sse2_comilt_sd: 06164 Opc = X86ISD::COMI; 06165 CC = ISD::SETLT; 06166 break; 06167 case Intrinsic::x86_sse_comile_ss: 06168 case Intrinsic::x86_sse2_comile_sd: 06169 Opc = X86ISD::COMI; 06170 CC = ISD::SETLE; 06171 break; 06172 case Intrinsic::x86_sse_comigt_ss: 06173 case Intrinsic::x86_sse2_comigt_sd: 06174 Opc = X86ISD::COMI; 06175 CC = ISD::SETGT; 06176 break; 06177 case Intrinsic::x86_sse_comige_ss: 06178 case Intrinsic::x86_sse2_comige_sd: 06179 Opc = X86ISD::COMI; 06180 CC = ISD::SETGE; 06181 break; 06182 case Intrinsic::x86_sse_comineq_ss: 06183 case Intrinsic::x86_sse2_comineq_sd: 06184 Opc = X86ISD::COMI; 06185 CC = ISD::SETNE; 06186 break; 06187 case Intrinsic::x86_sse_ucomieq_ss: 06188 case Intrinsic::x86_sse2_ucomieq_sd: 06189 Opc = X86ISD::UCOMI; 06190 CC = ISD::SETEQ; 06191 break; 06192 case Intrinsic::x86_sse_ucomilt_ss: 06193 case Intrinsic::x86_sse2_ucomilt_sd: 06194 Opc = X86ISD::UCOMI; 06195 CC = ISD::SETLT; 06196 break; 06197 case Intrinsic::x86_sse_ucomile_ss: 06198 case Intrinsic::x86_sse2_ucomile_sd: 06199 Opc = X86ISD::UCOMI; 06200 CC = ISD::SETLE; 06201 break; 06202 case Intrinsic::x86_sse_ucomigt_ss: 06203 case Intrinsic::x86_sse2_ucomigt_sd: 06204 Opc = X86ISD::UCOMI; 06205 CC = ISD::SETGT; 06206 break; 06207 case Intrinsic::x86_sse_ucomige_ss: 06208 case Intrinsic::x86_sse2_ucomige_sd: 06209 Opc = X86ISD::UCOMI; 06210 CC = ISD::SETGE; 06211 break; 06212 case Intrinsic::x86_sse_ucomineq_ss: 06213 case Intrinsic::x86_sse2_ucomineq_sd: 06214 Opc = X86ISD::UCOMI; 06215 CC = ISD::SETNE; 06216 break; 06217 } 06218 06219 SDValue LHS = Op.getOperand(1); 06220 SDValue RHS = Op.getOperand(2); 06221 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG); 06222 SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS); 06223 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, 06224 DAG.getConstant(X86CC, MVT::i8), Cond); 06225 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); 06226 } 06227 // ptest intrinsics. The intrinsic these come from are designed to return 06228 // an integer value, not just an instruction so lower it to the ptest 06229 // pattern and a setcc for the result. 06230 case Intrinsic::x86_sse41_ptestz: 06231 case Intrinsic::x86_sse41_ptestc: 06232 case Intrinsic::x86_sse41_ptestnzc:{ 06233 unsigned X86CC = 0; 06234 switch (IntNo) { 06235 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering."); 06236 case Intrinsic::x86_sse41_ptestz: 06237 // ZF = 1 06238 X86CC = X86::COND_E; 06239 break; 06240 case Intrinsic::x86_sse41_ptestc: 06241 // CF = 1 06242 X86CC = X86::COND_B; 06243 break; 06244 case Intrinsic::x86_sse41_ptestnzc: 06245 // ZF and CF = 0 06246 X86CC = X86::COND_A; 06247 break; 06248 } 06249 06250 SDValue LHS = Op.getOperand(1); 06251 SDValue RHS = Op.getOperand(2); 06252 SDValue Test = DAG.getNode(X86ISD::PTEST, dl, MVT::i32, LHS, RHS); 06253 SDValue CC = DAG.getConstant(X86CC, MVT::i8); 06254 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); 06255 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); 06256 } 06257 06258 // Fix vector shift instructions where the last operand is a non-immediate 06259 // i32 value. 06260 case Intrinsic::x86_sse2_pslli_w: 06261 case Intrinsic::x86_sse2_pslli_d: 06262 case Intrinsic::x86_sse2_pslli_q: 06263 case Intrinsic::x86_sse2_psrli_w: 06264 case Intrinsic::x86_sse2_psrli_d: 06265 case Intrinsic::x86_sse2_psrli_q: 06266 case Intrinsic::x86_sse2_psrai_w: 06267 case Intrinsic::x86_sse2_psrai_d: 06268 case Intrinsic::x86_mmx_pslli_w: 06269 case Intrinsic::x86_mmx_pslli_d: 06270 case Intrinsic::x86_mmx_pslli_q: 06271 case Intrinsic::x86_mmx_psrli_w: 06272 case Intrinsic::x86_mmx_psrli_d: 06273 case Intrinsic::x86_mmx_psrli_q: 06274 case Intrinsic::x86_mmx_psrai_w: 06275 case Intrinsic::x86_mmx_psrai_d: { 06276 SDValue ShAmt = Op.getOperand(2); 06277 if (isa<ConstantSDNode>(ShAmt)) 06278 return SDValue(); 06279 06280 unsigned NewIntNo = 0; 06281 EVT ShAmtVT = MVT::v4i32; 06282 switch (IntNo) { 06283 case Intrinsic::x86_sse2_pslli_w: 06284 NewIntNo = Intrinsic::x86_sse2_psll_w; 06285 break; 06286 case Intrinsic::x86_sse2_pslli_d: 06287 NewIntNo = Intrinsic::x86_sse2_psll_d; 06288 break; 06289 case Intrinsic::x86_sse2_pslli_q: 06290 NewIntNo = Intrinsic::x86_sse2_psll_q; 06291 break; 06292 case Intrinsic::x86_sse2_psrli_w: 06293 NewIntNo = Intrinsic::x86_sse2_psrl_w; 06294 break; 06295 case Intrinsic::x86_sse2_psrli_d: 06296 NewIntNo = Intrinsic::x86_sse2_psrl_d; 06297 break; 06298 case Intrinsic::x86_sse2_psrli_q: 06299 NewIntNo = Intrinsic::x86_sse2_psrl_q; 06300 break; 06301 case Intrinsic::x86_sse2_psrai_w: 06302 NewIntNo = Intrinsic::x86_sse2_psra_w; 06303 break; 06304 case Intrinsic::x86_sse2_psrai_d: 06305 NewIntNo = Intrinsic::x86_sse2_psra_d; 06306 break; 06307 default: { 06308 ShAmtVT = MVT::v2i32; 06309 switch (IntNo) { 06310 case Intrinsic::x86_mmx_pslli_w: 06311 NewIntNo = Intrinsic::x86_mmx_psll_w; 06312 break; 06313 case Intrinsic::x86_mmx_pslli_d: 06314 NewIntNo = Intrinsic::x86_mmx_psll_d; 06315 break; 06316 case Intrinsic::x86_mmx_pslli_q: 06317 NewIntNo = Intrinsic::x86_mmx_psll_q; 06318 break; 06319 case Intrinsic::x86_mmx_psrli_w: 06320 NewIntNo = Intrinsic::x86_mmx_psrl_w; 06321 break; 06322 case Intrinsic::x86_mmx_psrli_d: 06323 NewIntNo = Intrinsic::x86_mmx_psrl_d; 06324 break; 06325 case Intrinsic::x86_mmx_psrli_q: 06326 NewIntNo = Intrinsic::x86_mmx_psrl_q; 06327 break; 06328 case Intrinsic::x86_mmx_psrai_w: 06329 NewIntNo = Intrinsic::x86_mmx_psra_w; 06330 break; 06331 case Intrinsic::x86_mmx_psrai_d: 06332 NewIntNo = Intrinsic::x86_mmx_psra_d; 06333 break; 06334 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 06335 } 06336 break; 06337 } 06338 } 06339 EVT VT = Op.getValueType(); 06340 ShAmt = DAG.getNode(ISD::BIT_CONVERT, dl, VT, 06341 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShAmtVT, ShAmt)); 06342 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06343 DAG.getConstant(NewIntNo, MVT::i32), 06344 Op.getOperand(1), ShAmt); 06345 } 06346 } 06347 } 06348 06349 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) { 06350 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 06351 DebugLoc dl = Op.getDebugLoc(); 06352 06353 if (Depth > 0) { 06354 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 06355 SDValue Offset = 06356 DAG.getConstant(TD->getPointerSize(), 06357 Subtarget->is64Bit() ? MVT::i64 : MVT::i32); 06358 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 06359 DAG.getNode(ISD::ADD, dl, getPointerTy(), 06360 FrameAddr, Offset), 06361 NULL, 0); 06362 } 06363 06364 // Just load the return address. 06365 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); 06366 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), 06367 RetAddrFI, NULL, 0); 06368 } 06369 06370 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) { 06371 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 06372 MFI->setFrameAddressIsTaken(true); 06373 EVT VT = Op.getValueType(); 06374 DebugLoc dl = Op.getDebugLoc(); // FIXME probably not meaningful 06375 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 06376 unsigned FrameReg = Subtarget->is64Bit() ? X86::RBP : X86::EBP; 06377 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); 06378 while (Depth--) 06379 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, NULL, 0); 06380 return FrameAddr; 06381 } 06382 06383 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, 06384 SelectionDAG &DAG) { 06385 return DAG.getIntPtrConstant(2*TD->getPointerSize()); 06386 } 06387 06388 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) 06389 { 06390 MachineFunction &MF = DAG.getMachineFunction(); 06391 SDValue Chain = Op.getOperand(0); 06392 SDValue Offset = Op.getOperand(1); 06393 SDValue Handler = Op.getOperand(2); 06394 DebugLoc dl = Op.getDebugLoc(); 06395 06396 SDValue Frame = DAG.getRegister(Subtarget->is64Bit() ? X86::RBP : X86::EBP, 06397 getPointerTy()); 06398 unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX); 06399 06400 SDValue StoreAddr = DAG.getNode(ISD::SUB, dl, getPointerTy(), Frame, 06401 DAG.getIntPtrConstant(-TD->getPointerSize())); 06402 StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset); 06403 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, NULL, 0); 06404 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr); 06405 MF.getRegInfo().addLiveOut(StoreAddrReg); 06406 06407 return DAG.getNode(X86ISD::EH_RETURN, dl, 06408 MVT::Other, 06409 Chain, DAG.getRegister(StoreAddrReg, getPointerTy())); 06410 } 06411 06412 SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op, 06413 SelectionDAG &DAG) { 06414 SDValue Root = Op.getOperand(0); 06415 SDValue Trmp = Op.getOperand(1); // trampoline 06416 SDValue FPtr = Op.getOperand(2); // nested function 06417 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 06418 DebugLoc dl = Op.getDebugLoc(); 06419 06420 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 06421 06422 const X86InstrInfo *TII = 06423 ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); 06424 06425 if (Subtarget->is64Bit()) { 06426 SDValue OutChains[6]; 06427 06428 // Large code-model. 06429 06430 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); 06431 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); 06432 06433 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); 06434 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); 06435 06436 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix 06437 06438 // Load the pointer to the nested function into R11. 06439 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 06440 SDValue Addr = Trmp; 06441 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 06442 Addr, TrmpAddr, 0); 06443 06444 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 06445 DAG.getConstant(2, MVT::i64)); 06446 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, TrmpAddr, 2, false, 2); 06447 06448 // Load the 'nest' parameter value into R10. 06449 // R10 is specified in X86CallingConv.td 06450 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 06451 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 06452 DAG.getConstant(10, MVT::i64)); 06453 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 06454 Addr, TrmpAddr, 10); 06455 06456 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 06457 DAG.getConstant(12, MVT::i64)); 06458 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, TrmpAddr, 12, false, 2); 06459 06460 // Jump to the nested function. 06461 OpCode = (JMP64r << 8) | REX_WB; // jmpq *... 06462 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 06463 DAG.getConstant(20, MVT::i64)); 06464 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), 06465 Addr, TrmpAddr, 20); 06466 06467 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 06468 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, 06469 DAG.getConstant(22, MVT::i64)); 06470 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr, 06471 TrmpAddr, 22); 06472 06473 SDValue Ops[] = 06474 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6) }; 06475 return DAG.getMergeValues(Ops, 2, dl); 06476 } else { 06477 const Function *Func = 06478 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); 06479 unsigned CC = Func->getCallingConv(); 06480 unsigned NestReg; 06481 06482 switch (CC) { 06483 default: 06484 llvm_unreachable("Unsupported calling convention"); 06485 case CallingConv::C: 06486 case CallingConv::X86_StdCall: { 06487 // Pass 'nest' parameter in ECX. 06488 // Must be kept in sync with X86CallingConv.td 06489 NestReg = X86::ECX; 06490 06491 // Check that ECX wasn't needed by an 'inreg' parameter. 06492 const FunctionType *FTy = Func->getFunctionType(); 06493 const AttrListPtr &Attrs = Func->getAttributes(); 06494 06495 if (!Attrs.isEmpty() && !Func->isVarArg()) { 06496 unsigned InRegCount = 0; 06497 unsigned Idx = 1; 06498 06499 for (FunctionType::param_iterator I = FTy->param_begin(), 06500 E = FTy->param_end(); I != E; ++I, ++Idx) 06501 if (Attrs.paramHasAttr(Idx, Attribute::InReg)) 06502 // FIXME: should only count parameters that are lowered to integers. 06503 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32; 06504 06505 if (InRegCount > 2) { 06506 llvm_report_error("Nest register in use - reduce number of inreg parameters!"); 06507 } 06508 } 06509 break; 06510 } 06511 case CallingConv::X86_FastCall: 06512 case CallingConv::Fast: 06513 // Pass 'nest' parameter in EAX. 06514 // Must be kept in sync with X86CallingConv.td 06515 NestReg = X86::EAX; 06516 break; 06517 } 06518 06519 SDValue OutChains[4]; 06520 SDValue Addr, Disp; 06521 06522 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 06523 DAG.getConstant(10, MVT::i32)); 06524 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); 06525 06526 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); 06527 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); 06528 OutChains[0] = DAG.getStore(Root, dl, 06529 DAG.getConstant(MOV32ri|N86Reg, MVT::i8), 06530 Trmp, TrmpAddr, 0); 06531 06532 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 06533 DAG.getConstant(1, MVT::i32)); 06534 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, TrmpAddr, 1, false, 1); 06535 06536 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); 06537 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 06538 DAG.getConstant(5, MVT::i32)); 06539 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr, 06540 TrmpAddr, 5, false, 1); 06541 06542 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, 06543 DAG.getConstant(6, MVT::i32)); 06544 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, TrmpAddr, 6, false, 1); 06545 06546 SDValue Ops[] = 06547 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4) }; 06548 return DAG.getMergeValues(Ops, 2, dl); 06549 } 06550 } 06551 06552 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) { 06553 /* 06554 The rounding mode is in bits 11:10 of FPSR, and has the following 06555 settings: 06556 00 Round to nearest 06557 01 Round to -inf 06558 10 Round to +inf 06559 11 Round to 0 06560 06561 FLT_ROUNDS, on the other hand, expects the following: 06562 -1 Undefined 06563 0 Round to 0 06564 1 Round to nearest 06565 2 Round to +inf 06566 3 Round to -inf 06567 06568 To perform the conversion, we do: 06569 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) 06570 */ 06571 06572 MachineFunction &MF = DAG.getMachineFunction(); 06573 const TargetMachine &TM = MF.getTarget(); 06574 const TargetFrameInfo &TFI = *TM.getFrameInfo(); 06575 unsigned StackAlignment = TFI.getStackAlignment(); 06576 EVT VT = Op.getValueType(); 06577 DebugLoc dl = Op.getDebugLoc(); 06578 06579 // Save FP Control Word to stack slot 06580 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); 06581 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); 06582 06583 SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, dl, MVT::Other, 06584 DAG.getEntryNode(), StackSlot); 06585 06586 // Load FP Control Word from stack slot 06587 SDValue CWD = DAG.getLoad(MVT::i16, dl, Chain, StackSlot, NULL, 0); 06588 06589 // Transform as necessary 06590 SDValue CWD1 = 06591 DAG.getNode(ISD::SRL, dl, MVT::i16, 06592 DAG.getNode(ISD::AND, dl, MVT::i16, 06593 CWD, DAG.getConstant(0x800, MVT::i16)), 06594 DAG.getConstant(11, MVT::i8)); 06595 SDValue CWD2 = 06596 DAG.getNode(ISD::SRL, dl, MVT::i16, 06597 DAG.getNode(ISD::AND, dl, MVT::i16, 06598 CWD, DAG.getConstant(0x400, MVT::i16)), 06599 DAG.getConstant(9, MVT::i8)); 06600 06601 SDValue RetVal = 06602 DAG.getNode(ISD::AND, dl, MVT::i16, 06603 DAG.getNode(ISD::ADD, dl, MVT::i16, 06604 DAG.getNode(ISD::OR, dl, MVT::i16, CWD1, CWD2), 06605 DAG.getConstant(1, MVT::i16)), 06606 DAG.getConstant(3, MVT::i16)); 06607 06608 06609 return DAG.getNode((VT.getSizeInBits() < 16 ? 06610 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); 06611 } 06612 06613 SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) { 06614 EVT VT = Op.getValueType(); 06615 EVT OpVT = VT; 06616 unsigned NumBits = VT.getSizeInBits(); 06617 DebugLoc dl = Op.getDebugLoc(); 06618 06619 Op = Op.getOperand(0); 06620 if (VT == MVT::i8) { 06621 // Zero extend to i32 since there is not an i8 bsr. 06622 OpVT = MVT::i32; 06623 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); 06624 } 06625 06626 // Issue a bsr (scan bits in reverse) which also sets EFLAGS. 06627 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 06628 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); 06629 06630 // If src is zero (i.e. bsr sets ZF), returns NumBits. 06631 SmallVector<SDValue, 4> Ops; 06632 Ops.push_back(Op); 06633 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); 06634 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 06635 Ops.push_back(Op.getValue(1)); 06636 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, &Ops[0], 4); 06637 06638 // Finally xor with NumBits-1. 06639 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); 06640 06641 if (VT == MVT::i8) 06642 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); 06643 return Op; 06644 } 06645 06646 SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) { 06647 EVT VT = Op.getValueType(); 06648 EVT OpVT = VT; 06649 unsigned NumBits = VT.getSizeInBits(); 06650 DebugLoc dl = Op.getDebugLoc(); 06651 06652 Op = Op.getOperand(0); 06653 if (VT == MVT::i8) { 06654 OpVT = MVT::i32; 06655 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); 06656 } 06657 06658 // Issue a bsf (scan bits forward) which also sets EFLAGS. 06659 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); 06660 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op); 06661 06662 // If src is zero (i.e. bsf sets ZF), returns NumBits. 06663 SmallVector<SDValue, 4> Ops; 06664 Ops.push_back(Op); 06665 Ops.push_back(DAG.getConstant(NumBits, OpVT)); 06666 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); 06667 Ops.push_back(Op.getValue(1)); 06668 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, &Ops[0], 4); 06669 06670 if (VT == MVT::i8) 06671 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); 06672 return Op; 06673 } 06674 06675 SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) { 06676 EVT VT = Op.getValueType(); 06677 assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply"); 06678 DebugLoc dl = Op.getDebugLoc(); 06679 06680 // ulong2 Ahi = __builtin_ia32_psrlqi128( a, 32); 06681 // ulong2 Bhi = __builtin_ia32_psrlqi128( b, 32); 06682 // ulong2 AloBlo = __builtin_ia32_pmuludq128( a, b ); 06683 // ulong2 AloBhi = __builtin_ia32_pmuludq128( a, Bhi ); 06684 // ulong2 AhiBlo = __builtin_ia32_pmuludq128( Ahi, b ); 06685 // 06686 // AloBhi = __builtin_ia32_psllqi128( AloBhi, 32 ); 06687 // AhiBlo = __builtin_ia32_psllqi128( AhiBlo, 32 ); 06688 // return AloBlo + AloBhi + AhiBlo; 06689 06690 SDValue A = Op.getOperand(0); 06691 SDValue B = Op.getOperand(1); 06692 06693 SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06694 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 06695 A, DAG.getConstant(32, MVT::i32)); 06696 SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06697 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 06698 B, DAG.getConstant(32, MVT::i32)); 06699 SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06700 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), 06701 A, B); 06702 SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06703 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), 06704 A, Bhi); 06705 SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06706 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), 06707 Ahi, B); 06708 AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06709 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 06710 AloBhi, DAG.getConstant(32, MVT::i32)); 06711 AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, 06712 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 06713 AhiBlo, DAG.getConstant(32, MVT::i32)); 06714 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi); 06715 Res = DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo); 06716 return Res; 06717 } 06718 06719 06720 SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) { 06721 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus 06722 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering 06723 // looks for this combo and may remove the "setcc" instruction if the "setcc" 06724 // has only one use. 06725 SDNode *N = Op.getNode(); 06726 SDValue LHS = N->getOperand(0); 06727 SDValue RHS = N->getOperand(1); 06728 unsigned BaseOp = 0; 06729 unsigned Cond = 0; 06730 DebugLoc dl = Op.getDebugLoc(); 06731 06732 switch (Op.getOpcode()) { 06733 default: llvm_unreachable("Unknown ovf instruction!"); 06734 case ISD::SADDO: 06735 // A subtract of one will be selected as a INC. Note that INC doesn't 06736 // set CF, so we can't do this for UADDO. 06737 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 06738 if (C->getAPIntValue() == 1) { 06739 BaseOp = X86ISD::INC; 06740 Cond = X86::COND_O; 06741 break; 06742 } 06743 BaseOp = X86ISD::ADD; 06744 Cond = X86::COND_O; 06745 break; 06746 case ISD::UADDO: 06747 BaseOp = X86ISD::ADD; 06748 Cond = X86::COND_B; 06749 break; 06750 case ISD::SSUBO: 06751 // A subtract of one will be selected as a DEC. Note that DEC doesn't 06752 // set CF, so we can't do this for USUBO. 06753 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 06754 if (C->getAPIntValue() == 1) { 06755 BaseOp = X86ISD::DEC; 06756 Cond = X86::COND_O; 06757 break; 06758 } 06759 BaseOp = X86ISD::SUB; 06760 Cond = X86::COND_O; 06761 break; 06762 case ISD::USUBO: 06763 BaseOp = X86ISD::SUB; 06764 Cond = X86::COND_B; 06765 break; 06766 case ISD::SMULO: 06767 BaseOp = X86ISD::SMUL; 06768 Cond = X86::COND_O; 06769 break; 06770 case ISD::UMULO: 06771 BaseOp = X86ISD::UMUL; 06772 Cond = X86::COND_B; 06773 break; 06774 } 06775 06776 // Also sets EFLAGS. 06777 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); 06778 SDValue Sum = DAG.getNode(BaseOp, dl, VTs, LHS, RHS); 06779 06780 SDValue SetCC = 06781 DAG.getNode(X86ISD::SETCC, dl, N->getValueType(1), 06782 DAG.getConstant(Cond, MVT::i32), SDValue(Sum.getNode(), 1)); 06783 06784 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC); 06785 return Sum; 06786 } 06787 06788 SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) { 06789 EVT T = Op.getValueType(); 06790 DebugLoc dl = Op.getDebugLoc(); 06791 unsigned Reg = 0; 06792 unsigned size = 0; 06793 switch(T.getSimpleVT().SimpleTy) { 06794 default: 06795 assert(false && "Invalid value type!"); 06796 case MVT::i8: Reg = X86::AL; size = 1; break; 06797 case MVT::i16: Reg = X86::AX; size = 2; break; 06798 case MVT::i32: Reg = X86::EAX; size = 4; break; 06799 case MVT::i64: 06800 assert(Subtarget->is64Bit() && "Node not type legal!"); 06801 Reg = X86::RAX; size = 8; 06802 break; 06803 } 06804 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), dl, Reg, 06805 Op.getOperand(2), SDValue()); 06806 SDValue Ops[] = { cpIn.getValue(0), 06807 Op.getOperand(1), 06808 Op.getOperand(3), 06809 DAG.getTargetConstant(size, MVT::i8), 06810 cpIn.getValue(1) }; 06811 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 06812 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, dl, Tys, Ops, 5); 06813 SDValue cpOut = 06814 DAG.getCopyFromReg(Result.getValue(0), dl, Reg, T, Result.getValue(1)); 06815 return cpOut; 06816 } 06817 06818 SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op, 06819 SelectionDAG &DAG) { 06820 assert(Subtarget->is64Bit() && "Result not type legalized?"); 06821 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 06822 SDValue TheChain = Op.getOperand(0); 06823 DebugLoc dl = Op.getDebugLoc(); 06824 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); 06825 SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1)); 06826 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64, 06827 rax.getValue(2)); 06828 SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx, 06829 DAG.getConstant(32, MVT::i8)); 06830 SDValue Ops[] = { 06831 DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp), 06832 rdx.getValue(1) 06833 }; 06834 return DAG.getMergeValues(Ops, 2, dl); 06835 } 06836 06837 SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) { 06838 SDNode *Node = Op.getNode(); 06839 DebugLoc dl = Node->getDebugLoc(); 06840 EVT T = Node->getValueType(0); 06841 SDValue negOp = DAG.getNode(ISD::SUB, dl, T, 06842 DAG.getConstant(0, T), Node->getOperand(2)); 06843 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, 06844 cast<AtomicSDNode>(Node)->getMemoryVT(), 06845 Node->getOperand(0), 06846 Node->getOperand(1), negOp, 06847 cast<AtomicSDNode>(Node)->getSrcValue(), 06848 cast<AtomicSDNode>(Node)->getAlignment()); 06849 } 06850 06851 /// LowerOperation - Provide custom lowering hooks for some operations. 06852 /// 06853 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { 06854 switch (Op.getOpcode()) { 06855 default: llvm_unreachable("Should not custom lower this!"); 06856 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); 06857 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG); 06858 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 06859 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 06860 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 06861 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 06862 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 06863 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 06864 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 06865 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 06866 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); 06867 case ISD::SHL_PARTS: 06868 case ISD::SRA_PARTS: 06869 case ISD::SRL_PARTS: return LowerShift(Op, DAG); 06870 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 06871 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); 06872 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 06873 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); 06874 case ISD::FABS: return LowerFABS(Op, DAG); 06875 case ISD::FNEG: return LowerFNEG(Op, DAG); 06876 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); 06877 case ISD::SETCC: return LowerSETCC(Op, DAG); 06878 case ISD::VSETCC: return LowerVSETCC(Op, DAG); 06879 case ISD::SELECT: return LowerSELECT(Op, DAG); 06880 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 06881 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 06882 case ISD::VASTART: return LowerVASTART(Op, DAG); 06883 case ISD::VAARG: return LowerVAARG(Op, DAG); 06884 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 06885 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 06886 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 06887 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 06888 case ISD::FRAME_TO_ARGS_OFFSET: 06889 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); 06890 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 06891 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); 06892 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); 06893 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 06894 case ISD::CTLZ: return LowerCTLZ(Op, DAG); 06895 case ISD::CTTZ: return LowerCTTZ(Op, DAG); 06896 case ISD::MUL: return LowerMUL_V2I64(Op, DAG); 06897 case ISD::SADDO: 06898 case ISD::UADDO: 06899 case ISD::SSUBO: 06900 case ISD::USUBO: 06901 case ISD::SMULO: 06902 case ISD::UMULO: return LowerXALUO(Op, DAG); 06903 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); 06904 } 06905 } 06906 06907 void X86TargetLowering:: 06908 ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results, 06909 SelectionDAG &DAG, unsigned NewOp) { 06910 EVT T = Node->getValueType(0); 06911 DebugLoc dl = Node->getDebugLoc(); 06912 assert (T == MVT::i64 && "Only know how to expand i64 atomics"); 06913 06914 SDValue Chain = Node->getOperand(0); 06915 SDValue In1 = Node->getOperand(1); 06916 SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 06917 Node->getOperand(2), DAG.getIntPtrConstant(0)); 06918 SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, 06919 Node->getOperand(2), DAG.getIntPtrConstant(1)); 06920 // This is a generalized SDNode, not an AtomicSDNode, so it doesn't 06921 // have a MemOperand. Pass the info through as a normal operand. 06922 SDValue LSI = DAG.getMemOperand(cast<MemSDNode>(Node)->getMemOperand()); 06923 SDValue Ops[] = { Chain, In1, In2L, In2H, LSI }; 06924 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 06925 SDValue Result = DAG.getNode(NewOp, dl, Tys, Ops, 5); 06926 SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; 06927 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); 06928 Results.push_back(Result.getValue(2)); 06929 } 06930 06931 /// ReplaceNodeResults - Replace a node with an illegal result type 06932 /// with a new node built out of custom code. 06933 void X86TargetLowering::ReplaceNodeResults(SDNode *N, 06934 SmallVectorImpl<SDValue>&Results, 06935 SelectionDAG &DAG) { 06936 DebugLoc dl = N->getDebugLoc(); 06937 switch (N->getOpcode()) { 06938 default: 06939 assert(false && "Do not know how to custom type legalize this operation!"); 06940 return; 06941 case ISD::FP_TO_SINT: { 06942 std::pair<SDValue,SDValue> Vals = 06943 FP_TO_INTHelper(SDValue(N, 0), DAG, true); 06944 SDValue FIST = Vals.first, StackSlot = Vals.second; 06945 if (FIST.getNode() != 0) { 06946 EVT VT = N->getValueType(0); 06947 // Return a load from the stack slot. 06948 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot, NULL, 0)); 06949 } 06950 return; 06951 } 06952 case ISD::READCYCLECOUNTER: { 06953 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 06954 SDValue TheChain = N->getOperand(0); 06955 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); 06956 SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32, 06957 rd.getValue(1)); 06958 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32, 06959 eax.getValue(2)); 06960 // Use a buildpair to merge the two 32-bit values into a 64-bit one. 06961 SDValue Ops[] = { eax, edx }; 06962 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2)); 06963 Results.push_back(edx.getValue(1)); 06964 return; 06965 } 06966 case ISD::ATOMIC_CMP_SWAP: { 06967 EVT T = N->getValueType(0); 06968 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); 06969 SDValue cpInL, cpInH; 06970 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2), 06971 DAG.getConstant(0, MVT::i32)); 06972 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2), 06973 DAG.getConstant(1, MVT::i32)); 06974 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue()); 06975 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH, 06976 cpInL.getValue(1)); 06977 SDValue swapInL, swapInH; 06978 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3), 06979 DAG.getConstant(0, MVT::i32)); 06980 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3), 06981 DAG.getConstant(1, MVT::i32)); 06982 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL, 06983 cpInH.getValue(1)); 06984 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH, 06985 swapInL.getValue(1)); 06986 SDValue Ops[] = { swapInH.getValue(0), 06987 N->getOperand(1), 06988 swapInH.getValue(1) }; 06989 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); 06990 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, 3); 06991 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX, 06992 MVT::i32, Result.getValue(1)); 06993 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX, 06994 MVT::i32, cpOutL.getValue(2)); 06995 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; 06996 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); 06997 Results.push_back(cpOutH.getValue(1)); 06998 return; 06999 } 07000 case ISD::ATOMIC_LOAD_ADD: 07001 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG); 07002 return; 07003 case ISD::ATOMIC_LOAD_AND: 07004 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG); 07005 return; 07006 case ISD::ATOMIC_LOAD_NAND: 07007 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG); 07008 return; 07009 case ISD::ATOMIC_LOAD_OR: 07010 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG); 07011 return; 07012 case ISD::ATOMIC_LOAD_SUB: 07013 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG); 07014 return; 07015 case ISD::ATOMIC_LOAD_XOR: 07016 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG); 07017 return; 07018 case ISD::ATOMIC_SWAP: 07019 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG); 07020 return; 07021 } 07022 } 07023 07024 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { 07025 switch (Opcode) { 07026 default: return NULL; 07027 case X86ISD::BSF: return "X86ISD::BSF"; 07028 case X86ISD::BSR: return "X86ISD::BSR"; 07029 case X86ISD::SHLD: return "X86ISD::SHLD"; 07030 case X86ISD::SHRD: return "X86ISD::SHRD"; 07031 case X86ISD::FAND: return "X86ISD::FAND"; 07032 case X86ISD::FOR: return "X86ISD::FOR"; 07033 case X86ISD::FXOR: return "X86ISD::FXOR"; 07034 case X86ISD::FSRL: return "X86ISD::FSRL"; 07035 case X86ISD::FILD: return "X86ISD::FILD"; 07036 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; 07037 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; 07038 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; 07039 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; 07040 case X86ISD::FLD: return "X86ISD::FLD"; 07041 case X86ISD::FST: return "X86ISD::FST"; 07042 case X86ISD::CALL: return "X86ISD::CALL"; 07043 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; 07044 case X86ISD::BT: return "X86ISD::BT"; 07045 case X86ISD::CMP: return "X86ISD::CMP"; 07046 case X86ISD::COMI: return "X86ISD::COMI"; 07047 case X86ISD::UCOMI: return "X86ISD::UCOMI"; 07048 case X86ISD::SETCC: return "X86ISD::SETCC"; 07049 case X86ISD::CMOV: return "X86ISD::CMOV"; 07050 case X86ISD::BRCOND: return "X86ISD::BRCOND"; 07051 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; 07052 case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; 07053 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; 07054 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; 07055 case X86ISD::Wrapper: return "X86ISD::Wrapper"; 07056 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP"; 07057 case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; 07058 case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; 07059 case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; 07060 case X86ISD::PINSRB: return "X86ISD::PINSRB"; 07061 case X86ISD::PINSRW: return "X86ISD::PINSRW"; 07062 case X86ISD::PSHUFB: return "X86ISD::PSHUFB"; 07063 case X86ISD::FMAX: return "X86ISD::FMAX"; 07064 case X86ISD::FMIN: return "X86ISD::FMIN"; 07065 case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; 07066 case X86ISD::FRCP: return "X86ISD::FRCP"; 07067 case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; 07068 case X86ISD::SegmentBaseAddress: return "X86ISD::SegmentBaseAddress"; 07069 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; 07070 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; 07071 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; 07072 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; 07073 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; 07074 case X86ISD::ATOMADD64_DAG: return "X86ISD::ATOMADD64_DAG"; 07075 case X86ISD::ATOMSUB64_DAG: return "X86ISD::ATOMSUB64_DAG"; 07076 case X86ISD::ATOMOR64_DAG: return "X86ISD::ATOMOR64_DAG"; 07077 case X86ISD::ATOMXOR64_DAG: return "X86ISD::ATOMXOR64_DAG"; 07078 case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG"; 07079 case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG"; 07080 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; 07081 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; 07082 case X86ISD::VSHL: return "X86ISD::VSHL"; 07083 case X86ISD::VSRL: return "X86ISD::VSRL"; 07084 case X86ISD::CMPPD: return "X86ISD::CMPPD"; 07085 case X86ISD::CMPPS: return "X86ISD::CMPPS"; 07086 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; 07087 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; 07088 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; 07089 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; 07090 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; 07091 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; 07092 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; 07093 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; 07094 case X86ISD::ADD: return "X86ISD::ADD"; 07095 case X86ISD::SUB: return "X86ISD::SUB"; 07096 case X86ISD::SMUL: return "X86ISD::SMUL"; 07097 case X86ISD::UMUL: return "X86ISD::UMUL"; 07098 case X86ISD::INC: return "X86ISD::INC"; 07099 case X86ISD::DEC: return "X86ISD::DEC"; 07100 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM"; 07101 case X86ISD::PTEST: return "X86ISD::PTEST"; 07102 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS"; 07103 } 07104 } 07105 07106 // isLegalAddressingMode - Return true if the addressing mode represented 07107 // by AM is legal for this target, for a load/store of the specified type. 07108 bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, 07109 const Type *Ty) const { 07110 // X86 supports extremely general addressing modes. 07111 CodeModel::Model M = getTargetMachine().getCodeModel(); 07112 07113 // X86 allows a sign-extended 32-bit immediate field as a displacement. 07114 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != NULL)) 07115 return false; 07116 07117 if (AM.BaseGV) { 07118 unsigned GVFlags = 07119 Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine()); 07120 07121 // If a reference to this global requires an extra load, we can't fold it. 07122 if (isGlobalStubReference(GVFlags)) 07123 return false; 07124 07125 // If BaseGV requires a register for the PIC base, we cannot also have a 07126 // BaseReg specified. 07127 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags)) 07128 return false; 07129 07130 // If lower 4G is not available, then we must use rip-relative addressing. 07131 if (Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1)) 07132 return false; 07133 } 07134 07135 switch (AM.Scale) { 07136 case 0: 07137 case 1: 07138 case 2: 07139 case 4: 07140 case 8: 07141 // These scales always work. 07142 break; 07143 case 3: 07144 case 5: 07145 case 9: 07146 // These scales are formed with basereg+scalereg. Only accept if there is 07147 // no basereg yet. 07148 if (AM.HasBaseReg) 07149 return false; 07150 break; 07151 default: // Other stuff never works. 07152 return false; 07153 } 07154 07155 return true; 07156 } 07157 07158 07159 bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { 07160 if (!Ty1->isInteger() || !Ty2->isInteger()) 07161 return false; 07162 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 07163 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 07164 if (NumBits1 <= NumBits2) 07165 return false; 07166 return Subtarget->is64Bit() || NumBits1 < 64; 07167 } 07168 07169 bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 07170 if (!VT1.isInteger() || !VT2.isInteger()) 07171 return false; 07172 unsigned NumBits1 = VT1.getSizeInBits(); 07173 unsigned NumBits2 = VT2.getSizeInBits(); 07174 if (NumBits1 <= NumBits2) 07175 return false; 07176 return Subtarget->is64Bit() || NumBits1 < 64; 07177 } 07178 07179 bool X86TargetLowering::isZExtFree(const Type *Ty1, const Type *Ty2) const { 07180 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. 07181 return Ty1 == Type::getInt32Ty(Ty1->getContext()) && 07182 Ty2 == Type::getInt64Ty(Ty1->getContext()) && Subtarget->is64Bit(); 07183 } 07184 07185 bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const { 07186 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. 07187 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit(); 07188 } 07189 07190 bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const { 07191 // i16 instructions are longer (0x66 prefix) and potentially slower. 07192 return !(VT1 == MVT::i32 && VT2 == MVT::i16); 07193 } 07194 07195 /// isShuffleMaskLegal - Targets can use this to indicate that they only 07196 /// support *some* VECTOR_SHUFFLE operations, those with specific masks. 07197 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values 07198 /// are assumed to be legal. 07199 bool 07200 X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M, 07201 EVT VT) const { 07202 // Only do shuffles on 128-bit vector types for now. 07203 if (VT.getSizeInBits() == 64) 07204 return false; 07205 07206 // FIXME: pshufb, blends, palignr, shifts. 07207 return (VT.getVectorNumElements() == 2 || 07208 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || 07209 isMOVLMask(M, VT) || 07210 isSHUFPMask(M, VT) || 07211 isPSHUFDMask(M, VT) || 07212 isPSHUFHWMask(M, VT) || 07213 isPSHUFLWMask(M, VT) || 07214 isUNPCKLMask(M, VT) || 07215 isUNPCKHMask(M, VT) || 07216 isUNPCKL_v_undef_Mask(M, VT) || 07217 isUNPCKH_v_undef_Mask(M, VT)); 07218 } 07219 07220 bool 07221 X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask, 07222 EVT VT) const { 07223 unsigned NumElts = VT.getVectorNumElements(); 07224 // FIXME: This collection of masks seems suspect. 07225 if (NumElts == 2) 07226 return true; 07227 if (NumElts == 4 && VT.getSizeInBits() == 128) { 07228 return (isMOVLMask(Mask, VT) || 07229 isCommutedMOVLMask(Mask, VT, true) || 07230 isSHUFPMask(Mask, VT) || 07231 isCommutedSHUFPMask(Mask, VT)); 07232 } 07233 return false; 07234 } 07235 07236 //===----------------------------------------------------------------------===// 07237 // X86 Scheduler Hooks 07238 //===----------------------------------------------------------------------===// 07239 07240 // private utility function 07241 MachineBasicBlock * 07242 X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, 07243 MachineBasicBlock *MBB, 07244 unsigned regOpc, 07245 unsigned immOpc, 07246 unsigned LoadOpc, 07247 unsigned CXchgOpc, 07248 unsigned copyOpc, 07249 unsigned notOpc, 07250 unsigned EAXreg, 07251 TargetRegisterClass *RC, 07252 bool invSrc) const { 07253 // For the atomic bitwise operator, we generate 07254 // thisMBB: 07255 // newMBB: 07256 // ld t1 = [bitinstr.addr] 07257 // op t2 = t1, [bitinstr.val] 07258 // mov EAX = t1 07259 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 07260 // bz newMBB 07261 // fallthrough -->nextMBB 07262 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 07263 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 07264 MachineFunction::iterator MBBIter = MBB; 07265 ++MBBIter; 07266 07267 /// First build the CFG 07268 MachineFunction *F = MBB->getParent(); 07269 MachineBasicBlock *thisMBB = MBB; 07270 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 07271 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 07272 F->insert(MBBIter, newMBB); 07273 F->insert(MBBIter, nextMBB); 07274 07275 // Move all successors to thisMBB to nextMBB 07276 nextMBB->transferSuccessors(thisMBB); 07277 07278 // Update thisMBB to fall through to newMBB 07279 thisMBB->addSuccessor(newMBB); 07280 07281 // newMBB jumps to itself and fall through to nextMBB 07282 newMBB->addSuccessor(nextMBB); 07283 newMBB->addSuccessor(newMBB); 07284 07285 // Insert instructions into newMBB based on incoming instruction 07286 assert(bInstr->getNumOperands() < X86AddrNumOperands + 4 && 07287 "unexpected number of operands"); 07288 DebugLoc dl = bInstr->getDebugLoc(); 07289 MachineOperand& destOper = bInstr->getOperand(0); 07290 MachineOperand* argOpers[2 + X86AddrNumOperands]; 07291 int numArgs = bInstr->getNumOperands() - 1; 07292 for (int i=0; i < numArgs; ++i) 07293 argOpers[i] = &bInstr->getOperand(i+1); 07294 07295 // x86 address has 4 operands: base, index, scale, and displacement 07296 int lastAddrIndx = X86AddrNumOperands - 1; // [0,3] 07297 int valArgIndx = lastAddrIndx + 1; 07298 07299 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 07300 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(LoadOpc), t1); 07301 for (int i=0; i <= lastAddrIndx; ++i) 07302 (*MIB).addOperand(*argOpers[i]); 07303 07304 unsigned tt = F->getRegInfo().createVirtualRegister(RC); 07305 if (invSrc) { 07306 MIB = BuildMI(newMBB, dl, TII->get(notOpc), tt).addReg(t1); 07307 } 07308 else 07309 tt = t1; 07310 07311 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 07312 assert((argOpers[valArgIndx]->isReg() || 07313 argOpers[valArgIndx]->isImm()) && 07314 "invalid operand"); 07315 if (argOpers[valArgIndx]->isReg()) 07316 MIB = BuildMI(newMBB, dl, TII->get(regOpc), t2); 07317 else 07318 MIB = BuildMI(newMBB, dl, TII->get(immOpc), t2); 07319 MIB.addReg(tt); 07320 (*MIB).addOperand(*argOpers[valArgIndx]); 07321 07322 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), EAXreg); 07323 MIB.addReg(t1); 07324 07325 MIB = BuildMI(newMBB, dl, TII->get(CXchgOpc)); 07326 for (int i=0; i <= lastAddrIndx; ++i) 07327 (*MIB).addOperand(*argOpers[i]); 07328 MIB.addReg(t2); 07329 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 07330 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 07331 07332 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), destOper.getReg()); 07333 MIB.addReg(EAXreg); 07334 07335 // insert branch 07336 BuildMI(newMBB, dl, TII->get(X86::JNE)).addMBB(newMBB); 07337 07338 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 07339 return nextMBB; 07340 } 07341 07342 // private utility function: 64 bit atomics on 32 bit host. 07343 MachineBasicBlock * 07344 X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr, 07345 MachineBasicBlock *MBB, 07346 unsigned regOpcL, 07347 unsigned regOpcH, 07348 unsigned immOpcL, 07349 unsigned immOpcH, 07350 bool invSrc) const { 07351 // For the atomic bitwise operator, we generate 07352 // thisMBB (instructions are in pairs, except cmpxchg8b) 07353 // ld t1,t2 = [bitinstr.addr] 07354 // newMBB: 07355 // out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4) 07356 // op t5, t6 <- out1, out2, [bitinstr.val] 07357 // (for SWAP, substitute: mov t5, t6 <- [bitinstr.val]) 07358 // mov ECX, EBX <- t5, t6 07359 // mov EAX, EDX <- t1, t2 07360 // cmpxchg8b [bitinstr.addr] [EAX, EDX, EBX, ECX implicit] 07361 // mov t3, t4 <- EAX, EDX 07362 // bz newMBB 07363 // result in out1, out2 07364 // fallthrough -->nextMBB 07365 07366 const TargetRegisterClass *RC = X86::GR32RegisterClass; 07367 const unsigned LoadOpc = X86::MOV32rm; 07368 const unsigned copyOpc = X86::MOV32rr; 07369 const unsigned NotOpc = X86::NOT32r; 07370 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 07371 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 07372 MachineFunction::iterator MBBIter = MBB; 07373 ++MBBIter; 07374 07375 /// First build the CFG 07376 MachineFunction *F = MBB->getParent(); 07377 MachineBasicBlock *thisMBB = MBB; 07378 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 07379 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 07380 F->insert(MBBIter, newMBB); 07381 F->insert(MBBIter, nextMBB); 07382 07383 // Move all successors to thisMBB to nextMBB 07384 nextMBB->transferSuccessors(thisMBB); 07385 07386 // Update thisMBB to fall through to newMBB 07387 thisMBB->addSuccessor(newMBB); 07388 07389 // newMBB jumps to itself and fall through to nextMBB 07390 newMBB->addSuccessor(nextMBB); 07391 newMBB->addSuccessor(newMBB); 07392 07393 DebugLoc dl = bInstr->getDebugLoc(); 07394 // Insert instructions into newMBB based on incoming instruction 07395 // There are 8 "real" operands plus 9 implicit def/uses, ignored here. 07396 assert(bInstr->getNumOperands() < X86AddrNumOperands + 14 && 07397 "unexpected number of operands"); 07398 MachineOperand& dest1Oper = bInstr->getOperand(0); 07399 MachineOperand& dest2Oper = bInstr->getOperand(1); 07400 MachineOperand* argOpers[2 + X86AddrNumOperands]; 07401 for (int i=0; i < 2 + X86AddrNumOperands; ++i) 07402 argOpers[i] = &bInstr->getOperand(i+2); 07403 07404 // x86 address has 4 operands: base, index, scale, and displacement 07405 int lastAddrIndx = X86AddrNumOperands - 1; // [0,3] 07406 07407 unsigned t1 = F->getRegInfo().createVirtualRegister(RC); 07408 MachineInstrBuilder MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t1); 07409 for (int i=0; i <= lastAddrIndx; ++i) 07410 (*MIB).addOperand(*argOpers[i]); 07411 unsigned t2 = F->getRegInfo().createVirtualRegister(RC); 07412 MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2); 07413 // add 4 to displacement. 07414 for (int i=0; i <= lastAddrIndx-2; ++i) 07415 (*MIB).addOperand(*argOpers[i]); 07416 MachineOperand newOp3 = *(argOpers[3]); 07417 if (newOp3.isImm()) 07418 newOp3.setImm(newOp3.getImm()+4); 07419 else 07420 newOp3.setOffset(newOp3.getOffset()+4); 07421 (*MIB).addOperand(newOp3); 07422 (*MIB).addOperand(*argOpers[lastAddrIndx]); 07423 07424 // t3/4 are defined later, at the bottom of the loop 07425 unsigned t3 = F->getRegInfo().createVirtualRegister(RC); 07426 unsigned t4 = F->getRegInfo().createVirtualRegister(RC); 07427 BuildMI(newMBB, dl, TII->get(X86::PHI), dest1Oper.getReg()) 07428 .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB); 07429 BuildMI(newMBB, dl, TII->get(X86::PHI), dest2Oper.getReg()) 07430 .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB); 07431 07432 unsigned tt1 = F->getRegInfo().createVirtualRegister(RC); 07433 unsigned tt2 = F->getRegInfo().createVirtualRegister(RC); 07434 if (invSrc) { 07435 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), tt1).addReg(t1); 07436 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), tt2).addReg(t2); 07437 } else { 07438 tt1 = t1; 07439 tt2 = t2; 07440 } 07441 07442 int valArgIndx = lastAddrIndx + 1; 07443 assert((argOpers[valArgIndx]->isReg() || 07444 argOpers[valArgIndx]->isImm()) && 07445 "invalid operand"); 07446 unsigned t5 = F->getRegInfo().createVirtualRegister(RC); 07447 unsigned t6 = F->getRegInfo().createVirtualRegister(RC); 07448 if (argOpers[valArgIndx]->isReg()) 07449 MIB = BuildMI(newMBB, dl, TII->get(regOpcL), t5); 07450 else 07451 MIB = BuildMI(newMBB, dl, TII->get(immOpcL), t5); 07452 if (regOpcL != X86::MOV32rr) 07453 MIB.addReg(tt1); 07454 (*MIB).addOperand(*argOpers[valArgIndx]); 07455 assert(argOpers[valArgIndx + 1]->isReg() == 07456 argOpers[valArgIndx]->isReg()); 07457 assert(argOpers[valArgIndx + 1]->isImm() == 07458 argOpers[valArgIndx]->isImm()); 07459 if (argOpers[valArgIndx + 1]->isReg()) 07460 MIB = BuildMI(newMBB, dl, TII->get(regOpcH), t6); 07461 else 07462 MIB = BuildMI(newMBB, dl, TII->get(immOpcH), t6); 07463 if (regOpcH != X86::MOV32rr) 07464 MIB.addReg(tt2); 07465 (*MIB).addOperand(*argOpers[valArgIndx + 1]); 07466 07467 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EAX); 07468 MIB.addReg(t1); 07469 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EDX); 07470 MIB.addReg(t2); 07471 07472 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EBX); 07473 MIB.addReg(t5); 07474 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::ECX); 07475 MIB.addReg(t6); 07476 07477 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG8B)); 07478 for (int i=0; i <= lastAddrIndx; ++i) 07479 (*MIB).addOperand(*argOpers[i]); 07480 07481 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 07482 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); 07483 07484 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t3); 07485 MIB.addReg(X86::EAX); 07486 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t4); 07487 MIB.addReg(X86::EDX); 07488 07489 // insert branch 07490 BuildMI(newMBB, dl, TII->get(X86::JNE)).addMBB(newMBB); 07491 07492 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. 07493 return nextMBB; 07494 } 07495 07496 // private utility function 07497 MachineBasicBlock * 07498 X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, 07499 MachineBasicBlock *MBB, 07500 unsigned cmovOpc) const { 07501 // For the atomic min/max operator, we generate 07502 // thisMBB: 07503 // newMBB: 07504 // ld t1 = [min/max.addr] 07505 // mov t2 = [min/max.val] 07506 // cmp t1, t2 07507 // cmov[cond] t2 = t1 07508 // mov EAX = t1 07509 // lcs dest = [bitinstr.addr], t2 [EAX is implicit] 07510 // bz newMBB 07511 // fallthrough -->nextMBB 07512 // 07513 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 07514 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 07515 MachineFunction::iterator MBBIter = MBB; 07516 ++MBBIter; 07517 07518 /// First build the CFG 07519 MachineFunction *F = MBB->getParent(); 07520 MachineBasicBlock *thisMBB = MBB; 07521 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); 07522 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); 07523 F->insert(MBBIter, newMBB); 07524 F->insert(MBBIter, nextMBB); 07525 07526 // Move all successors of thisMBB to nextMBB 07527 nextMBB->transferSuccessors(thisMBB); 07528 07529 // Update thisMBB to fall through to newMBB 07530 thisMBB->addSuccessor(newMBB); 07531 07532 // newMBB jumps to newMBB and fall through to nextMBB 07533 newMBB->addSuccessor(nextMBB); 07534 newMBB->addSuccessor(newMBB); 07535 07536 DebugLoc dl = mInstr->getDebugLoc(); 07537 // Insert instructions into newMBB based on incoming instruction 07538 assert(mInstr->getNumOperands() < X86AddrNumOperands + 4 && 07539 "unexpected number of operands"); 07540 MachineOperand& destOper = mInstr->getOperand(0); 07541 MachineOperand* argOpers[2 + X86AddrNumOperands]; 07542 int numArgs = mInstr->getNumOperands() - 1; 07543 for (int i=0; i < numArgs; ++i) 07544 argOpers[i] = &mInstr->getOperand(i+1); 07545 07546 // x86 address has 4 operands: base, index, scale, and displacement 07547 int lastAddrIndx = X86AddrNumOperands - 1; // [0,3] 07548 int valArgIndx = lastAddrIndx + 1; 07549 07550 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 07551 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rm), t1); 07552 for (int i=0; i <= lastAddrIndx; ++i) 07553 (*MIB).addOperand(*argOpers[i]); 07554 07555 // We only support register and immediate values 07556 assert((argOpers[valArgIndx]->isReg() || 07557 argOpers[valArgIndx]->isImm()) && 07558 "invalid operand"); 07559 07560 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 07561 if (argOpers[valArgIndx]->isReg()) 07562 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2); 07563 else 07564 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2); 07565 (*MIB).addOperand(*argOpers[valArgIndx]); 07566 07567 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), X86::EAX); 07568 MIB.addReg(t1); 07569 07570 MIB = BuildMI(newMBB, dl, TII->get(X86::CMP32rr)); 07571 MIB.addReg(t1); 07572 MIB.addReg(t2); 07573 07574 // Generate movc 07575 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); 07576 MIB = BuildMI(newMBB, dl, TII->get(cmovOpc),t3); 07577 MIB.addReg(t2); 07578 MIB.addReg(t1); 07579 07580 // Cmp and exchange if none has modified the memory location 07581 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG32)); 07582 for (int i=0; i <= lastAddrIndx; ++i) 07583 (*MIB).addOperand(*argOpers[i]); 07584 MIB.addReg(t3); 07585 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); 07586 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); 07587 07588 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), destOper.getReg()); 07589 MIB.addReg(X86::EAX); 07590 07591 // insert branch 07592 BuildMI(newMBB, dl, TII->get(X86::JNE)).addMBB(newMBB); 07593 07594 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. 07595 return nextMBB; 07596 } 07597 07598 MachineBasicBlock * 07599 X86TargetLowering::EmitPCMP(MachineInstr *MI, MachineBasicBlock *BB, 07600 unsigned numArgs, bool memArg) const { 07601 07602 MachineFunction *F = BB->getParent(); 07603 DebugLoc dl = MI->getDebugLoc(); 07604 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 07605 07606 unsigned Opc; 07607 07608 if (memArg) { 07609 Opc = numArgs == 3 ? 07610 X86::PCMPISTRM128rm : 07611 X86::PCMPESTRM128rm; 07612 } else { 07613 Opc = numArgs == 3 ? 07614 X86::PCMPISTRM128rr : 07615 X86::PCMPESTRM128rr; 07616 } 07617 07618 MachineInstrBuilder MIB = BuildMI(BB, dl, TII->get(Opc)); 07619 07620 for (unsigned i = 0; i < numArgs; ++i) { 07621 MachineOperand &Op = MI->getOperand(i+1); 07622 07623 if (!(Op.isReg() && Op.isImplicit())) 07624 MIB.addOperand(Op); 07625 } 07626 07627 BuildMI(BB, dl, TII->get(X86::MOVAPSrr), MI->getOperand(0).getReg()) 07628 .addReg(X86::XMM0); 07629 07630 F->DeleteMachineInstr(MI); 07631 07632 return BB; 07633 } 07634 07635 MachineBasicBlock * 07636 X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( 07637 MachineInstr *MI, 07638 MachineBasicBlock *MBB) const { 07639 // Emit code to save XMM registers to the stack. The ABI says that the 07640 // number of registers to save is given in %al, so it's theoretically 07641 // possible to do an indirect jump trick to avoid saving all of them, 07642 // however this code takes a simpler approach and just executes all 07643 // of the stores if %al is non-zero. It's less code, and it's probably 07644 // easier on the hardware branch predictor, and stores aren't all that 07645 // expensive anyway. 07646 07647 // Create the new basic blocks. One block contains all the XMM stores, 07648 // and one block is the final destination regardless of whether any 07649 // stores were performed. 07650 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); 07651 MachineFunction *F = MBB->getParent(); 07652 MachineFunction::iterator MBBIter = MBB; 07653 ++MBBIter; 07654 MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB); 07655 MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB); 07656 F->insert(MBBIter, XMMSaveMBB); 07657 F->insert(MBBIter, EndMBB); 07658 07659 // Set up the CFG. 07660 // Move any original successors of MBB to the end block. 07661 EndMBB->transferSuccessors(MBB); 07662 // The original block will now fall through to the XMM save block. 07663 MBB->addSuccessor(XMMSaveMBB); 07664 // The XMMSaveMBB will fall through to the end block. 07665 XMMSaveMBB->addSuccessor(EndMBB); 07666 07667 // Now add the instructions. 07668 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 07669 DebugLoc DL = MI->getDebugLoc(); 07670 07671 unsigned CountReg = MI->getOperand(0).getReg(); 07672 int64_t RegSaveFrameIndex = MI->getOperand(1).getImm(); 07673 int64_t VarArgsFPOffset = MI->getOperand(2).getImm(); 07674 07675 if (!Subtarget->isTargetWin64()) { 07676 // If %al is 0, branch around the XMM save block. 07677 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg); 07678 BuildMI(MBB, DL, TII->get(X86::JE)).addMBB(EndMBB); 07679 MBB->addSuccessor(EndMBB); 07680 } 07681 07682 // In the XMM save block, save all the XMM argument registers. 07683 for (int i = 3, e = MI->getNumOperands(); i != e; ++i) { 07684 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset; 07685 BuildMI(XMMSaveMBB, DL, TII->get(X86::MOVAPSmr)) 07686 .addFrameIndex(RegSaveFrameIndex) 07687 .addImm(/*Scale=*/1) 07688 .addReg(/*IndexReg=*/0) 07689 .addImm(/*Disp=*/Offset) 07690 .addReg(/*Segment=*/0) 07691 .addReg(MI->getOperand(i).getReg()) 07692 .addMemOperand(MachineMemOperand( 07693 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 07694 MachineMemOperand::MOStore, Offset, 07695 /*Size=*/16, /*Align=*/16)); 07696 } 07697 07698 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 07699 07700 return EndMBB; 07701 } 07702 07703 MachineBasicBlock * 07704 X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 07705 MachineBasicBlock *BB) const { 07706 DebugLoc dl = MI->getDebugLoc(); 07707 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 07708 switch (MI->getOpcode()) { 07709 default: assert(false && "Unexpected instr type to insert"); 07710 case X86::CMOV_V1I64: 07711 case X86::CMOV_FR32: 07712 case X86::CMOV_FR64: 07713 case X86::CMOV_V4F32: 07714 case X86::CMOV_V2F64: 07715 case X86::CMOV_V2I64: { 07716 // To "insert" a SELECT_CC instruction, we actually have to insert the 07717 // diamond control-flow pattern. The incoming instruction knows the 07718 // destination vreg to set, the condition code register to branch on, the 07719 // true/false values to select between, and a branch opcode to use. 07720 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 07721 MachineFunction::iterator It = BB; 07722 ++It; 07723 07724 // thisMBB: 07725 // ... 07726 // TrueVal = ... 07727 // cmpTY ccX, r1, r2 07728 // bCC copy1MBB 07729 // fallthrough --> copy0MBB 07730 MachineBasicBlock *thisMBB = BB; 07731 MachineFunction *F = BB->getParent(); 07732 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 07733 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 07734 unsigned Opc = 07735 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); 07736 BuildMI(BB, dl, TII->get(Opc)).addMBB(sinkMBB); 07737 F->insert(It, copy0MBB); 07738 F->insert(It, sinkMBB); 07739 // Update machine-CFG edges by transferring all successors of the current 07740 // block to the new block which will contain the Phi node for the select. 07741 sinkMBB->transferSuccessors(BB); 07742 07743 // Add the true and fallthrough blocks as its successors. 07744 BB->addSuccessor(copy0MBB); 07745 BB->addSuccessor(sinkMBB); 07746 07747 // copy0MBB: 07748 // %FalseValue = ... 07749 // # fallthrough to sinkMBB 07750 BB = copy0MBB; 07751 07752 // Update machine-CFG edges 07753 BB->addSuccessor(sinkMBB); 07754 07755 // sinkMBB: 07756 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 07757 // ... 07758 BB = sinkMBB; 07759 BuildMI(BB, dl, TII->get(X86::PHI), MI->getOperand(0).getReg()) 07760 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) 07761 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); 07762 07763 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 07764 return BB; 07765 } 07766 07767 case X86::FP32_TO_INT16_IN_MEM: 07768 case X86::FP32_TO_INT32_IN_MEM: 07769 case X86::FP32_TO_INT64_IN_MEM: 07770 case X86::FP64_TO_INT16_IN_MEM: 07771 case X86::FP64_TO_INT32_IN_MEM: 07772 case X86::FP64_TO_INT64_IN_MEM: 07773 case X86::FP80_TO_INT16_IN_MEM: 07774 case X86::FP80_TO_INT32_IN_MEM: 07775 case X86::FP80_TO_INT64_IN_MEM: { 07776 // Change the floating point control register to use "round towards zero" 07777 // mode when truncating to an integer value. 07778 MachineFunction *F = BB->getParent(); 07779 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); 07780 addFrameReference(BuildMI(BB, dl, TII->get(X86::FNSTCW16m)), CWFrameIdx); 07781 07782 // Load the old value of the high byte of the control word... 07783 unsigned OldCW = 07784 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); 07785 addFrameReference(BuildMI(BB, dl, TII->get(X86::MOV16rm), OldCW), 07786 CWFrameIdx); 07787 07788 // Set the high part to be round to zero... 07789 addFrameReference(BuildMI(BB, dl, TII->get(X86::MOV16mi)), CWFrameIdx) 07790 .addImm(0xC7F); 07791 07792 // Reload the modified control word now... 07793 addFrameReference(BuildMI(BB, dl, TII->get(X86::FLDCW16m)), CWFrameIdx); 07794 07795 // Restore the memory image of control word to original value 07796 addFrameReference(BuildMI(BB, dl, TII->get(X86::MOV16mr)), CWFrameIdx) 07797 .addReg(OldCW); 07798 07799 // Get the X86 opcode to use. 07800 unsigned Opc; 07801 switch (MI->getOpcode()) { 07802 default: llvm_unreachable("illegal opcode!"); 07803 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; 07804 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; 07805 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; 07806 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; 07807 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; 07808 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; 07809 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; 07810 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; 07811 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; 07812 } 07813 07814 X86AddressMode AM; 07815 MachineOperand &Op = MI->getOperand(0); 07816 if (Op.isReg()) { 07817 AM.BaseType = X86AddressMode::RegBase; 07818 AM.Base.Reg = Op.getReg(); 07819 } else { 07820 AM.BaseType = X86AddressMode::FrameIndexBase; 07821 AM.Base.FrameIndex = Op.getIndex(); 07822 } 07823 Op = MI->getOperand(1); 07824 if (Op.isImm()) 07825 AM.Scale = Op.getImm(); 07826 Op = MI->getOperand(2); 07827 if (Op.isImm()) 07828 AM.IndexReg = Op.getImm(); 07829 Op = MI->getOperand(3); 07830 if (Op.isGlobal()) { 07831 AM.GV = Op.getGlobal(); 07832 } else { 07833 AM.Disp = Op.getImm(); 07834 } 07835 addFullAddress(BuildMI(BB, dl, TII->get(Opc)), AM) 07836 .addReg(MI->getOperand(X86AddrNumOperands).getReg()); 07837 07838 // Reload the original control word now. 07839 addFrameReference(BuildMI(BB, dl, TII->get(X86::FLDCW16m)), CWFrameIdx); 07840 07841 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. 07842 return BB; 07843 } 07844 // String/text processing lowering. 07845 case X86::PCMPISTRM128REG: 07846 return EmitPCMP(MI, BB, 3, false /* in-mem */); 07847 case X86::PCMPISTRM128MEM: 07848 return EmitPCMP(MI, BB, 3, true /* in-mem */); 07849 case X86::PCMPESTRM128REG: 07850 return EmitPCMP(MI, BB, 5, false /* in mem */); 07851 case X86::PCMPESTRM128MEM: 07852 return EmitPCMP(MI, BB, 5, true /* in mem */); 07853 07854 // Atomic Lowering. 07855 case X86::ATOMAND32: 07856 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 07857 X86::AND32ri, X86::MOV32rm, 07858 X86::LCMPXCHG32, X86::MOV32rr, 07859 X86::NOT32r, X86::EAX, 07860 X86::GR32RegisterClass); 07861 case X86::ATOMOR32: 07862 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, 07863 X86::OR32ri, X86::MOV32rm, 07864 X86::LCMPXCHG32, X86::MOV32rr, 07865 X86::NOT32r, X86::EAX, 07866 X86::GR32RegisterClass); 07867 case X86::ATOMXOR32: 07868 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, 07869 X86::XOR32ri, X86::MOV32rm, 07870 X86::LCMPXCHG32, X86::MOV32rr, 07871 X86::NOT32r, X86::EAX, 07872 X86::GR32RegisterClass); 07873 case X86::ATOMNAND32: 07874 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, 07875 X86::AND32ri, X86::MOV32rm, 07876 X86::LCMPXCHG32, X86::MOV32rr, 07877 X86::NOT32r, X86::EAX, 07878 X86::GR32RegisterClass, true); 07879 case X86::ATOMMIN32: 07880 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); 07881 case X86::ATOMMAX32: 07882 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); 07883 case X86::ATOMUMIN32: 07884 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); 07885 case X86::ATOMUMAX32: 07886 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); 07887 07888 case X86::ATOMAND16: 07889 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 07890 X86::AND16ri, X86::MOV16rm, 07891 X86::LCMPXCHG16, X86::MOV16rr, 07892 X86::NOT16r, X86::AX, 07893 X86::GR16RegisterClass); 07894 case X86::ATOMOR16: 07895 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, 07896 X86::OR16ri, X86::MOV16rm, 07897 X86::LCMPXCHG16, X86::MOV16rr, 07898 X86::NOT16r, X86::AX, 07899 X86::GR16RegisterClass); 07900 case X86::ATOMXOR16: 07901 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, 07902 X86::XOR16ri, X86::MOV16rm, 07903 X86::LCMPXCHG16, X86::MOV16rr, 07904 X86::NOT16r, X86::AX, 07905 X86::GR16RegisterClass); 07906 case X86::ATOMNAND16: 07907 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, 07908 X86::AND16ri, X86::MOV16rm, 07909 X86::LCMPXCHG16, X86::MOV16rr, 07910 X86::NOT16r, X86::AX, 07911 X86::GR16RegisterClass, true); 07912 case X86::ATOMMIN16: 07913 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); 07914 case X86::ATOMMAX16: 07915 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); 07916 case X86::ATOMUMIN16: 07917 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); 07918 case X86::ATOMUMAX16: 07919 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); 07920 07921 case X86::ATOMAND8: 07922 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 07923 X86::AND8ri, X86::MOV8rm, 07924 X86::LCMPXCHG8, X86::MOV8rr, 07925 X86::NOT8r, X86::AL, 07926 X86::GR8RegisterClass); 07927 case X86::ATOMOR8: 07928 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, 07929 X86::OR8ri, X86::MOV8rm, 07930 X86::LCMPXCHG8, X86::MOV8rr, 07931 X86::NOT8r, X86::AL, 07932 X86::GR8RegisterClass); 07933 case X86::ATOMXOR8: 07934 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, 07935 X86::XOR8ri, X86::MOV8rm, 07936 X86::LCMPXCHG8, X86::MOV8rr, 07937 X86::NOT8r, X86::AL, 07938 X86::GR8RegisterClass); 07939 case X86::ATOMNAND8: 07940 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, 07941 X86::AND8ri, X86::MOV8rm, 07942 X86::LCMPXCHG8, X86::MOV8rr, 07943 X86::NOT8r, X86::AL, 07944 X86::GR8RegisterClass, true); 07945 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. 07946 // This group is for 64-bit host. 07947 case X86::ATOMAND64: 07948 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 07949 X86::AND64ri32, X86::MOV64rm, 07950 X86::LCMPXCHG64, X86::MOV64rr, 07951 X86::NOT64r, X86::RAX, 07952 X86::GR64RegisterClass); 07953 case X86::ATOMOR64: 07954 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr, 07955 X86::OR64ri32, X86::MOV64rm, 07956 X86::LCMPXCHG64, X86::MOV64rr, 07957 X86::NOT64r, X86::RAX, 07958 X86::GR64RegisterClass); 07959 case X86::ATOMXOR64: 07960 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr, 07961 X86::XOR64ri32, X86::MOV64rm, 07962 X86::LCMPXCHG64, X86::MOV64rr, 07963 X86::NOT64r, X86::RAX, 07964 X86::GR64RegisterClass); 07965 case X86::ATOMNAND64: 07966 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, 07967 X86::AND64ri32, X86::MOV64rm, 07968 X86::LCMPXCHG64, X86::MOV64rr, 07969 X86::NOT64r, X86::RAX, 07970 X86::GR64RegisterClass, true); 07971 case X86::ATOMMIN64: 07972 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr); 07973 case X86::ATOMMAX64: 07974 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr); 07975 case X86::ATOMUMIN64: 07976 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr); 07977 case X86::ATOMUMAX64: 07978 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr); 07979 07980 // This group does 64-bit operations on a 32-bit host. 07981 case X86::ATOMAND6432: 07982 return EmitAtomicBit6432WithCustomInserter(MI, BB, 07983 X86::AND32rr, X86::AND32rr, 07984 X86::AND32ri, X86::AND32ri, 07985 false); 07986 case X86::ATOMOR6432: 07987 return EmitAtomicBit6432WithCustomInserter(MI, BB, 07988 X86::OR32rr, X86::OR32rr, 07989 X86::OR32ri, X86::OR32ri, 07990 false); 07991 case X86::ATOMXOR6432: 07992 return EmitAtomicBit6432WithCustomInserter(MI, BB, 07993 X86::XOR32rr, X86::XOR32rr, 07994 X86::XOR32ri, X86::XOR32ri, 07995 false); 07996 case X86::ATOMNAND6432: 07997 return EmitAtomicBit6432WithCustomInserter(MI, BB, 07998 X86::AND32rr, X86::AND32rr, 07999 X86::AND32ri, X86::AND32ri, 08000 true); 08001 case X86::ATOMADD6432: 08002 return EmitAtomicBit6432WithCustomInserter(MI, BB, 08003 X86::ADD32rr, X86::ADC32rr, 08004 X86::ADD32ri, X86::ADC32ri, 08005 false); 08006 case X86::ATOMSUB6432: 08007 return EmitAtomicBit6432WithCustomInserter(MI, BB, 08008 X86::SUB32rr, X86::SBB32rr, 08009 X86::SUB32ri, X86::SBB32ri, 08010 false); 08011 case X86::ATOMSWAP6432: 08012 return EmitAtomicBit6432WithCustomInserter(MI, BB, 08013 X86::MOV32rr, X86::MOV32rr, 08014 X86::MOV32ri, X86::MOV32ri, 08015 false); 08016 case X86::VASTART_SAVE_XMM_REGS: 08017 return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB); 08018 } 08019 } 08020 08021 //===----------------------------------------------------------------------===// 08022 // X86 Optimization Hooks 08023 //===----------------------------------------------------------------------===// 08024 08025 void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, 08026 const APInt &Mask, 08027 APInt &KnownZero, 08028 APInt &KnownOne, 08029 const SelectionDAG &DAG, 08030 unsigned Depth) const { 08031 unsigned Opc = Op.getOpcode(); 08032 assert((Opc >= ISD::BUILTIN_OP_END || 08033 Opc == ISD::INTRINSIC_WO_CHAIN || 08034 Opc == ISD::INTRINSIC_W_CHAIN || 08035 Opc == ISD::INTRINSIC_VOID) && 08036 "Should use MaskedValueIsZero if you don't know whether Op" 08037 " is a target node!"); 08038 08039 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. 08040 switch (Opc) { 08041 default: break; 08042 case X86ISD::ADD: 08043 case X86ISD::SUB: 08044 case X86ISD::SMUL: 08045 case X86ISD::UMUL: 08046 case X86ISD::INC: 08047 case X86ISD::DEC: 08048 // These nodes' second result is a boolean. 08049 if (Op.getResNo() == 0) 08050 break; 08051 // Fallthrough 08052 case X86ISD::SETCC: 08053 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), 08054 Mask.getBitWidth() - 1); 08055 break; 08056 } 08057 } 08058 08059 /// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the 08060 /// node is a GlobalAddress + offset. 08061 bool X86TargetLowering::isGAPlusOffset(SDNode *N, 08062 GlobalValue* &GA, int64_t &Offset) const{ 08063 if (N->getOpcode() == X86ISD::Wrapper) { 08064 if (isa<GlobalAddressSDNode>(N->getOperand(0))) { 08065 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); 08066 Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset(); 08067 return true; 08068 } 08069 } 08070 return TargetLowering::isGAPlusOffset(N, GA, Offset); 08071 } 08072 08073 static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, 08074 const TargetLowering &TLI) { 08075 GlobalValue *GV; 08076 int64_t Offset = 0; 08077 if (TLI.isGAPlusOffset(Base, GV, Offset)) 08078 return (GV->getAlignment() >= N && (Offset % N) == 0); 08079 // DAG combine handles the stack object case. 08080 return false; 08081 } 08082 08083 static bool EltsFromConsecutiveLoads(ShuffleVectorSDNode *N, unsigned NumElems, 08084 EVT EVT, LoadSDNode *&LDBase, 08085 unsigned &LastLoadedElt, 08086 SelectionDAG &DAG, MachineFrameInfo *MFI, 08087 const TargetLowering &TLI) { 08088 LDBase = NULL; 08089 LastLoadedElt = -1U; 08090 for (unsigned i = 0; i < NumElems; ++i) { 08091 if (N->getMaskElt(i) < 0) { 08092 if (!LDBase) 08093 return false; 08094 continue; 08095 } 08096 08097 SDValue Elt = DAG.getShuffleScalarElt(N, i); 08098 if (!Elt.getNode() || 08099 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) 08100 return false; 08101 if (!LDBase) { 08102 if (Elt.getNode()->getOpcode() == ISD::UNDEF) 08103 return false; 08104 LDBase = cast<LoadSDNode>(Elt.getNode()); 08105 LastLoadedElt = i; 08106 continue; 08107 } 08108 if (Elt.getOpcode() == ISD::UNDEF) 08109 continue; 08110 08111 LoadSDNode *LD = cast<LoadSDNode>(Elt); 08112 if (!TLI.isConsecutiveLoad(LD, LDBase, EVT.getSizeInBits()/8, i, MFI)) 08113 return false; 08114 LastLoadedElt = i; 08115 } 08116 return true; 08117 } 08118 08119 /// PerformShuffleCombine - Combine a vector_shuffle that is equal to 08120 /// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load 08121 /// if the load addresses are consecutive, non-overlapping, and in the right 08122 /// order. In the case of v2i64, it will see if it can rewrite the 08123 /// shuffle to be an appropriate build vector so it can take advantage of 08124 // performBuildVectorCombine. 08125 static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, 08126 const TargetLowering &TLI) { 08127 DebugLoc dl = N->getDebugLoc(); 08128 EVT VT = N->getValueType(0); 08129 EVT EVT = VT.getVectorElementType(); 08130 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 08131 unsigned NumElems = VT.getVectorNumElements(); 08132 08133 if (VT.getSizeInBits() != 128) 08134 return SDValue(); 08135 08136 // Try to combine a vector_shuffle into a 128-bit load. 08137 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 08138 LoadSDNode *LD = NULL; 08139 unsigned LastLoadedElt; 08140 if (!EltsFromConsecutiveLoads(SVN, NumElems, EVT, LD, LastLoadedElt, DAG, 08141 MFI, TLI)) 08142 return SDValue(); 08143 08144 if (LastLoadedElt == NumElems - 1) { 08145 if (isBaseAlignmentOfN(16, LD->getBasePtr().getNode(), TLI)) 08146 return DAG.getLoad(VT, dl, LD->getChain(), LD->getBasePtr(), 08147 LD->getSrcValue(), LD->getSrcValueOffset(), 08148 LD->isVolatile()); 08149 return DAG.getLoad(VT, dl, LD->getChain(), LD->getBasePtr(), 08150 LD->getSrcValue(), LD->getSrcValueOffset(), 08151 LD->isVolatile(), LD->getAlignment()); 08152 } else if (NumElems == 4 && LastLoadedElt == 1) { 08153 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); 08154 SDValue Ops[] = { LD->getChain(), LD->getBasePtr() }; 08155 SDValue ResNode = DAG.getNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, 2); 08156 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, ResNode); 08157 } 08158 return SDValue(); 08159 } 08160 08161 /// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. 08162 static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, 08163 const X86Subtarget *Subtarget) { 08164 DebugLoc DL = N->getDebugLoc(); 08165 SDValue Cond = N->getOperand(0); 08166 // Get the LHS/RHS of the select. 08167 SDValue LHS = N->getOperand(1); 08168 SDValue RHS = N->getOperand(2); 08169 08170 // If we have SSE[12] support, try to form min/max nodes. 08171 if (Subtarget->hasSSE2() && 08172 (LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64) && 08173 Cond.getOpcode() == ISD::SETCC) { 08174 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 08175 08176 unsigned Opcode = 0; 08177 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { 08178 switch (CC) { 08179 default: break; 08180 case ISD::SETOLE: // (X <= Y) ? X : Y -> min 08181 case ISD::SETULE: 08182 case ISD::SETLE: 08183 if (!UnsafeFPMath) break; 08184 // FALL THROUGH. 08185 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min 08186 case ISD::SETLT: 08187 Opcode = X86ISD::FMIN; 08188 break; 08189 08190 case ISD::SETOGT: // (X > Y) ? X : Y -> max 08191 case ISD::SETUGT: 08192 case ISD::SETGT: 08193 if (!UnsafeFPMath) break; 08194 // FALL THROUGH. 08195 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max 08196 case ISD::SETGE: 08197 Opcode = X86ISD::FMAX; 08198 break; 08199 } 08200 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { 08201 switch (CC) { 08202 default: break; 08203 case ISD::SETOGT: // (X > Y) ? Y : X -> min 08204 case ISD::SETUGT: 08205 case ISD::SETGT: 08206 if (!UnsafeFPMath) break; 08207 // FALL THROUGH. 08208 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min 08209 case ISD::SETGE: 08210 Opcode = X86ISD::FMIN; 08211 break; 08212 08213 case ISD::SETOLE: // (X <= Y) ? Y : X -> max 08214 case ISD::SETULE: 08215 case ISD::SETLE: 08216 if (!UnsafeFPMath) break; 08217 // FALL THROUGH. 08218 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max 08219 case ISD::SETLT: 08220 Opcode = X86ISD::FMAX; 08221 break; 08222 } 08223 } 08224 08225 if (Opcode) 08226 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); 08227 } 08228 08229 // If this is a select between two integer constants, try to do some 08230 // optimizations. 08231 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) { 08232 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS)) 08233 // Don't do this for crazy integer types. 08234 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) { 08235 // If this is efficiently invertible, canonicalize the LHSC/RHSC values 08236 // so that TrueC (the true value) is larger than FalseC. 08237 bool NeedsCondInvert = false; 08238 08239 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) && 08240 // Efficiently invertible. 08241 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible. 08242 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible. 08243 isa<ConstantSDNode>(Cond.getOperand(1))))) { 08244 NeedsCondInvert = true; 08245 std::swap(TrueC, FalseC); 08246 } 08247 08248 // Optimize C ? 8 : 0 -> zext(C) << 3. Likewise for any pow2/0. 08249 if (FalseC->getAPIntValue() == 0 && 08250 TrueC->getAPIntValue().isPowerOf2()) { 08251 if (NeedsCondInvert) // Invert the condition if needed. 08252 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 08253 DAG.getConstant(1, Cond.getValueType())); 08254 08255 // Zero extend the condition if needed. 08256 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond); 08257 08258 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); 08259 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond, 08260 DAG.getConstant(ShAmt, MVT::i8)); 08261 } 08262 08263 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. 08264 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { 08265 if (NeedsCondInvert) // Invert the condition if needed. 08266 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 08267 DAG.getConstant(1, Cond.getValueType())); 08268 08269 // Zero extend the condition if needed. 08270 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, 08271 FalseC->getValueType(0), Cond); 08272 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 08273 SDValue(FalseC, 0)); 08274 } 08275 08276 // Optimize cases that will turn into an LEA instruction. This requires 08277 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9). 08278 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { 08279 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue(); 08280 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; 08281 08282 bool isFastMultiplier = false; 08283 if (Diff < 10) { 08284 switch ((unsigned char)Diff) { 08285 default: break; 08286 case 1: // result = add base, cond 08287 case 2: // result = lea base( , cond*2) 08288 case 3: // result = lea base(cond, cond*2) 08289 case 4: // result = lea base( , cond*4) 08290 case 5: // result = lea base(cond, cond*4) 08291 case 8: // result = lea base( , cond*8) 08292 case 9: // result = lea base(cond, cond*8) 08293 isFastMultiplier = true; 08294 break; 08295 } 08296 } 08297 08298 if (isFastMultiplier) { 08299 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue(); 08300 if (NeedsCondInvert) // Invert the condition if needed. 08301 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, 08302 DAG.getConstant(1, Cond.getValueType())); 08303 08304 // Zero extend the condition if needed. 08305 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), 08306 Cond); 08307 // Scale the condition by the difference. 08308 if (Diff != 1) 08309 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, 08310 DAG.getConstant(Diff, Cond.getValueType())); 08311 08312 // Add the base if non-zero. 08313 if (FalseC->getAPIntValue() != 0) 08314 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 08315 SDValue(FalseC, 0)); 08316 return Cond; 08317 } 08318 } 08319 } 08320 } 08321 08322 return SDValue(); 08323 } 08324 08325 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL] 08326 static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG, 08327 TargetLowering::DAGCombinerInfo &DCI) { 08328 DebugLoc DL = N->getDebugLoc(); 08329 08330 // If the flag operand isn't dead, don't touch this CMOV. 08331 if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty()) 08332 return SDValue(); 08333 08334 // If this is a select between two integer constants, try to do some 08335 // optimizations. Note that the operands are ordered the opposite of SELECT 08336 // operands. 08337 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 08338 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 08339 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is 08340 // larger than FalseC (the false value). 08341 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2); 08342 08343 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) { 08344 CC = X86::GetOppositeBranchCondition(CC); 08345 std::swap(TrueC, FalseC); 08346 } 08347 08348 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0. 08349 // This is efficient for any integer data type (including i8/i16) and 08350 // shift amount. 08351 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) { 08352 SDValue Cond = N->getOperand(3); 08353 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 08354 DAG.getConstant(CC, MVT::i8), Cond); 08355 08356 // Zero extend the condition if needed. 08357 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); 08358 08359 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); 08360 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, 08361 DAG.getConstant(ShAmt, MVT::i8)); 08362 if (N->getNumValues() == 2) // Dead flag value? 08363 return DCI.CombineTo(N, Cond, SDValue()); 08364 return Cond; 08365 } 08366 08367 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient 08368 // for any integer data type, including i8/i16. 08369 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { 08370 SDValue Cond = N->getOperand(3); 08371 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 08372 DAG.getConstant(CC, MVT::i8), Cond); 08373 08374 // Zero extend the condition if needed. 08375 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, 08376 FalseC->getValueType(0), Cond); 08377 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 08378 SDValue(FalseC, 0)); 08379 08380 if (N->getNumValues() == 2) // Dead flag value? 08381 return DCI.CombineTo(N, Cond, SDValue()); 08382 return Cond; 08383 } 08384 08385 // Optimize cases that will turn into an LEA instruction. This requires 08386 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9). 08387 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { 08388 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue(); 08389 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; 08390 08391 bool isFastMultiplier = false; 08392 if (Diff < 10) { 08393 switch ((unsigned char)Diff) { 08394 default: break; 08395 case 1: // result = add base, cond 08396 case 2: // result = lea base( , cond*2) 08397 case 3: // result = lea base(cond, cond*2) 08398 case 4: // result = lea base( , cond*4) 08399 case 5: // result = lea base(cond, cond*4) 08400 case 8: // result = lea base( , cond*8) 08401 case 9: // result = lea base(cond, cond*8) 08402 isFastMultiplier = true; 08403 break; 08404 } 08405 } 08406 08407 if (isFastMultiplier) { 08408 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue(); 08409 SDValue Cond = N->getOperand(3); 08410 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, 08411 DAG.getConstant(CC, MVT::i8), Cond); 08412 // Zero extend the condition if needed. 08413 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), 08414 Cond); 08415 // Scale the condition by the difference. 08416 if (Diff != 1) 08417 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, 08418 DAG.getConstant(Diff, Cond.getValueType())); 08419 08420 // Add the base if non-zero. 08421 if (FalseC->getAPIntValue() != 0) 08422 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, 08423 SDValue(FalseC, 0)); 08424 if (N->getNumValues() == 2) // Dead flag value? 08425 return DCI.CombineTo(N, Cond, SDValue()); 08426 return Cond; 08427 } 08428 } 08429 } 08430 } 08431 return SDValue(); 08432 } 08433 08434 08435 /// PerformMulCombine - Optimize a single multiply with constant into two 08436 /// in order to implement it with two cheaper instructions, e.g. 08437 /// LEA + SHL, LEA + LEA. 08438 static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG, 08439 TargetLowering::DAGCombinerInfo &DCI) { 08440 if (DAG.getMachineFunction(). 08441 getFunction()->hasFnAttr(Attribute::OptimizeForSize)) 08442 return SDValue(); 08443 08444 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer()) 08445 return SDValue(); 08446 08447 EVT VT = N->getValueType(0); 08448 if (VT != MVT::i64) 08449 return SDValue(); 08450 08451 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 08452 if (!C) 08453 return SDValue(); 08454 uint64_t MulAmt = C->getZExtValue(); 08455 if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9) 08456 return SDValue(); 08457 08458 uint64_t MulAmt1 = 0; 08459 uint64_t MulAmt2 = 0; 08460 if ((MulAmt % 9) == 0) { 08461 MulAmt1 = 9; 08462 MulAmt2 = MulAmt / 9; 08463 } else if ((MulAmt % 5) == 0) { 08464 MulAmt1 = 5; 08465 MulAmt2 = MulAmt / 5; 08466 } else if ((MulAmt % 3) == 0) { 08467 MulAmt1 = 3; 08468 MulAmt2 = MulAmt / 3; 08469 } 08470 if (MulAmt2 && 08471 (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){ 08472 DebugLoc DL = N->getDebugLoc(); 08473 08474 if (isPowerOf2_64(MulAmt2) && 08475 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD)) 08476 // If second multiplifer is pow2, issue it first. We want the multiply by 08477 // 3, 5, or 9 to be folded into the addressing mode unless the lone use 08478 // is an add. 08479 std::swap(MulAmt1, MulAmt2); 08480 08481 SDValue NewMul; 08482 if (isPowerOf2_64(MulAmt1)) 08483 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 08484 DAG.getConstant(Log2_64(MulAmt1), MVT::i8)); 08485 else 08486 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), 08487 DAG.getConstant(MulAmt1, VT)); 08488 08489 if (isPowerOf2_64(MulAmt2)) 08490 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, 08491 DAG.getConstant(Log2_64(MulAmt2), MVT::i8)); 08492 else 08493 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, 08494 DAG.getConstant(MulAmt2, VT)); 08495 08496 // Do not add new nodes to DAG combiner worklist. 08497 DCI.CombineTo(N, NewMul, false); 08498 } 08499 return SDValue(); 08500 } 08501 08502 08503 /// PerformShiftCombine - Transforms vector shift nodes to use vector shifts 08504 /// when possible. 08505 static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG, 08506 const X86Subtarget *Subtarget) { 08507 // On X86 with SSE2 support, we can transform this to a vector shift if 08508 // all elements are shifted by the same amount. We can't do this in legalize 08509 // because the a constant vector is typically transformed to a constant pool 08510 // so we have no knowledge of the shift amount. 08511 if (!Subtarget->hasSSE2()) 08512 return SDValue(); 08513 08514 EVT VT = N->getValueType(0); 08515 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16) 08516 return SDValue(); 08517 08518 SDValue ShAmtOp = N->getOperand(1); 08519 EVT EltVT = VT.getVectorElementType(); 08520 DebugLoc DL = N->getDebugLoc(); 08521 SDValue BaseShAmt; 08522 if (ShAmtOp.getOpcode() == ISD::BUILD_VECTOR) { 08523 unsigned NumElts = VT.getVectorNumElements(); 08524 unsigned i = 0; 08525 for (; i != NumElts; ++i) { 08526 SDValue Arg = ShAmtOp.getOperand(i); 08527 if (Arg.getOpcode() == ISD::UNDEF) continue; 08528 BaseShAmt = Arg; 08529 break; 08530 } 08531 for (; i != NumElts; ++i) { 08532 SDValue Arg = ShAmtOp.getOperand(i); 08533 if (Arg.getOpcode() == ISD::UNDEF) continue; 08534 if (Arg != BaseShAmt) { 08535 return SDValue(); 08536 } 08537 } 08538 } else if (ShAmtOp.getOpcode() == ISD::VECTOR_SHUFFLE && 08539 cast<ShuffleVectorSDNode>(ShAmtOp)->isSplat()) { 08540 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ShAmtOp, 08541 DAG.getIntPtrConstant(0)); 08542 } else 08543 return SDValue(); 08544 08545 if (EltVT.bitsGT(MVT::i32)) 08546 BaseShAmt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, BaseShAmt); 08547 else if (EltVT.bitsLT(MVT::i32)) 08548 BaseShAmt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, BaseShAmt); 08549 08550 // The shift amount is identical so we can do a vector shift. 08551 SDValue ValOp = N->getOperand(0); 08552 switch (N->getOpcode()) { 08553 default: 08554 llvm_unreachable("Unknown shift opcode!"); 08555 break; 08556 case ISD::SHL: 08557 if (VT == MVT::v2i64) 08558 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08559 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), 08560 ValOp, BaseShAmt); 08561 if (VT == MVT::v4i32) 08562 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08563 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), 08564 ValOp, BaseShAmt); 08565 if (VT == MVT::v8i16) 08566 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08567 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), 08568 ValOp, BaseShAmt); 08569 break; 08570 case ISD::SRA: 08571 if (VT == MVT::v4i32) 08572 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08573 DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32), 08574 ValOp, BaseShAmt); 08575 if (VT == MVT::v8i16) 08576 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08577 DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32), 08578 ValOp, BaseShAmt); 08579 break; 08580 case ISD::SRL: 08581 if (VT == MVT::v2i64) 08582 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08583 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), 08584 ValOp, BaseShAmt); 08585 if (VT == MVT::v4i32) 08586 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08587 DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32), 08588 ValOp, BaseShAmt); 08589 if (VT == MVT::v8i16) 08590 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 08591 DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32), 08592 ValOp, BaseShAmt); 08593 break; 08594 } 08595 return SDValue(); 08596 } 08597 08598 /// PerformSTORECombine - Do target-specific dag combines on STORE nodes. 08599 static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, 08600 const X86Subtarget *Subtarget) { 08601 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering 08602 // the FP state in cases where an emms may be missing. 08603 // A preferable solution to the general problem is to figure out the right 08604 // places to insert EMMS. This qualifies as a quick hack. 08605 08606 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode. 08607 StoreSDNode *St = cast<StoreSDNode>(N); 08608 EVT VT = St->getValue().getValueType(); 08609 if (VT.getSizeInBits() != 64) 08610 return SDValue(); 08611 08612 const Function *F = DAG.getMachineFunction().getFunction(); 08613 bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat); 08614 bool F64IsLegal = !UseSoftFloat && !NoImplicitFloatOps 08615 && Subtarget->hasSSE2(); 08616 if ((VT.isVector() || 08617 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) && 08618 isa<LoadSDNode>(St->getValue()) && 08619 !cast<LoadSDNode>(St->getValue())->isVolatile() && 08620 St->getChain().hasOneUse() && !St->isVolatile()) { 08621 SDNode* LdVal = St->getValue().getNode(); 08622 LoadSDNode *Ld = 0; 08623 int TokenFactorIndex = -1; 08624 SmallVector<SDValue, 8> Ops; 08625 SDNode* ChainVal = St->getChain().getNode(); 08626 // Must be a store of a load. We currently handle two cases: the load 08627 // is a direct child, and it's under an intervening TokenFactor. It is 08628 // possible to dig deeper under nested TokenFactors. 08629 if (ChainVal == LdVal) 08630 Ld = cast<LoadSDNode>(St->getChain()); 08631 else if (St->getValue().hasOneUse() && 08632 ChainVal->getOpcode() == ISD::TokenFactor) { 08633 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { 08634 if (ChainVal->getOperand(i).getNode() == LdVal) { 08635 TokenFactorIndex = i; 08636 Ld = cast<LoadSDNode>(St->getValue()); 08637 } else 08638 Ops.push_back(ChainVal->getOperand(i)); 08639 } 08640 } 08641 08642 if (!Ld || !ISD::isNormalLoad(Ld)) 08643 return SDValue(); 08644 08645 // If this is not the MMX case, i.e. we are just turning i64 load/store 08646 // into f64 load/store, avoid the transformation if there are multiple 08647 // uses of the loaded value. 08648 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0)) 08649 return SDValue(); 08650 08651 DebugLoc LdDL = Ld->getDebugLoc(); 08652 DebugLoc StDL = N->getDebugLoc(); 08653 // If we are a 64-bit capable x86, lower to a single movq load/store pair. 08654 // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store 08655 // pair instead. 08656 if (Subtarget->is64Bit() || F64IsLegal) { 08657 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64; 08658 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), 08659 Ld->getBasePtr(), Ld->getSrcValue(), 08660 Ld->getSrcValueOffset(), Ld->isVolatile(), 08661 Ld->getAlignment()); 08662 SDValue NewChain = NewLd.getValue(1); 08663 if (TokenFactorIndex != -1) { 08664 Ops.push_back(NewChain); 08665 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], 08666 Ops.size()); 08667 } 08668 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(), 08669 St->getSrcValue(), St->getSrcValueOffset(), 08670 St->isVolatile(), St->getAlignment()); 08671 } 08672 08673 // Otherwise, lower to two pairs of 32-bit loads / stores. 08674 SDValue LoAddr = Ld->getBasePtr(); 08675 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, 08676 DAG.getConstant(4, MVT::i32)); 08677 08678 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, 08679 Ld->getSrcValue(), Ld->getSrcValueOffset(), 08680 Ld->isVolatile(), Ld->getAlignment()); 08681 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, 08682 Ld->getSrcValue(), Ld->getSrcValueOffset()+4, 08683 Ld->isVolatile(), 08684 MinAlign(Ld->getAlignment(), 4)); 08685 08686 SDValue NewChain = LoLd.getValue(1); 08687 if (TokenFactorIndex != -1) { 08688 Ops.push_back(LoLd); 08689 Ops.push_back(HiLd); 08690 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], 08691 Ops.size()); 08692 } 08693 08694 LoAddr = St->getBasePtr(); 08695 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, 08696 DAG.getConstant(4, MVT::i32)); 08697 08698 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr, 08699 St->getSrcValue(), St->getSrcValueOffset(), 08700 St->isVolatile(), St->getAlignment()); 08701 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr, 08702 St->getSrcValue(), 08703 St->getSrcValueOffset() + 4, 08704 St->isVolatile(), 08705 MinAlign(St->getAlignment(), 4)); 08706 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); 08707 } 08708 return SDValue(); 08709 } 08710 08711 /// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and 08712 /// X86ISD::FXOR nodes. 08713 static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { 08714 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); 08715 // F[X]OR(0.0, x) -> x 08716 // F[X]OR(x, 0.0) -> x 08717 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 08718 if (C->getValueAPF().isPosZero()) 08719 return N->getOperand(1); 08720 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 08721 if (C->getValueAPF().isPosZero()) 08722 return N->getOperand(0); 08723 return SDValue(); 08724 } 08725 08726 /// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. 08727 static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { 08728 // FAND(0.0, x) -> 0.0 08729 // FAND(x, 0.0) -> 0.0 08730 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) 08731 if (C->getValueAPF().isPosZero()) 08732 return N->getOperand(0); 08733 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) 08734 if (C->getValueAPF().isPosZero()) 08735 return N->getOperand(1); 08736 return SDValue(); 08737 } 08738 08739 static SDValue PerformBTCombine(SDNode *N, 08740 SelectionDAG &DAG, 08741 TargetLowering::DAGCombinerInfo &DCI) { 08742 // BT ignores high bits in the bit index operand. 08743 SDValue Op1 = N->getOperand(1); 08744 if (Op1.hasOneUse()) { 08745 unsigned BitWidth = Op1.getValueSizeInBits(); 08746 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth)); 08747 APInt KnownZero, KnownOne; 08748 TargetLowering::TargetLoweringOpt TLO(DAG); 08749 TargetLowering &TLI = DAG.getTargetLoweringInfo(); 08750 if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) || 08751 TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO)) 08752 DCI.CommitTargetLoweringOpt(TLO); 08753 } 08754 return SDValue(); 08755 } 08756 08757 static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) { 08758 SDValue Op = N->getOperand(0); 08759 if (Op.getOpcode() == ISD::BIT_CONVERT) 08760 Op = Op.getOperand(0); 08761 EVT VT = N->getValueType(0), OpVT = Op.getValueType(); 08762 if (Op.getOpcode() == X86ISD::VZEXT_LOAD && 08763 VT.getVectorElementType().getSizeInBits() == 08764 OpVT.getVectorElementType().getSizeInBits()) { 08765 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), VT, Op); 08766 } 08767 return SDValue(); 08768 } 08769 08770 // On X86 and X86-64, atomic operations are lowered to locked instructions. 08771 // Locked instructions, in turn, have implicit fence semantics (all memory 08772 // operations are flushed before issuing the locked instruction, and the 08773 // are not buffered), so we can fold away the common pattern of 08774 // fence-atomic-fence. 08775 static SDValue PerformMEMBARRIERCombine(SDNode* N, SelectionDAG &DAG) { 08776 SDValue atomic = N->getOperand(0); 08777 switch (atomic.getOpcode()) { 08778 case ISD::ATOMIC_CMP_SWAP: 08779 case ISD::ATOMIC_SWAP: 08780 case ISD::ATOMIC_LOAD_ADD: 08781 case ISD::ATOMIC_LOAD_SUB: 08782 case ISD::ATOMIC_LOAD_AND: 08783 case ISD::ATOMIC_LOAD_OR: 08784 case ISD::ATOMIC_LOAD_XOR: 08785 case ISD::ATOMIC_LOAD_NAND: 08786 case ISD::ATOMIC_LOAD_MIN: 08787 case ISD::ATOMIC_LOAD_MAX: 08788 case ISD::ATOMIC_LOAD_UMIN: 08789 case ISD::ATOMIC_LOAD_UMAX: 08790 break; 08791 default: 08792 return SDValue(); 08793 } 08794 08795 SDValue fence = atomic.getOperand(0); 08796 if (fence.getOpcode() != ISD::MEMBARRIER) 08797 return SDValue(); 08798 08799 switch (atomic.getOpcode()) { 08800 case ISD::ATOMIC_CMP_SWAP: 08801 return DAG.UpdateNodeOperands(atomic, fence.getOperand(0), 08802 atomic.getOperand(1), atomic.getOperand(2), 08803 atomic.getOperand(3)); 08804 case ISD::ATOMIC_SWAP: 08805 case ISD::ATOMIC_LOAD_ADD: 08806 case ISD::ATOMIC_LOAD_SUB: 08807 case ISD::ATOMIC_LOAD_AND: 08808 case ISD::ATOMIC_LOAD_OR: 08809 case ISD::ATOMIC_LOAD_XOR: 08810 case ISD::ATOMIC_LOAD_NAND: 08811 case ISD::ATOMIC_LOAD_MIN: 08812 case ISD::ATOMIC_LOAD_MAX: 08813 case ISD::ATOMIC_LOAD_UMIN: 08814 case ISD::ATOMIC_LOAD_UMAX: 08815 return DAG.UpdateNodeOperands(atomic, fence.getOperand(0), 08816 atomic.getOperand(1), atomic.getOperand(2)); 08817 default: 08818 return SDValue(); 08819 } 08820 } 08821 08822 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, 08823 DAGCombinerInfo &DCI) const { 08824 SelectionDAG &DAG = DCI.DAG; 08825 switch (N->getOpcode()) { 08826 default: break; 08827 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); 08828 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); 08829 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI); 08830 case ISD::MUL: return PerformMulCombine(N, DAG, DCI); 08831 case ISD::SHL: 08832 case ISD::SRA: 08833 case ISD::SRL: return PerformShiftCombine(N, DAG, Subtarget); 08834 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); 08835 case X86ISD::FXOR: 08836 case X86ISD::FOR: return PerformFORCombine(N, DAG); 08837 case X86ISD::FAND: return PerformFANDCombine(N, DAG); 08838 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI); 08839 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG); 08840 case ISD::MEMBARRIER: return PerformMEMBARRIERCombine(N, DAG); 08841 } 08842 08843 return SDValue(); 08844 } 08845 08846 //===----------------------------------------------------------------------===// 08847 // X86 Inline Assembly Support 08848 //===----------------------------------------------------------------------===// 08849 08850 static bool LowerToBSwap(CallInst *CI) { 08851 // FIXME: this should verify that we are targetting a 486 or better. If not, 08852 // we will turn this bswap into something that will be lowered to logical ops 08853 // instead of emitting the bswap asm. For now, we don't support 486 or lower 08854 // so don't worry about this. 08855 08856 // Verify this is a simple bswap. 08857 if (CI->getNumOperands() != 2 || 08858 CI->getType() != CI->getOperand(1)->getType() || 08859 !CI->getType()->isInteger()) 08860 return false; 08861 08862 const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); 08863 if (!Ty || Ty->getBitWidth() % 16 != 0) 08864 return false; 08865 08866 // Okay, we can do this xform, do so now. 08867 const Type *Tys[] = { Ty }; 08868 Module *M = CI->getParent()->getParent()->getParent(); 08869 Constant *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Tys, 1); 08870 08871 Value *Op = CI->getOperand(1); 08872 Op = CallInst::Create(Int, Op, CI->getName(), CI); 08873 08874 CI->replaceAllUsesWith(Op); 08875 CI->eraseFromParent(); 08876 return true; 08877 } 08878 08879 bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const { 08880 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue()); 08881 std::vector<InlineAsm::ConstraintInfo> Constraints = IA->ParseConstraints(); 08882 08883 std::string AsmStr = IA->getAsmString(); 08884 08885 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a" 08886 std::vector<std::string> AsmPieces; 08887 SplitString(AsmStr, AsmPieces, "\n"); // ; as separator? 08888 08889 switch (AsmPieces.size()) { 08890 default: return false; 08891 case 1: 08892 AsmStr = AsmPieces[0]; 08893 AsmPieces.clear(); 08894 SplitString(AsmStr, AsmPieces, " \t"); // Split with whitespace. 08895 08896 // bswap $0 08897 if (AsmPieces.size() == 2 && 08898 (AsmPieces[0] == "bswap" || 08899 AsmPieces[0] == "bswapq" || 08900 AsmPieces[0] == "bswapl") && 08901 (AsmPieces[1] == "$0" || 08902 AsmPieces[1] == "${0:q}")) { 08903 // No need to check constraints, nothing other than the equivalent of 08904 // "=r,0" would be valid here. 08905 return LowerToBSwap(CI); 08906 } 08907 // rorw $$8, ${0:w} --> llvm.bswap.i16 08908 if (CI->getType() == Type::getInt16Ty(CI->getContext()) && 08909 AsmPieces.size() == 3 && 08910 AsmPieces[0] == "rorw" && 08911 AsmPieces[1] == "$$8," && 08912 AsmPieces[2] == "${0:w}" && 08913 IA->getConstraintString() == "=r,0,~{dirflag},~{fpsr},~{flags},~{cc}") { 08914 return LowerToBSwap(CI); 08915 } 08916 break; 08917 case 3: 08918 if (CI->getType() == Type::getInt64Ty(CI->getContext()) && 08919 Constraints.size() >= 2 && 08920 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" && 08921 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") { 08922 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64 08923 std::vector<std::string> Words; 08924 SplitString(AsmPieces[0], Words, " \t"); 08925 if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%eax") { 08926 Words.clear(); 08927 SplitString(AsmPieces[1], Words, " \t"); 08928 if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%edx") { 08929 Words.clear(); 08930 SplitString(AsmPieces[2], Words, " \t,"); 08931 if (Words.size() == 3 && Words[0] == "xchgl" && Words[1] == "%eax" && 08932 Words[2] == "%edx") { 08933 return LowerToBSwap(CI); 08934 } 08935 } 08936 } 08937 } 08938 break; 08939 } 08940 return false; 08941 } 08942 08943 08944 08945 /// getConstraintType - Given a constraint letter, return the type of 08946 /// constraint it is for this target. 08947 X86TargetLowering::ConstraintType 08948 X86TargetLowering::getConstraintType(const std::string &Constraint) const { 08949 if (Constraint.size() == 1) { 08950 switch (Constraint[0]) { 08951 case 'A': 08952 return C_Register; 08953 case 'f': 08954 case 'r': 08955 case 'R': 08956 case 'l': 08957 case 'q': 08958 case 'Q': 08959 case 'x': 08960 case 'y': 08961 case 'Y': 08962 return C_RegisterClass; 08963 case 'e': 08964 case 'Z': 08965 return C_Other; 08966 default: 08967 break; 08968 } 08969 } 08970 return TargetLowering::getConstraintType(Constraint); 08971 } 08972 08973 /// LowerXConstraint - try to replace an X constraint, which matches anything, 08974 /// with another that has more specific requirements based on the type of the 08975 /// corresponding operand. 08976 const char *X86TargetLowering:: 08977 LowerXConstraint(EVT ConstraintVT) const { 08978 // FP X constraints get lowered to SSE1/2 registers if available, otherwise 08979 // 'f' like normal targets. 08980 if (ConstraintVT.isFloatingPoint()) { 08981 if (Subtarget->hasSSE2()) 08982 return "Y"; 08983 if (Subtarget->hasSSE1()) 08984 return "x"; 08985 } 08986 08987 return TargetLowering::LowerXConstraint(ConstraintVT); 08988 } 08989 08990 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 08991 /// vector. If it is invalid, don't add anything to Ops. 08992 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, 08993 char Constraint, 08994 bool hasMemory, 08995 std::vector<SDValue>&Ops, 08996 SelectionDAG &DAG) const { 08997 SDValue Result(0, 0); 08998 08999 switch (Constraint) { 09000 default: break; 09001 case 'I': 09002 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 09003 if (C->getZExtValue() <= 31) { 09004 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 09005 break; 09006 } 09007 } 09008 return; 09009 case 'J': 09010 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 09011 if (C->getZExtValue() <= 63) { 09012 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 09013 break; 09014 } 09015 } 09016 return; 09017 case 'K': 09018 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 09019 if ((int8_t)C->getSExtValue() == C->getSExtValue()) { 09020 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 09021 break; 09022 } 09023 } 09024 return; 09025 case 'N': 09026 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 09027 if (C->getZExtValue() <= 255) { 09028 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 09029 break; 09030 } 09031 } 09032 return; 09033 case 'e': { 09034 // 32-bit signed value 09035 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 09036 const ConstantInt *CI = C->getConstantIntValue(); 09037 if (CI->isValueValidForType(Type::getInt32Ty(*DAG.getContext()), 09038 C->getSExtValue())) { 09039 // Widen to 64 bits here to get it sign extended. 09040 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64); 09041 break; 09042 } 09043 // FIXME gcc accepts some relocatable values here too, but only in certain 09044 // memory models; it's complicated. 09045 } 09046 return; 09047 } 09048 case 'Z': { 09049 // 32-bit unsigned value 09050 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 09051 const ConstantInt *CI = C->getConstantIntValue(); 09052 if (CI->isValueValidForType(Type::getInt32Ty(*DAG.getContext()), 09053 C->getZExtValue())) { 09054 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); 09055 break; 09056 } 09057 } 09058 // FIXME gcc accepts some relocatable values here too, but only in certain 09059 // memory models; it's complicated. 09060 return; 09061 } 09062 case 'i': { 09063 // Literal immediates are always ok. 09064 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { 09065 // Widen to 64 bits here to get it sign extended. 09066 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64); 09067 break; 09068 } 09069 09070 // If we are in non-pic codegen mode, we allow the address of a global (with 09071 // an optional displacement) to be used with 'i'. 09072 GlobalAddressSDNode *GA = 0; 09073 int64_t Offset = 0; 09074 09075 // Match either (GA), (GA+C), (GA+C1+C2), etc. 09076 while (1) { 09077 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) { 09078 Offset += GA->getOffset(); 09079 break; 09080 } else if (Op.getOpcode() == ISD::ADD) { 09081 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 09082 Offset += C->getZExtValue(); 09083 Op = Op.getOperand(0); 09084 continue; 09085 } 09086 } else if (Op.getOpcode() == ISD::SUB) { 09087 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 09088 Offset += -C->getZExtValue(); 09089 Op = Op.getOperand(0); 09090 continue; 09091 } 09092 } 09093 09094 // Otherwise, this isn't something we can handle, reject it. 09095 return; 09096 } 09097 09098 GlobalValue *GV = GA->getGlobal(); 09099 // If we require an extra load to get this address, as in PIC mode, we 09100 // can't accept it. 09101 if (isGlobalStubReference(Subtarget->ClassifyGlobalReference(GV, 09102 getTargetMachine()))) 09103 return; 09104 09105 if (hasMemory) 09106 Op = LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG); 09107 else 09108 Op = DAG.getTargetGlobalAddress(GV, GA->getValueType(0), Offset); 09109 Result = Op; 09110 break; 09111 } 09112 } 09113 09114 if (Result.getNode()) { 09115 Ops.push_back(Result); 09116 return; 09117 } 09118 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, hasMemory, 09119 Ops, DAG); 09120 } 09121 09122 std::vector<unsigned> X86TargetLowering:: 09123 getRegClassForInlineAsmConstraint(const std::string &Constraint, 09124 EVT VT) const { 09125 if (Constraint.size() == 1) { 09126 // FIXME: not handling fp-stack yet! 09127 switch (Constraint[0]) { // GCC X86 Constraint Letters 09128 default: break; // Unknown constraint letter 09129 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode. 09130 if (Subtarget->is64Bit()) { 09131 if (VT == MVT::i32) 09132 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 09133 X86::ESI, X86::EDI, X86::R8D, X86::R9D, 09134 X86::R10D,X86::R11D,X86::R12D, 09135 X86::R13D,X86::R14D,X86::R15D, 09136 X86::EBP, X86::ESP, 0); 09137 else if (VT == MVT::i16) 09138 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 09139 X86::SI, X86::DI, X86::R8W,X86::R9W, 09140 X86::R10W,X86::R11W,X86::R12W, 09141 X86::R13W,X86::R14W,X86::R15W, 09142 X86::BP, X86::SP, 0); 09143 else if (VT == MVT::i8) 09144 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 09145 X86::SIL, X86::DIL, X86::R8B,X86::R9B, 09146 X86::R10B,X86::R11B,X86::R12B, 09147 X86::R13B,X86::R14B,X86::R15B, 09148 X86::BPL, X86::SPL, 0); 09149 09150 else if (VT == MVT::i64) 09151 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 09152 X86::RSI, X86::RDI, X86::R8, X86::R9, 09153 X86::R10, X86::R11, X86::R12, 09154 X86::R13, X86::R14, X86::R15, 09155 X86::RBP, X86::RSP, 0); 09156 09157 break; 09158 } 09159 // 32-bit fallthrough 09160 case 'Q': // Q_REGS 09161 if (VT == MVT::i32) 09162 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); 09163 else if (VT == MVT::i16) 09164 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); 09165 else if (VT == MVT::i8) 09166 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); 09167 else if (VT == MVT::i64) 09168 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); 09169 break; 09170 } 09171 } 09172 09173 return std::vector<unsigned>(); 09174 } 09175 09176 std::pair<unsigned, const TargetRegisterClass*> 09177 X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, 09178 EVT VT) const { 09179 // First, see if this is a constraint that directly corresponds to an LLVM 09180 // register class. 09181 if (Constraint.size() == 1) { 09182 // GCC Constraint Letters 09183 switch (Constraint[0]) { 09184 default: break; 09185 case 'r': // GENERAL_REGS 09186 case 'R': // LEGACY_REGS 09187 case 'l': // INDEX_REGS 09188 if (VT == MVT::i8) 09189 return std::make_pair(0U, X86::GR8RegisterClass); 09190 if (VT == MVT::i16) 09191 return std::make_pair(0U, X86::GR16RegisterClass); 09192 if (VT == MVT::i32 || !Subtarget->is64Bit()) 09193 return std::make_pair(0U, X86::GR32RegisterClass); 09194 return std::make_pair(0U, X86::GR64RegisterClass); 09195 case 'f': // FP Stack registers. 09196 // If SSE is enabled for this VT, use f80 to ensure the isel moves the 09197 // value to the correct fpstack register class. 09198 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) 09199 return std::make_pair(0U, X86::RFP32RegisterClass); 09200 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) 09201 return std::make_pair(0U, X86::RFP64RegisterClass); 09202 return std::make_pair(0U, X86::RFP80RegisterClass); 09203 case 'y': // MMX_REGS if MMX allowed. 09204 if (!Subtarget->hasMMX()) break; 09205 return std::make_pair(0U, X86::VR64RegisterClass); 09206 case 'Y': // SSE_REGS if SSE2 allowed 09207 if (!Subtarget->hasSSE2()) break; 09208 // FALL THROUGH. 09209 case 'x': // SSE_REGS if SSE1 allowed 09210 if (!Subtarget->hasSSE1()) break; 09211 09212 switch (VT.getSimpleVT().SimpleTy) { 09213 default: break; 09214 // Scalar SSE types. 09215 case MVT::f32: 09216 case MVT::i32: 09217 return std::make_pair(0U, X86::FR32RegisterClass); 09218 case MVT::f64: 09219 case MVT::i64: 09220 return std::make_pair(0U, X86::FR64RegisterClass); 09221 // Vector types. 09222 case MVT::v16i8: 09223 case MVT::v8i16: 09224 case MVT::v4i32: 09225 case MVT::v2i64: 09226 case MVT::v4f32: 09227 case MVT::v2f64: 09228 return std::make_pair(0U, X86::VR128RegisterClass); 09229 } 09230 break; 09231 } 09232 } 09233 09234 // Use the default implementation in TargetLowering to convert the register 09235 // constraint into a member of a register class. 09236 std::pair<unsigned, const TargetRegisterClass*> Res; 09237 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 09238 09239 // Not found as a standard register? 09240 if (Res.second == 0) { 09241 // GCC calls "st(0)" just plain "st". 09242 if (StringsEqualNoCase("{st}", Constraint)) { 09243 Res.first = X86::ST0; 09244 Res.second = X86::RFP80RegisterClass; 09245 } 09246 // 'A' means EAX + EDX. 09247 if (Constraint == "A") { 09248 Res.first = X86::EAX; 09249 Res.second = X86::GR32_ADRegisterClass; 09250 } 09251 return Res; 09252 } 09253 09254 // Otherwise, check to see if this is a register class of the wrong value 09255 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to 09256 // turn into {ax},{dx}. 09257 if (Res.second->hasType(VT)) 09258 return Res; // Correct type already, nothing to do. 09259 09260 // All of the single-register GCC register classes map their values onto 09261 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we 09262 // really want an 8-bit or 32-bit register, map to the appropriate register 09263 // class and return the appropriate register. 09264 if (Res.second == X86::GR16RegisterClass) { 09265 if (VT == MVT::i8) { 09266 unsigned DestReg = 0; 09267 switch (Res.first) { 09268 default: break; 09269 case X86::AX: DestReg = X86::AL; break; 09270 case X86::DX: DestReg = X86::DL; break; 09271 case X86::CX: DestReg = X86::CL; break; 09272 case X86::BX: DestReg = X86::BL; break; 09273 } 09274 if (DestReg) { 09275 Res.first = DestReg; 09276 Res.second = X86::GR8RegisterClass; 09277 } 09278 } else if (VT == MVT::i32) { 09279 unsigned DestReg = 0; 09280 switch (Res.first) { 09281 default: break; 09282 case X86::AX: DestReg = X86::EAX; break; 09283 case X86::DX: DestReg = X86::EDX; break; 09284 case X86::CX: DestReg = X86::ECX; break; 09285 case X86::BX: DestReg = X86::EBX; break; 09286 case X86::SI: DestReg = X86::ESI; break; 09287 case X86::DI: DestReg = X86::EDI; break; 09288 case X86::BP: DestReg = X86::EBP; break; 09289 case X86::SP: DestReg = X86::ESP; break; 09290 } 09291 if (DestReg) { 09292 Res.first = DestReg; 09293 Res.second = X86::GR32RegisterClass; 09294 } 09295 } else if (VT == MVT::i64) { 09296 unsigned DestReg = 0; 09297 switch (Res.first) { 09298 default: break; 09299 case X86::AX: DestReg = X86::RAX; break; 09300 case X86::DX: DestReg = X86::RDX; break; 09301 case X86::CX: DestReg = X86::RCX; break; 09302 case X86::BX: DestReg = X86::RBX; break; 09303 case X86::SI: DestReg = X86::RSI; break; 09304 case X86::DI: DestReg = X86::RDI; break; 09305 case X86::BP: DestReg = X86::RBP; break; 09306 case X86::SP: DestReg = X86::RSP; break; 09307 } 09308 if (DestReg) { 09309 Res.first = DestReg; 09310 Res.second = X86::GR64RegisterClass; 09311 } 09312 } 09313 } else if (Res.second == X86::FR32RegisterClass || 09314 Res.second == X86::FR64RegisterClass || 09315 Res.second == X86::VR128RegisterClass) { 09316 // Handle references to XMM physical registers that got mapped into the 09317 // wrong class. This can happen with constraints like {xmm0} where the 09318 // target independent register mapper will just pick the first match it can 09319 // find, ignoring the required type. 09320 if (VT == MVT::f32) 09321 Res.second = X86::FR32RegisterClass; 09322 else if (VT == MVT::f64) 09323 Res.second = X86::FR64RegisterClass; 09324 else if (X86::VR128RegisterClass->hasType(VT)) 09325 Res.second = X86::VR128RegisterClass; 09326 } 09327 09328 return Res; 09329 } 09330 09331 //===----------------------------------------------------------------------===// 09332 // X86 Widen vector type 09333 //===----------------------------------------------------------------------===// 09334 09335 /// getWidenVectorType: given a vector type, returns the type to widen 09336 /// to (e.g., v7i8 to v8i8). If the vector type is legal, it returns itself. 09337 /// If there is no vector type that we want to widen to, returns MVT::Other 09338 /// When and where to widen is target dependent based on the cost of 09339 /// scalarizing vs using the wider vector type. 09340 09341 EVT X86TargetLowering::getWidenVectorType(EVT VT) const { 09342 assert(VT.isVector()); 09343 if (isTypeLegal(VT)) 09344 return VT; 09345 09346 // TODO: In computeRegisterProperty, we can compute the list of legal vector 09347 // type based on element type. This would speed up our search (though 09348 // it may not be worth it since the size of the list is relatively 09349 // small). 09350 EVT EltVT = VT.getVectorElementType(); 09351 unsigned NElts = VT.getVectorNumElements(); 09352 09353 // On X86, it make sense to widen any vector wider than 1 09354 if (NElts <= 1) 09355 return MVT::Other; 09356 09357 for (unsigned nVT = MVT::FIRST_VECTOR_VALUETYPE; 09358 nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { 09359 EVT SVT = (MVT::SimpleValueType)nVT; 09360 09361 if (isTypeLegal(SVT) && 09362 SVT.getVectorElementType() == EltVT && 09363 SVT.getVectorNumElements() > NElts) 09364 return SVT; 09365 } 09366 return MVT::Other; 09367 }