LLVM API Documentation

ARMISelLowering.cpp

Go to the documentation of this file.
00001 //===-- ARMISelLowering.cpp - ARM DAG Lowering Implementation -------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines the interfaces that ARM uses to lower LLVM code into a
00011 // selection DAG.
00012 //
00013 //===----------------------------------------------------------------------===//
00014 
00015 #include "ARM.h"
00016 #include "ARMAddressingModes.h"
00017 #include "ARMConstantPoolValue.h"
00018 #include "ARMISelLowering.h"
00019 #include "ARMMachineFunctionInfo.h"
00020 #include "ARMPerfectShuffle.h"
00021 #include "ARMRegisterInfo.h"
00022 #include "ARMSubtarget.h"
00023 #include "ARMTargetMachine.h"
00024 #include "ARMTargetObjectFile.h"
00025 #include "llvm/CallingConv.h"
00026 #include "llvm/Constants.h"
00027 #include "llvm/Function.h"
00028 #include "llvm/Instruction.h"
00029 #include "llvm/Intrinsics.h"
00030 #include "llvm/GlobalValue.h"
00031 #include "llvm/CodeGen/CallingConvLower.h"
00032 #include "llvm/CodeGen/MachineBasicBlock.h"
00033 #include "llvm/CodeGen/MachineFrameInfo.h"
00034 #include "llvm/CodeGen/MachineFunction.h"
00035 #include "llvm/CodeGen/MachineInstrBuilder.h"
00036 #include "llvm/CodeGen/MachineRegisterInfo.h"
00037 #include "llvm/CodeGen/PseudoSourceValue.h"
00038 #include "llvm/CodeGen/SelectionDAG.h"
00039 #include "llvm/Target/TargetOptions.h"
00040 #include "llvm/ADT/VectorExtras.h"
00041 #include "llvm/Support/ErrorHandling.h"
00042 #include "llvm/Support/MathExtras.h"
00043 using namespace llvm;
00044 
00045 static bool CC_ARM_APCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00046                                    CCValAssign::LocInfo &LocInfo,
00047                                    ISD::ArgFlagsTy &ArgFlags,
00048                                    CCState &State);
00049 static bool CC_ARM_AAPCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00050                                     CCValAssign::LocInfo &LocInfo,
00051                                     ISD::ArgFlagsTy &ArgFlags,
00052                                     CCState &State);
00053 static bool RetCC_ARM_APCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00054                                       CCValAssign::LocInfo &LocInfo,
00055                                       ISD::ArgFlagsTy &ArgFlags,
00056                                       CCState &State);
00057 static bool RetCC_ARM_AAPCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00058                                        CCValAssign::LocInfo &LocInfo,
00059                                        ISD::ArgFlagsTy &ArgFlags,
00060                                        CCState &State);
00061 
00062 void ARMTargetLowering::addTypeForNEON(EVT VT, EVT PromotedLdStVT,
00063                                        EVT PromotedBitwiseVT) {
00064   if (VT != PromotedLdStVT) {
00065     setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote);
00066     AddPromotedToType (ISD::LOAD, VT.getSimpleVT(),
00067                        PromotedLdStVT.getSimpleVT());
00068 
00069     setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote);
00070     AddPromotedToType (ISD::STORE, VT.getSimpleVT(),
00071                        PromotedLdStVT.getSimpleVT());
00072   }
00073 
00074   EVT ElemTy = VT.getVectorElementType();
00075   if (ElemTy != MVT::i64 && ElemTy != MVT::f64)
00076     setOperationAction(ISD::VSETCC, VT.getSimpleVT(), Custom);
00077   if (ElemTy == MVT::i8 || ElemTy == MVT::i16)
00078     setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT.getSimpleVT(), Custom);
00079   setOperationAction(ISD::BUILD_VECTOR, VT.getSimpleVT(), Custom);
00080   setOperationAction(ISD::VECTOR_SHUFFLE, VT.getSimpleVT(), Custom);
00081   setOperationAction(ISD::SCALAR_TO_VECTOR, VT.getSimpleVT(), Custom);
00082   setOperationAction(ISD::CONCAT_VECTORS, VT.getSimpleVT(), Custom);
00083   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT.getSimpleVT(), Expand);
00084   if (VT.isInteger()) {
00085     setOperationAction(ISD::SHL, VT.getSimpleVT(), Custom);
00086     setOperationAction(ISD::SRA, VT.getSimpleVT(), Custom);
00087     setOperationAction(ISD::SRL, VT.getSimpleVT(), Custom);
00088   }
00089 
00090   // Promote all bit-wise operations.
00091   if (VT.isInteger() && VT != PromotedBitwiseVT) {
00092     setOperationAction(ISD::AND, VT.getSimpleVT(), Promote);
00093     AddPromotedToType (ISD::AND, VT.getSimpleVT(),
00094                        PromotedBitwiseVT.getSimpleVT());
00095     setOperationAction(ISD::OR,  VT.getSimpleVT(), Promote);
00096     AddPromotedToType (ISD::OR,  VT.getSimpleVT(),
00097                        PromotedBitwiseVT.getSimpleVT());
00098     setOperationAction(ISD::XOR, VT.getSimpleVT(), Promote);
00099     AddPromotedToType (ISD::XOR, VT.getSimpleVT(),
00100                        PromotedBitwiseVT.getSimpleVT());
00101   }
00102 }
00103 
00104 void ARMTargetLowering::addDRTypeForNEON(EVT VT) {
00105   addRegisterClass(VT, ARM::DPRRegisterClass);
00106   addTypeForNEON(VT, MVT::f64, MVT::v2i32);
00107 }
00108 
00109 void ARMTargetLowering::addQRTypeForNEON(EVT VT) {
00110   addRegisterClass(VT, ARM::QPRRegisterClass);
00111   addTypeForNEON(VT, MVT::v2f64, MVT::v4i32);
00112 }
00113 
00114 static TargetLoweringObjectFile *createTLOF(TargetMachine &TM) {
00115   if (TM.getSubtarget<ARMSubtarget>().isTargetDarwin())
00116     return new TargetLoweringObjectFileMachO();
00117   return new ARMElfTargetObjectFile();
00118 }
00119 
00120 ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
00121     : TargetLowering(TM, createTLOF(TM)), ARMPCLabelIndex(0) {
00122   Subtarget = &TM.getSubtarget<ARMSubtarget>();
00123 
00124   if (Subtarget->isTargetDarwin()) {
00125     // Uses VFP for Thumb libfuncs if available.
00126     if (Subtarget->isThumb() && Subtarget->hasVFP2()) {
00127       // Single-precision floating-point arithmetic.
00128       setLibcallName(RTLIB::ADD_F32, "__addsf3vfp");
00129       setLibcallName(RTLIB::SUB_F32, "__subsf3vfp");
00130       setLibcallName(RTLIB::MUL_F32, "__mulsf3vfp");
00131       setLibcallName(RTLIB::DIV_F32, "__divsf3vfp");
00132 
00133       // Double-precision floating-point arithmetic.
00134       setLibcallName(RTLIB::ADD_F64, "__adddf3vfp");
00135       setLibcallName(RTLIB::SUB_F64, "__subdf3vfp");
00136       setLibcallName(RTLIB::MUL_F64, "__muldf3vfp");
00137       setLibcallName(RTLIB::DIV_F64, "__divdf3vfp");
00138 
00139       // Single-precision comparisons.
00140       setLibcallName(RTLIB::OEQ_F32, "__eqsf2vfp");
00141       setLibcallName(RTLIB::UNE_F32, "__nesf2vfp");
00142       setLibcallName(RTLIB::OLT_F32, "__ltsf2vfp");
00143       setLibcallName(RTLIB::OLE_F32, "__lesf2vfp");
00144       setLibcallName(RTLIB::OGE_F32, "__gesf2vfp");
00145       setLibcallName(RTLIB::OGT_F32, "__gtsf2vfp");
00146       setLibcallName(RTLIB::UO_F32,  "__unordsf2vfp");
00147       setLibcallName(RTLIB::O_F32,   "__unordsf2vfp");
00148 
00149       setCmpLibcallCC(RTLIB::OEQ_F32, ISD::SETNE);
00150       setCmpLibcallCC(RTLIB::UNE_F32, ISD::SETNE);
00151       setCmpLibcallCC(RTLIB::OLT_F32, ISD::SETNE);
00152       setCmpLibcallCC(RTLIB::OLE_F32, ISD::SETNE);
00153       setCmpLibcallCC(RTLIB::OGE_F32, ISD::SETNE);
00154       setCmpLibcallCC(RTLIB::OGT_F32, ISD::SETNE);
00155       setCmpLibcallCC(RTLIB::UO_F32,  ISD::SETNE);
00156       setCmpLibcallCC(RTLIB::O_F32,   ISD::SETEQ);
00157 
00158       // Double-precision comparisons.
00159       setLibcallName(RTLIB::OEQ_F64, "__eqdf2vfp");
00160       setLibcallName(RTLIB::UNE_F64, "__nedf2vfp");
00161       setLibcallName(RTLIB::OLT_F64, "__ltdf2vfp");
00162       setLibcallName(RTLIB::OLE_F64, "__ledf2vfp");
00163       setLibcallName(RTLIB::OGE_F64, "__gedf2vfp");
00164       setLibcallName(RTLIB::OGT_F64, "__gtdf2vfp");
00165       setLibcallName(RTLIB::UO_F64,  "__unorddf2vfp");
00166       setLibcallName(RTLIB::O_F64,   "__unorddf2vfp");
00167 
00168       setCmpLibcallCC(RTLIB::OEQ_F64, ISD::SETNE);
00169       setCmpLibcallCC(RTLIB::UNE_F64, ISD::SETNE);
00170       setCmpLibcallCC(RTLIB::OLT_F64, ISD::SETNE);
00171       setCmpLibcallCC(RTLIB::OLE_F64, ISD::SETNE);
00172       setCmpLibcallCC(RTLIB::OGE_F64, ISD::SETNE);
00173       setCmpLibcallCC(RTLIB::OGT_F64, ISD::SETNE);
00174       setCmpLibcallCC(RTLIB::UO_F64,  ISD::SETNE);
00175       setCmpLibcallCC(RTLIB::O_F64,   ISD::SETEQ);
00176 
00177       // Floating-point to integer conversions.
00178       // i64 conversions are done via library routines even when generating VFP
00179       // instructions, so use the same ones.
00180       setLibcallName(RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp");
00181       setLibcallName(RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp");
00182       setLibcallName(RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp");
00183       setLibcallName(RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp");
00184 
00185       // Conversions between floating types.
00186       setLibcallName(RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp");
00187       setLibcallName(RTLIB::FPEXT_F32_F64,   "__extendsfdf2vfp");
00188 
00189       // Integer to floating-point conversions.
00190       // i64 conversions are done via library routines even when generating VFP
00191       // instructions, so use the same ones.
00192       // FIXME: There appears to be some naming inconsistency in ARM libgcc:
00193       // e.g., __floatunsidf vs. __floatunssidfvfp.
00194       setLibcallName(RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp");
00195       setLibcallName(RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp");
00196       setLibcallName(RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp");
00197       setLibcallName(RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp");
00198     }
00199   }
00200 
00201   // These libcalls are not available in 32-bit.
00202   setLibcallName(RTLIB::SHL_I128, 0);
00203   setLibcallName(RTLIB::SRL_I128, 0);
00204   setLibcallName(RTLIB::SRA_I128, 0);
00205 
00206   // Libcalls should use the AAPCS base standard ABI, even if hard float
00207   // is in effect, as per the ARM RTABI specification, section 4.1.2.
00208   if (Subtarget->isAAPCS_ABI()) {
00209     for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i) {
00210       setLibcallCallingConv(static_cast<RTLIB::Libcall>(i),
00211                             CallingConv::ARM_AAPCS);
00212     }
00213   }
00214 
00215   if (Subtarget->isThumb1Only())
00216     addRegisterClass(MVT::i32, ARM::tGPRRegisterClass);
00217   else
00218     addRegisterClass(MVT::i32, ARM::GPRRegisterClass);
00219   if (!UseSoftFloat && Subtarget->hasVFP2() && !Subtarget->isThumb1Only()) {
00220     addRegisterClass(MVT::f32, ARM::SPRRegisterClass);
00221     addRegisterClass(MVT::f64, ARM::DPRRegisterClass);
00222 
00223     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
00224   }
00225 
00226   if (Subtarget->hasNEON()) {
00227     addDRTypeForNEON(MVT::v2f32);
00228     addDRTypeForNEON(MVT::v8i8);
00229     addDRTypeForNEON(MVT::v4i16);
00230     addDRTypeForNEON(MVT::v2i32);
00231     addDRTypeForNEON(MVT::v1i64);
00232 
00233     addQRTypeForNEON(MVT::v4f32);
00234     addQRTypeForNEON(MVT::v2f64);
00235     addQRTypeForNEON(MVT::v16i8);
00236     addQRTypeForNEON(MVT::v8i16);
00237     addQRTypeForNEON(MVT::v4i32);
00238     addQRTypeForNEON(MVT::v2i64);
00239 
00240     setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
00241     setTargetDAGCombine(ISD::SHL);
00242     setTargetDAGCombine(ISD::SRL);
00243     setTargetDAGCombine(ISD::SRA);
00244     setTargetDAGCombine(ISD::SIGN_EXTEND);
00245     setTargetDAGCombine(ISD::ZERO_EXTEND);
00246     setTargetDAGCombine(ISD::ANY_EXTEND);
00247   }
00248 
00249   computeRegisterProperties();
00250 
00251   // ARM does not have f32 extending load.
00252   setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
00253 
00254   // ARM does not have i1 sign extending load.
00255   setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
00256 
00257   // ARM supports all 4 flavors of integer indexed load / store.
00258   if (!Subtarget->isThumb1Only()) {
00259     for (unsigned im = (unsigned)ISD::PRE_INC;
00260          im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
00261       setIndexedLoadAction(im,  MVT::i1,  Legal);
00262       setIndexedLoadAction(im,  MVT::i8,  Legal);
00263       setIndexedLoadAction(im,  MVT::i16, Legal);
00264       setIndexedLoadAction(im,  MVT::i32, Legal);
00265       setIndexedStoreAction(im, MVT::i1,  Legal);
00266       setIndexedStoreAction(im, MVT::i8,  Legal);
00267       setIndexedStoreAction(im, MVT::i16, Legal);
00268       setIndexedStoreAction(im, MVT::i32, Legal);
00269     }
00270   }
00271 
00272   // i64 operation support.
00273   if (Subtarget->isThumb1Only()) {
00274     setOperationAction(ISD::MUL,     MVT::i64, Expand);
00275     setOperationAction(ISD::MULHU,   MVT::i32, Expand);
00276     setOperationAction(ISD::MULHS,   MVT::i32, Expand);
00277     setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
00278     setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
00279   } else {
00280     setOperationAction(ISD::MUL,     MVT::i64, Expand);
00281     setOperationAction(ISD::MULHU,   MVT::i32, Expand);
00282     if (!Subtarget->hasV6Ops())
00283       setOperationAction(ISD::MULHS, MVT::i32, Expand);
00284   }
00285   setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
00286   setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
00287   setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
00288   setOperationAction(ISD::SRL,       MVT::i64, Custom);
00289   setOperationAction(ISD::SRA,       MVT::i64, Custom);
00290 
00291   // ARM does not have ROTL.
00292   setOperationAction(ISD::ROTL,  MVT::i32, Expand);
00293   setOperationAction(ISD::CTTZ,  MVT::i32, Expand);
00294   setOperationAction(ISD::CTPOP, MVT::i32, Expand);
00295   if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only())
00296     setOperationAction(ISD::CTLZ, MVT::i32, Expand);
00297 
00298   // Only ARMv6 has BSWAP.
00299   if (!Subtarget->hasV6Ops())
00300     setOperationAction(ISD::BSWAP, MVT::i32, Expand);
00301 
00302   // These are expanded into libcalls.
00303   setOperationAction(ISD::SDIV,  MVT::i32, Expand);
00304   setOperationAction(ISD::UDIV,  MVT::i32, Expand);
00305   setOperationAction(ISD::SREM,  MVT::i32, Expand);
00306   setOperationAction(ISD::UREM,  MVT::i32, Expand);
00307   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
00308   setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
00309 
00310   // Support label based line numbers.
00311   setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand);
00312   setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
00313 
00314   setOperationAction(ISD::GlobalAddress, MVT::i32,   Custom);
00315   setOperationAction(ISD::ConstantPool,  MVT::i32,   Custom);
00316   setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom);
00317   setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
00318 
00319   // Use the default implementation.
00320   setOperationAction(ISD::VASTART,            MVT::Other, Custom);
00321   setOperationAction(ISD::VAARG,              MVT::Other, Expand);
00322   setOperationAction(ISD::VACOPY,             MVT::Other, Expand);
00323   setOperationAction(ISD::VAEND,              MVT::Other, Expand);
00324   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
00325   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
00326   setOperationAction(ISD::EHSELECTION,        MVT::i32,   Expand);
00327   // FIXME: Shouldn't need this, since no register is used, but the legalizer
00328   // doesn't yet know how to not do that for SjLj.
00329   setExceptionSelectorRegister(ARM::R0);
00330   if (Subtarget->isThumb())
00331     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
00332   else
00333     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
00334   setOperationAction(ISD::MEMBARRIER,         MVT::Other, Expand);
00335 
00336   if (!Subtarget->hasV6Ops() && !Subtarget->isThumb2()) {
00337     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
00338     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8,  Expand);
00339   }
00340   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
00341 
00342   if (!UseSoftFloat && Subtarget->hasVFP2() && !Subtarget->isThumb1Only())
00343     // Turn f64->i64 into FMRRD, i64 -> f64 to FMDRR iff target supports vfp2.
00344     setOperationAction(ISD::BIT_CONVERT, MVT::i64, Custom);
00345 
00346   // We want to custom lower some of our intrinsics.
00347   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
00348   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
00349   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
00350 
00351   setOperationAction(ISD::SETCC,     MVT::i32, Expand);
00352   setOperationAction(ISD::SETCC,     MVT::f32, Expand);
00353   setOperationAction(ISD::SETCC,     MVT::f64, Expand);
00354   setOperationAction(ISD::SELECT,    MVT::i32, Expand);
00355   setOperationAction(ISD::SELECT,    MVT::f32, Expand);
00356   setOperationAction(ISD::SELECT,    MVT::f64, Expand);
00357   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
00358   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
00359   setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
00360 
00361   setOperationAction(ISD::BRCOND,    MVT::Other, Expand);
00362   setOperationAction(ISD::BR_CC,     MVT::i32,   Custom);
00363   setOperationAction(ISD::BR_CC,     MVT::f32,   Custom);
00364   setOperationAction(ISD::BR_CC,     MVT::f64,   Custom);
00365   setOperationAction(ISD::BR_JT,     MVT::Other, Custom);
00366 
00367   // We don't support sin/cos/fmod/copysign/pow
00368   setOperationAction(ISD::FSIN,      MVT::f64, Expand);
00369   setOperationAction(ISD::FSIN,      MVT::f32, Expand);
00370   setOperationAction(ISD::FCOS,      MVT::f32, Expand);
00371   setOperationAction(ISD::FCOS,      MVT::f64, Expand);
00372   setOperationAction(ISD::FREM,      MVT::f64, Expand);
00373   setOperationAction(ISD::FREM,      MVT::f32, Expand);
00374   if (!UseSoftFloat && Subtarget->hasVFP2() && !Subtarget->isThumb1Only()) {
00375     setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
00376     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
00377   }
00378   setOperationAction(ISD::FPOW,      MVT::f64, Expand);
00379   setOperationAction(ISD::FPOW,      MVT::f32, Expand);
00380 
00381   // int <-> fp are custom expanded into bit_convert + ARMISD ops.
00382   if (!UseSoftFloat && Subtarget->hasVFP2() && !Subtarget->isThumb1Only()) {
00383     setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
00384     setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
00385     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
00386     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
00387   }
00388 
00389   // We have target-specific dag combine patterns for the following nodes:
00390   // ARMISD::FMRRD  - No need to call setTargetDAGCombine
00391   setTargetDAGCombine(ISD::ADD);
00392   setTargetDAGCombine(ISD::SUB);
00393 
00394   setStackPointerRegisterToSaveRestore(ARM::SP);
00395   setSchedulingPreference(SchedulingForRegPressure);
00396 
00397   // FIXME: If-converter should use instruction latency to determine
00398   // profitability rather than relying on fixed limits.
00399   if (Subtarget->getCPUString() == "generic") {
00400     // Generic (and overly aggressive) if-conversion limits.
00401     setIfCvtBlockSizeLimit(10);
00402     setIfCvtDupBlockSizeLimit(2);
00403   } else if (Subtarget->hasV6Ops()) {
00404     setIfCvtBlockSizeLimit(2);
00405     setIfCvtDupBlockSizeLimit(1);
00406   } else {
00407     setIfCvtBlockSizeLimit(3);
00408     setIfCvtDupBlockSizeLimit(2);
00409   }
00410 
00411   maxStoresPerMemcpy = 1;   //// temporary - rewrite interface to use type
00412   // Do not enable CodePlacementOpt for now: it currently runs after the
00413   // ARMConstantIslandPass and messes up branch relaxation and placement
00414   // of constant islands.
00415   // benefitFromCodePlacementOpt = true;
00416 }
00417 
00418 const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
00419   switch (Opcode) {
00420   default: return 0;
00421   case ARMISD::Wrapper:       return "ARMISD::Wrapper";
00422   case ARMISD::WrapperJT:     return "ARMISD::WrapperJT";
00423   case ARMISD::CALL:          return "ARMISD::CALL";
00424   case ARMISD::CALL_PRED:     return "ARMISD::CALL_PRED";
00425   case ARMISD::CALL_NOLINK:   return "ARMISD::CALL_NOLINK";
00426   case ARMISD::tCALL:         return "ARMISD::tCALL";
00427   case ARMISD::BRCOND:        return "ARMISD::BRCOND";
00428   case ARMISD::BR_JT:         return "ARMISD::BR_JT";
00429   case ARMISD::BR2_JT:        return "ARMISD::BR2_JT";
00430   case ARMISD::RET_FLAG:      return "ARMISD::RET_FLAG";
00431   case ARMISD::PIC_ADD:       return "ARMISD::PIC_ADD";
00432   case ARMISD::CMP:           return "ARMISD::CMP";
00433   case ARMISD::CMPZ:          return "ARMISD::CMPZ";
00434   case ARMISD::CMPFP:         return "ARMISD::CMPFP";
00435   case ARMISD::CMPFPw0:       return "ARMISD::CMPFPw0";
00436   case ARMISD::FMSTAT:        return "ARMISD::FMSTAT";
00437   case ARMISD::CMOV:          return "ARMISD::CMOV";
00438   case ARMISD::CNEG:          return "ARMISD::CNEG";
00439 
00440   case ARMISD::FTOSI:         return "ARMISD::FTOSI";
00441   case ARMISD::FTOUI:         return "ARMISD::FTOUI";
00442   case ARMISD::SITOF:         return "ARMISD::SITOF";
00443   case ARMISD::UITOF:         return "ARMISD::UITOF";
00444 
00445   case ARMISD::SRL_FLAG:      return "ARMISD::SRL_FLAG";
00446   case ARMISD::SRA_FLAG:      return "ARMISD::SRA_FLAG";
00447   case ARMISD::RRX:           return "ARMISD::RRX";
00448 
00449   case ARMISD::FMRRD:         return "ARMISD::FMRRD";
00450   case ARMISD::FMDRR:         return "ARMISD::FMDRR";
00451 
00452   case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER";
00453 
00454   case ARMISD::DYN_ALLOC:     return "ARMISD::DYN_ALLOC";
00455 
00456   case ARMISD::VCEQ:          return "ARMISD::VCEQ";
00457   case ARMISD::VCGE:          return "ARMISD::VCGE";
00458   case ARMISD::VCGEU:         return "ARMISD::VCGEU";
00459   case ARMISD::VCGT:          return "ARMISD::VCGT";
00460   case ARMISD::VCGTU:         return "ARMISD::VCGTU";
00461   case ARMISD::VTST:          return "ARMISD::VTST";
00462 
00463   case ARMISD::VSHL:          return "ARMISD::VSHL";
00464   case ARMISD::VSHRs:         return "ARMISD::VSHRs";
00465   case ARMISD::VSHRu:         return "ARMISD::VSHRu";
00466   case ARMISD::VSHLLs:        return "ARMISD::VSHLLs";
00467   case ARMISD::VSHLLu:        return "ARMISD::VSHLLu";
00468   case ARMISD::VSHLLi:        return "ARMISD::VSHLLi";
00469   case ARMISD::VSHRN:         return "ARMISD::VSHRN";
00470   case ARMISD::VRSHRs:        return "ARMISD::VRSHRs";
00471   case ARMISD::VRSHRu:        return "ARMISD::VRSHRu";
00472   case ARMISD::VRSHRN:        return "ARMISD::VRSHRN";
00473   case ARMISD::VQSHLs:        return "ARMISD::VQSHLs";
00474   case ARMISD::VQSHLu:        return "ARMISD::VQSHLu";
00475   case ARMISD::VQSHLsu:       return "ARMISD::VQSHLsu";
00476   case ARMISD::VQSHRNs:       return "ARMISD::VQSHRNs";
00477   case ARMISD::VQSHRNu:       return "ARMISD::VQSHRNu";
00478   case ARMISD::VQSHRNsu:      return "ARMISD::VQSHRNsu";
00479   case ARMISD::VQRSHRNs:      return "ARMISD::VQRSHRNs";
00480   case ARMISD::VQRSHRNu:      return "ARMISD::VQRSHRNu";
00481   case ARMISD::VQRSHRNsu:     return "ARMISD::VQRSHRNsu";
00482   case ARMISD::VGETLANEu:     return "ARMISD::VGETLANEu";
00483   case ARMISD::VGETLANEs:     return "ARMISD::VGETLANEs";
00484   case ARMISD::VDUP:          return "ARMISD::VDUP";
00485   case ARMISD::VDUPLANE:      return "ARMISD::VDUPLANE";
00486   case ARMISD::VLD2D:         return "ARMISD::VLD2D";
00487   case ARMISD::VLD3D:         return "ARMISD::VLD3D";
00488   case ARMISD::VLD4D:         return "ARMISD::VLD4D";
00489   case ARMISD::VST2D:         return "ARMISD::VST2D";
00490   case ARMISD::VST3D:         return "ARMISD::VST3D";
00491   case ARMISD::VST4D:         return "ARMISD::VST4D";
00492   case ARMISD::VEXT:          return "ARMISD::VEXT";
00493   case ARMISD::VREV64:        return "ARMISD::VREV64";
00494   case ARMISD::VREV32:        return "ARMISD::VREV32";
00495   case ARMISD::VREV16:        return "ARMISD::VREV16";
00496   case ARMISD::VZIP:          return "ARMISD::VZIP";
00497   case ARMISD::VUZP:          return "ARMISD::VUZP";
00498   case ARMISD::VTRN:          return "ARMISD::VTRN";
00499   }
00500 }
00501 
00502 /// getFunctionAlignment - Return the Log2 alignment of this function.
00503 unsigned ARMTargetLowering::getFunctionAlignment(const Function *F) const {
00504   return getTargetMachine().getSubtarget<ARMSubtarget>().isThumb() ? 1 : 2;
00505 }
00506 
00507 //===----------------------------------------------------------------------===//
00508 // Lowering Code
00509 //===----------------------------------------------------------------------===//
00510 
00511 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
00512 static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
00513   switch (CC) {
00514   default: llvm_unreachable("Unknown condition code!");
00515   case ISD::SETNE:  return ARMCC::NE;
00516   case ISD::SETEQ:  return ARMCC::EQ;
00517   case ISD::SETGT:  return ARMCC::GT;
00518   case ISD::SETGE:  return ARMCC::GE;
00519   case ISD::SETLT:  return ARMCC::LT;
00520   case ISD::SETLE:  return ARMCC::LE;
00521   case ISD::SETUGT: return ARMCC::HI;
00522   case ISD::SETUGE: return ARMCC::HS;
00523   case ISD::SETULT: return ARMCC::LO;
00524   case ISD::SETULE: return ARMCC::LS;
00525   }
00526 }
00527 
00528 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC. It
00529 /// returns true if the operands should be inverted to form the proper
00530 /// comparison.
00531 static bool FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
00532                         ARMCC::CondCodes &CondCode2) {
00533   bool Invert = false;
00534   CondCode2 = ARMCC::AL;
00535   switch (CC) {
00536   default: llvm_unreachable("Unknown FP condition!");
00537   case ISD::SETEQ:
00538   case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
00539   case ISD::SETGT:
00540   case ISD::SETOGT: CondCode = ARMCC::GT; break;
00541   case ISD::SETGE:
00542   case ISD::SETOGE: CondCode = ARMCC::GE; break;
00543   case ISD::SETOLT: CondCode = ARMCC::MI; break;
00544   case ISD::SETOLE: CondCode = ARMCC::GT; Invert = true; break;
00545   case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
00546   case ISD::SETO:   CondCode = ARMCC::VC; break;
00547   case ISD::SETUO:  CondCode = ARMCC::VS; break;
00548   case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
00549   case ISD::SETUGT: CondCode = ARMCC::HI; break;
00550   case ISD::SETUGE: CondCode = ARMCC::PL; break;
00551   case ISD::SETLT:
00552   case ISD::SETULT: CondCode = ARMCC::LT; break;
00553   case ISD::SETLE:
00554   case ISD::SETULE: CondCode = ARMCC::LE; break;
00555   case ISD::SETNE:
00556   case ISD::SETUNE: CondCode = ARMCC::NE; break;
00557   }
00558   return Invert;
00559 }
00560 
00561 //===----------------------------------------------------------------------===//
00562 //                      Calling Convention Implementation
00563 //===----------------------------------------------------------------------===//
00564 
00565 #include "ARMGenCallingConv.inc"
00566 
00567 // APCS f64 is in register pairs, possibly split to stack
00568 static bool f64AssignAPCS(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00569                           CCValAssign::LocInfo &LocInfo,
00570                           CCState &State, bool CanFail) {
00571   static const unsigned RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
00572 
00573   // Try to get the first register.
00574   if (unsigned Reg = State.AllocateReg(RegList, 4))
00575     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
00576   else {
00577     // For the 2nd half of a v2f64, do not fail.
00578     if (CanFail)
00579       return false;
00580 
00581     // Put the whole thing on the stack.
00582     State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
00583                                            State.AllocateStack(8, 4),
00584                                            LocVT, LocInfo));
00585     return true;
00586   }
00587 
00588   // Try to get the second register.
00589   if (unsigned Reg = State.AllocateReg(RegList, 4))
00590     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
00591   else
00592     State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
00593                                            State.AllocateStack(4, 4),
00594                                            LocVT, LocInfo));
00595   return true;
00596 }
00597 
00598 static bool CC_ARM_APCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00599                                    CCValAssign::LocInfo &LocInfo,
00600                                    ISD::ArgFlagsTy &ArgFlags,
00601                                    CCState &State) {
00602   if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
00603     return false;
00604   if (LocVT == MVT::v2f64 &&
00605       !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
00606     return false;
00607   return true;  // we handled it
00608 }
00609 
00610 // AAPCS f64 is in aligned register pairs
00611 static bool f64AssignAAPCS(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00612                            CCValAssign::LocInfo &LocInfo,
00613                            CCState &State, bool CanFail) {
00614   static const unsigned HiRegList[] = { ARM::R0, ARM::R2 };
00615   static const unsigned LoRegList[] = { ARM::R1, ARM::R3 };
00616 
00617   unsigned Reg = State.AllocateReg(HiRegList, LoRegList, 2);
00618   if (Reg == 0) {
00619     // For the 2nd half of a v2f64, do not just fail.
00620     if (CanFail)
00621       return false;
00622 
00623     // Put the whole thing on the stack.
00624     State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
00625                                            State.AllocateStack(8, 8),
00626                                            LocVT, LocInfo));
00627     return true;
00628   }
00629 
00630   unsigned i;
00631   for (i = 0; i < 2; ++i)
00632     if (HiRegList[i] == Reg)
00633       break;
00634 
00635   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
00636   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
00637                                          LocVT, LocInfo));
00638   return true;
00639 }
00640 
00641 static bool CC_ARM_AAPCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00642                                     CCValAssign::LocInfo &LocInfo,
00643                                     ISD::ArgFlagsTy &ArgFlags,
00644                                     CCState &State) {
00645   if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
00646     return false;
00647   if (LocVT == MVT::v2f64 &&
00648       !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
00649     return false;
00650   return true;  // we handled it
00651 }
00652 
00653 static bool f64RetAssign(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00654                          CCValAssign::LocInfo &LocInfo, CCState &State) {
00655   static const unsigned HiRegList[] = { ARM::R0, ARM::R2 };
00656   static const unsigned LoRegList[] = { ARM::R1, ARM::R3 };
00657 
00658   unsigned Reg = State.AllocateReg(HiRegList, LoRegList, 2);
00659   if (Reg == 0)
00660     return false; // we didn't handle it
00661 
00662   unsigned i;
00663   for (i = 0; i < 2; ++i)
00664     if (HiRegList[i] == Reg)
00665       break;
00666 
00667   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
00668   State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
00669                                          LocVT, LocInfo));
00670   return true;
00671 }
00672 
00673 static bool RetCC_ARM_APCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00674                                       CCValAssign::LocInfo &LocInfo,
00675                                       ISD::ArgFlagsTy &ArgFlags,
00676                                       CCState &State) {
00677   if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
00678     return false;
00679   if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
00680     return false;
00681   return true;  // we handled it
00682 }
00683 
00684 static bool RetCC_ARM_AAPCS_Custom_f64(unsigned &ValNo, EVT &ValVT, EVT &LocVT,
00685                                        CCValAssign::LocInfo &LocInfo,
00686                                        ISD::ArgFlagsTy &ArgFlags,
00687                                        CCState &State) {
00688   return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
00689                                    State);
00690 }
00691 
00692 /// CCAssignFnForNode - Selects the correct CCAssignFn for a the
00693 /// given CallingConvention value.
00694 CCAssignFn *ARMTargetLowering::CCAssignFnForNode(unsigned CC,
00695                                                  bool Return,
00696                                                  bool isVarArg) const {
00697   switch (CC) {
00698   default:
00699     llvm_unreachable("Unsupported calling convention");
00700   case CallingConv::C:
00701   case CallingConv::Fast:
00702     // Use target triple & subtarget features to do actual dispatch.
00703     if (Subtarget->isAAPCS_ABI()) {
00704       if (Subtarget->hasVFP2() &&
00705           FloatABIType == FloatABI::Hard && !isVarArg)
00706         return (Return ? RetCC_ARM_AAPCS_VFP: CC_ARM_AAPCS_VFP);
00707       else
00708         return (Return ? RetCC_ARM_AAPCS: CC_ARM_AAPCS);
00709     } else
00710         return (Return ? RetCC_ARM_APCS: CC_ARM_APCS);
00711   case CallingConv::ARM_AAPCS_VFP:
00712     return (Return ? RetCC_ARM_AAPCS_VFP: CC_ARM_AAPCS_VFP);
00713   case CallingConv::ARM_AAPCS:
00714     return (Return ? RetCC_ARM_AAPCS: CC_ARM_AAPCS);
00715   case CallingConv::ARM_APCS:
00716     return (Return ? RetCC_ARM_APCS: CC_ARM_APCS);
00717   }
00718 }
00719 
00720 /// LowerCallResult - Lower the result values of a call into the
00721 /// appropriate copies out of appropriate physical registers.
00722 SDValue
00723 ARMTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
00724                                    unsigned CallConv, bool isVarArg,
00725                                    const SmallVectorImpl<ISD::InputArg> &Ins,
00726                                    DebugLoc dl, SelectionDAG &DAG,
00727                                    SmallVectorImpl<SDValue> &InVals) {
00728 
00729   // Assign locations to each value returned by this call.
00730   SmallVector<CCValAssign, 16> RVLocs;
00731   CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
00732                  RVLocs, *DAG.getContext());
00733   CCInfo.AnalyzeCallResult(Ins,
00734                            CCAssignFnForNode(CallConv, /* Return*/ true,
00735                                              isVarArg));
00736 
00737   // Copy all of the result registers out of their specified physreg.
00738   for (unsigned i = 0; i != RVLocs.size(); ++i) {
00739     CCValAssign VA = RVLocs[i];
00740 
00741     SDValue Val;
00742     if (VA.needsCustom()) {
00743       // Handle f64 or half of a v2f64.
00744       SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
00745                                       InFlag);
00746       Chain = Lo.getValue(1);
00747       InFlag = Lo.getValue(2);
00748       VA = RVLocs[++i]; // skip ahead to next loc
00749       SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
00750                                       InFlag);
00751       Chain = Hi.getValue(1);
00752       InFlag = Hi.getValue(2);
00753       Val = DAG.getNode(ARMISD::FMDRR, dl, MVT::f64, Lo, Hi);
00754 
00755       if (VA.getLocVT() == MVT::v2f64) {
00756         SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
00757         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
00758                           DAG.getConstant(0, MVT::i32));
00759 
00760         VA = RVLocs[++i]; // skip ahead to next loc
00761         Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
00762         Chain = Lo.getValue(1);
00763         InFlag = Lo.getValue(2);
00764         VA = RVLocs[++i]; // skip ahead to next loc
00765         Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
00766         Chain = Hi.getValue(1);
00767         InFlag = Hi.getValue(2);
00768         Val = DAG.getNode(ARMISD::FMDRR, dl, MVT::f64, Lo, Hi);
00769         Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
00770                           DAG.getConstant(1, MVT::i32));
00771       }
00772     } else {
00773       Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
00774                                InFlag);
00775       Chain = Val.getValue(1);
00776       InFlag = Val.getValue(2);
00777     }
00778 
00779     switch (VA.getLocInfo()) {
00780     default: llvm_unreachable("Unknown loc info!");
00781     case CCValAssign::Full: break;
00782     case CCValAssign::BCvt:
00783       Val = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), Val);
00784       break;
00785     }
00786 
00787     InVals.push_back(Val);
00788   }
00789 
00790   return Chain;
00791 }
00792 
00793 /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
00794 /// by "Src" to address "Dst" of size "Size".  Alignment information is
00795 /// specified by the specific parameter attribute.  The copy will be passed as
00796 /// a byval function parameter.
00797 /// Sometimes what we are copying is the end of a larger object, the part that
00798 /// does not fit in registers.
00799 static SDValue
00800 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
00801                           ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
00802                           DebugLoc dl) {
00803   SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
00804   return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
00805                        /*AlwaysInline=*/false, NULL, 0, NULL, 0);
00806 }
00807 
00808 /// LowerMemOpCallTo - Store the argument to the stack.
00809 SDValue
00810 ARMTargetLowering::LowerMemOpCallTo(SDValue Chain,
00811                                     SDValue StackPtr, SDValue Arg,
00812                                     DebugLoc dl, SelectionDAG &DAG,
00813                                     const CCValAssign &VA,
00814                                     ISD::ArgFlagsTy Flags) {
00815   unsigned LocMemOffset = VA.getLocMemOffset();
00816   SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
00817   PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
00818   if (Flags.isByVal()) {
00819     return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
00820   }
00821   return DAG.getStore(Chain, dl, Arg, PtrOff,
00822                       PseudoSourceValue::getStack(), LocMemOffset);
00823 }
00824 
00825 void ARMTargetLowering::PassF64ArgInRegs(DebugLoc dl, SelectionDAG &DAG,
00826                                          SDValue Chain, SDValue &Arg,
00827                                          RegsToPassVector &RegsToPass,
00828                                          CCValAssign &VA, CCValAssign &NextVA,
00829                                          SDValue &StackPtr,
00830                                          SmallVector<SDValue, 8> &MemOpChains,
00831                                          ISD::ArgFlagsTy Flags) {
00832 
00833   SDValue fmrrd = DAG.getNode(ARMISD::FMRRD, dl,
00834                               DAG.getVTList(MVT::i32, MVT::i32), Arg);
00835   RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd));
00836 
00837   if (NextVA.isRegLoc())
00838     RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1)));
00839   else {
00840     assert(NextVA.isMemLoc());
00841     if (StackPtr.getNode() == 0)
00842       StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy());
00843 
00844     MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1),
00845                                            dl, DAG, NextVA,
00846                                            Flags));
00847   }
00848 }
00849 
00850 /// LowerCall - Lowering a call into a callseq_start <-
00851 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
00852 /// nodes.
00853 SDValue
00854 ARMTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
00855                              unsigned CallConv, bool isVarArg,
00856                              bool isTailCall,
00857                              const SmallVectorImpl<ISD::OutputArg> &Outs,
00858                              const SmallVectorImpl<ISD::InputArg> &Ins,
00859                              DebugLoc dl, SelectionDAG &DAG,
00860                              SmallVectorImpl<SDValue> &InVals) {
00861 
00862   // Analyze operands of the call, assigning locations to each operand.
00863   SmallVector<CCValAssign, 16> ArgLocs;
00864   CCState CCInfo(CallConv, isVarArg, getTargetMachine(), ArgLocs,
00865                  *DAG.getContext());
00866   CCInfo.AnalyzeCallOperands(Outs,
00867                              CCAssignFnForNode(CallConv, /* Return*/ false,
00868                                                isVarArg));
00869 
00870   // Get a count of how many bytes are to be pushed on the stack.
00871   unsigned NumBytes = CCInfo.getNextStackOffset();
00872 
00873   // Adjust the stack pointer for the new arguments...
00874   // These operations are automatically eliminated by the prolog/epilog pass
00875   Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true));
00876 
00877   SDValue StackPtr = DAG.getRegister(ARM::SP, MVT::i32);
00878 
00879   RegsToPassVector RegsToPass;
00880   SmallVector<SDValue, 8> MemOpChains;
00881 
00882   // Walk the register/memloc assignments, inserting copies/loads.  In the case
00883   // of tail call optimization, arguments are handled later.
00884   for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
00885        i != e;
00886        ++i, ++realArgIdx) {
00887     CCValAssign &VA = ArgLocs[i];
00888     SDValue Arg = Outs[realArgIdx].Val;
00889     ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
00890 
00891     // Promote the value if needed.
00892     switch (VA.getLocInfo()) {
00893     default: llvm_unreachable("Unknown loc info!");
00894     case CCValAssign::Full: break;
00895     case CCValAssign::SExt:
00896       Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
00897       break;
00898     case CCValAssign::ZExt:
00899       Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
00900       break;
00901     case CCValAssign::AExt:
00902       Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
00903       break;
00904     case CCValAssign::BCvt:
00905       Arg = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getLocVT(), Arg);
00906       break;
00907     }
00908 
00909     // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
00910     if (VA.needsCustom()) {
00911       if (VA.getLocVT() == MVT::v2f64) {
00912         SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
00913                                   DAG.getConstant(0, MVT::i32));
00914         SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
00915                                   DAG.getConstant(1, MVT::i32));
00916 
00917         PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass,
00918                          VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
00919 
00920         VA = ArgLocs[++i]; // skip ahead to next loc
00921         if (VA.isRegLoc()) {
00922           PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass,
00923                            VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
00924         } else {
00925           assert(VA.isMemLoc());
00926           if (StackPtr.getNode() == 0)
00927             StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy());
00928 
00929           MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Op1,
00930                                                  dl, DAG, VA, Flags));
00931         }
00932       } else {
00933         PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
00934                          StackPtr, MemOpChains, Flags);
00935       }
00936     } else if (VA.isRegLoc()) {
00937       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
00938     } else {
00939       assert(VA.isMemLoc());
00940       if (StackPtr.getNode() == 0)
00941         StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy());
00942 
00943       MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
00944                                              dl, DAG, VA, Flags));
00945     }
00946   }
00947 
00948   if (!MemOpChains.empty())
00949     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
00950                         &MemOpChains[0], MemOpChains.size());
00951 
00952   // Build a sequence of copy-to-reg nodes chained together with token chain
00953   // and flag operands which copy the outgoing args into the appropriate regs.
00954   SDValue InFlag;
00955   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
00956     Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
00957                              RegsToPass[i].second, InFlag);
00958     InFlag = Chain.getValue(1);
00959   }
00960 
00961   // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
00962   // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
00963   // node so that legalize doesn't hack it.
00964   bool isDirect = false;
00965   bool isARMFunc = false;
00966   bool isLocalARMFunc = false;
00967   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
00968     GlobalValue *GV = G->getGlobal();
00969     isDirect = true;
00970     bool isExt = GV->isDeclaration() || GV->isWeakForLinker();
00971     bool isStub = (isExt && Subtarget->isTargetDarwin()) &&
00972                    getTargetMachine().getRelocationModel() != Reloc::Static;
00973     isARMFunc = !Subtarget->isThumb() || isStub;
00974     // ARM call to a local ARM function is predicable.
00975     isLocalARMFunc = !Subtarget->isThumb() && !isExt;
00976     // tBX takes a register source operand.
00977     if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
00978       ARMConstantPoolValue *CPV = new ARMConstantPoolValue(GV,
00979                                                            ARMPCLabelIndex, 4);
00980       SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
00981       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
00982       Callee = DAG.getLoad(getPointerTy(), dl,
00983                            DAG.getEntryNode(), CPAddr, NULL, 0);
00984       SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
00985       Callee = DAG.getNode(ARMISD::PIC_ADD, dl,
00986                            getPointerTy(), Callee, PICLabel);
00987    } else
00988       Callee = DAG.getTargetGlobalAddress(GV, getPointerTy());
00989   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
00990     isDirect = true;
00991     bool isStub = Subtarget->isTargetDarwin() &&
00992                   getTargetMachine().getRelocationModel() != Reloc::Static;
00993     isARMFunc = !Subtarget->isThumb() || isStub;
00994     // tBX takes a register source operand.
00995     const char *Sym = S->getSymbol();
00996     if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
00997       ARMConstantPoolValue *CPV = new ARMConstantPoolValue(*DAG.getContext(),
00998                                                        Sym, ARMPCLabelIndex, 4);
00999       SDValue CPAddr = DAG.getTargetConstantPool(CPV, getPointerTy(), 4);
01000       CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
01001       Callee = DAG.getLoad(getPointerTy(), dl,
01002                            DAG.getEntryNode(), CPAddr, NULL, 0);
01003       SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
01004       Callee = DAG.getNode(ARMISD::PIC_ADD, dl,
01005                            getPointerTy(), Callee, PICLabel);
01006     } else
01007       Callee = DAG.getTargetExternalSymbol(Sym, getPointerTy());
01008   }
01009 
01010   // FIXME: handle tail calls differently.
01011   unsigned CallOpc;
01012   if (Subtarget->isThumb()) {
01013     if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
01014       CallOpc = ARMISD::CALL_NOLINK;
01015     else
01016       CallOpc = isARMFunc ? ARMISD::CALL : ARMISD::tCALL;
01017   } else {
01018     CallOpc = (isDirect || Subtarget->hasV5TOps())
01019       ? (isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL)
01020       : ARMISD::CALL_NOLINK;
01021   }
01022   if (CallOpc == ARMISD::CALL_NOLINK && !Subtarget->isThumb1Only()) {
01023     // implicit def LR - LR mustn't be allocated as GRP:$dst of CALL_NOLINK
01024     Chain = DAG.getCopyToReg(Chain, dl, ARM::LR, DAG.getUNDEF(MVT::i32),InFlag);
01025     InFlag = Chain.getValue(1);
01026   }
01027 
01028   std::vector<SDValue> Ops;
01029   Ops.push_back(Chain);
01030   Ops.push_back(Callee);
01031 
01032   // Add argument registers to the end of the list so that they are known live
01033   // into the call.
01034   for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
01035     Ops.push_back(DAG.getRegister(RegsToPass[i].first,
01036                                   RegsToPass[i].second.getValueType()));
01037 
01038   if (InFlag.getNode())
01039     Ops.push_back(InFlag);
01040   // Returns a chain and a flag for retval copy to use.
01041   Chain = DAG.getNode(CallOpc, dl, DAG.getVTList(MVT::Other, MVT::Flag),
01042                       &Ops[0], Ops.size());
01043   InFlag = Chain.getValue(1);
01044 
01045   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
01046                              DAG.getIntPtrConstant(0, true), InFlag);
01047   if (!Ins.empty())
01048     InFlag = Chain.getValue(1);
01049 
01050   // Handle result values, copying them out of physregs into vregs that we
01051   // return.
01052   return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins,
01053                          dl, DAG, InVals);
01054 }
01055 
01056 SDValue
01057 ARMTargetLowering::LowerReturn(SDValue Chain,
01058                                unsigned CallConv, bool isVarArg,
01059                                const SmallVectorImpl<ISD::OutputArg> &Outs,
01060                                DebugLoc dl, SelectionDAG &DAG) {
01061 
01062   // CCValAssign - represent the assignment of the return value to a location.
01063   SmallVector<CCValAssign, 16> RVLocs;
01064 
01065   // CCState - Info about the registers and stack slots.
01066   CCState CCInfo(CallConv, isVarArg, getTargetMachine(), RVLocs,
01067                  *DAG.getContext());
01068 
01069   // Analyze outgoing return values.
01070   CCInfo.AnalyzeReturn(Outs, CCAssignFnForNode(CallConv, /* Return */ true,
01071                                                isVarArg));
01072 
01073   // If this is the first return lowered for this function, add
01074   // the regs to the liveout set for the function.
01075   if (DAG.getMachineFunction().getRegInfo().liveout_empty()) {
01076     for (unsigned i = 0; i != RVLocs.size(); ++i)
01077       if (RVLocs[i].isRegLoc())
01078         DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg());
01079   }
01080 
01081   SDValue Flag;
01082 
01083   // Copy the result values into the output registers.
01084   for (unsigned i = 0, realRVLocIdx = 0;
01085        i != RVLocs.size();
01086        ++i, ++realRVLocIdx) {
01087     CCValAssign &VA = RVLocs[i];
01088     assert(VA.isRegLoc() && "Can only return in registers!");
01089 
01090     SDValue Arg = Outs[realRVLocIdx].Val;
01091 
01092     switch (VA.getLocInfo()) {
01093     default: llvm_unreachable("Unknown loc info!");
01094     case CCValAssign::Full: break;
01095     case CCValAssign::BCvt:
01096       Arg = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getLocVT(), Arg);
01097       break;
01098     }
01099 
01100     if (VA.needsCustom()) {
01101       if (VA.getLocVT() == MVT::v2f64) {
01102         // Extract the first half and return it in two registers.
01103         SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
01104                                    DAG.getConstant(0, MVT::i32));
01105         SDValue HalfGPRs = DAG.getNode(ARMISD::FMRRD, dl,
01106                                        DAG.getVTList(MVT::i32, MVT::i32), Half);
01107 
01108         Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), HalfGPRs, Flag);
01109         Flag = Chain.getValue(1);
01110         VA = RVLocs[++i]; // skip ahead to next loc
01111         Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
01112                                  HalfGPRs.getValue(1), Flag);
01113         Flag = Chain.getValue(1);
01114         VA = RVLocs[++i]; // skip ahead to next loc
01115 
01116         // Extract the 2nd half and fall through to handle it as an f64 value.
01117         Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
01118                           DAG.getConstant(1, MVT::i32));
01119       }
01120       // Legalize ret f64 -> ret 2 x i32.  We always have fmrrd if f64 is
01121       // available.
01122       SDValue fmrrd = DAG.getNode(ARMISD::FMRRD, dl,
01123                                   DAG.getVTList(MVT::i32, MVT::i32), &Arg, 1);
01124       Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), fmrrd, Flag);
01125       Flag = Chain.getValue(1);
01126       VA = RVLocs[++i]; // skip ahead to next loc
01127       Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), fmrrd.getValue(1),
01128                                Flag);
01129     } else
01130       Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
01131 
01132     // Guarantee that all emitted copies are
01133     // stuck together, avoiding something bad.
01134     Flag = Chain.getValue(1);
01135   }
01136 
01137   SDValue result;
01138   if (Flag.getNode())
01139     result = DAG.getNode(ARMISD::RET_FLAG, dl, MVT::Other, Chain, Flag);
01140   else // Return Void
01141     result = DAG.getNode(ARMISD::RET_FLAG, dl, MVT::Other, Chain);
01142 
01143   return result;
01144 }
01145 
01146 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
01147 // their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
01148 // one of the above mentioned nodes. It has to be wrapped because otherwise
01149 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
01150 // be used to form addressing mode. These wrapped nodes will be selected
01151 // into MOVi.
01152 static SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) {
01153   EVT PtrVT = Op.getValueType();
01154   // FIXME there is no actual debug info here
01155   DebugLoc dl = Op.getDebugLoc();
01156   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
01157   SDValue Res;
01158   if (CP->isMachineConstantPoolEntry())
01159     Res = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT,
01160                                     CP->getAlignment());
01161   else
01162     Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT,
01163                                     CP->getAlignment());
01164   return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
01165 }
01166 
01167 // Lower ISD::GlobalTLSAddress using the "general dynamic" model
01168 SDValue
01169 ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
01170                                                  SelectionDAG &DAG) {
01171   DebugLoc dl = GA->getDebugLoc();
01172   EVT PtrVT = getPointerTy();
01173   unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
01174   ARMConstantPoolValue *CPV =
01175     new ARMConstantPoolValue(GA->getGlobal(), ARMPCLabelIndex,
01176                              PCAdj, "tlsgd", true);
01177   SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01178   Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
01179   Argument = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Argument, NULL, 0);
01180   SDValue Chain = Argument.getValue(1);
01181 
01182   SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
01183   Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
01184 
01185   // call __tls_get_addr.
01186   ArgListTy Args;
01187   ArgListEntry Entry;
01188   Entry.Node = Argument;
01189   Entry.Ty = (const Type *) Type::getInt32Ty(*DAG.getContext());
01190   Args.push_back(Entry);
01191   // FIXME: is there useful debug info available here?
01192   std::pair<SDValue, SDValue> CallResult =
01193     LowerCallTo(Chain, (const Type *) Type::getInt32Ty(*DAG.getContext()),
01194                 false, false, false, false,
01195                 0, CallingConv::C, false, /*isReturnValueUsed=*/true,
01196                 DAG.getExternalSymbol("__tls_get_addr", PtrVT), Args, DAG, dl);
01197   return CallResult.first;
01198 }
01199 
01200 // Lower ISD::GlobalTLSAddress using the "initial exec" or
01201 // "local exec" model.
01202 SDValue
01203 ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
01204                                         SelectionDAG &DAG) {
01205   GlobalValue *GV = GA->getGlobal();
01206   DebugLoc dl = GA->getDebugLoc();
01207   SDValue Offset;
01208   SDValue Chain = DAG.getEntryNode();
01209   EVT PtrVT = getPointerTy();
01210   // Get the Thread Pointer
01211   SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
01212 
01213   if (GV->isDeclaration()) {
01214     // initial exec model
01215     unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
01216     ARMConstantPoolValue *CPV =
01217       new ARMConstantPoolValue(GA->getGlobal(), ARMPCLabelIndex,
01218                                PCAdj, "gottpoff", true);
01219     Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01220     Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
01221     Offset = DAG.getLoad(PtrVT, dl, Chain, Offset, NULL, 0);
01222     Chain = Offset.getValue(1);
01223 
01224     SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
01225     Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
01226 
01227     Offset = DAG.getLoad(PtrVT, dl, Chain, Offset, NULL, 0);
01228   } else {
01229     // local exec model
01230     ARMConstantPoolValue *CPV = new ARMConstantPoolValue(GV, "tpoff");
01231     Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01232     Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
01233     Offset = DAG.getLoad(PtrVT, dl, Chain, Offset, NULL, 0);
01234   }
01235 
01236   // The address of the thread local variable is the add of the thread
01237   // pointer with the offset of the variable.
01238   return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
01239 }
01240 
01241 SDValue
01242 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) {
01243   // TODO: implement the "local dynamic" model
01244   assert(Subtarget->isTargetELF() &&
01245          "TLS not implemented for non-ELF targets");
01246   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
01247   // If the relocation model is PIC, use the "General Dynamic" TLS Model,
01248   // otherwise use the "Local Exec" TLS Model
01249   if (getTargetMachine().getRelocationModel() == Reloc::PIC_)
01250     return LowerToTLSGeneralDynamicModel(GA, DAG);
01251   else
01252     return LowerToTLSExecModels(GA, DAG);
01253 }
01254 
01255 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
01256                                                  SelectionDAG &DAG) {
01257   EVT PtrVT = getPointerTy();
01258   DebugLoc dl = Op.getDebugLoc();
01259   GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
01260   Reloc::Model RelocM = getTargetMachine().getRelocationModel();
01261   if (RelocM == Reloc::PIC_) {
01262     bool UseGOTOFF = GV->hasLocalLinkage() || GV->hasHiddenVisibility();
01263     ARMConstantPoolValue *CPV =
01264       new ARMConstantPoolValue(GV, UseGOTOFF ? "GOTOFF" : "GOT");
01265     SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01266     CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
01267     SDValue Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
01268                                  CPAddr, NULL, 0);
01269     SDValue Chain = Result.getValue(1);
01270     SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
01271     Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result, GOT);
01272     if (!UseGOTOFF)
01273       Result = DAG.getLoad(PtrVT, dl, Chain, Result, NULL, 0);
01274     return Result;
01275   } else {
01276     SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4);
01277     CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
01278     return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr, NULL, 0);
01279   }
01280 }
01281 
01282 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
01283                                                     SelectionDAG &DAG) {
01284   EVT PtrVT = getPointerTy();
01285   DebugLoc dl = Op.getDebugLoc();
01286   GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
01287   Reloc::Model RelocM = getTargetMachine().getRelocationModel();
01288   SDValue CPAddr;
01289   if (RelocM == Reloc::Static)
01290     CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4);
01291   else {
01292     unsigned PCAdj = (RelocM != Reloc::PIC_) ? 0 : (Subtarget->isThumb()?4:8);
01293     ARMConstantPoolValue *CPV =
01294       new ARMConstantPoolValue(GV, ARMPCLabelIndex, PCAdj);
01295     CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01296   }
01297   CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
01298 
01299   SDValue Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr, NULL, 0);
01300   SDValue Chain = Result.getValue(1);
01301 
01302   if (RelocM == Reloc::PIC_) {
01303     SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
01304     Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
01305   }
01306 
01307   if (Subtarget->GVIsIndirectSymbol(GV, RelocM == Reloc::Static))
01308     Result = DAG.getLoad(PtrVT, dl, Chain, Result, NULL, 0);
01309 
01310   return Result;
01311 }
01312 
01313 SDValue ARMTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op,
01314                                                     SelectionDAG &DAG){
01315   assert(Subtarget->isTargetELF() &&
01316          "GLOBAL OFFSET TABLE not implemented for non-ELF targets");
01317   EVT PtrVT = getPointerTy();
01318   DebugLoc dl = Op.getDebugLoc();
01319   unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
01320   ARMConstantPoolValue *CPV = new ARMConstantPoolValue(*DAG.getContext(),
01321                                                        "_GLOBAL_OFFSET_TABLE_",
01322                                                        ARMPCLabelIndex, PCAdj);
01323   SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01324   CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
01325   SDValue Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr, NULL, 0);
01326   SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
01327   return DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
01328 }
01329 
01330 static SDValue LowerNeonVLDIntrinsic(SDValue Op, SelectionDAG &DAG,
01331                                      unsigned Opcode) {
01332   SDNode *Node = Op.getNode();
01333   EVT VT = Node->getValueType(0);
01334   DebugLoc dl = Op.getDebugLoc();
01335 
01336   if (!VT.is64BitVector())
01337     return SDValue(); // unimplemented
01338 
01339   SDValue Ops[] = { Node->getOperand(0),
01340                     Node->getOperand(2) };
01341   return DAG.getNode(Opcode, dl, Node->getVTList(), Ops, 2);
01342 }
01343 
01344 static SDValue LowerNeonVSTIntrinsic(SDValue Op, SelectionDAG &DAG,
01345                                      unsigned Opcode, unsigned NumVecs) {
01346   SDNode *Node = Op.getNode();
01347   EVT VT = Node->getOperand(3).getValueType();
01348   DebugLoc dl = Op.getDebugLoc();
01349 
01350   if (!VT.is64BitVector())
01351     return SDValue(); // unimplemented
01352 
01353   SmallVector<SDValue, 6> Ops;
01354   Ops.push_back(Node->getOperand(0));
01355   Ops.push_back(Node->getOperand(2));
01356   for (unsigned N = 0; N < NumVecs; ++N)
01357     Ops.push_back(Node->getOperand(N + 3));
01358   return DAG.getNode(Opcode, dl, MVT::Other, Ops.data(), Ops.size());
01359 }
01360 
01361 SDValue
01362 ARMTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) {
01363   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
01364   switch (IntNo) {
01365   case Intrinsic::arm_neon_vld2:
01366     return LowerNeonVLDIntrinsic(Op, DAG, ARMISD::VLD2D);
01367   case Intrinsic::arm_neon_vld3:
01368     return LowerNeonVLDIntrinsic(Op, DAG, ARMISD::VLD3D);
01369   case Intrinsic::arm_neon_vld4:
01370     return LowerNeonVLDIntrinsic(Op, DAG, ARMISD::VLD4D);
01371   case Intrinsic::arm_neon_vst2:
01372     return LowerNeonVSTIntrinsic(Op, DAG, ARMISD::VST2D, 2);
01373   case Intrinsic::arm_neon_vst3:
01374     return LowerNeonVSTIntrinsic(Op, DAG, ARMISD::VST3D, 3);
01375   case Intrinsic::arm_neon_vst4:
01376     return LowerNeonVSTIntrinsic(Op, DAG, ARMISD::VST4D, 4);
01377   default: return SDValue();    // Don't custom lower most intrinsics.
01378   }
01379 }
01380 
01381 SDValue
01382 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) {
01383   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
01384   DebugLoc dl = Op.getDebugLoc();
01385   switch (IntNo) {
01386   default: return SDValue();    // Don't custom lower most intrinsics.
01387   case Intrinsic::arm_thread_pointer: {
01388     EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
01389     return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
01390   }
01391   case Intrinsic::eh_sjlj_lsda: {
01392     // blah. horrible, horrible hack with the forced magic name.
01393     // really need to clean this up. It belongs in the target-independent
01394     // layer somehow that doesn't require the coupling with the asm
01395     // printer.
01396     MachineFunction &MF = DAG.getMachineFunction();
01397     EVT PtrVT = getPointerTy();
01398     DebugLoc dl = Op.getDebugLoc();
01399     Reloc::Model RelocM = getTargetMachine().getRelocationModel();
01400     SDValue CPAddr;
01401     unsigned PCAdj = (RelocM != Reloc::PIC_)
01402       ? 0 : (Subtarget->isThumb() ? 4 : 8);
01403     // Save off the LSDA name for the AsmPrinter to use when it's time
01404     // to emit the table
01405     std::string LSDAName = "L_lsda_";
01406     LSDAName += MF.getFunction()->getName();
01407     ARMConstantPoolValue *CPV =
01408       new ARMConstantPoolValue(*DAG.getContext(), LSDAName.c_str(), 
01409                                ARMPCLabelIndex, PCAdj);
01410     CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
01411     CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
01412     SDValue Result =
01413       DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), CPAddr, NULL, 0);
01414     SDValue Chain = Result.getValue(1);
01415 
01416     if (RelocM == Reloc::PIC_) {
01417       SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex++, MVT::i32);
01418       Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
01419     }
01420     return Result;
01421   }
01422   case Intrinsic::eh_sjlj_setjmp:
01423     return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl, MVT::i32, Op.getOperand(1));
01424   }
01425 }
01426 
01427 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG,
01428                             unsigned VarArgsFrameIndex) {
01429   // vastart just stores the address of the VarArgsFrameIndex slot into the
01430   // memory location argument.
01431   DebugLoc dl = Op.getDebugLoc();
01432   EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
01433   SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, PtrVT);
01434   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
01435   return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), SV, 0);
01436 }
01437 
01438 SDValue
01439 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) {
01440   SDNode *Node = Op.getNode();
01441   DebugLoc dl = Node->getDebugLoc();
01442   EVT VT = Node->getValueType(0);
01443   SDValue Chain = Op.getOperand(0);
01444   SDValue Size  = Op.getOperand(1);
01445   SDValue Align = Op.getOperand(2);
01446 
01447   // Chain the dynamic stack allocation so that it doesn't modify the stack
01448   // pointer when other instructions are using the stack.
01449   Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true));
01450 
01451   unsigned AlignVal = cast<ConstantSDNode>(Align)->getZExtValue();
01452   unsigned StackAlign = getTargetMachine().getFrameInfo()->getStackAlignment();
01453   if (AlignVal > StackAlign)
01454     // Do this now since selection pass cannot introduce new target
01455     // independent node.
01456     Align = DAG.getConstant(-(uint64_t)AlignVal, VT);
01457 
01458   // In Thumb1 mode, there isn't a "sub r, sp, r" instruction, we will end up
01459   // using a "add r, sp, r" instead. Negate the size now so we don't have to
01460   // do even more horrible hack later.
01461   MachineFunction &MF = DAG.getMachineFunction();
01462   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
01463   if (AFI->isThumb1OnlyFunction()) {
01464     bool Negate = true;
01465     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Size);
01466     if (C) {
01467       uint32_t Val = C->getZExtValue();
01468       if (Val <= 508 && ((Val & 3) == 0))
01469         Negate = false;
01470     }
01471     if (Negate)
01472       Size = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, VT), Size);
01473   }
01474 
01475   SDVTList VTList = DAG.getVTList(VT, MVT::Other);
01476   SDValue Ops1[] = { Chain, Size, Align };
01477   SDValue Res = DAG.getNode(ARMISD::DYN_ALLOC, dl, VTList, Ops1, 3);
01478   Chain = Res.getValue(1);
01479   Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true),
01480                              DAG.getIntPtrConstant(0, true), SDValue());
01481   SDValue Ops2[] = { Res, Chain };
01482   return DAG.getMergeValues(Ops2, 2, dl);
01483 }
01484 
01485 SDValue
01486 ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA, CCValAssign &NextVA,
01487                                         SDValue &Root, SelectionDAG &DAG,
01488                                         DebugLoc dl) {
01489   MachineFunction &MF = DAG.getMachineFunction();
01490   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
01491 
01492   TargetRegisterClass *RC;
01493   if (AFI->isThumb1OnlyFunction())
01494     RC = ARM::tGPRRegisterClass;
01495   else
01496     RC = ARM::GPRRegisterClass;
01497 
01498   // Transform the arguments stored in physical registers into virtual ones.
01499   unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
01500   SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
01501 
01502   SDValue ArgValue2;
01503   if (NextVA.isMemLoc()) {
01504     unsigned ArgSize = NextVA.getLocVT().getSizeInBits()/8;
01505     MachineFrameInfo *MFI = MF.getFrameInfo();
01506     int FI = MFI->CreateFixedObject(ArgSize, NextVA.getLocMemOffset());
01507 
01508     // Create load node to retrieve arguments from the stack.
01509     SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
01510     ArgValue2 = DAG.getLoad(MVT::i32, dl, Root, FIN, NULL, 0);
01511   } else {
01512     Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
01513     ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
01514   }
01515 
01516   return DAG.getNode(ARMISD::FMDRR, dl, MVT::f64, ArgValue, ArgValue2);
01517 }
01518 
01519 SDValue
01520 ARMTargetLowering::LowerFormalArguments(SDValue Chain,
01521                                         unsigned CallConv, bool isVarArg,
01522                                         const SmallVectorImpl<ISD::InputArg>
01523                                           &Ins,
01524                                         DebugLoc dl, SelectionDAG &DAG,
01525                                         SmallVectorImpl<SDValue> &InVals) {
01526 
01527   MachineFunction &MF = DAG.getMachineFunction();
01528   MachineFrameInfo *MFI = MF.getFrameInfo();
01529 
01530   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
01531 
01532   // Assign locations to all of the incoming arguments.
01533   SmallVector<CCValAssign, 16> ArgLocs;
01534   CCState CCInfo(CallConv, isVarArg, getTargetMachine(), ArgLocs,
01535                  *DAG.getContext());
01536   CCInfo.AnalyzeFormalArguments(Ins,
01537                                 CCAssignFnForNode(CallConv, /* Return*/ false,
01538                                                   isVarArg));
01539 
01540   SmallVector<SDValue, 16> ArgValues;
01541 
01542   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
01543     CCValAssign &VA = ArgLocs[i];
01544 
01545     // Arguments stored in registers.
01546     if (VA.isRegLoc()) {
01547       EVT RegVT = VA.getLocVT();
01548 
01549       SDValue ArgValue;
01550       if (VA.needsCustom()) {
01551         // f64 and vector types are split up into multiple registers or
01552         // combinations of registers and stack slots.
01553         RegVT = MVT::i32;
01554 
01555         if (VA.getLocVT() == MVT::v2f64) {
01556           SDValue ArgValue1 = GetF64FormalArgument(VA, ArgLocs[++i],
01557                                                    Chain, DAG, dl);
01558           VA = ArgLocs[++i]; // skip ahead to next loc
01559           SDValue ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i],
01560                                                    Chain, DAG, dl);
01561           ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
01562           ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
01563                                  ArgValue, ArgValue1, DAG.getIntPtrConstant(0));
01564           ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
01565                                  ArgValue, ArgValue2, DAG.getIntPtrConstant(1));
01566         } else
01567           ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
01568 
01569       } else {
01570         TargetRegisterClass *RC;
01571 
01572         if (RegVT == MVT::f32)
01573           RC = ARM::SPRRegisterClass;
01574         else if (RegVT == MVT::f64)
01575           RC = ARM::DPRRegisterClass;
01576         else if (RegVT == MVT::v2f64)
01577           RC = ARM::QPRRegisterClass;
01578         else if (RegVT == MVT::i32)
01579           RC = (AFI->isThumb1OnlyFunction() ?
01580                 ARM::tGPRRegisterClass : ARM::GPRRegisterClass);
01581         else
01582           llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
01583 
01584         // Transform the arguments in physical registers into virtual ones.
01585         unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
01586         ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
01587       }
01588 
01589       // If this is an 8 or 16-bit value, it is really passed promoted
01590       // to 32 bits.  Insert an assert[sz]ext to capture this, then
01591       // truncate to the right size.
01592       switch (VA.getLocInfo()) {
01593       default: llvm_unreachable("Unknown loc info!");
01594       case CCValAssign::Full: break;
01595       case CCValAssign::BCvt:
01596         ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue);
01597         break;
01598       case CCValAssign::SExt:
01599         ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
01600                                DAG.getValueType(VA.getValVT()));
01601         ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
01602         break;
01603       case CCValAssign::ZExt:
01604         ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
01605                                DAG.getValueType(VA.getValVT()));
01606         ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
01607         break;
01608       }
01609 
01610       InVals.push_back(ArgValue);
01611 
01612     } else { // VA.isRegLoc()
01613 
01614       // sanity check
01615       assert(VA.isMemLoc());
01616       assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
01617 
01618       unsigned ArgSize = VA.getLocVT().getSizeInBits()/8;
01619       int FI = MFI->CreateFixedObject(ArgSize, VA.getLocMemOffset());
01620 
01621       // Create load nodes to retrieve arguments from the stack.
01622       SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
01623       InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN, NULL, 0));
01624     }
01625   }
01626 
01627   // varargs
01628   if (isVarArg) {
01629     static const unsigned GPRArgRegs[] = {
01630       ARM::R0, ARM::R1, ARM::R2, ARM::R3
01631     };
01632 
01633     unsigned NumGPRs = CCInfo.getFirstUnallocated
01634       (GPRArgRegs, sizeof(GPRArgRegs) / sizeof(GPRArgRegs[0]));
01635 
01636     unsigned Align = MF.getTarget().getFrameInfo()->getStackAlignment();
01637     unsigned VARegSize = (4 - NumGPRs) * 4;
01638     unsigned VARegSaveSize = (VARegSize + Align - 1) & ~(Align - 1);
01639     unsigned ArgOffset = 0;
01640     if (VARegSaveSize) {
01641       // If this function is vararg, store any remaining integer argument regs
01642       // to their spots on the stack so that they may be loaded by deferencing
01643       // the result of va_next.
01644       AFI->setVarArgsRegSaveSize(VARegSaveSize);
01645       ArgOffset = CCInfo.getNextStackOffset();
01646       VarArgsFrameIndex = MFI->CreateFixedObject(VARegSaveSize, ArgOffset +
01647                                                  VARegSaveSize - VARegSize);
01648       SDValue FIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
01649 
01650       SmallVector<SDValue, 4> MemOps;
01651       for (; NumGPRs < 4; ++NumGPRs) {
01652         TargetRegisterClass *RC;
01653         if (AFI->isThumb1OnlyFunction())
01654           RC = ARM::tGPRRegisterClass;
01655         else
01656           RC = ARM::GPRRegisterClass;
01657 
01658         unsigned VReg = MF.addLiveIn(GPRArgRegs[NumGPRs], RC);
01659         SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
01660         SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, NULL, 0);
01661         MemOps.push_back(Store);
01662         FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), FIN,
01663                           DAG.getConstant(4, getPointerTy()));
01664       }
01665       if (!MemOps.empty())
01666         Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
01667                             &MemOps[0], MemOps.size());
01668     } else
01669       // This will point to the next argument passed via stack.
01670       VarArgsFrameIndex = MFI->CreateFixedObject(4, ArgOffset);
01671   }
01672 
01673   return Chain;
01674 }
01675 
01676 /// isFloatingPointZero - Return true if this is +0.0.
01677 static bool isFloatingPointZero(SDValue Op) {
01678   if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
01679     return CFP->getValueAPF().isPosZero();
01680   else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
01681     // Maybe this has already been legalized into the constant pool?
01682     if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
01683       SDValue WrapperOp = Op.getOperand(1).getOperand(0);
01684       if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp))
01685         if (ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
01686           return CFP->getValueAPF().isPosZero();
01687     }
01688   }
01689   return false;
01690 }
01691 
01692 static bool isLegalCmpImmediate(unsigned C, bool isThumb1Only) {
01693   return ( isThumb1Only && (C & ~255U) == 0) ||
01694          (!isThumb1Only && ARM_AM::getSOImmVal(C) != -1);
01695 }
01696 
01697 /// Returns appropriate ARM CMP (cmp) and corresponding condition code for
01698 /// the given operands.
01699 static SDValue getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
01700                          SDValue &ARMCC, SelectionDAG &DAG, bool isThumb1Only,
01701                          DebugLoc dl) {
01702   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
01703     unsigned C = RHSC->getZExtValue();
01704     if (!isLegalCmpImmediate(C, isThumb1Only)) {
01705       // Constant does not fit, try adjusting it by one?
01706       switch (CC) {
01707       default: break;
01708       case ISD::SETLT:
01709       case ISD::SETGE:
01710         if (isLegalCmpImmediate(C-1, isThumb1Only)) {
01711           CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
01712           RHS = DAG.getConstant(C-1, MVT::i32);
01713         }
01714         break;
01715       case ISD::SETULT:
01716       case ISD::SETUGE:
01717         if (C > 0 && isLegalCmpImmediate(C-1, isThumb1Only)) {
01718           CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
01719           RHS = DAG.getConstant(C-1, MVT::i32);
01720         }
01721         break;
01722       case ISD::SETLE:
01723       case ISD::SETGT:
01724         if (isLegalCmpImmediate(C+1, isThumb1Only)) {
01725           CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
01726           RHS = DAG.getConstant(C+1, MVT::i32);
01727         }
01728         break;
01729       case ISD::SETULE:
01730       case ISD::SETUGT:
01731         if (C < 0xffffffff && isLegalCmpImmediate(C+1, isThumb1Only)) {
01732           CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
01733           RHS = DAG.getConstant(C+1, MVT::i32);
01734         }
01735         break;
01736       }
01737     }
01738   }
01739 
01740   ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
01741   ARMISD::NodeType CompareType;
01742   switch (CondCode) {
01743   default:
01744     CompareType = ARMISD::CMP;
01745     break;
01746   case ARMCC::EQ:
01747   case ARMCC::NE:
01748     // Uses only Z Flag
01749     CompareType = ARMISD::CMPZ;
01750     break;
01751   }
01752   ARMCC = DAG.getConstant(CondCode, MVT::i32);
01753   return DAG.getNode(CompareType, dl, MVT::Flag, LHS, RHS);
01754 }
01755 
01756 /// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
01757 static SDValue getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
01758                          DebugLoc dl) {
01759   SDValue Cmp;
01760   if (!isFloatingPointZero(RHS))
01761     Cmp = DAG.getNode(ARMISD::CMPFP, dl, MVT::Flag, LHS, RHS);
01762   else
01763     Cmp = DAG.getNode(ARMISD::CMPFPw0, dl, MVT::Flag, LHS);
01764   return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Flag, Cmp);
01765 }
01766 
01767 static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG,
01768                               const ARMSubtarget *ST) {
01769   EVT VT = Op.getValueType();
01770   SDValue LHS = Op.getOperand(0);
01771   SDValue RHS = Op.getOperand(1);
01772   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
01773   SDValue TrueVal = Op.getOperand(2);
01774   SDValue FalseVal = Op.getOperand(3);
01775   DebugLoc dl = Op.getDebugLoc();
01776 
01777   if (LHS.getValueType() == MVT::i32) {
01778     SDValue ARMCC;
01779     SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
01780     SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMCC, DAG, ST->isThumb1Only(), dl);
01781     return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMCC, CCR,Cmp);
01782   }
01783 
01784   ARMCC::CondCodes CondCode, CondCode2;
01785   if (FPCCToARMCC(CC, CondCode, CondCode2))
01786     std::swap(TrueVal, FalseVal);
01787 
01788   SDValue ARMCC = DAG.getConstant(CondCode, MVT::i32);
01789   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
01790   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
01791   SDValue Result = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
01792                                  ARMCC, CCR, Cmp);
01793   if (CondCode2 != ARMCC::AL) {
01794     SDValue ARMCC2 = DAG.getConstant(CondCode2, MVT::i32);
01795     // FIXME: Needs another CMP because flag can have but one use.
01796     SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
01797     Result = DAG.getNode(ARMISD::CMOV, dl, VT,
01798                          Result, TrueVal, ARMCC2, CCR, Cmp2);
01799   }
01800   return Result;
01801 }
01802 
01803 static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
01804                           const ARMSubtarget *ST) {
01805   SDValue  Chain = Op.getOperand(0);
01806   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
01807   SDValue    LHS = Op.getOperand(2);
01808   SDValue    RHS = Op.getOperand(3);
01809   SDValue   Dest = Op.getOperand(4);
01810   DebugLoc dl = Op.getDebugLoc();
01811 
01812   if (LHS.getValueType() == MVT::i32) {
01813     SDValue ARMCC;
01814     SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
01815     SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMCC, DAG, ST->isThumb1Only(), dl);
01816     return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
01817                        Chain, Dest, ARMCC, CCR,Cmp);
01818   }
01819 
01820   assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
01821   ARMCC::CondCodes CondCode, CondCode2;
01822   if (FPCCToARMCC(CC, CondCode, CondCode2))
01823     // Swap the LHS/RHS of the comparison if needed.
01824     std::swap(LHS, RHS);
01825 
01826   SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
01827   SDValue ARMCC = DAG.getConstant(CondCode, MVT::i32);
01828   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
01829   SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Flag);
01830   SDValue Ops[] = { Chain, Dest, ARMCC, CCR, Cmp };
01831   SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops, 5);
01832   if (CondCode2 != ARMCC::AL) {
01833     ARMCC = DAG.getConstant(CondCode2, MVT::i32);
01834     SDValue Ops[] = { Res, Dest, ARMCC, CCR, Res.getValue(1) };
01835     Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops, 5);
01836   }
01837   return Res;
01838 }
01839 
01840 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) {
01841   SDValue Chain = Op.getOperand(0);
01842   SDValue Table = Op.getOperand(1);
01843   SDValue Index = Op.getOperand(2);
01844   DebugLoc dl = Op.getDebugLoc();
01845 
01846   EVT PTy = getPointerTy();
01847   JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
01848   ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
01849   SDValue UId = DAG.getConstant(AFI->createJumpTableUId(), PTy);
01850   SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
01851   Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI, UId);
01852   Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, PTy));
01853   SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
01854   if (Subtarget->isThumb2()) {
01855     // Thumb2 uses a two-level jump. That is, it jumps into the jump table
01856     // which does another jump to the destination. This also makes it easier
01857     // to translate it to TBB / TBH later.
01858     // FIXME: This might not work if the function is extremely large.
01859     return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
01860                        Addr, Op.getOperand(2), JTI, UId);
01861   }
01862   if (getTargetMachine().getRelocationModel() == Reloc::PIC_) {
01863     Addr = DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr, NULL, 0);
01864     Chain = Addr.getValue(1);
01865     Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, Table);
01866     return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI, UId);
01867   } else {
01868     Addr = DAG.getLoad(PTy, dl, Chain, Addr, NULL, 0);
01869     Chain = Addr.getValue(1);
01870     return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI, UId);
01871   }
01872 }
01873 
01874 static SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
01875   DebugLoc dl = Op.getDebugLoc();
01876   unsigned Opc =
01877     Op.getOpcode() == ISD::FP_TO_SINT ? ARMISD::FTOSI : ARMISD::FTOUI;
01878   Op = DAG.getNode(Opc, dl, MVT::f32, Op.getOperand(0));
01879   return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op);
01880 }
01881 
01882 static SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
01883   EVT VT = Op.getValueType();
01884   DebugLoc dl = Op.getDebugLoc();
01885   unsigned Opc =
01886     Op.getOpcode() == ISD::SINT_TO_FP ? ARMISD::SITOF : ARMISD::UITOF;
01887 
01888   Op = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, Op.getOperand(0));
01889   return DAG.getNode(Opc, dl, VT, Op);
01890 }
01891 
01892 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
01893   // Implement fcopysign with a fabs and a conditional fneg.
01894   SDValue Tmp0 = Op.getOperand(0);
01895   SDValue Tmp1 = Op.getOperand(1);
01896   DebugLoc dl = Op.getDebugLoc();
01897   EVT VT = Op.getValueType();
01898   EVT SrcVT = Tmp1.getValueType();
01899   SDValue AbsVal = DAG.getNode(ISD::FABS, dl, VT, Tmp0);
01900   SDValue Cmp = getVFPCmp(Tmp1, DAG.getConstantFP(0.0, SrcVT), DAG, dl);
01901   SDValue ARMCC = DAG.getConstant(ARMCC::LT, MVT::i32);
01902   SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
01903   return DAG.getNode(ARMISD::CNEG, dl, VT, AbsVal, AbsVal, ARMCC, CCR, Cmp);
01904 }
01905 
01906 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) {
01907   MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
01908   MFI->setFrameAddressIsTaken(true);
01909   EVT VT = Op.getValueType();
01910   DebugLoc dl = Op.getDebugLoc();  // FIXME probably not meaningful
01911   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
01912   unsigned FrameReg = (Subtarget->isThumb() || Subtarget->isTargetDarwin())
01913     ? ARM::R7 : ARM::R11;
01914   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
01915   while (Depth--)
01916     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, NULL, 0);
01917   return FrameAddr;
01918 }
01919 
01920 SDValue
01921 ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
01922                                            SDValue Chain,
01923                                            SDValue Dst, SDValue Src,
01924                                            SDValue Size, unsigned Align,
01925                                            bool AlwaysInline,
01926                                          const Value *DstSV, uint64_t DstSVOff,
01927                                          const Value *SrcSV, uint64_t SrcSVOff){
01928   // Do repeated 4-byte loads and stores. To be improved.
01929   // This requires 4-byte alignment.
01930   if ((Align & 3) != 0)
01931     return SDValue();
01932   // This requires the copy size to be a constant, preferrably
01933   // within a subtarget-specific limit.
01934   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
01935   if (!ConstantSize)
01936     return SDValue();
01937   uint64_t SizeVal = ConstantSize->getZExtValue();
01938   if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold())
01939     return SDValue();
01940 
01941   unsigned BytesLeft = SizeVal & 3;
01942   unsigned NumMemOps = SizeVal >> 2;
01943   unsigned EmittedNumMemOps = 0;
01944   EVT VT = MVT::i32;
01945   unsigned VTSize = 4;
01946   unsigned i = 0;
01947   const unsigned MAX_LOADS_IN_LDM = 6;
01948   SDValue TFOps[MAX_LOADS_IN_LDM];
01949   SDValue Loads[MAX_LOADS_IN_LDM];
01950   uint64_t SrcOff = 0, DstOff = 0;
01951 
01952   // Emit up to MAX_LOADS_IN_LDM loads, then a TokenFactor barrier, then the
01953   // same number of stores.  The loads and stores will get combined into
01954   // ldm/stm later on.
01955   while (EmittedNumMemOps < NumMemOps) {
01956     for (i = 0;
01957          i < MAX_LOADS_IN_LDM && EmittedNumMemOps + i < NumMemOps; ++i) {
01958       Loads[i] = DAG.getLoad(VT, dl, Chain,
01959                              DAG.getNode(ISD::ADD, dl, MVT::i32, Src,
01960                                          DAG.getConstant(SrcOff, MVT::i32)),
01961                              SrcSV, SrcSVOff + SrcOff);
01962       TFOps[i] = Loads[i].getValue(1);
01963       SrcOff += VTSize;
01964     }
01965     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
01966 
01967     for (i = 0;
01968          i < MAX_LOADS_IN_LDM && EmittedNumMemOps + i < NumMemOps; ++i) {
01969       TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
01970                            DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,
01971                                        DAG.getConstant(DstOff, MVT::i32)),
01972                            DstSV, DstSVOff + DstOff);
01973       DstOff += VTSize;
01974     }
01975     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
01976 
01977     EmittedNumMemOps += i;
01978   }
01979 
01980   if (BytesLeft == 0)
01981     return Chain;
01982 
01983   // Issue loads / stores for the trailing (1 - 3) bytes.
01984   unsigned BytesLeftSave = BytesLeft;
01985   i = 0;
01986   while (BytesLeft) {
01987     if (BytesLeft >= 2) {
01988       VT = MVT::i16;
01989       VTSize = 2;
01990     } else {
01991       VT = MVT::i8;
01992       VTSize = 1;
01993     }
01994 
01995     Loads[i] = DAG.getLoad(VT, dl, Chain,
01996                            DAG.getNode(ISD::ADD, dl, MVT::i32, Src,
01997                                        DAG.getConstant(SrcOff, MVT::i32)),
01998                            SrcSV, SrcSVOff + SrcOff);
01999     TFOps[i] = Loads[i].getValue(1);
02000     ++i;
02001     SrcOff += VTSize;
02002     BytesLeft -= VTSize;
02003   }
02004   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
02005 
02006   i = 0;
02007   BytesLeft = BytesLeftSave;
02008   while (BytesLeft) {
02009     if (BytesLeft >= 2) {
02010       VT = MVT::i16;
02011       VTSize = 2;
02012     } else {
02013       VT = MVT::i8;
02014       VTSize = 1;
02015     }
02016 
02017     TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
02018                             DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,
02019                                         DAG.getConstant(DstOff, MVT::i32)),
02020                             DstSV, DstSVOff + DstOff);
02021     ++i;
02022     DstOff += VTSize;
02023     BytesLeft -= VTSize;
02024   }
02025   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &TFOps[0], i);
02026 }
02027 
02028 static SDValue ExpandBIT_CONVERT(SDNode *N, SelectionDAG &DAG) {
02029   SDValue Op = N->getOperand(0);
02030   DebugLoc dl = N->getDebugLoc();
02031   if (N->getValueType(0) == MVT::f64) {
02032     // Turn i64->f64 into FMDRR.
02033     SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
02034                              DAG.getConstant(0, MVT::i32));
02035     SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
02036                              DAG.getConstant(1, MVT::i32));
02037     return DAG.getNode(ARMISD::FMDRR, dl, MVT::f64, Lo, Hi);
02038   }
02039 
02040   // Turn f64->i64 into FMRRD.
02041   SDValue Cvt = DAG.getNode(ARMISD::FMRRD, dl,
02042                             DAG.getVTList(MVT::i32, MVT::i32), &Op, 1);
02043 
02044   // Merge the pieces into a single i64 value.
02045   return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
02046 }
02047 
02048 /// getZeroVector - Returns a vector of specified type with all zero elements.
02049 ///
02050 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) {
02051   assert(VT.isVector() && "Expected a vector type");
02052 
02053   // Zero vectors are used to represent vector negation and in those cases
02054   // will be implemented with the NEON VNEG instruction.  However, VNEG does
02055   // not support i64 elements, so sometimes the zero vectors will need to be
02056   // explicitly constructed.  For those cases, and potentially other uses in
02057   // the future, always build zero vectors as <4 x i32> or <2 x i32> bitcasted
02058   // to their dest type.  This ensures they get CSE'd.
02059   SDValue Vec;
02060   SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
02061   if (VT.getSizeInBits() == 64)
02062     Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst);
02063   else
02064     Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
02065 
02066   return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec);
02067 }
02068 
02069 /// getOnesVector - Returns a vector of specified type with all bits set.
02070 ///
02071 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) {
02072   assert(VT.isVector() && "Expected a vector type");
02073 
02074   // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest
02075   // type.  This ensures they get CSE'd.
02076   SDValue Vec;
02077   SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32);
02078   if (VT.getSizeInBits() == 64)
02079     Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst);
02080   else
02081     Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
02082 
02083   return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec);
02084 }
02085 
02086 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
02087                           const ARMSubtarget *ST) {
02088   EVT VT = N->getValueType(0);
02089   DebugLoc dl = N->getDebugLoc();
02090 
02091   // Lower vector shifts on NEON to use VSHL.
02092   if (VT.isVector()) {
02093     assert(ST->hasNEON() && "unexpected vector shift");
02094 
02095     // Left shifts translate directly to the vshiftu intrinsic.
02096     if (N->getOpcode() == ISD::SHL)
02097       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
02098                          DAG.getConstant(Intrinsic::arm_neon_vshiftu, MVT::i32),
02099                          N->getOperand(0), N->getOperand(1));
02100 
02101     assert((N->getOpcode() == ISD::SRA ||
02102             N->getOpcode() == ISD::SRL) && "unexpected vector shift opcode");
02103 
02104     // NEON uses the same intrinsics for both left and right shifts.  For
02105     // right shifts, the shift amounts are negative, so negate the vector of
02106     // shift amounts.
02107     EVT ShiftVT = N->getOperand(1).getValueType();
02108     SDValue NegatedCount = DAG.getNode(ISD::SUB, dl, ShiftVT,
02109                                        getZeroVector(ShiftVT, DAG, dl),
02110                                        N->getOperand(1));
02111     Intrinsic::ID vshiftInt = (N->getOpcode() == ISD::SRA ?
02112                                Intrinsic::arm_neon_vshifts :
02113                                Intrinsic::arm_neon_vshiftu);
02114     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
02115                        DAG.getConstant(vshiftInt, MVT::i32),
02116                        N->getOperand(0), NegatedCount);
02117   }
02118 
02119   // We can get here for a node like i32 = ISD::SHL i32, i64
02120   if (VT != MVT::i64)
02121     return SDValue();
02122 
02123   assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
02124          "Unknown shift to lower!");
02125 
02126   // We only lower SRA, SRL of 1 here, all others use generic lowering.
02127   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
02128       cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 1)
02129     return SDValue();
02130 
02131   // If we are in thumb mode, we don't have RRX.
02132   if (ST->isThumb1Only()) return SDValue();
02133 
02134   // Okay, we have a 64-bit SRA or SRL of 1.  Lower this to an RRX expr.
02135   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
02136                              DAG.getConstant(0, MVT::i32));
02137   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
02138                              DAG.getConstant(1, MVT::i32));
02139 
02140   // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and
02141   // captures the result into a carry flag.
02142   unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG;
02143   Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Flag), &Hi, 1);
02144 
02145   // The low part is an ARMISD::RRX operand, which shifts the carry in.
02146   Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
02147 
02148   // Merge the pieces into a single i64 value.
02149  return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
02150 }
02151 
02152 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) {
02153   SDValue TmpOp0, TmpOp1;
02154   bool Invert = false;
02155   bool Swap = false;
02156   unsigned Opc = 0;
02157 
02158   SDValue Op0 = Op.getOperand(0);
02159   SDValue Op1 = Op.getOperand(1);
02160   SDValue CC = Op.getOperand(2);
02161   EVT VT = Op.getValueType();
02162   ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
02163   DebugLoc dl = Op.getDebugLoc();
02164 
02165   if (Op.getOperand(1).getValueType().isFloatingPoint()) {
02166     switch (SetCCOpcode) {
02167     default: llvm_unreachable("Illegal FP comparison"); break;
02168     case ISD::SETUNE:
02169     case ISD::SETNE:  Invert = true; // Fallthrough
02170     case ISD::SETOEQ:
02171     case ISD::SETEQ:  Opc = ARMISD::VCEQ; break;
02172     case ISD::SETOLT:
02173     case ISD::SETLT: Swap = true; // Fallthrough
02174     case ISD::SETOGT:
02175     case ISD::SETGT:  Opc = ARMISD::VCGT; break;
02176     case ISD::SETOLE:
02177     case ISD::SETLE:  Swap = true; // Fallthrough
02178     case ISD::SETOGE:
02179     case ISD::SETGE: Opc = ARMISD::VCGE; break;
02180     case ISD::SETUGE: Swap = true; // Fallthrough
02181     case ISD::SETULE: Invert = true; Opc = ARMISD::VCGT; break;
02182     case ISD::SETUGT: Swap = true; // Fallthrough
02183     case ISD::SETULT: Invert = true; Opc = ARMISD::VCGE; break;
02184     case ISD::SETUEQ: Invert = true; // Fallthrough
02185     case ISD::SETONE:
02186       // Expand this to (OLT | OGT).
02187       TmpOp0 = Op0;
02188       TmpOp1 = Op1;
02189       Opc = ISD::OR;
02190       Op0 = DAG.getNode(ARMISD::VCGT, dl, VT, TmpOp1, TmpOp0);
02191       Op1 = DAG.getNode(ARMISD::VCGT, dl, VT, TmpOp0, TmpOp1);
02192       break;
02193     case ISD::SETUO: Invert = true; // Fallthrough
02194     case ISD::SETO:
02195       // Expand this to (OLT | OGE).
02196       TmpOp0 = Op0;
02197       TmpOp1 = Op1;
02198       Opc = ISD::OR;
02199       Op0 = DAG.getNode(ARMISD::VCGT, dl, VT, TmpOp1, TmpOp0);
02200       Op1 = DAG.getNode(ARMISD::VCGE, dl, VT, TmpOp0, TmpOp1);
02201       break;
02202     }
02203   } else {
02204     // Integer comparisons.
02205     switch (SetCCOpcode) {
02206     default: llvm_unreachable("Illegal integer comparison"); break;
02207     case ISD::SETNE:  Invert = true;
02208     case ISD::SETEQ:  Opc = ARMISD::VCEQ; break;
02209     case ISD::SETLT:  Swap = true;
02210     case ISD::SETGT:  Opc = ARMISD::VCGT; break;
02211     case ISD::SETLE:  Swap = true;
02212     case ISD::SETGE:  Opc = ARMISD::VCGE; break;
02213     case ISD::SETULT: Swap = true;
02214     case ISD::SETUGT: Opc = ARMISD::VCGTU; break;
02215     case ISD::SETULE: Swap = true;
02216     case ISD::SETUGE: Opc = ARMISD::VCGEU; break;
02217     }
02218 
02219     // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
02220     if (Opc == ARMISD::VCEQ) {
02221 
02222       SDValue AndOp;
02223       if (ISD::isBuildVectorAllZeros(Op1.getNode()))
02224         AndOp = Op0;
02225       else if (ISD::isBuildVectorAllZeros(Op0.getNode()))
02226         AndOp = Op1;
02227 
02228       // Ignore bitconvert.
02229       if (AndOp.getNode() && AndOp.getOpcode() == ISD::BIT_CONVERT)
02230         AndOp = AndOp.getOperand(0);
02231 
02232       if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
02233         Opc = ARMISD::VTST;
02234         Op0 = DAG.getNode(ISD::BIT_CONVERT, dl, VT, AndOp.getOperand(0));
02235         Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, VT, AndOp.getOperand(1));
02236         Invert = !Invert;
02237       }
02238     }
02239   }
02240 
02241   if (Swap)
02242     std::swap(Op0, Op1);
02243 
02244   SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
02245 
02246   if (Invert)
02247     Result = DAG.getNOT(dl, Result, VT);
02248 
02249   return Result;
02250 }
02251 
02252 /// isVMOVSplat - Check if the specified splat value corresponds to an immediate
02253 /// VMOV instruction, and if so, return the constant being splatted.
02254 static SDValue isVMOVSplat(uint64_t SplatBits, uint64_t SplatUndef,
02255                            unsigned SplatBitSize, SelectionDAG &DAG) {
02256   switch (SplatBitSize) {
02257   case 8:
02258     // Any 1-byte value is OK.
02259     assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
02260     return DAG.getTargetConstant(SplatBits, MVT::i8);
02261 
02262   case 16:
02263     // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
02264     if ((SplatBits & ~0xff) == 0 ||
02265         (SplatBits & ~0xff00) == 0)
02266       return DAG.getTargetConstant(SplatBits, MVT::i16);
02267     break;
02268 
02269   case 32:
02270     // NEON's 32-bit VMOV supports splat values where:
02271     // * only one byte is nonzero, or
02272     // * the least significant byte is 0xff and the second byte is nonzero, or
02273     // * the least significant 2 bytes are 0xff and the third is nonzero.
02274     if ((SplatBits & ~0xff) == 0 ||
02275         (SplatBits & ~0xff00) == 0 ||
02276         (SplatBits & ~0xff0000) == 0 ||
02277         (SplatBits & ~0xff000000) == 0)
02278       return DAG.getTargetConstant(SplatBits, MVT::i32);
02279 
02280     if ((SplatBits & ~0xffff) == 0 &&
02281         ((SplatBits | SplatUndef) & 0xff) == 0xff)
02282       return DAG.getTargetConstant(SplatBits | 0xff, MVT::i32);
02283 
02284     if ((SplatBits & ~0xffffff) == 0 &&
02285         ((SplatBits | SplatUndef) & 0xffff) == 0xffff)
02286       return DAG.getTargetConstant(SplatBits | 0xffff, MVT::i32);
02287 
02288     // Note: there are a few 32-bit splat values (specifically: 00ffff00,
02289     // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
02290     // VMOV.I32.  A (very) minor optimization would be to replicate the value
02291     // and fall through here to test for a valid 64-bit splat.  But, then the
02292     // caller would also need to check and handle the change in size.
02293     break;
02294 
02295   case 64: {
02296     // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
02297     uint64_t BitMask = 0xff;
02298     uint64_t Val = 0;
02299     for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
02300       if (((SplatBits | SplatUndef) & BitMask) == BitMask)
02301         Val |= BitMask;
02302       else if ((SplatBits & BitMask) != 0)
02303         return SDValue();
02304       BitMask <<= 8;
02305     }
02306     return DAG.getTargetConstant(Val, MVT::i64);
02307   }
02308 
02309   default:
02310     llvm_unreachable("unexpected size for isVMOVSplat");
02311     break;
02312   }
02313 
02314   return SDValue();
02315 }
02316 
02317 /// getVMOVImm - If this is a build_vector of constants which can be
02318 /// formed by using a VMOV instruction of the specified element size,
02319 /// return the constant being splatted.  The ByteSize field indicates the
02320 /// number of bytes of each element [1248].
02321 SDValue ARM::getVMOVImm(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
02322   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N);
02323   APInt SplatBits, SplatUndef;
02324   unsigned SplatBitSize;
02325   bool HasAnyUndefs;
02326   if (! BVN || ! BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
02327                                       HasAnyUndefs, ByteSize * 8))
02328     return SDValue();
02329 
02330   if (SplatBitSize > ByteSize * 8)
02331     return SDValue();
02332 
02333   return isVMOVSplat(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
02334                      SplatBitSize, DAG);
02335 }
02336 
02337 static bool isVEXTMask(const SmallVectorImpl<int> &M, EVT VT,
02338                        bool &ReverseVEXT, unsigned &Imm) {
02339   unsigned NumElts = VT.getVectorNumElements();
02340   ReverseVEXT = false;
02341   Imm = M[0];
02342 
02343   // If this is a VEXT shuffle, the immediate value is the index of the first
02344   // element.  The other shuffle indices must be the successive elements after
02345   // the first one.
02346   unsigned ExpectedElt = Imm;
02347   for (unsigned i = 1; i < NumElts; ++i) {
02348     // Increment the expected index.  If it wraps around, it may still be
02349     // a VEXT but the source vectors must be swapped.
02350     ExpectedElt += 1;
02351     if (ExpectedElt == NumElts * 2) {
02352       ExpectedElt = 0;
02353       ReverseVEXT = true;
02354     }
02355 
02356     if (ExpectedElt != static_cast<unsigned>(M[i]))
02357       return false;
02358   }
02359 
02360   // Adjust the index value if the source operands will be swapped.
02361   if (ReverseVEXT)
02362     Imm -= NumElts;
02363 
02364   return true;
02365 }
02366 
02367 /// isVREVMask - Check if a vector shuffle corresponds to a VREV
02368 /// instruction with the specified blocksize.  (The order of the elements
02369 /// within each block of the vector is reversed.)
02370 static bool isVREVMask(const SmallVectorImpl<int> &M, EVT VT,
02371                        unsigned BlockSize) {
02372   assert((BlockSize==16 || BlockSize==32 || BlockSize==64) &&
02373          "Only possible block sizes for VREV are: 16, 32, 64");
02374 
02375   unsigned NumElts = VT.getVectorNumElements();
02376   unsigned EltSz = VT.getVectorElementType().getSizeInBits();
02377   unsigned BlockElts = M[0] + 1;
02378 
02379   if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
02380     return false;
02381 
02382   for (unsigned i = 0; i < NumElts; ++i) {
02383     if ((unsigned) M[i] !=
02384         (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts))
02385       return false;
02386   }
02387 
02388   return true;
02389 }
02390 
02391 static bool isVTRNMask(const SmallVectorImpl<int> &M, EVT VT,
02392                        unsigned &WhichResult) {
02393   unsigned NumElts = VT.getVectorNumElements();
02394   WhichResult = (M[0] == 0 ? 0 : 1);
02395   for (unsigned i = 0; i < NumElts; i += 2) {
02396     if ((unsigned) M[i] != i + WhichResult ||
02397         (unsigned) M[i+1] != i + NumElts + WhichResult)
02398       return false;
02399   }
02400   return true;
02401 }
02402 
02403 static bool isVUZPMask(const SmallVectorImpl<int> &M, EVT VT,
02404                        unsigned &WhichResult) {
02405   unsigned NumElts = VT.getVectorNumElements();
02406   WhichResult = (M[0] == 0 ? 0 : 1);
02407   for (unsigned i = 0; i != NumElts; ++i) {
02408     if ((unsigned) M[i] != 2 * i + WhichResult)
02409       return false;
02410   }
02411 
02412   // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
02413   if (VT.is64BitVector() && VT.getVectorElementType().getSizeInBits() == 32)
02414     return false;
02415 
02416   return true;
02417 }
02418 
02419 static bool isVZIPMask(const SmallVectorImpl<int> &M, EVT VT,
02420                        unsigned &WhichResult) {
02421   unsigned NumElts = VT.getVectorNumElements();
02422   WhichResult = (M[0] == 0 ? 0 : 1);
02423   unsigned Idx = WhichResult * NumElts / 2;
02424   for (unsigned i = 0; i != NumElts; i += 2) {
02425     if ((unsigned) M[i] != Idx ||
02426         (unsigned) M[i+1] != Idx + NumElts)
02427       return false;
02428     Idx += 1;
02429   }
02430 
02431   // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
02432   if (VT.is64BitVector() && VT.getVectorElementType().getSizeInBits() == 32)
02433     return false;
02434 
02435   return true;
02436 }
02437 
02438 static SDValue BuildSplat(SDValue Val, EVT VT, SelectionDAG &DAG, DebugLoc dl) {
02439   // Canonicalize all-zeros and all-ones vectors.
02440   ConstantSDNode *ConstVal = cast<ConstantSDNode>(Val.getNode());
02441   if (ConstVal->isNullValue())
02442     return getZeroVector(VT, DAG, dl);
02443   if (ConstVal->isAllOnesValue())
02444     return getOnesVector(VT, DAG, dl);
02445 
02446   EVT CanonicalVT;
02447   if (VT.is64BitVector()) {
02448     switch (Val.getValueType().getSizeInBits()) {
02449     case 8:  CanonicalVT = MVT::v8i8; break;
02450     case 16: CanonicalVT = MVT::v4i16; break;
02451     case 32: CanonicalVT = MVT::v2i32; break;
02452     case 64: CanonicalVT = MVT::v1i64; break;
02453     default: llvm_unreachable("unexpected splat element type"); break;
02454     }
02455   } else {
02456     assert(VT.is128BitVector() && "unknown splat vector size");
02457     switch (Val.getValueType().getSizeInBits()) {
02458     case 8:  CanonicalVT = MVT::v16i8; break;
02459     case 16: CanonicalVT = MVT::v8i16; break;
02460     case 32: CanonicalVT = MVT::v4i32; break;
02461     case 64: CanonicalVT = MVT::v2i64; break;
02462     default: llvm_unreachable("unexpected splat element type"); break;
02463     }
02464   }
02465 
02466   // Build a canonical splat for this value.
02467   SmallVector<SDValue, 8> Ops;
02468   Ops.assign(CanonicalVT.getVectorNumElements(), Val);
02469   SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, CanonicalVT, &Ops[0],
02470                             Ops.size());
02471   return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Res);
02472 }
02473 
02474 // If this is a case we can't handle, return null and let the default
02475 // expansion code take care of it.
02476 static SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) {
02477   BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
02478   DebugLoc dl = Op.getDebugLoc();
02479   EVT VT = Op.getValueType();
02480 
02481   APInt SplatBits, SplatUndef;
02482   unsigned SplatBitSize;
02483   bool HasAnyUndefs;
02484   if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
02485     SDValue Val = isVMOVSplat(SplatBits.getZExtValue(),
02486                               SplatUndef.getZExtValue(), SplatBitSize, DAG);
02487     if (Val.getNode())
02488       return BuildSplat(Val, VT, DAG, dl);
02489   }
02490 
02491   // If there are only 2 elements in a 128-bit vector, insert them into an
02492   // undef vector.  This handles the common case for 128-bit vector argument
02493   // passing, where the insertions should be translated to subreg accesses
02494   // with no real instructions.
02495   if (VT.is128BitVector() && Op.getNumOperands() == 2) {
02496     SDValue Val = DAG.getUNDEF(VT);
02497     SDValue Op0 = Op.getOperand(0);
02498     SDValue Op1 = Op.getOperand(1);
02499     if (Op0.getOpcode() != ISD::UNDEF)
02500       Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, Op0,
02501                         DAG.getIntPtrConstant(0));
02502     if (Op1.getOpcode() != ISD::UNDEF)
02503       Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, Op1,
02504                         DAG.getIntPtrConstant(1));
02505     return Val;
02506   }
02507 
02508   return SDValue();
02509 }
02510 
02511 /// isShuffleMaskLegal - Targets can use this to indicate that they only
02512 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
02513 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
02514 /// are assumed to be legal.
02515 bool
02516 ARMTargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
02517                                       EVT VT) const {
02518   if (VT.getVectorNumElements() == 4 &&
02519       (VT.is128BitVector() || VT.is64BitVector())) {
02520     unsigned PFIndexes[4];
02521     for (unsigned i = 0; i != 4; ++i) {
02522       if (M[i] < 0)
02523         PFIndexes[i] = 8;
02524       else
02525         PFIndexes[i] = M[i];
02526     }
02527 
02528     // Compute the index in the perfect shuffle table.
02529     unsigned PFTableIndex =
02530       PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
02531     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
02532     unsigned Cost = (PFEntry >> 30);
02533 
02534     if (Cost <= 4)
02535       return true;
02536   }
02537 
02538   bool ReverseVEXT;
02539   unsigned Imm, WhichResult;
02540 
02541   return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
02542           isVREVMask(M, VT, 64) ||
02543           isVREVMask(M, VT, 32) ||
02544           isVREVMask(M, VT, 16) ||
02545           isVEXTMask(M, VT, ReverseVEXT, Imm) ||
02546           isVTRNMask(M, VT, WhichResult) ||
02547           isVUZPMask(M, VT, WhichResult) ||
02548           isVZIPMask(M, VT, WhichResult));
02549 }
02550 
02551 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
02552 /// the specified operations to build the shuffle.
02553 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
02554                                       SDValue RHS, SelectionDAG &DAG,
02555                                       DebugLoc dl) {
02556   unsigned OpNum = (PFEntry >> 26) & 0x0F;
02557   unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
02558   unsigned RHSID = (PFEntry >>  0) & ((1 << 13)-1);
02559 
02560   enum {
02561     OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
02562     OP_VREV,
02563     OP_VDUP0,
02564     OP_VDUP1,
02565     OP_VDUP2,
02566     OP_VDUP3,
02567     OP_VEXT1,
02568     OP_VEXT2,
02569     OP_VEXT3,
02570     OP_VUZPL, // VUZP, left result
02571     OP_VUZPR, // VUZP, right result
02572     OP_VZIPL, // VZIP, left result
02573     OP_VZIPR, // VZIP, right result
02574     OP_VTRNL, // VTRN, left result
02575     OP_VTRNR  // VTRN, right result
02576   };
02577 
02578   if (OpNum == OP_COPY) {
02579     if (LHSID == (1*9+2)*9+3) return LHS;
02580     assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
02581     return RHS;
02582   }
02583 
02584   SDValue OpLHS, OpRHS;
02585   OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
02586   OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
02587   EVT VT = OpLHS.getValueType();
02588 
02589   switch (OpNum) {
02590   default: llvm_unreachable("Unknown shuffle opcode!");
02591   case OP_VREV:
02592     return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
02593   case OP_VDUP0:
02594   case OP_VDUP1:
02595   case OP_VDUP2:
02596   case OP_VDUP3:
02597     return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
02598                        OpLHS, DAG.getConstant(OpNum-OP_VDUP0, MVT::i32));
02599   case OP_VEXT1:
02600   case OP_VEXT2:
02601   case OP_VEXT3:
02602     return DAG.getNode(ARMISD::VEXT, dl, VT,
02603                        OpLHS, OpRHS,
02604                        DAG.getConstant(OpNum-OP_VEXT1+1, MVT::i32));
02605   case OP_VUZPL:
02606   case OP_VUZPR:
02607     return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
02608                        OpLHS, OpRHS).getValue(OpNum-OP_VUZPL);
02609   case OP_VZIPL:
02610   case OP_VZIPR:
02611     return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
02612                        OpLHS, OpRHS).getValue(OpNum-OP_VZIPL);
02613   case OP_VTRNL:
02614   case OP_VTRNR:
02615     return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
02616                        OpLHS, OpRHS).getValue(OpNum-OP_VTRNL);
02617   }
02618 }
02619 
02620 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
02621   SDValue V1 = Op.getOperand(0);
02622   SDValue V2 = Op.getOperand(1);
02623   DebugLoc dl = Op.getDebugLoc();
02624   EVT VT = Op.getValueType();
02625   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
02626   SmallVector<int, 8> ShuffleMask;
02627 
02628   // Convert shuffles that are directly supported on NEON to target-specific
02629   // DAG nodes, instead of keeping them as shuffles and matching them again
02630   // during code selection.  This is more efficient and avoids the possibility
02631   // of inconsistencies between legalization and selection.
02632   // FIXME: floating-point vectors should be canonicalized to integer vectors
02633   // of the same time so that they get CSEd properly.
02634   SVN->getMask(ShuffleMask);
02635 
02636   if (ShuffleVectorSDNode::isSplatMask(&ShuffleMask[0], VT)) {
02637     int Lane = SVN->getSplatIndex();
02638     if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
02639       return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
02640     }
02641     return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
02642                        DAG.getConstant(Lane, MVT::i32));
02643   }
02644 
02645   bool ReverseVEXT;
02646   unsigned Imm;
02647   if (isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
02648     if (ReverseVEXT)
02649       std::swap(V1, V2);
02650     return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
02651                        DAG.getConstant(Imm, MVT::i32));
02652   }
02653 
02654   if (isVREVMask(ShuffleMask, VT, 64))
02655     return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
02656   if (isVREVMask(ShuffleMask, VT, 32))
02657     return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
02658   if (isVREVMask(ShuffleMask, VT, 16))
02659     return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
02660 
02661   // Check for Neon shuffles that modify both input vectors in place.
02662   // If both results are used, i.e., if there are two shuffles with the same
02663   // source operands and with masks corresponding to both results of one of
02664   // these operations, DAG memoization will ensure that a single node is
02665   // used for both shuffles.
02666   unsigned WhichResult;
02667   if (isVTRNMask(ShuffleMask, VT, WhichResult))
02668     return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
02669                        V1, V2).getValue(WhichResult);
02670   if (isVUZPMask(ShuffleMask, VT, WhichResult))
02671     return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
02672                        V1, V2).getValue(WhichResult);
02673   if (isVZIPMask(ShuffleMask, VT, WhichResult))
02674     return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
02675                        V1, V2).getValue(WhichResult);
02676 
02677   // If the shuffle is not directly supported and it has 4 elements, use
02678   // the PerfectShuffle-generated table to synthesize it from other shuffles.
02679   if (VT.getVectorNumElements() == 4 &&
02680       (VT.is128BitVector() || VT.is64BitVector())) {
02681     unsigned PFIndexes[4];
02682     for (unsigned i = 0; i != 4; ++i) {
02683       if (ShuffleMask[i] < 0)
02684         PFIndexes[i] = 8;
02685       else
02686         PFIndexes[i] = ShuffleMask[i];
02687     }
02688 
02689     // Compute the index in the perfect shuffle table.
02690     unsigned PFTableIndex =
02691       PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
02692 
02693     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
02694     unsigned Cost = (PFEntry >> 30);
02695 
02696     if (Cost <= 4)
02697       return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
02698   }
02699 
02700   return SDValue();
02701 }
02702 
02703 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
02704   return Op;
02705 }
02706 
02707 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
02708   EVT VT = Op.getValueType();
02709   DebugLoc dl = Op.getDebugLoc();
02710   assert((VT == MVT::i8 || VT == MVT::i16) &&
02711          "unexpected type for custom-lowering vector extract");
02712   SDValue Vec = Op.getOperand(0);
02713   SDValue Lane = Op.getOperand(1);
02714   Op = DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
02715   Op = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Op, DAG.getValueType(VT));
02716   return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
02717 }
02718 
02719 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
02720   // The only time a CONCAT_VECTORS operation can have legal types is when
02721   // two 64-bit vectors are concatenated to a 128-bit vector.
02722   assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
02723          "unexpected CONCAT_VECTORS");
02724   DebugLoc dl = Op.getDebugLoc();
02725   SDValue Val = DAG.getUNDEF(MVT::v2f64);
02726   SDValue Op0 = Op.getOperand(0);
02727   SDValue Op1 = Op.getOperand(1);
02728   if (Op0.getOpcode() != ISD::UNDEF)
02729     Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
02730                       DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f64, Op0),
02731                       DAG.getIntPtrConstant(0));
02732   if (Op1.getOpcode() != ISD::UNDEF)
02733     Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
02734                       DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f64, Op1),
02735                       DAG.getIntPtrConstant(1));
02736   return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(), Val);
02737 }
02738 
02739 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
02740   switch (Op.getOpcode()) {
02741   default: llvm_unreachable("Don't know how to custom lower this!");
02742   case ISD::ConstantPool:  return LowerConstantPool(Op, DAG);
02743   case ISD::GlobalAddress:
02744     return Subtarget->isTargetDarwin() ? LowerGlobalAddressDarwin(Op, DAG) :
02745       LowerGlobalAddressELF(Op, DAG);
02746   case ISD::GlobalTLSAddress:   return LowerGlobalTLSAddress(Op, DAG);
02747   case ISD::SELECT_CC:     return LowerSELECT_CC(Op, DAG, Subtarget);
02748   case ISD::BR_CC:         return LowerBR_CC(Op, DAG, Subtarget);
02749   case ISD::BR_JT:         return LowerBR_JT(Op, DAG);
02750   case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
02751   case ISD::VASTART:       return LowerVASTART(Op, DAG, VarArgsFrameIndex);
02752   case ISD::SINT_TO_FP:
02753   case ISD::UINT_TO_FP:    return LowerINT_TO_FP(Op, DAG);
02754   case ISD::FP_TO_SINT:
02755   case ISD::FP_TO_UINT:    return LowerFP_TO_INT(Op, DAG);
02756   case ISD::FCOPYSIGN:     return LowerFCOPYSIGN(Op, DAG);
02757   case ISD::RETURNADDR:    break;
02758   case ISD::FRAMEADDR:     return LowerFRAMEADDR(Op, DAG);
02759   case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
02760   case ISD::INTRINSIC_VOID:
02761   case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG);
02762   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
02763   case ISD::BIT_CONVERT:   return ExpandBIT_CONVERT(Op.getNode(), DAG);
02764   case ISD::SHL:
02765   case ISD::SRL:
02766   case ISD::SRA:           return LowerShift(Op.getNode(), DAG, Subtarget);
02767   case ISD::VSETCC:        return LowerVSETCC(Op, DAG);
02768   case ISD::BUILD_VECTOR:  return LowerBUILD_VECTOR(Op, DAG);
02769   case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
02770   case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
02771   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
02772   case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
02773   }
02774   return SDValue();
02775 }
02776 
02777 /// ReplaceNodeResults - Replace the results of node with an illegal result
02778 /// type with new values built out of custom code.
02779 void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
02780                                            SmallVectorImpl<SDValue>&Results,
02781                                            SelectionDAG &DAG) {
02782   switch (N->getOpcode()) {
02783   default:
02784     llvm_unreachable("Don't know how to custom expand this!");
02785     return;
02786   case ISD::BIT_CONVERT:
02787     Results.push_back(ExpandBIT_CONVERT(N, DAG));
02788     return;
02789   case ISD::SRL:
02790   case ISD::SRA: {
02791     SDValue Res = LowerShift(N, DAG, Subtarget);
02792     if (Res.getNode())
02793       Results.push_back(Res);
02794     return;
02795   }
02796   }
02797 }
02798 
02799 //===----------------------------------------------------------------------===//
02800 //                           ARM Scheduler Hooks
02801 //===----------------------------------------------------------------------===//
02802 
02803 MachineBasicBlock *
02804 ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
02805                                                MachineBasicBlock *BB) const {
02806   const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
02807   DebugLoc dl = MI->getDebugLoc();
02808   switch (MI->getOpcode()) {
02809   default:
02810     llvm_unreachable("Unexpected instr type to insert");
02811   case ARM::tMOVCCr_pseudo: {
02812     // To "insert" a SELECT_CC instruction, we actually have to insert the
02813     // diamond control-flow pattern.  The incoming instruction knows the
02814     // destination vreg to set, the condition code register to branch on, the
02815     // true/false values to select between, and a branch opcode to use.
02816     const BasicBlock *LLVM_BB = BB->getBasicBlock();
02817     MachineFunction::iterator It = BB;
02818     ++It;
02819 
02820     //  thisMBB:
02821     //  ...
02822     //   TrueVal = ...
02823     //   cmpTY ccX, r1, r2
02824     //   bCC copy1MBB
02825     //   fallthrough --> copy0MBB
02826     MachineBasicBlock *thisMBB  = BB;
02827     MachineFunction *F = BB->getParent();
02828     MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
02829     MachineBasicBlock *sinkMBB  = F->CreateMachineBasicBlock(LLVM_BB);
02830     BuildMI(BB, dl, TII->get(ARM::tBcc)).addMBB(sinkMBB)
02831       .addImm(MI->getOperand(3).getImm()).addReg(MI->getOperand(4).getReg());
02832     F->insert(It, copy0MBB);
02833     F->insert(It, sinkMBB);
02834     // Update machine-CFG edges by first adding all successors of the current
02835     // block to the new block which will contain the Phi node for the select.
02836     for(MachineBasicBlock::succ_iterator i = BB->succ_begin(),
02837         e = BB->succ_end(); i != e; ++i)
02838       sinkMBB->addSuccessor(*i);
02839     // Next, remove all successors of the current block, and add the true
02840     // and fallthrough blocks as its successors.
02841     while(!BB->succ_empty())
02842       BB->removeSuccessor(BB->succ_begin());
02843     BB->addSuccessor(copy0MBB);
02844     BB->addSuccessor(sinkMBB);
02845 
02846     //  copy0MBB:
02847     //   %FalseValue = ...
02848     //   # fallthrough to sinkMBB
02849     BB = copy0MBB;
02850 
02851     // Update machine-CFG edges
02852     BB->addSuccessor(sinkMBB);
02853 
02854     //  sinkMBB:
02855     //   %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
02856     //  ...
02857     BB = sinkMBB;
02858     BuildMI(BB, dl, TII->get(ARM::PHI), MI->getOperand(0).getReg())
02859       .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
02860       .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
02861 
02862     F->DeleteMachineInstr(MI);   // The pseudo instruction is gone now.
02863     return BB;
02864   }
02865 
02866   case ARM::tANDsp:
02867   case ARM::tADDspr_:
02868   case ARM::tSUBspi_:
02869   case ARM::t2SUBrSPi_:
02870   case ARM::t2SUBrSPi12_:
02871   case ARM::t2SUBrSPs_: {
02872     MachineFunction *MF = BB->getParent();
02873     unsigned DstReg = MI->getOperand(0).getReg();
02874     unsigned SrcReg = MI->getOperand(1).getReg();
02875     bool DstIsDead = MI->getOperand(0).isDead();
02876     bool SrcIsKill = MI->getOperand(1).isKill();
02877 
02878     if (SrcReg != ARM::SP) {
02879       // Copy the source to SP from virtual register.
02880       const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(SrcReg);
02881       unsigned CopyOpc = (RC == ARM::tGPRRegisterClass)
02882         ? ARM::tMOVtgpr2gpr : ARM::tMOVgpr2gpr;
02883       BuildMI(BB, dl, TII->get(CopyOpc), ARM::SP)
02884         .addReg(SrcReg, getKillRegState(SrcIsKill));
02885     }
02886 
02887     unsigned OpOpc = 0;
02888     bool NeedPred = false, NeedCC = false, NeedOp3 = false;
02889     switch (MI->getOpcode()) {
02890     default:
02891       llvm_unreachable("Unexpected pseudo instruction!");
02892     case ARM::tANDsp:
02893       OpOpc = ARM::tAND;
02894       NeedPred = true;
02895       break;
02896     case ARM::tADDspr_:
02897       OpOpc = ARM::tADDspr;
02898       break;
02899     case ARM::tSUBspi_:
02900       OpOpc = ARM::tSUBspi;
02901       break;
02902     case ARM::t2SUBrSPi_:
02903       OpOpc = ARM::t2SUBrSPi;
02904       NeedPred = true; NeedCC = true;
02905       break;
02906     case ARM::t2SUBrSPi12_:
02907       OpOpc = ARM::t2SUBrSPi12;
02908       NeedPred = true;
02909       break;
02910     case ARM::t2SUBrSPs_:
02911       OpOpc = ARM::t2SUBrSPs;
02912       NeedPred = true; NeedCC = true; NeedOp3 = true;
02913       break;
02914     }
02915     MachineInstrBuilder MIB = BuildMI(BB, dl, TII->get(OpOpc), ARM::SP);
02916     if (OpOpc == ARM::tAND)
02917       AddDefaultT1CC(MIB);
02918     MIB.addReg(ARM::SP);
02919     MIB.addOperand(MI->getOperand(2));
02920     if (NeedOp3)
02921       MIB.addOperand(MI->getOperand(3));
02922     if (NeedPred)
02923       AddDefaultPred(MIB);
02924     if (NeedCC)
02925       AddDefaultCC(MIB);
02926 
02927     // Copy the result from SP to virtual register.
02928     const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(DstReg);
02929     unsigned CopyOpc = (RC == ARM::tGPRRegisterClass)
02930       ? ARM::tMOVgpr2tgpr : ARM::tMOVgpr2gpr;
02931     BuildMI(BB, dl, TII->get(CopyOpc))
02932       .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstIsDead))
02933       .addReg(ARM::SP);
02934     MF->DeleteMachineInstr(MI);   // The pseudo instruction is gone now.
02935     return BB;
02936   }
02937   }
02938 }
02939 
02940 //===----------------------------------------------------------------------===//
02941 //                           ARM Optimization Hooks
02942 //===----------------------------------------------------------------------===//
02943 
02944 static
02945 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
02946                             TargetLowering::DAGCombinerInfo &DCI) {
02947   SelectionDAG &DAG = DCI.DAG;
02948   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02949   EVT VT = N->getValueType(0);
02950   unsigned Opc = N->getOpcode();
02951   bool isSlctCC = Slct.getOpcode() == ISD::SELECT_CC;
02952   SDValue LHS = isSlctCC ? Slct.getOperand(2) : Slct.getOperand(1);
02953   SDValue RHS = isSlctCC ? Slct.getOperand(3) : Slct.getOperand(2);
02954   ISD::CondCode CC = ISD::SETCC_INVALID;
02955 
02956   if (isSlctCC) {
02957     CC = cast<CondCodeSDNode>(Slct.getOperand(4))->get();
02958   } else {
02959     SDValue CCOp = Slct.getOperand(0);
02960     if (CCOp.getOpcode() == ISD::SETCC)
02961       CC = cast<CondCodeSDNode>(CCOp.getOperand(2))->get();
02962   }
02963 
02964   bool DoXform = false;
02965   bool InvCC = false;
02966   assert ((Opc == ISD::ADD || (Opc == ISD::SUB && Slct == N->getOperand(1))) &&
02967           "Bad input!");
02968 
02969   if (LHS.getOpcode() == ISD::Constant &&
02970       cast<ConstantSDNode>(LHS)->isNullValue()) {
02971     DoXform = true;
02972   } else if (CC != ISD::SETCC_INVALID &&
02973              RHS.getOpcode() == ISD::Constant &&
02974              cast<ConstantSDNode>(RHS)->isNullValue()) {
02975     std::swap(LHS, RHS);
02976     SDValue Op0 = Slct.getOperand(0);
02977     EVT OpVT = isSlctCC ? Op0.getValueType() :
02978                           Op0.getOperand(0).getValueType();
02979     bool isInt = OpVT.isInteger();
02980     CC = ISD::getSetCCInverse(CC, isInt);
02981 
02982     if (!TLI.isCondCodeLegal(CC, OpVT))
02983       return SDValue();         // Inverse operator isn't legal.
02984 
02985     DoXform = true;
02986     InvCC = true;
02987   }
02988 
02989   if (DoXform) {
02990     SDValue Result = DAG.getNode(Opc, RHS.getDebugLoc(), VT, OtherOp, RHS);
02991     if (isSlctCC)
02992       return DAG.getSelectCC(N->getDebugLoc(), OtherOp, Result,
02993                              Slct.getOperand(0), Slct.getOperand(1), CC);
02994     SDValue CCOp = Slct.getOperand(0);
02995     if (InvCC)
02996       CCOp = DAG.getSetCC(Slct.getDebugLoc(), CCOp.getValueType(),
02997                           CCOp.getOperand(0), CCOp.getOperand(1), CC);
02998     return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT,
02999                        CCOp, OtherOp, Result);
03000   }
03001   return SDValue();
03002 }
03003 
03004 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
03005 static SDValue PerformADDCombine(SDNode *N,
03006                                  TargetLowering::DAGCombinerInfo &DCI) {
03007   // added by evan in r37685 with no testcase.
03008   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
03009 
03010   // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
03011   if (N0.getOpcode() == ISD::SELECT && N0.getNode()->hasOneUse()) {
03012     SDValue Result = combineSelectAndUse(N, N0, N1, DCI);
03013     if (Result.getNode()) return Result;
03014   }
03015   if (N1.getOpcode() == ISD::SELECT && N1.getNode()->hasOneUse()) {
03016     SDValue Result = combineSelectAndUse(N, N1, N0, DCI);
03017     if (Result.getNode()) return Result;
03018   }
03019 
03020   return SDValue();
03021 }
03022 
03023 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
03024 static SDValue PerformSUBCombine(SDNode *N,
03025                                  TargetLowering::DAGCombinerInfo &DCI) {
03026   // added by evan in r37685 with no testcase.
03027   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
03028 
03029   // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
03030   if (N1.getOpcode() == ISD::SELECT && N1.getNode()->hasOneUse()) {
03031     SDValue Result = combineSelectAndUse(N, N1, N0, DCI);
03032     if (Result.getNode()) return Result;
03033   }
03034 
03035   return SDValue();
03036 }
03037 
03038 
03039 /// PerformFMRRDCombine - Target-specific dag combine xforms for ARMISD::FMRRD.
03040 static SDValue PerformFMRRDCombine(SDNode *N,
03041                                    TargetLowering::DAGCombinerInfo &DCI) {
03042   // fmrrd(fmdrr x, y) -> x,y
03043   SDValue InDouble = N->getOperand(0);
03044   if (InDouble.getOpcode() == ARMISD::FMDRR)
03045     return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1));
03046   return SDValue();
03047 }
03048 
03049 /// getVShiftImm - Check if this is a valid build_vector for the immediate
03050 /// operand of a vector shift operation, where all the elements of the
03051 /// build_vector must have the same constant integer value.
03052 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
03053   // Ignore bit_converts.
03054   while (Op.getOpcode() == ISD::BIT_CONVERT)
03055     Op = Op.getOperand(0);
03056   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
03057   APInt SplatBits, SplatUndef;
03058   unsigned SplatBitSize;
03059   bool HasAnyUndefs;
03060   if (! BVN || ! BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
03061                                       HasAnyUndefs, ElementBits) ||
03062       SplatBitSize > ElementBits)
03063     return false;
03064   Cnt = SplatBits.getSExtValue();
03065   return true;
03066 }
03067 
03068 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
03069 /// operand of a vector shift left operation.  That value must be in the range:
03070 ///   0 <= Value < ElementBits for a left shift; or
03071 ///   0 <= Value <= ElementBits for a long left shift.
03072 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
03073   assert(VT.isVector() && "vector shift count is not a vector type");
03074   unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
03075   if (! getVShiftImm(Op, ElementBits, Cnt))
03076     return false;
03077   return (Cnt >= 0 && (isLong ? Cnt-1 : Cnt) < ElementBits);
03078 }
03079 
03080 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
03081 /// operand of a vector shift right operation.  For a shift opcode, the value
03082 /// is positive, but for an intrinsic the value count must be negative. The
03083 /// absolute value must be in the range:
03084 ///   1 <= |Value| <= ElementBits for a right shift; or
03085 ///   1 <= |Value| <= ElementBits/2 for a narrow right shift.
03086 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
03087                          int64_t &Cnt) {
03088   assert(VT.isVector() && "vector shift count is not a vector type");
03089   unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
03090   if (! getVShiftImm(Op, ElementBits, Cnt))
03091     return false;
03092   if (isIntrinsic)
03093     Cnt = -Cnt;
03094   return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits/2 : ElementBits));
03095 }
03096 
03097 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
03098 static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
03099   unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
03100   switch (IntNo) {
03101   default:
03102     // Don't do anything for most intrinsics.
03103     break;
03104 
03105   // Vector shifts: check for immediate versions and lower them.
03106   // Note: This is done during DAG combining instead of DAG legalizing because
03107   // the build_vectors for 64-bit vector element shift counts are generally
03108   // not legal, and it is hard to see their values after they get legalized to
03109   // loads from a constant pool.
03110   case Intrinsic::arm_neon_vshifts:
03111   case Intrinsic::arm_neon_vshiftu:
03112   case Intrinsic::arm_neon_vshiftls:
03113   case Intrinsic::arm_neon_vshiftlu:
03114   case Intrinsic::arm_neon_vshiftn:
03115   case Intrinsic::arm_neon_vrshifts:
03116   case Intrinsic::arm_neon_vrshiftu:
03117   case Intrinsic::arm_neon_vrshiftn:
03118   case Intrinsic::arm_neon_vqshifts:
03119   case Intrinsic::arm_neon_vqshiftu:
03120   case Intrinsic::arm_neon_vqshiftsu:
03121   case Intrinsic::arm_neon_vqshiftns:
03122   case Intrinsic::arm_neon_vqshiftnu:
03123   case Intrinsic::arm_neon_vqshiftnsu:
03124   case Intrinsic::arm_neon_vqrshiftns:
03125   case Intrinsic::arm_neon_vqrshiftnu:
03126   case Intrinsic::arm_neon_vqrshiftnsu: {
03127     EVT VT = N->getOperand(1).getValueType();
03128     int64_t Cnt;
03129     unsigned VShiftOpc = 0;
03130 
03131     switch (IntNo) {
03132     case Intrinsic::arm_neon_vshifts:
03133     case Intrinsic::arm_neon_vshiftu:
03134       if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) {
03135         VShiftOpc = ARMISD::VSHL;
03136         break;
03137       }
03138       if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) {
03139         VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ?
03140                      ARMISD::VSHRs : ARMISD::VSHRu);
03141         break;
03142       }
03143       return SDValue();
03144 
03145     case Intrinsic::arm_neon_vshiftls:
03146     case Intrinsic::arm_neon_vshiftlu:
03147       if (isVShiftLImm(N->getOperand(2), VT, true, Cnt))
03148         break;
03149       llvm_unreachable("invalid shift count for vshll intrinsic");
03150 
03151     case Intrinsic::arm_neon_vrshifts:
03152     case Intrinsic::arm_neon_vrshiftu:
03153       if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt))
03154         break;
03155       return SDValue();
03156 
03157     case Intrinsic::arm_neon_vqshifts:
03158     case Intrinsic::arm_neon_vqshiftu:
03159       if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
03160         break;
03161       return SDValue();
03162 
03163     case Intrinsic::arm_neon_vqshiftsu:
03164       if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
03165         break;
03166       llvm_unreachable("invalid shift count for vqshlu intrinsic");
03167 
03168     case Intrinsic::arm_neon_vshiftn:
03169     case Intrinsic::arm_neon_vrshiftn:
03170     case Intrinsic::arm_neon_vqshiftns:
03171     case Intrinsic::arm_neon_vqshiftnu:
03172     case Intrinsic::arm_neon_vqshiftnsu:
03173     case Intrinsic::arm_neon_vqrshiftns:
03174     case Intrinsic::arm_neon_vqrshiftnu:
03175     case Intrinsic::arm_neon_vqrshiftnsu:
03176       // Narrowing shifts require an immediate right shift.
03177       if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt))
03178         break;
03179       llvm_unreachable("invalid shift count for narrowing vector shift intrinsic");
03180 
03181     default:
03182       llvm_unreachable("unhandled vector shift");
03183     }
03184 
03185     switch (IntNo) {
03186     case Intrinsic::arm_neon_vshifts:
03187     case Intrinsic::arm_neon_vshiftu:
03188       // Opcode already set above.
03189       break;
03190     case Intrinsic::arm_neon_vshiftls:
03191     case Intrinsic::arm_neon_vshiftlu:
03192       if (Cnt == VT.getVectorElementType().getSizeInBits())
03193         VShiftOpc = ARMISD::VSHLLi;
03194       else
03195         VShiftOpc = (IntNo == Intrinsic::arm_neon_vshiftls ?
03196                      ARMISD::VSHLLs : ARMISD::VSHLLu);
03197       break;
03198     case Intrinsic::arm_neon_vshiftn:
03199       VShiftOpc = ARMISD::VSHRN; break;
03200     case Intrinsic::arm_neon_vrshifts:
03201       VShiftOpc = ARMISD::VRSHRs; break;
03202     case Intrinsic::arm_neon_vrshiftu:
03203       VShiftOpc = ARMISD::VRSHRu; break;
03204     case Intrinsic::arm_neon_vrshiftn:
03205       VShiftOpc = ARMISD::VRSHRN; break;
03206     case Intrinsic::arm_neon_vqshifts:
03207       VShiftOpc = ARMISD::VQSHLs; break;
03208     case Intrinsic::arm_neon_vqshiftu:
03209       VShiftOpc = ARMISD::VQSHLu; break;
03210     case Intrinsic::arm_neon_vqshiftsu:
03211       VShiftOpc = ARMISD::VQSHLsu; break;
03212     case Intrinsic::arm_neon_vqshiftns:
03213       VShiftOpc = ARMISD::VQSHRNs; break;
03214     case Intrinsic::arm_neon_vqshiftnu:
03215       VShiftOpc = ARMISD::VQSHRNu; break;
03216     case Intrinsic::arm_neon_vqshiftnsu:
03217       VShiftOpc = ARMISD::VQSHRNsu; break;
03218     case Intrinsic::arm_neon_vqrshiftns:
03219       VShiftOpc = ARMISD::VQRSHRNs; break;
03220     case Intrinsic::arm_neon_vqrshiftnu:
03221       VShiftOpc = ARMISD::VQRSHRNu; break;
03222     case Intrinsic::arm_neon_vqrshiftnsu:
03223       VShiftOpc = ARMISD::VQRSHRNsu; break;
03224     }
03225 
03226     return DAG.getNode(VShiftOpc, N->getDebugLoc(), N->getValueType(0),
03227                        N->getOperand(1), DAG.getConstant(Cnt, MVT::i32));
03228   }
03229 
03230   case Intrinsic::arm_neon_vshiftins: {
03231     EVT VT = N->getOperand(1).getValueType();
03232     int64_t Cnt;
03233     unsigned VShiftOpc = 0;
03234 
03235     if (isVShiftLImm(N->getOperand(3), VT, false, Cnt))
03236       VShiftOpc = ARMISD::VSLI;
03237     else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt))
03238       VShiftOpc = ARMISD::VSRI;
03239     else {
03240       llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
03241     }
03242 
03243     return DAG.getNode(VShiftOpc, N->getDebugLoc(), N->getValueType(0),
03244                        N->getOperand(1), N->getOperand(2),
03245                        DAG.getConstant(Cnt, MVT::i32));
03246   }
03247 
03248   case Intrinsic::arm_neon_vqrshifts:
03249   case Intrinsic::arm_neon_vqrshiftu:
03250     // No immediate versions of these to check for.
03251     break;
03252   }
03253 
03254   return SDValue();
03255 }
03256 
03257 /// PerformShiftCombine - Checks for immediate versions of vector shifts and
03258 /// lowers them.  As with the vector shift intrinsics, this is done during DAG
03259 /// combining instead of DAG legalizing because the build_vectors for 64-bit
03260 /// vector element shift counts are generally not legal, and it is hard to see
03261 /// their values after they get legalized to loads from a constant pool.
03262 static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
03263                                    const ARMSubtarget *ST) {
03264   EVT VT = N->getValueType(0);
03265 
03266   // Nothing to be done for scalar shifts.
03267   if (! VT.isVector())
03268     return SDValue();
03269 
03270   assert(ST->hasNEON() && "unexpected vector shift");
03271   int64_t Cnt;
03272 
03273   switch (N->getOpcode()) {
03274   default: llvm_unreachable("unexpected shift opcode");
03275 
03276   case ISD::SHL:
03277     if (isVShiftLImm(N->getOperand(1), VT, false, Cnt))
03278       return DAG.getNode(ARMISD::VSHL, N->getDebugLoc(), VT, N->getOperand(0),
03279                          DAG.getConstant(Cnt, MVT::i32));
03280     break;
03281 
03282   case ISD::SRA:
03283   case ISD::SRL:
03284     if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
03285       unsigned VShiftOpc = (N->getOpcode() == ISD::SRA ?
03286                             ARMISD::VSHRs : ARMISD::VSHRu);
03287       return DAG.getNode(VShiftOpc, N->getDebugLoc(), VT, N->getOperand(0),
03288                          DAG.getConstant(Cnt, MVT::i32));
03289     }
03290   }
03291   return SDValue();
03292 }
03293 
03294 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
03295 /// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
03296 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
03297                                     const ARMSubtarget *ST) {
03298   SDValue N0 = N->getOperand(0);
03299 
03300   // Check for sign- and zero-extensions of vector extract operations of 8-
03301   // and 16-bit vector elements.  NEON supports these directly.  They are
03302   // handled during DAG combining because type legalization will promote them
03303   // to 32-bit types and it is messy to recognize the operations after that.
03304   if (ST->hasNEON() && N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
03305     SDValue Vec = N0.getOperand(0);
03306     SDValue Lane = N0.getOperand(1);
03307     EVT VT = N->getValueType(0);
03308     EVT EltVT = N0.getValueType();
03309     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03310 
03311     if (VT == MVT::i32 &&
03312         (EltVT == MVT::i8 || EltVT == MVT::i16) &&
03313         TLI.isTypeLegal(Vec.getValueType())) {
03314 
03315       unsigned Opc = 0;
03316       switch (N->getOpcode()) {
03317       default: llvm_unreachable("unexpected opcode");
03318       case ISD::SIGN_EXTEND:
03319         Opc = ARMISD::VGETLANEs;
03320         break;
03321       case ISD::ZERO_EXTEND:
03322       case ISD::ANY_EXTEND:
03323         Opc = ARMISD::VGETLANEu;
03324         break;
03325       }
03326       return DAG.getNode(Opc, N->getDebugLoc(), VT, Vec, Lane);
03327     }
03328   }
03329 
03330   return SDValue();
03331 }
03332 
03333 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
03334                                              DAGCombinerInfo &DCI) const {
03335   switch (N->getOpcode()) {
03336   default: break;
03337   case ISD::ADD:      return PerformADDCombine(N, DCI);
03338   case ISD::SUB:      return PerformSUBCombine(N, DCI);
03339   case ARMISD::FMRRD: return PerformFMRRDCombine(N, DCI);
03340   case ISD::INTRINSIC_WO_CHAIN:
03341     return PerformIntrinsicCombine(N, DCI.DAG);
03342   case ISD::SHL:
03343   case ISD::SRA:
03344   case ISD::SRL:
03345     return PerformShiftCombine(N, DCI.DAG, Subtarget);
03346   case ISD::SIGN_EXTEND:
03347   case ISD::ZERO_EXTEND:
03348   case ISD::ANY_EXTEND:
03349     return PerformExtendCombine(N, DCI.DAG, Subtarget);
03350   }
03351   return SDValue();
03352 }
03353 
03354 bool ARMTargetLowering::allowsUnalignedMemoryAccesses(EVT VT) const {
03355   if (!Subtarget->hasV6Ops())
03356     // Pre-v6 does not support unaligned mem access.
03357     return false;
03358   else if (!Subtarget->hasV6Ops()) {
03359     // v6 may or may not support unaligned mem access.
03360     if (!Subtarget->isTargetDarwin())
03361       return false;
03362   }
03363 
03364   switch (VT.getSimpleVT().SimpleTy) {
03365   default:
03366     return false;
03367   case MVT::i8:
03368   case MVT::i16:
03369   case MVT::i32:
03370     return true;
03371   // FIXME: VLD1 etc with standard alignment is legal.
03372   }
03373 }
03374 
03375 static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
03376   if (V < 0)
03377     return false;
03378 
03379   unsigned Scale = 1;
03380   switch (VT.getSimpleVT().SimpleTy) {
03381   default: return false;
03382   case MVT::i1:
03383   case MVT::i8:
03384     // Scale == 1;
03385     break;
03386   case MVT::i16:
03387     // Scale == 2;
03388     Scale = 2;
03389     break;
03390   case MVT::i32:
03391     // Scale == 4;
03392     Scale = 4;
03393     break;
03394   }
03395 
03396   if ((V & (Scale - 1)) != 0)
03397     return false;
03398   V /= Scale;
03399   return V == (V & ((1LL << 5) - 1));
03400 }
03401 
03402 static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
03403                                       const ARMSubtarget *Subtarget) {
03404   bool isNeg = false;
03405   if (V < 0) {
03406     isNeg = true;
03407     V = - V;
03408   }
03409 
03410   switch (VT.getSimpleVT().SimpleTy) {
03411   default: return false;
03412   case MVT::i1:
03413   case MVT::i8:
03414   case MVT::i16:
03415   case MVT::i32:
03416     // + imm12 or - imm8
03417     if (isNeg)
03418       return V == (V & ((1LL << 8) - 1));
03419     return V == (V & ((1LL << 12) - 1));
03420   case MVT::f32:
03421   case MVT::f64:
03422     // Same as ARM mode. FIXME: NEON?
03423     if (!Subtarget->hasVFP2())
03424       return false;
03425     if ((V & 3) != 0)
03426       return false;
03427     V >>= 2;
03428     return V == (V & ((1LL << 8) - 1));
03429   }
03430 }
03431 
03432 /// isLegalAddressImmediate - Return true if the integer value can be used
03433 /// as the offset of the target addressing mode for load / store of the
03434 /// given type.
03435 static bool isLegalAddressImmediate(int64_t V, EVT VT,
03436                                     const ARMSubtarget *Subtarget) {
03437   if (V == 0)
03438     return true;
03439 
03440   if (!VT.isSimple())
03441     return false;
03442 
03443   if (Subtarget->isThumb1Only())
03444     return isLegalT1AddressImmediate(V, VT);
03445   else if (Subtarget->isThumb2())
03446     return isLegalT2AddressImmediate(V, VT, Subtarget);
03447 
03448   // ARM mode.
03449   if (V < 0)
03450     V = - V;
03451   switch (VT.getSimpleVT().SimpleTy) {
03452   default: return false;
03453   case MVT::i1:
03454   case MVT::i8:
03455   case MVT::i32:
03456     // +- imm12
03457     return V == (V & ((1LL << 12) - 1));
03458   case MVT::i16:
03459     // +- imm8
03460     return V == (V & ((1LL << 8) - 1));
03461   case MVT::f32:
03462   case MVT::f64:
03463     if (!Subtarget->hasVFP2()) // FIXME: NEON?
03464       return false;
03465     if ((V & 3) != 0)
03466       return false;
03467     V >>= 2;
03468     return V == (V & ((1LL << 8) - 1));
03469   }
03470 }
03471 
03472 bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
03473                                                       EVT VT) const {
03474   int Scale = AM.Scale;
03475   if (Scale < 0)
03476     return false;
03477 
03478   switch (VT.getSimpleVT().SimpleTy) {
03479   default: return false;
03480   case MVT::i1:
03481   case MVT::i8:
03482   case MVT::i16:
03483   case MVT::i32:
03484     if (Scale == 1)
03485       return true;
03486     // r + r << imm
03487     Scale = Scale & ~1;
03488     return Scale == 2 || Scale == 4 || Scale == 8;
03489   case MVT::i64:
03490     // r + r
03491     if (((unsigned)AM.HasBaseReg + Scale) <= 2)
03492       return true;
03493     return false;
03494   case MVT::isVoid:
03495     // Note, we allow "void" uses (basically, uses that aren't loads or
03496     // stores), because arm allows folding a scale into many arithmetic
03497     // operations.  This should be made more precise and revisited later.
03498 
03499     // Allow r << imm, but the imm has to be a multiple of two.
03500     if (Scale & 1) return false;
03501     return isPowerOf2_32(Scale);
03502   }
03503 }
03504 
03505 /// isLegalAddressingMode - Return true if the addressing mode represented
03506 /// by AM is legal for this target, for a load/store of the specified type.
03507 bool ARMTargetLowering::isLegalAddressingMode(const AddrMode &AM,
03508                                               const Type *Ty) const {
03509   EVT VT = getValueType(Ty, true);
03510   if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
03511     return false;
03512 
03513   // Can never fold addr of global into load/store.
03514   if (AM.BaseGV)
03515     return false;
03516 
03517   switch (AM.Scale) {
03518   case 0:  // no scale reg, must be "r+i" or "r", or "i".
03519     break;
03520   case 1:
03521     if (Subtarget->isThumb1Only())
03522       return false;
03523     // FALL THROUGH.
03524   default:
03525     // ARM doesn't support any R+R*scale+imm addr modes.
03526     if (AM.BaseOffs)
03527       return false;
03528 
03529     if (!VT.isSimple())
03530       return false;
03531 
03532     if (Subtarget->isThumb2())
03533       return isLegalT2ScaledAddressingMode(AM, VT);
03534 
03535     int Scale = AM.Scale;
03536     switch (VT.getSimpleVT().SimpleTy) {
03537     default: return false;
03538     case MVT::i1:
03539     case MVT::i8:
03540     case MVT::i32:
03541       if (Scale < 0) Scale = -Scale;
03542       if (Scale == 1)
03543         return true;
03544       // r + r << imm
03545       return isPowerOf2_32(Scale & ~1);
03546     case MVT::i16:
03547     case MVT::i64:
03548       // r + r
03549       if (((unsigned)AM.HasBaseReg + Scale) <= 2)
03550         return true;
03551       return false;
03552 
03553     case MVT::isVoid:
03554       // Note, we allow "void" uses (basically, uses that aren't loads or
03555       // stores), because arm allows folding a scale into many arithmetic
03556       // operations.  This should be made more precise and revisited later.
03557 
03558       // Allow r << imm, but the imm has to be a multiple of two.
03559       if (Scale & 1) return false;
03560       return isPowerOf2_32(Scale);
03561     }
03562     break;
03563   }
03564   return true;
03565 }
03566 
03567 static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
03568                                       bool isSEXTLoad, SDValue &Base,
03569                                       SDValue &Offset, bool &isInc,
03570                                       SelectionDAG &DAG) {
03571   if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
03572     return false;
03573 
03574   if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
03575     // AddressingMode 3
03576     Base = Ptr->getOperand(0);
03577     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
03578       int RHSC = (int)RHS->getZExtValue();
03579       if (RHSC < 0 && RHSC > -256) {
03580         assert(Ptr->getOpcode() == ISD::ADD);
03581         isInc = false;
03582         Offset = DAG.getConstant(-RHSC, RHS->getValueType(0));
03583         return true;
03584       }
03585     }
03586     isInc = (Ptr->getOpcode() == ISD::ADD);
03587     Offset = Ptr->getOperand(1);
03588     return true;
03589   } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
03590     // AddressingMode 2
03591     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
03592       int RHSC = (int)RHS->getZExtValue();
03593       if (RHSC < 0 && RHSC > -0x1000) {
03594         assert(Ptr->getOpcode() == ISD::ADD);
03595         isInc = false;
03596         Offset = DAG.getConstant(-RHSC, RHS->getValueType(0));
03597         Base = Ptr->getOperand(0);
03598         return true;
03599       }
03600     }
03601 
03602     if (Ptr->getOpcode() == ISD::ADD) {
03603       isInc = true;
03604       ARM_AM::ShiftOpc ShOpcVal= ARM_AM::getShiftOpcForNode(Ptr->getOperand(0));
03605       if (ShOpcVal != ARM_AM::no_shift) {
03606         Base = Ptr->getOperand(1);
03607         Offset = Ptr->getOperand(0);
03608       } else {
03609         Base = Ptr->getOperand(0);
03610         Offset = Ptr->getOperand(1);
03611       }
03612       return true;
03613     }
03614 
03615     isInc = (Ptr->getOpcode() == ISD::ADD);
03616     Base = Ptr->getOperand(0);
03617     Offset = Ptr->getOperand(1);
03618     return true;
03619   }
03620 
03621   // FIXME: Use FLDM / FSTM to emulate indexed FP load / store.
03622   return false;
03623 }
03624 
03625 static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
03626                                      bool isSEXTLoad, SDValue &Base,
03627                                      SDValue &Offset, bool &isInc,
03628                                      SelectionDAG &DAG) {
03629   if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
03630     return false;
03631 
03632   Base = Ptr->getOperand(0);
03633   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
03634     int RHSC = (int)RHS->getZExtValue();
03635     if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
03636       assert(Ptr->getOpcode() == ISD::ADD);
03637       isInc = false;
03638       Offset = DAG.getConstant(-RHSC, RHS->getValueType(0));
03639       return true;
03640     } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
03641       isInc = Ptr->getOpcode() == ISD::ADD;
03642       Offset = DAG.getConstant(RHSC, RHS->getValueType(0));
03643       return true;
03644     }
03645   }
03646 
03647   return false;
03648 }
03649 
03650 /// getPreIndexedAddressParts - returns true by value, base pointer and
03651 /// offset pointer and addressing mode by reference if the node's address
03652 /// can be legally represented as pre-indexed load / store address.
03653 bool
03654 ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
03655                                              SDValue &Offset,
03656                                              ISD::MemIndexedMode &AM,
03657                                              SelectionDAG &DAG) const {
03658   if (Subtarget->isThumb1Only())
03659     return false;
03660 
03661   EVT VT;
03662   SDValue Ptr;
03663   bool isSEXTLoad = false;
03664   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
03665     Ptr = LD->getBasePtr();
03666     VT  = LD->getMemoryVT();
03667     isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
03668   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
03669     Ptr = ST->getBasePtr();
03670     VT  = ST->getMemoryVT();
03671   } else
03672     return false;
03673 
03674   bool isInc;
03675   bool isLegal = false;
03676   if (Subtarget->isThumb2())
03677     isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
03678                                        Offset, isInc, DAG);
03679   else
03680     isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
03681                                         Offset, isInc, DAG);
03682   if (!isLegal)
03683     return false;
03684 
03685   AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
03686   return true;
03687 }
03688 
03689 /// getPostIndexedAddressParts - returns true by value, base pointer and
03690 /// offset pointer and addressing mode by reference if this node can be
03691 /// combined with a load / store to form a post-indexed load / store.
03692 bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
03693                                                    SDValue &Base,
03694                                                    SDValue &Offset,
03695                                                    ISD::MemIndexedMode &AM,
03696                                                    SelectionDAG &DAG) const {
03697   if (Subtarget->isThumb1Only())
03698     return false;
03699 
03700   EVT VT;
03701   SDValue Ptr;
03702   bool isSEXTLoad = false;
03703   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
03704     VT  = LD->getMemoryVT();
03705     isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
03706   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
03707     VT  = ST->getMemoryVT();
03708   } else
03709     return false;
03710 
03711   bool isInc;
03712   bool isLegal = false;
03713   if (Subtarget->isThumb2())
03714     isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
03715                                         isInc, DAG);
03716   else
03717     isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
03718                                         isInc, DAG);
03719   if (!isLegal)
03720     return false;
03721 
03722   AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
03723   return true;
03724 }
03725 
03726 void ARMTargetLowering::computeMaskedBitsForTargetNode(const SDValue Op,
03727                                                        const APInt &Mask,
03728                                                        APInt &KnownZero,
03729                                                        APInt &KnownOne,
03730                                                        const SelectionDAG &DAG,
03731                                                        unsigned Depth) const {
03732   KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0);
03733   switch (Op.getOpcode()) {
03734   default: break;
03735   case ARMISD::CMOV: {
03736     // Bits are known zero/one if known on the LHS and RHS.
03737     DAG.ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1);
03738     if (KnownZero == 0 && KnownOne == 0) return;
03739 
03740     APInt KnownZeroRHS, KnownOneRHS;
03741     DAG.ComputeMaskedBits(Op.getOperand(1), Mask,
03742                           KnownZeroRHS, KnownOneRHS, Depth+1);
03743     KnownZero &= KnownZeroRHS;
03744     KnownOne  &= KnownOneRHS;
03745     return;
03746   }
03747   }
03748 }
03749 
03750 //===----------------------------------------------------------------------===//
03751 //                           ARM Inline Assembly Support
03752 //===----------------------------------------------------------------------===//
03753 
03754 /// getConstraintType - Given a constraint letter, return the type of
03755 /// constraint it is for this target.
03756 ARMTargetLowering::ConstraintType
03757 ARMTargetLowering::getConstraintType(const std::string &Constraint) const {
03758   if (Constraint.size() == 1) {
03759     switch (Constraint[0]) {
03760     default:  break;
03761     case 'l': return C_RegisterClass;
03762     case 'w': return C_RegisterClass;
03763     }
03764   }
03765   return TargetLowering::getConstraintType(Constraint);
03766 }
03767 
03768 std::pair<unsigned, const TargetRegisterClass*>
03769 ARMTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
03770                                                 EVT VT) const {
03771   if (Constraint.size() == 1) {
03772     // GCC RS6000 Constraint Letters
03773     switch (Constraint[0]) {
03774     case 'l':
03775       if (Subtarget->isThumb1Only())
03776         return std::make_pair(0U, ARM::tGPRRegisterClass);
03777       else
03778         return std::make_pair(0U, ARM::GPRRegisterClass);
03779     case 'r':
03780       return std::make_pair(0U, ARM::GPRRegisterClass);
03781     case 'w':
03782       if (VT == MVT::f32)
03783         return std::make_pair(0U, ARM::SPRRegisterClass);
03784       if (VT == MVT::f64)
03785         return std::make_pair(0U, ARM::DPRRegisterClass);
03786       break;
03787     }
03788   }
03789   return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
03790 }
03791 
03792 std::vector<unsigned> ARMTargetLowering::
03793 getRegClassForInlineAsmConstraint(const std::string &Constraint,
03794                                   EVT VT) const {
03795   if (Constraint.size() != 1)
03796     return std::vector<unsigned>();
03797 
03798   switch (Constraint[0]) {      // GCC ARM Constraint Letters
03799   default: break;
03800   case 'l':
03801     return make_vector<unsigned>(ARM::R0, ARM::R1, ARM::R2, ARM::R3,
03802                                  ARM::R4, ARM::R5, ARM::R6, ARM::R7,
03803                                  0);
03804   case 'r':
03805     return make_vector<unsigned>(ARM::R0, ARM::R1, ARM::R2, ARM::R3,
03806                                  ARM::R4, ARM::R5, ARM::R6, ARM::R7,
03807                                  ARM::R8, ARM::R9, ARM::R10, ARM::R11,
03808                                  ARM::R12, ARM::LR, 0);
03809   case 'w':
03810     if (VT == MVT::f32)
03811       return make_vector<unsigned>(ARM::S0, ARM::S1, ARM::S2, ARM::S3,
03812                                    ARM::S4, ARM::S5, ARM::S6, ARM::S7,
03813                                    ARM::S8, ARM::S9, ARM::S10, ARM::S11,
03814                                    ARM::S12,ARM::S13,ARM::S14,ARM::S15,
03815                                    ARM::S16,ARM::S17,ARM::S18,ARM::S19,
03816                                    ARM::S20,ARM::S21,ARM::S22,ARM::S23,
03817                                    ARM::S24,ARM::S25,ARM::S26,ARM::S27,
03818                                    ARM::S28,ARM::S29,ARM::S30,ARM::S31, 0);
03819     if (VT == MVT::f64)
03820       return make_vector<unsigned>(ARM::D0, ARM::D1, ARM::D2, ARM::D3,
03821                                    ARM::D4, ARM::D5, ARM::D6, ARM::D7,
03822                                    ARM::D8, ARM::D9, ARM::D10,ARM::D11,
03823                                    ARM::D12,ARM::D13,ARM::D14,ARM::D15, 0);
03824       break;
03825   }
03826 
03827   return std::vector<unsigned>();
03828 }
03829 
03830 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
03831 /// vector.  If it is invalid, don't add anything to Ops.
03832 void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
03833                                                      char Constraint,
03834                                                      bool hasMemory,
03835                                                      std::vector<SDValue>&Ops,
03836                                                      SelectionDAG &DAG) const {
03837   SDValue Result(0, 0);
03838 
03839   switch (Constraint) {
03840   default: break;
03841   case 'I': case 'J': case 'K': case 'L':
03842   case 'M': case 'N': case 'O':
03843     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
03844     if (!C)
03845       return;
03846 
03847     int64_t CVal64 = C->getSExtValue();
03848     int CVal = (int) CVal64;
03849     // None of these constraints allow values larger than 32 bits.  Check
03850     // that the value fits in an int.
03851     if (CVal != CVal64)
03852       return;
03853 
03854     switch (Constraint) {
03855       case 'I':
03856         if (Subtarget->isThumb1Only()) {
03857           // This must be a constant between 0 and 255, for ADD
03858           // immediates.
03859           if (CVal >= 0 && CVal <= 255)
03860             break;
03861         } else if (Subtarget->isThumb2()) {
03862           // A constant that can be used as an immediate value in a
03863           // data-processing instruction.
03864           if (ARM_AM::getT2SOImmVal(CVal) != -1)
03865             break;
03866         } else {
03867           // A constant that can be used as an immediate value in a
03868           // data-processing instruction.
03869           if (ARM_AM::getSOImmVal(CVal) != -1)
03870             break;
03871         }
03872         return;
03873 
03874       case 'J':
03875         if (Subtarget->isThumb()) {  // FIXME thumb2
03876           // This must be a constant between -255 and -1, for negated ADD
03877           // immediates. This can be used in GCC with an "n" modifier that
03878           // prints the negated value, for use with SUB instructions. It is
03879           // not useful otherwise but is implemented for compatibility.
03880           if (CVal >= -255 && CVal <= -1)
03881             break;
03882         } else {
03883           // This must be a constant between -4095 and 4095. It is not clear
03884           // what this constraint is intended for. Implemented for
03885           // compatibility with GCC.
03886           if (CVal >= -4095 && CVal <= 4095)
03887             break;
03888         }
03889         return;
03890 
03891       case 'K':
03892         if (Subtarget->isThumb1Only()) {
03893           // A 32-bit value where only one byte has a nonzero value. Exclude
03894           // zero to match GCC. This constraint is used by GCC internally for
03895           // constants that can be loaded with a move/shift combination.
03896           // It is not useful otherwise but is implemented for compatibility.
03897           if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal))
03898             break;
03899         } else if (Subtarget->isThumb2()) {
03900           // A constant whose bitwise inverse can be used as an immediate
03901           // value in a data-processing instruction. This can be used in GCC
03902           // with a "B" modifier that prints the inverted value, for use with
03903           // BIC and MVN instructions. It is not useful otherwise but is
03904           // implemented for compatibility.
03905           if (ARM_AM::getT2SOImmVal(~CVal) != -1)
03906             break;
03907         } else {
03908           // A constant whose bitwise inverse can be used as an immediate
03909           // value in a data-processing instruction. This can be used in GCC
03910           // with a "B" modifier that prints the inverted value, for use with
03911           // BIC and MVN instructions. It is not useful otherwise but is
03912           // implemented for compatibility.
03913           if (ARM_AM::getSOImmVal(~CVal) != -1)
03914             break;
03915         }
03916         return;
03917 
03918       case 'L':
03919         if (Subtarget->isThumb1Only()) {
03920           // This must be a constant between -7 and 7,
03921           // for 3-operand ADD/SUB immediate instructions.
03922           if (CVal >= -7 && CVal < 7)
03923             break;
03924         } else if (Subtarget->isThumb2()) {
03925           // A constant whose negation can be used as an immediate value in a
03926           // data-processing instruction. This can be used in GCC with an "n"
03927           // modifier that prints the negated value, for use with SUB
03928           // instructions. It is not useful otherwise but is implemented for
03929           // compatibility.
03930           if (ARM_AM::getT2SOImmVal(-CVal) != -1)
03931             break;
03932         } else {
03933           // A constant whose negation can be used as an immediate value in a
03934           // data-processing instruction. This can be used in GCC with an "n"
03935           // modifier that prints the negated value, for use with SUB
03936           // instructions. It is not useful otherwise but is implemented for
03937           // compatibility.
03938           if (ARM_AM::getSOImmVal(-CVal) != -1)
03939             break;
03940         }
03941         return;
03942 
03943       case 'M':
03944         if (Subtarget->isThumb()) { // FIXME thumb2
03945           // This must be a multiple of 4 between 0 and 1020, for
03946           // ADD sp + immediate.
03947           if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
03948             break;
03949         } else {
03950           // A power of two or a constant between 0 and 32.  This is used in
03951           // GCC for the shift amount on shifted register operands, but it is
03952           // useful in general for any shift amounts.
03953           if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
03954             break;
03955         }
03956         return;
03957 
03958       case 'N':
03959         if (Subtarget->isThumb()) {  // FIXME thumb2
03960           // This must be a constant between 0 and 31, for shift amounts.
03961           if (CVal >= 0 && CVal <= 31)
03962             break;
03963         }
03964         return;
03965 
03966       case 'O':
03967         if (Subtarget->isThumb()) {  // FIXME thumb2
03968           // This must be a multiple of 4 between -508 and 508, for
03969           // ADD/SUB sp = sp + immediate.
03970           if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
03971             break;
03972         }
03973         return;
03974     }
03975     Result = DAG.getTargetConstant(CVal, Op.getValueType());
03976     break;
03977   }
03978 
03979   if (Result.getNode()) {
03980     Ops.push_back(Result);
03981     return;
03982   }
03983   return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, hasMemory,
03984                                                       Ops, DAG);
03985 }