LLVM API Documentation

LegalizeVectorTypes.cpp

Go to the documentation of this file.
00001 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file performs vector type splitting and scalarization for LegalizeTypes.
00011 // Scalarization is the act of changing a computation in an illegal one-element
00012 // vector type to be a computation in its scalar element type.  For example,
00013 // implementing <1 x f32> arithmetic in a scalar f32 register.  This is needed
00014 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
00015 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
00016 // types.
00017 // Splitting is the act of changing a computation in an invalid vector type to
00018 // be a computation in two vectors of half the size.  For example, implementing
00019 // <128 x f32> operations in terms of two <64 x f32> operations.
00020 //
00021 //===----------------------------------------------------------------------===//
00022 
00023 #include "LegalizeTypes.h"
00024 #include "llvm/CodeGen/PseudoSourceValue.h"
00025 #include "llvm/Target/TargetData.h"
00026 #include "llvm/Support/ErrorHandling.h"
00027 using namespace llvm;
00028 
00029 //===----------------------------------------------------------------------===//
00030 //  Result Vector Scalarization: <1 x ty> -> ty.
00031 //===----------------------------------------------------------------------===//
00032 
00033 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
00034   DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
00035         cerr << "\n");
00036   SDValue R = SDValue();
00037 
00038   switch (N->getOpcode()) {
00039   default:
00040 #ifndef NDEBUG
00041     cerr << "ScalarizeVectorResult #" << ResNo << ": ";
00042     N->dump(&DAG); cerr << "\n";
00043 #endif
00044     llvm_unreachable("Do not know how to scalarize the result of this operator!");
00045 
00046   case ISD::BIT_CONVERT:       R = ScalarizeVecRes_BIT_CONVERT(N); break;
00047   case ISD::BUILD_VECTOR:      R = N->getOperand(0); break;
00048   case ISD::CONVERT_RNDSAT:    R = ScalarizeVecRes_CONVERT_RNDSAT(N); break;
00049   case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
00050   case ISD::FPOWI:             R = ScalarizeVecRes_FPOWI(N); break;
00051   case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
00052   case ISD::LOAD:           R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
00053   case ISD::SCALAR_TO_VECTOR:  R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
00054   case ISD::SELECT:            R = ScalarizeVecRes_SELECT(N); break;
00055   case ISD::SELECT_CC:         R = ScalarizeVecRes_SELECT_CC(N); break;
00056   case ISD::SETCC:             R = ScalarizeVecRes_SETCC(N); break;
00057   case ISD::UNDEF:             R = ScalarizeVecRes_UNDEF(N); break;
00058   case ISD::VECTOR_SHUFFLE:    R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
00059   case ISD::VSETCC:            R = ScalarizeVecRes_VSETCC(N); break;
00060 
00061   case ISD::CTLZ:
00062   case ISD::CTPOP:
00063   case ISD::CTTZ:
00064   case ISD::FABS:
00065   case ISD::FCOS:
00066   case ISD::FNEG:
00067   case ISD::FP_TO_SINT:
00068   case ISD::FP_TO_UINT:
00069   case ISD::FSIN:
00070   case ISD::FSQRT:
00071   case ISD::FTRUNC:
00072   case ISD::FFLOOR:
00073   case ISD::FCEIL:
00074   case ISD::FRINT:
00075   case ISD::FNEARBYINT:
00076   case ISD::UINT_TO_FP:
00077   case ISD::SINT_TO_FP:
00078   case ISD::TRUNCATE:
00079   case ISD::SIGN_EXTEND:
00080   case ISD::ZERO_EXTEND:
00081   case ISD::ANY_EXTEND:
00082     R = ScalarizeVecRes_UnaryOp(N);
00083     break;
00084 
00085   case ISD::ADD:
00086   case ISD::AND:
00087   case ISD::FADD:
00088   case ISD::FDIV:
00089   case ISD::FMUL:
00090   case ISD::FPOW:
00091   case ISD::FREM:
00092   case ISD::FSUB:
00093   case ISD::MUL:
00094   case ISD::OR:
00095   case ISD::SDIV:
00096   case ISD::SREM:
00097   case ISD::SUB:
00098   case ISD::UDIV:
00099   case ISD::UREM:
00100   case ISD::XOR:
00101   case ISD::SHL:
00102   case ISD::SRA:
00103   case ISD::SRL:
00104     R = ScalarizeVecRes_BinOp(N);
00105     break;
00106   }
00107 
00108   // If R is null, the sub-method took care of registering the result.
00109   if (R.getNode())
00110     SetScalarizedVector(SDValue(N, ResNo), R);
00111 }
00112 
00113 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
00114   SDValue LHS = GetScalarizedVector(N->getOperand(0));
00115   SDValue RHS = GetScalarizedVector(N->getOperand(1));
00116   return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
00117                      LHS.getValueType(), LHS, RHS);
00118 }
00119 
00120 SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) {
00121   EVT NewVT = N->getValueType(0).getVectorElementType();
00122   return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
00123                      NewVT, N->getOperand(0));
00124 }
00125 
00126 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) {
00127   EVT NewVT = N->getValueType(0).getVectorElementType();
00128   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
00129   return DAG.getConvertRndSat(NewVT, N->getDebugLoc(),
00130                               Op0, DAG.getValueType(NewVT),
00131                               DAG.getValueType(Op0.getValueType()),
00132                               N->getOperand(3),
00133                               N->getOperand(4),
00134                               cast<CvtRndSatSDNode>(N)->getCvtCode());
00135 }
00136 
00137 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
00138   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
00139                      N->getValueType(0).getVectorElementType(),
00140                      N->getOperand(0), N->getOperand(1));
00141 }
00142 
00143 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
00144   SDValue Op = GetScalarizedVector(N->getOperand(0));
00145   return DAG.getNode(ISD::FPOWI, N->getDebugLoc(),
00146                      Op.getValueType(), Op, N->getOperand(1));
00147 }
00148 
00149 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
00150   // The value to insert may have a wider type than the vector element type,
00151   // so be sure to truncate it to the element type if necessary.
00152   SDValue Op = N->getOperand(1);
00153   EVT EltVT = N->getValueType(0).getVectorElementType();
00154   if (Op.getValueType() != EltVT)
00155     // FIXME: Can this happen for floating point types?
00156     Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op);
00157   return Op;
00158 }
00159 
00160 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
00161   assert(N->isUnindexed() && "Indexed vector load?");
00162 
00163   SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(),
00164                                N->getExtensionType(),
00165                                N->getValueType(0).getVectorElementType(),
00166                                N->getChain(), N->getBasePtr(),
00167                                DAG.getUNDEF(N->getBasePtr().getValueType()),
00168                                N->getSrcValue(), N->getSrcValueOffset(),
00169                                N->getMemoryVT().getVectorElementType(),
00170                                N->isVolatile(), N->getAlignment());
00171 
00172   // Legalized the chain result - switch anything that used the old chain to
00173   // use the new one.
00174   ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
00175   return Result;
00176 }
00177 
00178 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
00179   // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
00180   EVT DestVT = N->getValueType(0).getVectorElementType();
00181   SDValue Op = GetScalarizedVector(N->getOperand(0));
00182   return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op);
00183 }
00184 
00185 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
00186   // If the operand is wider than the vector element type then it is implicitly
00187   // truncated.  Make that explicit here.
00188   EVT EltVT = N->getValueType(0).getVectorElementType();
00189   SDValue InOp = N->getOperand(0);
00190   if (InOp.getValueType() != EltVT)
00191     return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp);
00192   return InOp;
00193 }
00194 
00195 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
00196   SDValue LHS = GetScalarizedVector(N->getOperand(1));
00197   return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
00198                      LHS.getValueType(), N->getOperand(0), LHS,
00199                      GetScalarizedVector(N->getOperand(2)));
00200 }
00201 
00202 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
00203   SDValue LHS = GetScalarizedVector(N->getOperand(2));
00204   return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(),
00205                      N->getOperand(0), N->getOperand(1),
00206                      LHS, GetScalarizedVector(N->getOperand(3)),
00207                      N->getOperand(4));
00208 }
00209 
00210 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
00211   SDValue LHS = GetScalarizedVector(N->getOperand(0));
00212   SDValue RHS = GetScalarizedVector(N->getOperand(1));
00213   DebugLoc DL = N->getDebugLoc();
00214 
00215   // Turn it into a scalar SETCC.
00216   return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
00217 }
00218 
00219 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
00220   return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
00221 }
00222 
00223 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
00224   // Figure out if the scalar is the LHS or RHS and return it.
00225   SDValue Arg = N->getOperand(2).getOperand(0);
00226   if (Arg.getOpcode() == ISD::UNDEF)
00227     return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
00228   unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
00229   return GetScalarizedVector(N->getOperand(Op));
00230 }
00231 
00232 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
00233   SDValue LHS = GetScalarizedVector(N->getOperand(0));
00234   SDValue RHS = GetScalarizedVector(N->getOperand(1));
00235   EVT NVT = N->getValueType(0).getVectorElementType();
00236   EVT SVT = TLI.getSetCCResultType(LHS.getValueType());
00237   DebugLoc DL = N->getDebugLoc();
00238 
00239   // Turn it into a scalar SETCC.
00240   SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2));
00241 
00242   // VSETCC always returns a sign-extended value, while SETCC may not.  The
00243   // SETCC result type may not match the vector element type.  Correct these.
00244   if (NVT.bitsLE(SVT)) {
00245     // The SETCC result type is bigger than the vector element type.
00246     // Ensure the SETCC result is sign-extended.
00247     if (TLI.getBooleanContents() !=
00248         TargetLowering::ZeroOrNegativeOneBooleanContent)
00249       Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res,
00250                         DAG.getValueType(MVT::i1));
00251     // Truncate to the final type.
00252     return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res);
00253   }
00254 
00255   // The SETCC result type is smaller than the vector element type.
00256   // If the SetCC result is not sign-extended, chop it down to MVT::i1.
00257   if (TLI.getBooleanContents() !=
00258         TargetLowering::ZeroOrNegativeOneBooleanContent)
00259     Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res);
00260   // Sign extend to the final type.
00261   return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res);
00262 }
00263 
00264 
00265 //===----------------------------------------------------------------------===//
00266 //  Operand Vector Scalarization <1 x ty> -> ty.
00267 //===----------------------------------------------------------------------===//
00268 
00269 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
00270   DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
00271         cerr << "\n");
00272   SDValue Res = SDValue();
00273 
00274   if (Res.getNode() == 0) {
00275     switch (N->getOpcode()) {
00276     default:
00277 #ifndef NDEBUG
00278       cerr << "ScalarizeVectorOperand Op #" << OpNo << ": ";
00279       N->dump(&DAG); cerr << "\n";
00280 #endif
00281       llvm_unreachable("Do not know how to scalarize this operator's operand!");
00282     case ISD::BIT_CONVERT:
00283       Res = ScalarizeVecOp_BIT_CONVERT(N);
00284       break;
00285     case ISD::CONCAT_VECTORS:
00286       Res = ScalarizeVecOp_CONCAT_VECTORS(N);
00287       break;
00288     case ISD::EXTRACT_VECTOR_ELT:
00289       Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
00290       break;
00291     case ISD::STORE:
00292       Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
00293       break;
00294     }
00295   }
00296 
00297   // If the result is null, the sub-method took care of registering results etc.
00298   if (!Res.getNode()) return false;
00299 
00300   // If the result is N, the sub-method updated N in place.  Tell the legalizer
00301   // core about this.
00302   if (Res.getNode() == N)
00303     return true;
00304 
00305   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
00306          "Invalid operand expansion");
00307 
00308   ReplaceValueWith(SDValue(N, 0), Res);
00309   return false;
00310 }
00311 
00312 /// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs
00313 /// to be scalarized, it must be <1 x ty>.  Convert the element instead.
00314 SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) {
00315   SDValue Elt = GetScalarizedVector(N->getOperand(0));
00316   return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
00317                      N->getValueType(0), Elt);
00318 }
00319 
00320 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
00321 /// use a BUILD_VECTOR instead.
00322 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
00323   SmallVector<SDValue, 8> Ops(N->getNumOperands());
00324   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
00325     Ops[i] = GetScalarizedVector(N->getOperand(i));
00326   return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0),
00327                      &Ops[0], Ops.size());
00328 }
00329 
00330 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
00331 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
00332 /// index.
00333 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
00334   SDValue Res = GetScalarizedVector(N->getOperand(0));
00335   if (Res.getValueType() != N->getValueType(0))
00336     Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0),
00337                       Res);
00338   return Res;
00339 }
00340 
00341 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
00342 /// scalarized, it must be <1 x ty>.  Just store the element.
00343 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
00344   assert(N->isUnindexed() && "Indexed store of one-element vector?");
00345   assert(OpNo == 1 && "Do not know how to scalarize this operand!");
00346   DebugLoc dl = N->getDebugLoc();
00347 
00348   if (N->isTruncatingStore())
00349     return DAG.getTruncStore(N->getChain(), dl,
00350                              GetScalarizedVector(N->getOperand(1)),
00351                              N->getBasePtr(),
00352                              N->getSrcValue(), N->getSrcValueOffset(),
00353                              N->getMemoryVT().getVectorElementType(),
00354                              N->isVolatile(), N->getAlignment());
00355 
00356   return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
00357                       N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(),
00358                       N->isVolatile(), N->getAlignment());
00359 }
00360 
00361 
00362 //===----------------------------------------------------------------------===//
00363 //  Result Vector Splitting
00364 //===----------------------------------------------------------------------===//
00365 
00366 /// SplitVectorResult - This method is called when the specified result of the
00367 /// specified node is found to need vector splitting.  At this point, the node
00368 /// may also have invalid operands or may have other results that need
00369 /// legalization, we just know that (at least) one result needs vector
00370 /// splitting.
00371 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
00372   DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n");
00373   SDValue Lo, Hi;
00374 
00375   switch (N->getOpcode()) {
00376   default:
00377 #ifndef NDEBUG
00378     cerr << "SplitVectorResult #" << ResNo << ": ";
00379     N->dump(&DAG); cerr << "\n";
00380 #endif
00381     llvm_unreachable("Do not know how to split the result of this operator!");
00382 
00383   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break;
00384   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
00385   case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
00386   case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
00387 
00388   case ISD::BIT_CONVERT:       SplitVecRes_BIT_CONVERT(N, Lo, Hi); break;
00389   case ISD::BUILD_VECTOR:      SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
00390   case ISD::CONCAT_VECTORS:    SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
00391   case ISD::CONVERT_RNDSAT:    SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break;
00392   case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
00393   case ISD::FPOWI:             SplitVecRes_FPOWI(N, Lo, Hi); break;
00394   case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
00395   case ISD::SCALAR_TO_VECTOR:  SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
00396   case ISD::LOAD:
00397     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
00398     break;
00399   case ISD::SETCC:
00400   case ISD::VSETCC:
00401     SplitVecRes_SETCC(N, Lo, Hi);
00402     break;
00403   case ISD::VECTOR_SHUFFLE:
00404     SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
00405     break;
00406 
00407   case ISD::CTTZ:
00408   case ISD::CTLZ:
00409   case ISD::CTPOP:
00410   case ISD::FNEG:
00411   case ISD::FABS:
00412   case ISD::FSQRT:
00413   case ISD::FSIN:
00414   case ISD::FCOS:
00415   case ISD::FTRUNC:
00416   case ISD::FFLOOR:
00417   case ISD::FCEIL:
00418   case ISD::FRINT:
00419   case ISD::FNEARBYINT:
00420   case ISD::FP_TO_SINT:
00421   case ISD::FP_TO_UINT:
00422   case ISD::SINT_TO_FP:
00423   case ISD::UINT_TO_FP:
00424   case ISD::TRUNCATE:
00425   case ISD::SIGN_EXTEND:
00426   case ISD::ZERO_EXTEND:
00427   case ISD::ANY_EXTEND:
00428     SplitVecRes_UnaryOp(N, Lo, Hi);
00429     break;
00430 
00431   case ISD::ADD:
00432   case ISD::SUB:
00433   case ISD::MUL:
00434   case ISD::FADD:
00435   case ISD::FSUB:
00436   case ISD::FMUL:
00437   case ISD::SDIV:
00438   case ISD::UDIV:
00439   case ISD::FDIV:
00440   case ISD::FPOW:
00441   case ISD::AND:
00442   case ISD::OR:
00443   case ISD::XOR:
00444   case ISD::SHL:
00445   case ISD::SRA:
00446   case ISD::SRL:
00447   case ISD::UREM:
00448   case ISD::SREM:
00449   case ISD::FREM:
00450     SplitVecRes_BinOp(N, Lo, Hi);
00451     break;
00452   }
00453 
00454   // If Lo/Hi is null, the sub-method took care of registering results etc.
00455   if (Lo.getNode())
00456     SetSplitVector(SDValue(N, ResNo), Lo, Hi);
00457 }
00458 
00459 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
00460                                          SDValue &Hi) {
00461   SDValue LHSLo, LHSHi;
00462   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
00463   SDValue RHSLo, RHSHi;
00464   GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
00465   DebugLoc dl = N->getDebugLoc();
00466 
00467   Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo);
00468   Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi);
00469 }
00470 
00471 void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
00472                                                SDValue &Hi) {
00473   // We know the result is a vector.  The input may be either a vector or a
00474   // scalar value.
00475   EVT LoVT, HiVT;
00476   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00477   DebugLoc dl = N->getDebugLoc();
00478 
00479   SDValue InOp = N->getOperand(0);
00480   EVT InVT = InOp.getValueType();
00481 
00482   // Handle some special cases efficiently.
00483   switch (getTypeAction(InVT)) {
00484   default:
00485     assert(false && "Unknown type action!");
00486   case Legal:
00487   case PromoteInteger:
00488   case SoftenFloat:
00489   case ScalarizeVector:
00490     break;
00491   case ExpandInteger:
00492   case ExpandFloat:
00493     // A scalar to vector conversion, where the scalar needs expansion.
00494     // If the vector is being split in two then we can just convert the
00495     // expanded pieces.
00496     if (LoVT == HiVT) {
00497       GetExpandedOp(InOp, Lo, Hi);
00498       if (TLI.isBigEndian())
00499         std::swap(Lo, Hi);
00500       Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
00501       Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
00502       return;
00503     }
00504     break;
00505   case SplitVector:
00506     // If the input is a vector that needs to be split, convert each split
00507     // piece of the input now.
00508     GetSplitVector(InOp, Lo, Hi);
00509     Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
00510     Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
00511     return;
00512   }
00513 
00514   // In the general case, convert the input to an integer and split it by hand.
00515   EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
00516   EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
00517   if (TLI.isBigEndian())
00518     std::swap(LoIntVT, HiIntVT);
00519 
00520   SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
00521 
00522   if (TLI.isBigEndian())
00523     std::swap(Lo, Hi);
00524   Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
00525   Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
00526 }
00527 
00528 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
00529                                                 SDValue &Hi) {
00530   EVT LoVT, HiVT;
00531   DebugLoc dl = N->getDebugLoc();
00532   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00533   unsigned LoNumElts = LoVT.getVectorNumElements();
00534   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
00535   Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size());
00536 
00537   SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
00538   Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size());
00539 }
00540 
00541 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
00542                                                   SDValue &Hi) {
00543   assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
00544   DebugLoc dl = N->getDebugLoc();
00545   unsigned NumSubvectors = N->getNumOperands() / 2;
00546   if (NumSubvectors == 1) {
00547     Lo = N->getOperand(0);
00548     Hi = N->getOperand(1);
00549     return;
00550   }
00551 
00552   EVT LoVT, HiVT;
00553   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00554 
00555   SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
00556   Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size());
00557 
00558   SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
00559   Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size());
00560 }
00561 
00562 void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo,
00563                                                   SDValue &Hi) {
00564   EVT LoVT, HiVT;
00565   DebugLoc dl = N->getDebugLoc();
00566   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00567 
00568   SDValue DTyOpLo =  DAG.getValueType(LoVT);
00569   SDValue DTyOpHi =  DAG.getValueType(HiVT);
00570 
00571   SDValue RndOp = N->getOperand(3);
00572   SDValue SatOp = N->getOperand(4);
00573   ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
00574 
00575   // Split the input.
00576   SDValue VLo, VHi;
00577   EVT InVT = N->getOperand(0).getValueType();
00578   switch (getTypeAction(InVT)) {
00579   default: llvm_unreachable("Unexpected type action!");
00580   case Legal: {
00581     EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
00582                                  LoVT.getVectorNumElements());
00583     VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
00584                       DAG.getIntPtrConstant(0));
00585     VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
00586                       DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
00587     break;
00588   }
00589   case SplitVector:
00590     GetSplitVector(N->getOperand(0), VLo, VHi);
00591     break;
00592   case WidenVector: {
00593     // If the result needs to be split and the input needs to be widened,
00594     // the two types must have different lengths. Use the widened result
00595     // and extract from it to do the split.
00596     SDValue InOp = GetWidenedVector(N->getOperand(0));
00597     EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
00598                                  LoVT.getVectorNumElements());
00599     VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
00600                      DAG.getIntPtrConstant(0));
00601     VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
00602                      DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
00603     break;
00604   }
00605   }
00606 
00607   SDValue STyOpLo =  DAG.getValueType(VLo.getValueType());
00608   SDValue STyOpHi =  DAG.getValueType(VHi.getValueType());
00609 
00610   Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp,
00611                             CvtCode);
00612   Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp,
00613                             CvtCode);
00614 }
00615 
00616 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
00617                                                      SDValue &Hi) {
00618   SDValue Vec = N->getOperand(0);
00619   SDValue Idx = N->getOperand(1);
00620   EVT IdxVT = Idx.getValueType();
00621   DebugLoc dl = N->getDebugLoc();
00622 
00623   EVT LoVT, HiVT;
00624   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00625 
00626   Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
00627   Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx,
00628                     DAG.getConstant(LoVT.getVectorNumElements(), IdxVT));
00629   Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx);
00630 }
00631 
00632 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
00633                                          SDValue &Hi) {
00634   DebugLoc dl = N->getDebugLoc();
00635   GetSplitVector(N->getOperand(0), Lo, Hi);
00636   Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
00637   Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
00638 }
00639 
00640 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
00641                                                      SDValue &Hi) {
00642   SDValue Vec = N->getOperand(0);
00643   SDValue Elt = N->getOperand(1);
00644   SDValue Idx = N->getOperand(2);
00645   DebugLoc dl = N->getDebugLoc();
00646   GetSplitVector(Vec, Lo, Hi);
00647 
00648   if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
00649     unsigned IdxVal = CIdx->getZExtValue();
00650     unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
00651     if (IdxVal < LoNumElts)
00652       Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
00653                        Lo.getValueType(), Lo, Elt, Idx);
00654     else
00655       Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
00656                        DAG.getIntPtrConstant(IdxVal - LoNumElts));
00657     return;
00658   }
00659 
00660   // Spill the vector to the stack.
00661   EVT VecVT = Vec.getValueType();
00662   EVT EltVT = VecVT.getVectorElementType();
00663   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
00664   SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0);
00665 
00666   // Store the new element.  This may be larger than the vector element type,
00667   // so use a truncating store.
00668   SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
00669   unsigned Alignment =
00670     TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext()));
00671   Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT);
00672 
00673   // Load the Lo part from the stack slot.
00674   Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0);
00675 
00676   // Increment the pointer to the other part.
00677   unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
00678   StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
00679                          DAG.getIntPtrConstant(IncrementSize));
00680 
00681   // Load the Hi part from the stack slot.
00682   Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false,
00683                    MinAlign(Alignment, IncrementSize));
00684 }
00685 
00686 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
00687                                                     SDValue &Hi) {
00688   EVT LoVT, HiVT;
00689   DebugLoc dl = N->getDebugLoc();
00690   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00691   Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
00692   Hi = DAG.getUNDEF(HiVT);
00693 }
00694 
00695 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
00696                                         SDValue &Hi) {
00697   assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
00698   EVT LoVT, HiVT;
00699   DebugLoc dl = LD->getDebugLoc();
00700   GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT);
00701 
00702   ISD::LoadExtType ExtType = LD->getExtensionType();
00703   SDValue Ch = LD->getChain();
00704   SDValue Ptr = LD->getBasePtr();
00705   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
00706   const Value *SV = LD->getSrcValue();
00707   int SVOffset = LD->getSrcValueOffset();
00708   EVT MemoryVT = LD->getMemoryVT();
00709   unsigned Alignment = LD->getAlignment();
00710   bool isVolatile = LD->isVolatile();
00711 
00712   EVT LoMemVT, HiMemVT;
00713   GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
00714 
00715   Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset,
00716                    SV, SVOffset, LoMemVT, isVolatile, Alignment);
00717 
00718   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
00719   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
00720                     DAG.getIntPtrConstant(IncrementSize));
00721   SVOffset += IncrementSize;
00722   Alignment = MinAlign(Alignment, IncrementSize);
00723   Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset,
00724                    SV, SVOffset, HiMemVT, isVolatile, Alignment);
00725 
00726   // Build a factor node to remember that this load is independent of the
00727   // other one.
00728   Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
00729                    Hi.getValue(1));
00730 
00731   // Legalized the chain result - switch anything that used the old chain to
00732   // use the new one.
00733   ReplaceValueWith(SDValue(LD, 1), Ch);
00734 }
00735 
00736 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
00737   EVT LoVT, HiVT;
00738   DebugLoc DL = N->getDebugLoc();
00739   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00740 
00741   // Split the input.
00742   EVT InVT = N->getOperand(0).getValueType();
00743   SDValue LL, LH, RL, RH;
00744   EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
00745                                LoVT.getVectorNumElements());
00746   LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0),
00747                    DAG.getIntPtrConstant(0));
00748   LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0),
00749                    DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
00750 
00751   RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1),
00752                    DAG.getIntPtrConstant(0));
00753   RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1),
00754                    DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
00755 
00756   Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
00757   Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
00758 }
00759 
00760 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
00761                                            SDValue &Hi) {
00762   // Get the dest types - they may not match the input types, e.g. int_to_fp.
00763   EVT LoVT, HiVT;
00764   DebugLoc dl = N->getDebugLoc();
00765   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
00766 
00767   // Split the input.
00768   EVT InVT = N->getOperand(0).getValueType();
00769   switch (getTypeAction(InVT)) {
00770   default: llvm_unreachable("Unexpected type action!");
00771   case Legal: {
00772     EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
00773                                  LoVT.getVectorNumElements());
00774     Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
00775                      DAG.getIntPtrConstant(0));
00776     Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
00777                      DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
00778     break;
00779   }
00780   case SplitVector:
00781     GetSplitVector(N->getOperand(0), Lo, Hi);
00782     break;
00783   case WidenVector: {
00784     // If the result needs to be split and the input needs to be widened,
00785     // the two types must have different lengths. Use the widened result
00786     // and extract from it to do the split.
00787     SDValue InOp = GetWidenedVector(N->getOperand(0));
00788     EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
00789                                  LoVT.getVectorNumElements());
00790     Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
00791                      DAG.getIntPtrConstant(0));
00792     Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
00793                      DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
00794     break;
00795   }
00796   }
00797 
00798   Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
00799   Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
00800 }
00801 
00802 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
00803                                                   SDValue &Lo, SDValue &Hi) {
00804   // The low and high parts of the original input give four input vectors.
00805   SDValue Inputs[4];
00806   DebugLoc dl = N->getDebugLoc();
00807   GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
00808   GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
00809   EVT NewVT = Inputs[0].getValueType();
00810   unsigned NewElts = NewVT.getVectorNumElements();
00811 
00812   // If Lo or Hi uses elements from at most two of the four input vectors, then
00813   // express it as a vector shuffle of those two inputs.  Otherwise extract the
00814   // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
00815   SmallVector<int, 16> Ops;
00816   for (unsigned High = 0; High < 2; ++High) {
00817     SDValue &Output = High ? Hi : Lo;
00818 
00819     // Build a shuffle mask for the output, discovering on the fly which
00820     // input vectors to use as shuffle operands (recorded in InputUsed).
00821     // If building a suitable shuffle vector proves too hard, then bail
00822     // out with useBuildVector set.
00823     unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
00824     unsigned FirstMaskIdx = High * NewElts;
00825     bool useBuildVector = false;
00826     for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
00827       // The mask element.  This indexes into the input.
00828       int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
00829 
00830       // The input vector this mask element indexes into.
00831       unsigned Input = (unsigned)Idx / NewElts;
00832 
00833       if (Input >= array_lengthof(Inputs)) {
00834         // The mask element does not index into any input vector.
00835         Ops.push_back(-1);
00836         continue;
00837       }
00838 
00839       // Turn the index into an offset from the start of the input vector.
00840       Idx -= Input * NewElts;
00841 
00842       // Find or create a shuffle vector operand to hold this input.
00843       unsigned OpNo;
00844       for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
00845         if (InputUsed[OpNo] == Input) {
00846           // This input vector is already an operand.
00847           break;
00848         } else if (InputUsed[OpNo] == -1U) {
00849           // Create a new operand for this input vector.
00850           InputUsed[OpNo] = Input;
00851           break;
00852         }
00853       }
00854 
00855       if (OpNo >= array_lengthof(InputUsed)) {
00856         // More than two input vectors used!  Give up on trying to create a
00857         // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
00858         useBuildVector = true;
00859         break;
00860       }
00861 
00862       // Add the mask index for the new shuffle vector.
00863       Ops.push_back(Idx + OpNo * NewElts);
00864     }
00865 
00866     if (useBuildVector) {
00867       EVT EltVT = NewVT.getVectorElementType();
00868       SmallVector<SDValue, 16> SVOps;
00869 
00870       // Extract the input elements by hand.
00871       for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
00872         // The mask element.  This indexes into the input.
00873         int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
00874 
00875         // The input vector this mask element indexes into.
00876         unsigned Input = (unsigned)Idx / NewElts;
00877 
00878         if (Input >= array_lengthof(Inputs)) {
00879           // The mask element is "undef" or indexes off the end of the input.
00880           SVOps.push_back(DAG.getUNDEF(EltVT));
00881           continue;
00882         }
00883 
00884         // Turn the index into an offset from the start of the input vector.
00885         Idx -= Input * NewElts;
00886 
00887         // Extract the vector element by hand.
00888         SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
00889                                     Inputs[Input], DAG.getIntPtrConstant(Idx)));
00890       }
00891 
00892       // Construct the Lo/Hi output using a BUILD_VECTOR.
00893       Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size());
00894     } else if (InputUsed[0] == -1U) {
00895       // No input vectors were used!  The result is undefined.
00896       Output = DAG.getUNDEF(NewVT);
00897     } else {
00898       SDValue Op0 = Inputs[InputUsed[0]];
00899       // If only one input was used, use an undefined vector for the other.
00900       SDValue Op1 = InputUsed[1] == -1U ?
00901         DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
00902       // At least one input vector was used.  Create a new shuffle vector.
00903       Output =  DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]);
00904     }
00905 
00906     Ops.clear();
00907   }
00908 }
00909 
00910 
00911 //===----------------------------------------------------------------------===//
00912 //  Operand Vector Splitting
00913 //===----------------------------------------------------------------------===//
00914 
00915 /// SplitVectorOperand - This method is called when the specified operand of the
00916 /// specified node is found to need vector splitting.  At this point, all of the
00917 /// result types of the node are known to be legal, but other operands of the
00918 /// node may need legalization as well as the specified one.
00919 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
00920   DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n");
00921   SDValue Res = SDValue();
00922 
00923   if (Res.getNode() == 0) {
00924     switch (N->getOpcode()) {
00925     default:
00926 #ifndef NDEBUG
00927       cerr << "SplitVectorOperand Op #" << OpNo << ": ";
00928       N->dump(&DAG); cerr << "\n";
00929 #endif
00930       llvm_unreachable("Do not know how to split this operator's operand!");
00931 
00932     case ISD::BIT_CONVERT:       Res = SplitVecOp_BIT_CONVERT(N); break;
00933     case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
00934     case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
00935     case ISD::STORE:
00936       Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
00937       break;
00938 
00939     case ISD::CTTZ:
00940     case ISD::CTLZ:
00941     case ISD::CTPOP:
00942     case ISD::FP_TO_SINT:
00943     case ISD::FP_TO_UINT:
00944     case ISD::SINT_TO_FP:
00945     case ISD::UINT_TO_FP:
00946     case ISD::TRUNCATE:
00947     case ISD::SIGN_EXTEND:
00948     case ISD::ZERO_EXTEND:
00949     case ISD::ANY_EXTEND:
00950       Res = SplitVecOp_UnaryOp(N);
00951       break;
00952     }
00953   }
00954 
00955   // If the result is null, the sub-method took care of registering results etc.
00956   if (!Res.getNode()) return false;
00957 
00958   // If the result is N, the sub-method updated N in place.  Tell the legalizer
00959   // core about this.
00960   if (Res.getNode() == N)
00961     return true;
00962 
00963   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
00964          "Invalid operand expansion");
00965 
00966   ReplaceValueWith(SDValue(N, 0), Res);
00967   return false;
00968 }
00969 
00970 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
00971   // The result has a legal vector type, but the input needs splitting.
00972   EVT ResVT = N->getValueType(0);
00973   SDValue Lo, Hi;
00974   DebugLoc dl = N->getDebugLoc();
00975   GetSplitVector(N->getOperand(0), Lo, Hi);
00976   EVT InVT = Lo.getValueType();
00977 
00978   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
00979                                InVT.getVectorNumElements());
00980 
00981   Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
00982   Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
00983 
00984   return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
00985 }
00986 
00987 SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) {
00988   // For example, i64 = BIT_CONVERT v4i16 on alpha.  Typically the vector will
00989   // end up being split all the way down to individual components.  Convert the
00990   // split pieces into integers and reassemble.
00991   SDValue Lo, Hi;
00992   GetSplitVector(N->getOperand(0), Lo, Hi);
00993   Lo = BitConvertToInteger(Lo);
00994   Hi = BitConvertToInteger(Hi);
00995 
00996   if (TLI.isBigEndian())
00997     std::swap(Lo, Hi);
00998 
00999   return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0),
01000                      JoinIntegers(Lo, Hi));
01001 }
01002 
01003 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
01004   // We know that the extracted result type is legal.  For now, assume the index
01005   // is a constant.
01006   EVT SubVT = N->getValueType(0);
01007   SDValue Idx = N->getOperand(1);
01008   DebugLoc dl = N->getDebugLoc();
01009   SDValue Lo, Hi;
01010   GetSplitVector(N->getOperand(0), Lo, Hi);
01011 
01012   uint64_t LoElts = Lo.getValueType().getVectorNumElements();
01013   uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
01014 
01015   if (IdxVal < LoElts) {
01016     assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
01017            "Extracted subvector crosses vector split!");
01018     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
01019   } else {
01020     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
01021                        DAG.getConstant(IdxVal - LoElts, Idx.getValueType()));
01022   }
01023 }
01024 
01025 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
01026   SDValue Vec = N->getOperand(0);
01027   SDValue Idx = N->getOperand(1);
01028   EVT VecVT = Vec.getValueType();
01029 
01030   if (isa<ConstantSDNode>(Idx)) {
01031     uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
01032     assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
01033 
01034     SDValue Lo, Hi;
01035     GetSplitVector(Vec, Lo, Hi);
01036 
01037     uint64_t LoElts = Lo.getValueType().getVectorNumElements();
01038 
01039     if (IdxVal < LoElts)
01040       return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx);
01041     return DAG.UpdateNodeOperands(SDValue(N, 0), Hi,
01042                                   DAG.getConstant(IdxVal - LoElts,
01043                                                   Idx.getValueType()));
01044   }
01045 
01046   // Store the vector to the stack.
01047   EVT EltVT = VecVT.getVectorElementType();
01048   DebugLoc dl = N->getDebugLoc();
01049   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
01050   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
01051   const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
01052   SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0);
01053 
01054   // Load back the required element.
01055   StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
01056   return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
01057                         SV, 0, EltVT);
01058 }
01059 
01060 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
01061   assert(N->isUnindexed() && "Indexed store of vector?");
01062   assert(OpNo == 1 && "Can only split the stored value");
01063   DebugLoc dl = N->getDebugLoc();
01064 
01065   bool isTruncating = N->isTruncatingStore();
01066   SDValue Ch  = N->getChain();
01067   SDValue Ptr = N->getBasePtr();
01068   int SVOffset = N->getSrcValueOffset();
01069   EVT MemoryVT = N->getMemoryVT();
01070   unsigned Alignment = N->getAlignment();
01071   bool isVol = N->isVolatile();
01072   SDValue Lo, Hi;
01073   GetSplitVector(N->getOperand(1), Lo, Hi);
01074 
01075   EVT LoMemVT, HiMemVT;
01076   GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
01077 
01078   unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
01079 
01080   if (isTruncating)
01081     Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset,
01082                            LoMemVT, isVol, Alignment);
01083   else
01084     Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset,
01085                       isVol, Alignment);
01086 
01087   // Increment the pointer to the other half.
01088   Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
01089                     DAG.getIntPtrConstant(IncrementSize));
01090 
01091   if (isTruncating)
01092     Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
01093                            N->getSrcValue(), SVOffset+IncrementSize,
01094                            HiMemVT,
01095                            isVol, MinAlign(Alignment, IncrementSize));
01096   else
01097     Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
01098                       isVol, MinAlign(Alignment, IncrementSize));
01099 
01100   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
01101 }
01102 
01103 
01104 //===----------------------------------------------------------------------===//
01105 //  Result Vector Widening
01106 //===----------------------------------------------------------------------===//
01107 
01108 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
01109   DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG);
01110         cerr << "\n");
01111   SDValue Res = SDValue();
01112 
01113   switch (N->getOpcode()) {
01114   default:
01115 #ifndef NDEBUG
01116     cerr << "WidenVectorResult #" << ResNo << ": ";
01117     N->dump(&DAG); cerr << "\n";
01118 #endif
01119     llvm_unreachable("Do not know how to widen the result of this operator!");
01120 
01121   case ISD::BIT_CONVERT:       Res = WidenVecRes_BIT_CONVERT(N); break;
01122   case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
01123   case ISD::CONCAT_VECTORS:    Res = WidenVecRes_CONCAT_VECTORS(N); break;
01124   case ISD::CONVERT_RNDSAT:    Res = WidenVecRes_CONVERT_RNDSAT(N); break;
01125   case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
01126   case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
01127   case ISD::LOAD:              Res = WidenVecRes_LOAD(N); break;
01128   case ISD::SCALAR_TO_VECTOR:  Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
01129   case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
01130   case ISD::SELECT_CC:         Res = WidenVecRes_SELECT_CC(N); break;
01131   case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
01132   case ISD::VECTOR_SHUFFLE:
01133     Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
01134     break;
01135   case ISD::VSETCC:
01136     Res = WidenVecRes_VSETCC(N);
01137     break;
01138 
01139   case ISD::ADD:
01140   case ISD::AND:
01141   case ISD::BSWAP:
01142   case ISD::FADD:
01143   case ISD::FCOPYSIGN:
01144   case ISD::FDIV:
01145   case ISD::FMUL:
01146   case ISD::FPOW:
01147   case ISD::FPOWI:
01148   case ISD::FREM:
01149   case ISD::FSUB:
01150   case ISD::MUL:
01151   case ISD::MULHS:
01152   case ISD::MULHU:
01153   case ISD::OR:
01154   case ISD::SDIV:
01155   case ISD::SREM:
01156   case ISD::UDIV:
01157   case ISD::UREM:
01158   case ISD::SUB:
01159   case ISD::XOR:
01160     Res = WidenVecRes_Binary(N);
01161     break;
01162 
01163   case ISD::SHL:
01164   case ISD::SRA:
01165   case ISD::SRL:
01166     Res = WidenVecRes_Shift(N);
01167     break;
01168 
01169   case ISD::FP_ROUND:
01170   case ISD::FP_TO_SINT:
01171   case ISD::FP_TO_UINT:
01172   case ISD::SINT_TO_FP:
01173   case ISD::UINT_TO_FP:
01174   case ISD::TRUNCATE:
01175   case ISD::SIGN_EXTEND:
01176   case ISD::ZERO_EXTEND:
01177   case ISD::ANY_EXTEND:
01178     Res = WidenVecRes_Convert(N);
01179     break;
01180 
01181   case ISD::CTLZ:
01182   case ISD::CTPOP:
01183   case ISD::CTTZ:
01184   case ISD::FABS:
01185   case ISD::FCOS:
01186   case ISD::FNEG:
01187   case ISD::FSIN:
01188   case ISD::FSQRT:
01189     Res = WidenVecRes_Unary(N);
01190     break;
01191   }
01192 
01193   // If Res is null, the sub-method took care of registering the result.
01194   if (Res.getNode())
01195     SetWidenedVector(SDValue(N, ResNo), Res);
01196 }
01197 
01198 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
01199   // Binary op widening.
01200   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01201   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
01202   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
01203   return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2);
01204 }
01205 
01206 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
01207   SDValue InOp = N->getOperand(0);
01208   DebugLoc dl = N->getDebugLoc();
01209 
01210   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01211   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01212 
01213   EVT InVT = InOp.getValueType();
01214   EVT InEltVT = InVT.getVectorElementType();
01215   EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
01216 
01217   unsigned Opcode = N->getOpcode();
01218   unsigned InVTNumElts = InVT.getVectorNumElements();
01219 
01220   if (getTypeAction(InVT) == WidenVector) {
01221     InOp = GetWidenedVector(N->getOperand(0));
01222     InVT = InOp.getValueType();
01223     InVTNumElts = InVT.getVectorNumElements();
01224     if (InVTNumElts == WidenNumElts)
01225       return DAG.getNode(Opcode, dl, WidenVT, InOp);
01226   }
01227 
01228   if (TLI.isTypeLegal(InWidenVT)) {
01229     // Because the result and the input are different vector types, widening
01230     // the result could create a legal type but widening the input might make
01231     // it an illegal type that might lead to repeatedly splitting the input
01232     // and then widening it. To avoid this, we widen the input only if
01233     // it results in a legal type.
01234     if (WidenNumElts % InVTNumElts == 0) {
01235       // Widen the input and call convert on the widened input vector.
01236       unsigned NumConcat = WidenNumElts/InVTNumElts;
01237       SmallVector<SDValue, 16> Ops(NumConcat);
01238       Ops[0] = InOp;
01239       SDValue UndefVal = DAG.getUNDEF(InVT);
01240       for (unsigned i = 1; i != NumConcat; ++i)
01241         Ops[i] = UndefVal;
01242       return DAG.getNode(Opcode, dl, WidenVT,
01243                          DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT,
01244                          &Ops[0], NumConcat));
01245     }
01246 
01247     if (InVTNumElts % WidenNumElts == 0) {
01248       // Extract the input and convert the shorten input vector.
01249       return DAG.getNode(Opcode, dl, WidenVT,
01250                          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT,
01251                                      InOp, DAG.getIntPtrConstant(0)));
01252     }
01253   }
01254 
01255   // Otherwise unroll into some nasty scalar code and rebuild the vector.
01256   SmallVector<SDValue, 16> Ops(WidenNumElts);
01257   EVT EltVT = WidenVT.getVectorElementType();
01258   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
01259   unsigned i;
01260   for (i=0; i < MinElts; ++i)
01261     Ops[i] = DAG.getNode(Opcode, dl, EltVT,
01262                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
01263                                      DAG.getIntPtrConstant(i)));
01264 
01265   SDValue UndefVal = DAG.getUNDEF(EltVT);
01266   for (; i < WidenNumElts; ++i)
01267     Ops[i] = UndefVal;
01268 
01269   return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
01270 }
01271 
01272 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
01273   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01274   SDValue InOp = GetWidenedVector(N->getOperand(0));
01275   SDValue ShOp = N->getOperand(1);
01276 
01277   EVT ShVT = ShOp.getValueType();
01278   if (getTypeAction(ShVT) == WidenVector) {
01279     ShOp = GetWidenedVector(ShOp);
01280     ShVT = ShOp.getValueType();
01281   }
01282   EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(),
01283                                    WidenVT.getVectorNumElements());
01284   if (ShVT != ShWidenVT)
01285     ShOp = ModifyToType(ShOp, ShWidenVT);
01286 
01287   return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp);
01288 }
01289 
01290 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
01291   // Unary op widening.
01292   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01293   SDValue InOp = GetWidenedVector(N->getOperand(0));
01294   return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp);
01295 }
01296 
01297 SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) {
01298   SDValue InOp = N->getOperand(0);
01299   EVT InVT = InOp.getValueType();
01300   EVT VT = N->getValueType(0);
01301   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
01302   DebugLoc dl = N->getDebugLoc();
01303 
01304   switch (getTypeAction(InVT)) {
01305   default:
01306     assert(false && "Unknown type action!");
01307     break;
01308   case Legal:
01309     break;
01310   case PromoteInteger:
01311     // If the InOp is promoted to the same size, convert it.  Otherwise,
01312     // fall out of the switch and widen the promoted input.
01313     InOp = GetPromotedInteger(InOp);
01314     InVT = InOp.getValueType();
01315     if (WidenVT.bitsEq(InVT))
01316       return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp);
01317     break;
01318   case SoftenFloat:
01319   case ExpandInteger:
01320   case ExpandFloat:
01321   case ScalarizeVector:
01322   case SplitVector:
01323     break;
01324   case WidenVector:
01325     // If the InOp is widened to the same size, convert it.  Otherwise, fall
01326     // out of the switch and widen the widened input.
01327     InOp = GetWidenedVector(InOp);
01328     InVT = InOp.getValueType();
01329     if (WidenVT.bitsEq(InVT))
01330       // The input widens to the same size. Convert to the widen value.
01331       return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp);
01332     break;
01333   }
01334 
01335   unsigned WidenSize = WidenVT.getSizeInBits();
01336   unsigned InSize = InVT.getSizeInBits();
01337   if (WidenSize % InSize == 0) {
01338     // Determine new input vector type.  The new input vector type will use
01339     // the same element type (if its a vector) or use the input type as a
01340     // vector.  It is the same size as the type to widen to.
01341     EVT NewInVT;
01342     unsigned NewNumElts = WidenSize / InSize;
01343     if (InVT.isVector()) {
01344       EVT InEltVT = InVT.getVectorElementType();
01345       NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits());
01346     } else {
01347       NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
01348     }
01349 
01350     if (TLI.isTypeLegal(NewInVT)) {
01351       // Because the result and the input are different vector types, widening
01352       // the result could create a legal type but widening the input might make
01353       // it an illegal type that might lead to repeatedly splitting the input
01354       // and then widening it. To avoid this, we widen the input only if
01355       // it results in a legal type.
01356       SmallVector<SDValue, 16> Ops(NewNumElts);
01357       SDValue UndefVal = DAG.getUNDEF(InVT);
01358       Ops[0] = InOp;
01359       for (unsigned i = 1; i < NewNumElts; ++i)
01360         Ops[i] = UndefVal;
01361 
01362       SDValue NewVec;
01363       if (InVT.isVector())
01364         NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl,
01365                              NewInVT, &Ops[0], NewNumElts);
01366       else
01367         NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
01368                              NewInVT, &Ops[0], NewNumElts);
01369       return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec);
01370     }
01371   }
01372 
01373   return CreateStackStoreLoad(InOp, WidenVT);
01374 }
01375 
01376 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
01377   DebugLoc dl = N->getDebugLoc();
01378   // Build a vector with undefined for the new nodes.
01379   EVT VT = N->getValueType(0);
01380   EVT EltVT = VT.getVectorElementType();
01381   unsigned NumElts = VT.getVectorNumElements();
01382 
01383   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
01384   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01385 
01386   SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
01387   NewOps.reserve(WidenNumElts);
01388   for (unsigned i = NumElts; i < WidenNumElts; ++i)
01389     NewOps.push_back(DAG.getUNDEF(EltVT));
01390 
01391   return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size());
01392 }
01393 
01394 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
01395   EVT InVT = N->getOperand(0).getValueType();
01396   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01397   DebugLoc dl = N->getDebugLoc();
01398   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01399   unsigned NumOperands = N->getNumOperands();
01400 
01401   bool InputWidened = false; // Indicates we need to widen the input.
01402   if (getTypeAction(InVT) != WidenVector) {
01403     if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
01404       // Add undef vectors to widen to correct length.
01405       unsigned NumConcat = WidenVT.getVectorNumElements() /
01406                            InVT.getVectorNumElements();
01407       SDValue UndefVal = DAG.getUNDEF(InVT);
01408       SmallVector<SDValue, 16> Ops(NumConcat);
01409       for (unsigned i=0; i < NumOperands; ++i)
01410         Ops[i] = N->getOperand(i);
01411       for (unsigned i = NumOperands; i != NumConcat; ++i)
01412         Ops[i] = UndefVal;
01413       return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat);
01414     }
01415   } else {
01416     InputWidened = true;
01417     if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
01418       // The inputs and the result are widen to the same value.
01419       unsigned i;
01420       for (i=1; i < NumOperands; ++i)
01421         if (N->getOperand(i).getOpcode() != ISD::UNDEF)
01422           break;
01423 
01424       if (i > NumOperands)
01425         // Everything but the first operand is an UNDEF so just return the
01426         // widened first operand.
01427         return GetWidenedVector(N->getOperand(0));
01428 
01429       if (NumOperands == 2) {
01430         // Replace concat of two operands with a shuffle.
01431         SmallVector<int, 16> MaskOps(WidenNumElts);
01432         for (unsigned i=0; i < WidenNumElts/2; ++i) {
01433           MaskOps[i] = i;
01434           MaskOps[i+WidenNumElts/2] = i+WidenNumElts;
01435         }
01436         return DAG.getVectorShuffle(WidenVT, dl,
01437                                     GetWidenedVector(N->getOperand(0)),
01438                                     GetWidenedVector(N->getOperand(1)),
01439                                     &MaskOps[0]);
01440       }
01441     }
01442   }
01443 
01444   // Fall back to use extracts and build vector.
01445   EVT EltVT = WidenVT.getVectorElementType();
01446   unsigned NumInElts = InVT.getVectorNumElements();
01447   SmallVector<SDValue, 16> Ops(WidenNumElts);
01448   unsigned Idx = 0;
01449   for (unsigned i=0; i < NumOperands; ++i) {
01450     SDValue InOp = N->getOperand(i);
01451     if (InputWidened)
01452       InOp = GetWidenedVector(InOp);
01453     for (unsigned j=0; j < NumInElts; ++j)
01454         Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
01455                                  DAG.getIntPtrConstant(j));
01456   }
01457   SDValue UndefVal = DAG.getUNDEF(EltVT);
01458   for (; Idx < WidenNumElts; ++Idx)
01459     Ops[Idx] = UndefVal;
01460   return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
01461 }
01462 
01463 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
01464   DebugLoc dl = N->getDebugLoc();
01465   SDValue InOp  = N->getOperand(0);
01466   SDValue RndOp = N->getOperand(3);
01467   SDValue SatOp = N->getOperand(4);
01468 
01469   EVT      WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01470   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01471 
01472   EVT InVT = InOp.getValueType();
01473   EVT InEltVT = InVT.getVectorElementType();
01474   EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
01475 
01476   SDValue DTyOp = DAG.getValueType(WidenVT);
01477   SDValue STyOp = DAG.getValueType(InWidenVT);
01478   ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
01479 
01480   unsigned InVTNumElts = InVT.getVectorNumElements();
01481   if (getTypeAction(InVT) == WidenVector) {
01482     InOp = GetWidenedVector(InOp);
01483     InVT = InOp.getValueType();
01484     InVTNumElts = InVT.getVectorNumElements();
01485     if (InVTNumElts == WidenNumElts)
01486       return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
01487                                   SatOp, CvtCode);
01488   }
01489 
01490   if (TLI.isTypeLegal(InWidenVT)) {
01491     // Because the result and the input are different vector types, widening
01492     // the result could create a legal type but widening the input might make
01493     // it an illegal type that might lead to repeatedly splitting the input
01494     // and then widening it. To avoid this, we widen the input only if
01495     // it results in a legal type.
01496     if (WidenNumElts % InVTNumElts == 0) {
01497       // Widen the input and call convert on the widened input vector.
01498       unsigned NumConcat = WidenNumElts/InVTNumElts;
01499       SmallVector<SDValue, 16> Ops(NumConcat);
01500       Ops[0] = InOp;
01501       SDValue UndefVal = DAG.getUNDEF(InVT);
01502       for (unsigned i = 1; i != NumConcat; ++i) {
01503         Ops[i] = UndefVal;
01504       }
01505       InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat);
01506       return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
01507                                   SatOp, CvtCode);
01508     }
01509 
01510     if (InVTNumElts % WidenNumElts == 0) {
01511       // Extract the input and convert the shorten input vector.
01512       InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
01513                          DAG.getIntPtrConstant(0));
01514       return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
01515                                 SatOp, CvtCode);
01516     }
01517   }
01518 
01519   // Otherwise unroll into some nasty scalar code and rebuild the vector.
01520   SmallVector<SDValue, 16> Ops(WidenNumElts);
01521   EVT EltVT = WidenVT.getVectorElementType();
01522   DTyOp = DAG.getValueType(EltVT);
01523   STyOp = DAG.getValueType(InEltVT);
01524 
01525   unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
01526   unsigned i;
01527   for (i=0; i < MinElts; ++i) {
01528     SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
01529                                  DAG.getIntPtrConstant(i));
01530     Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp,
01531                                         SatOp, CvtCode);
01532   }
01533 
01534   SDValue UndefVal = DAG.getUNDEF(EltVT);
01535   for (; i < WidenNumElts; ++i)
01536     Ops[i] = UndefVal;
01537 
01538   return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
01539 }
01540 
01541 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
01542   EVT      VT = N->getValueType(0);
01543   EVT      WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
01544   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01545   SDValue  InOp = N->getOperand(0);
01546   SDValue  Idx  = N->getOperand(1);
01547   DebugLoc dl = N->getDebugLoc();
01548 
01549   if (getTypeAction(InOp.getValueType()) == WidenVector)
01550     InOp = GetWidenedVector(InOp);
01551 
01552   EVT InVT = InOp.getValueType();
01553 
01554   ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx);
01555   if (CIdx) {
01556     unsigned IdxVal = CIdx->getZExtValue();
01557     // Check if we can just return the input vector after widening.
01558     if (IdxVal == 0 && InVT == WidenVT)
01559       return InOp;
01560 
01561     // Check if we can extract from the vector.
01562     unsigned InNumElts = InVT.getVectorNumElements();
01563     if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
01564         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
01565   }
01566 
01567   // We could try widening the input to the right length but for now, extract
01568   // the original elements, fill the rest with undefs and build a vector.
01569   SmallVector<SDValue, 16> Ops(WidenNumElts);
01570   EVT EltVT = VT.getVectorElementType();
01571   EVT IdxVT = Idx.getValueType();
01572   unsigned NumElts = VT.getVectorNumElements();
01573   unsigned i;
01574   if (CIdx) {
01575     unsigned IdxVal = CIdx->getZExtValue();
01576     for (i=0; i < NumElts; ++i)
01577       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
01578                            DAG.getConstant(IdxVal+i, IdxVT));
01579   } else {
01580     Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx);
01581     for (i=1; i < NumElts; ++i) {
01582       SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
01583                                    DAG.getConstant(i, IdxVT));
01584       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx);
01585     }
01586   }
01587 
01588   SDValue UndefVal = DAG.getUNDEF(EltVT);
01589   for (; i < WidenNumElts; ++i)
01590     Ops[i] = UndefVal;
01591   return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
01592 }
01593 
01594 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
01595   SDValue InOp = GetWidenedVector(N->getOperand(0));
01596   return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(),
01597                      InOp.getValueType(), InOp,
01598                      N->getOperand(1), N->getOperand(2));
01599 }
01600 
01601 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
01602   LoadSDNode *LD = cast<LoadSDNode>(N);
01603   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
01604   EVT LdVT    = LD->getMemoryVT();
01605   DebugLoc dl = N->getDebugLoc();
01606   assert(LdVT.isVector() && WidenVT.isVector());
01607 
01608   // Load information
01609   SDValue   Chain = LD->getChain();
01610   SDValue   BasePtr = LD->getBasePtr();
01611   int       SVOffset = LD->getSrcValueOffset();
01612   unsigned  Align    = LD->getAlignment();
01613   bool      isVolatile = LD->isVolatile();
01614   const Value *SV = LD->getSrcValue();
01615   ISD::LoadExtType ExtType = LD->getExtensionType();
01616 
01617   SDValue Result;
01618   SmallVector<SDValue, 16> LdChain;  // Chain for the series of load
01619   if (ExtType != ISD::NON_EXTLOAD) {
01620     // For extension loads, we can not play the tricks of chopping legal
01621     // vector types and bit cast it to the right type.  Instead, we unroll
01622     // the load and build a vector.
01623     EVT EltVT = WidenVT.getVectorElementType();
01624     EVT LdEltVT = LdVT.getVectorElementType();
01625     unsigned NumElts = LdVT.getVectorNumElements();
01626 
01627     // Load each element and widen
01628     unsigned WidenNumElts = WidenVT.getVectorNumElements();
01629     SmallVector<SDValue, 16> Ops(WidenNumElts);
01630     unsigned Increment = LdEltVT.getSizeInBits() / 8;
01631     Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset,
01632                             LdEltVT, isVolatile, Align);
01633     LdChain.push_back(Ops[0].getValue(1));
01634     unsigned i = 0, Offset = Increment;
01635     for (i=1; i < NumElts; ++i, Offset += Increment) {
01636       SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
01637                                        BasePtr, DAG.getIntPtrConstant(Offset));
01638       Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV,
01639                               SVOffset + Offset, LdEltVT, isVolatile, Align);
01640       LdChain.push_back(Ops[i].getValue(1));
01641     }
01642 
01643     // Fill the rest with undefs
01644     SDValue UndefVal = DAG.getUNDEF(EltVT);
01645     for (; i != WidenNumElts; ++i)
01646       Ops[i] = UndefVal;
01647 
01648     Result =  DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size());
01649   } else {
01650     assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
01651     unsigned int LdWidth = LdVT.getSizeInBits();
01652     Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset,
01653                                  Align, isVolatile, LdWidth, WidenVT, dl);
01654   }
01655 
01656  // If we generate a single load, we can use that for the chain.  Otherwise,
01657  // build a factor node to remember the multiple loads are independent and
01658  // chain to that.
01659  SDValue NewChain;
01660  if (LdChain.size() == 1)
01661    NewChain = LdChain[0];
01662  else
01663    NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0],
01664                           LdChain.size());
01665 
01666   // Modified the chain - switch anything that used the old chain to use
01667   // the new one.
01668   ReplaceValueWith(SDValue(N, 1), NewChain);
01669 
01670   return Result;
01671 }
01672 
01673 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
01674   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01675   return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(),
01676                      WidenVT, N->getOperand(0));
01677 }
01678 
01679 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
01680   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01681   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01682 
01683   SDValue Cond1 = N->getOperand(0);
01684   EVT CondVT = Cond1.getValueType();
01685   if (CondVT.isVector()) {
01686     EVT CondEltVT = CondVT.getVectorElementType();
01687     EVT CondWidenVT =  EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts);
01688     if (getTypeAction(CondVT) == WidenVector)
01689       Cond1 = GetWidenedVector(Cond1);
01690 
01691     if (Cond1.getValueType() != CondWidenVT)
01692        Cond1 = ModifyToType(Cond1, CondWidenVT);
01693   }
01694 
01695   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
01696   SDValue InOp2 = GetWidenedVector(N->getOperand(2));
01697   assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
01698   return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
01699                      WidenVT, Cond1, InOp1, InOp2);
01700 }
01701 
01702 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
01703   SDValue InOp1 = GetWidenedVector(N->getOperand(2));
01704   SDValue InOp2 = GetWidenedVector(N->getOperand(3));
01705   return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(),
01706                      InOp1.getValueType(), N->getOperand(0),
01707                      N->getOperand(1), InOp1, InOp2, N->getOperand(4));
01708 }
01709 
01710 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
01711  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01712  return DAG.getUNDEF(WidenVT);
01713 }
01714 
01715 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
01716   EVT VT = N->getValueType(0);
01717   DebugLoc dl = N->getDebugLoc();
01718 
01719   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
01720   unsigned NumElts = VT.getVectorNumElements();
01721   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01722 
01723   SDValue InOp1 = GetWidenedVector(N->getOperand(0));
01724   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
01725 
01726   // Adjust mask based on new input vector length.
01727   SmallVector<int, 16> NewMask;
01728   for (unsigned i = 0; i != NumElts; ++i) {
01729     int Idx = N->getMaskElt(i);
01730     if (Idx < (int)NumElts)
01731       NewMask.push_back(Idx);
01732     else
01733       NewMask.push_back(Idx - NumElts + WidenNumElts);
01734   }
01735   for (unsigned i = NumElts; i != WidenNumElts; ++i)
01736     NewMask.push_back(-1);
01737   return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]);
01738 }
01739 
01740 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
01741   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
01742   unsigned WidenNumElts = WidenVT.getVectorNumElements();
01743 
01744   SDValue InOp1 = N->getOperand(0);
01745   EVT InVT = InOp1.getValueType();
01746   assert(InVT.isVector() && "can not widen non vector type");
01747   EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts);
01748   InOp1 = GetWidenedVector(InOp1);
01749   SDValue InOp2 = GetWidenedVector(N->getOperand(1));
01750 
01751   // Assume that the input and output will be widen appropriately.  If not,
01752   // we will have to unroll it at some point.
01753   assert(InOp1.getValueType() == WidenInVT &&
01754          InOp2.getValueType() == WidenInVT &&
01755          "Input not widened to expected type!");
01756   return DAG.getNode(ISD::VSETCC, N->getDebugLoc(),
01757                      WidenVT, InOp1, InOp2, N->getOperand(2));
01758 }
01759 
01760 
01761 //===----------------------------------------------------------------------===//
01762 // Widen Vector Operand
01763 //===----------------------------------------------------------------------===//
01764 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) {
01765   DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG);
01766         cerr << "\n");
01767   SDValue Res = SDValue();
01768 
01769   switch (N->getOpcode()) {
01770   default:
01771 #ifndef NDEBUG
01772     cerr << "WidenVectorOperand op #" << ResNo << ": ";
01773     N->dump(&DAG); cerr << "\n";
01774 #endif
01775     llvm_unreachable("Do not know how to widen this operator's operand!");
01776 
01777   case ISD::BIT_CONVERT:        Res = WidenVecOp_BIT_CONVERT(N); break;
01778   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
01779   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
01780   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
01781 
01782   case ISD::FP_ROUND:
01783   case ISD::FP_TO_SINT:
01784   case ISD::FP_TO_UINT:
01785   case ISD::SINT_TO_FP:
01786   case ISD::UINT_TO_FP:
01787   case ISD::TRUNCATE:
01788   case ISD::SIGN_EXTEND:
01789   case ISD::ZERO_EXTEND:
01790   case ISD::ANY_EXTEND:
01791     Res = WidenVecOp_Convert(N);
01792     break;
01793   }
01794 
01795   // If Res is null, the sub-method took care of registering the result.
01796   if (!Res.getNode()) return false;
01797 
01798   // If the result is N, the sub-method updated N in place.  Tell the legalizer
01799   // core about this.
01800   if (Res.getNode() == N)
01801     return true;
01802 
01803 
01804   assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
01805          "Invalid operand expansion");
01806 
01807   ReplaceValueWith(SDValue(N, 0), Res);
01808   return false;
01809 }
01810 
01811 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
01812   // Since the result is legal and the input is illegal, it is unlikely
01813   // that we can fix the input to a legal type so unroll the convert
01814   // into some scalar code and create a nasty build vector.
01815   EVT VT = N->getValueType(0);
01816   EVT EltVT = VT.getVectorElementType();
01817   DebugLoc dl = N->getDebugLoc();
01818   unsigned NumElts = VT.getVectorNumElements();
01819   SDValue InOp = N->getOperand(0);
01820   if (getTypeAction(InOp.getValueType()) == WidenVector)
01821     InOp = GetWidenedVector(InOp);
01822   EVT InVT = InOp.getValueType();
01823   EVT InEltVT = InVT.getVectorElementType();
01824 
01825   unsigned Opcode = N->getOpcode();
01826   SmallVector<SDValue, 16> Ops(NumElts);
01827   for (unsigned i=0; i < NumElts; ++i)
01828     Ops[i] = DAG.getNode(Opcode, dl, EltVT,
01829                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
01830                                      DAG.getIntPtrConstant(i)));
01831 
01832   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
01833 }
01834 
01835 SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) {
01836   EVT VT = N->getValueType(0);
01837   SDValue InOp = GetWidenedVector(N->getOperand(0));
01838   EVT InWidenVT = InOp.getValueType();
01839   DebugLoc dl = N->getDebugLoc();
01840 
01841   // Check if we can convert between two legal vector types and extract.
01842   unsigned InWidenSize = InWidenVT.getSizeInBits();
01843   unsigned Size = VT.getSizeInBits();
01844   if (InWidenSize % Size == 0 && !VT.isVector()) {
01845     unsigned NewNumElts = InWidenSize / Size;
01846     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
01847     if (TLI.isTypeLegal(NewVT)) {
01848       SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp);
01849       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
01850                          DAG.getIntPtrConstant(0));
01851     }
01852   }
01853 
01854   return CreateStackStoreLoad(InOp, VT);
01855 }
01856 
01857 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
01858   // If the input vector is not legal, it is likely that we will not find a
01859   // legal vector of the same size. Replace the concatenate vector with a
01860   // nasty build vector.
01861   EVT VT = N->getValueType(0);
01862   EVT EltVT = VT.getVectorElementType();
01863   DebugLoc dl = N->getDebugLoc();
01864   unsigned NumElts = VT.getVectorNumElements();
01865   SmallVector<SDValue, 16> Ops(NumElts);
01866 
01867   EVT InVT = N->getOperand(0).getValueType();
01868   unsigned NumInElts = InVT.getVectorNumElements();
01869 
01870   unsigned Idx = 0;
01871   unsigned NumOperands = N->getNumOperands();
01872   for (unsigned i=0; i < NumOperands; ++i) {
01873     SDValue InOp = N->getOperand(i);
01874     if (getTypeAction(InOp.getValueType()) == WidenVector)
01875       InOp = GetWidenedVector(InOp);
01876     for (unsigned j=0; j < NumInElts; ++j)
01877       Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
01878                                DAG.getIntPtrConstant(j));
01879   }
01880   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
01881 }
01882 
01883 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
01884   SDValue InOp = GetWidenedVector(N->getOperand(0));
01885   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
01886                      N->getValueType(0), InOp, N->getOperand(1));
01887 }
01888 
01889 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
01890   // We have to widen the value but we want only to store the original
01891   // vector type.
01892   StoreSDNode *ST = cast<StoreSDNode>(N);
01893   SDValue  Chain = ST->getChain();
01894   SDValue  BasePtr = ST->getBasePtr();
01895   const    Value *SV = ST->getSrcValue();
01896   int      SVOffset = ST->getSrcValueOffset();
01897   unsigned Align = ST->getAlignment();
01898   bool     isVolatile = ST->isVolatile();
01899   SDValue  ValOp = GetWidenedVector(ST->getValue());
01900   DebugLoc dl = N->getDebugLoc();
01901 
01902   EVT StVT = ST->getMemoryVT();
01903   EVT ValVT = ValOp.getValueType();
01904   // It must be true that we the widen vector type is bigger than where
01905   // we need to store.
01906   assert(StVT.isVector() && ValOp.getValueType().isVector());
01907   assert(StVT.bitsLT(ValOp.getValueType()));
01908 
01909   SmallVector<SDValue, 16> StChain;
01910   if (ST->isTruncatingStore()) {
01911     // For truncating stores, we can not play the tricks of chopping legal
01912     // vector types and bit cast it to the right type.  Instead, we unroll
01913     // the store.
01914     EVT StEltVT  = StVT.getVectorElementType();
01915     EVT ValEltVT = ValVT.getVectorElementType();
01916     unsigned Increment = ValEltVT.getSizeInBits() / 8;
01917     unsigned NumElts = StVT.getVectorNumElements();
01918     SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
01919                               DAG.getIntPtrConstant(0));
01920     StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV,
01921                                         SVOffset, StEltVT,
01922                                         isVolatile, Align));
01923     unsigned Offset = Increment;
01924     for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
01925       SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
01926                                        BasePtr, DAG.getIntPtrConstant(Offset));
01927       SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
01928                               DAG.getIntPtrConstant(0));
01929       StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV,
01930                                           SVOffset + Offset, StEltVT,
01931                                           isVolatile, MinAlign(Align, Offset)));
01932     }
01933   }
01934   else {
01935     assert(StVT.getVectorElementType() == ValVT.getVectorElementType());
01936     // Store value
01937     GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset,
01938                          Align, isVolatile, ValOp, StVT.getSizeInBits(), dl);
01939   }
01940   if (StChain.size() == 1)
01941     return StChain[0];
01942   else
01943     return DAG.getNode(ISD::TokenFactor, dl,
01944                        MVT::Other,&StChain[0],StChain.size());
01945 }
01946 
01947 //===----------------------------------------------------------------------===//
01948 // Vector Widening Utilities
01949 //===----------------------------------------------------------------------===//
01950 
01951 
01952 // Utility function to find a vector type and its associated element
01953 // type from a preferred width and whose vector type must be the same size
01954 // as the VecVT.
01955 //  TLI:   Target lowering used to determine legal types.
01956 //  Width: Preferred width to store.
01957 //  VecVT: Vector value type whose size we must match.
01958 // Returns NewVecVT and NewEltVT - the vector type and its associated
01959 // element type.
01960 static void FindAssocWidenVecType(SelectionDAG& DAG,
01961                                   const TargetLowering &TLI, unsigned Width,
01962                                   EVT VecVT,
01963                                   EVT& NewEltVT, EVT& NewVecVT) {
01964   unsigned EltWidth = Width + 1;
01965   if (TLI.isTypeLegal(VecVT)) {
01966     // We start with the preferred with, making it a power of 2 and find a
01967     // legal vector type of that width.  If not, we reduce it by another of 2.
01968     // For incoming type is legal, this process will end as a vector of the
01969     // smallest loadable type should always be legal.
01970     do {
01971       assert(EltWidth > 0);
01972       EltWidth = 1 << Log2_32(EltWidth - 1);
01973       NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
01974       unsigned NumElts = VecVT.getSizeInBits() / EltWidth;
01975       NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts);
01976     } while (!TLI.isTypeLegal(NewVecVT) ||
01977              VecVT.getSizeInBits() != NewVecVT.getSizeInBits());
01978   } else {
01979     // The incoming vector type is illegal and is the result of widening
01980     // a vector to a power of 2. In this case, we will use the preferred
01981     // with as long as it is a multiple of the incoming vector length.
01982     // The legalization process will eventually make this into a legal type
01983     // and remove the illegal bit converts (which would turn to stack converts
01984     // if they are allow to exist).
01985      do {
01986       assert(EltWidth > 0);
01987       EltWidth = 1 << Log2_32(EltWidth - 1);
01988       NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
01989       unsigned NumElts = VecVT.getSizeInBits() / EltWidth;
01990       NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts);
01991     } while (!TLI.isTypeLegal(NewEltVT) ||
01992              VecVT.getSizeInBits() != NewVecVT.getSizeInBits());
01993   }
01994 }
01995 
01996 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain,
01997                                               SDValue      Chain,
01998                                               SDValue      BasePtr,
01999                                               const Value *SV,
02000                                               int          SVOffset,
02001                                               unsigned     Alignment,
02002                                               bool         isVolatile,
02003                                               unsigned     LdWidth,
02004                                               EVT          ResType,
02005                                               DebugLoc     dl) {
02006   // The strategy assumes that we can efficiently load powers of two widths.
02007   // The routines chops the vector into the largest power of 2 load and
02008   // can be inserted into a legal vector and then cast the result into the
02009   // vector type we want.  This avoids unnecessary stack converts.
02010 
02011   // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and
02012   //       the load is nonvolatile, we an use a wider load for the value.
02013 
02014   // Find the vector type that can load from.
02015   EVT NewEltVT, NewVecVT;
02016   unsigned NewEltVTWidth;
02017   FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT);
02018   NewEltVTWidth = NewEltVT.getSizeInBits();
02019 
02020   SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset,
02021                              isVolatile, Alignment);
02022   SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
02023   LdChain.push_back(LdOp.getValue(1));
02024 
02025   // Check if we can load the element with one instruction
02026   if (LdWidth == NewEltVTWidth) {
02027     return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp);
02028   }
02029 
02030   unsigned Idx = 1;
02031   LdWidth -= NewEltVTWidth;
02032   unsigned Offset = 0;
02033 
02034   while (LdWidth > 0) {
02035     unsigned Increment = NewEltVTWidth / 8;
02036     Offset += Increment;
02037     BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
02038                           DAG.getIntPtrConstant(Increment));
02039 
02040     if (LdWidth < NewEltVTWidth) {
02041       // Our current type we are using is too large, use a smaller size by
02042       // using a smaller power of 2
02043       unsigned oNewEltVTWidth = NewEltVTWidth;
02044       FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT);
02045       NewEltVTWidth = NewEltVT.getSizeInBits();
02046       // Readjust position and vector position based on new load type
02047       Idx = Idx * (oNewEltVTWidth/NewEltVTWidth);
02048       VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp);
02049     }
02050 
02051     SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV,
02052                                  SVOffset+Offset, isVolatile,
02053                                  MinAlign(Alignment, Offset));
02054     LdChain.push_back(LdOp.getValue(1));
02055     VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp,
02056                         DAG.getIntPtrConstant(Idx++));
02057 
02058     LdWidth -= NewEltVTWidth;
02059   }
02060 
02061   return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp);
02062 }
02063 
02064 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain,
02065                                             SDValue   Chain,
02066                                             SDValue   BasePtr,
02067                                             const Value *SV,
02068                                             int         SVOffset,
02069                                             unsigned    Alignment,
02070                                             bool        isVolatile,
02071                                             SDValue     ValOp,
02072                                             unsigned    StWidth,
02073                                             DebugLoc    dl) {
02074   // Breaks the stores into a series of power of 2 width stores.  For any
02075   // width, we convert the vector to the vector of element size that we
02076   // want to store.  This avoids requiring a stack convert.
02077 
02078   // Find a width of the element type we can store with
02079   EVT WidenVT = ValOp.getValueType();
02080   EVT NewEltVT, NewVecVT;
02081 
02082   FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT);
02083   unsigned NewEltVTWidth = NewEltVT.getSizeInBits();
02084 
02085   SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp);
02086   SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp,
02087                             DAG.getIntPtrConstant(0));
02088   SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset,
02089                                isVolatile, Alignment);
02090   StChain.push_back(StOp);
02091 
02092   // Check if we are done
02093   if (StWidth == NewEltVTWidth) {
02094     return;
02095   }
02096 
02097   unsigned Idx = 1;
02098   StWidth -= NewEltVTWidth;
02099   unsigned Offset = 0;
02100 
02101   while (StWidth > 0) {
02102     unsigned Increment = NewEltVTWidth / 8;
02103     Offset += Increment;
02104     BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
02105                           DAG.getIntPtrConstant(Increment));
02106 
02107     if (StWidth < NewEltVTWidth) {
02108       // Our current type we are using is too large, use a smaller size by
02109       // using a smaller power of 2
02110       unsigned oNewEltVTWidth = NewEltVTWidth;
02111       FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT);
02112       NewEltVTWidth = NewEltVT.getSizeInBits();
02113       // Readjust position and vector position based on new load type
02114       Idx = Idx * (oNewEltVTWidth/NewEltVTWidth);
02115       VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp);
02116     }
02117 
02118     EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp,
02119                       DAG.getIntPtrConstant(Idx++));
02120     StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV,
02121                                    SVOffset + Offset, isVolatile,
02122                                    MinAlign(Alignment, Offset)));
02123     StWidth -= NewEltVTWidth;
02124   }
02125 }
02126 
02127 /// Modifies a vector input (widen or narrows) to a vector of NVT.  The
02128 /// input vector must have the same element type as NVT.
02129 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) {
02130   // Note that InOp might have been widened so it might already have
02131   // the right width or it might need be narrowed.
02132   EVT InVT = InOp.getValueType();
02133   assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
02134          "input and widen element type must match");
02135   DebugLoc dl = InOp.getDebugLoc();
02136 
02137   // Check if InOp already has the right width.
02138   if (InVT == NVT)
02139     return InOp;
02140 
02141   unsigned InNumElts = InVT.getVectorNumElements();
02142   unsigned WidenNumElts = NVT.getVectorNumElements();
02143   if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
02144     unsigned NumConcat = WidenNumElts / InNumElts;
02145     SmallVector<SDValue, 16> Ops(NumConcat);
02146     SDValue UndefVal = DAG.getUNDEF(InVT);
02147     Ops[0] = InOp;
02148     for (unsigned i = 1; i != NumConcat; ++i)
02149       Ops[i] = UndefVal;
02150 
02151     return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat);
02152   }
02153 
02154   if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
02155     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
02156                        DAG.getIntPtrConstant(0));
02157 
02158   // Fall back to extract and build.
02159   SmallVector<SDValue, 16> Ops(WidenNumElts);
02160   EVT EltVT = NVT.getVectorElementType();
02161   unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
02162   unsigned Idx;
02163   for (Idx = 0; Idx < MinNumElts; ++Idx)
02164     Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
02165                            DAG.getIntPtrConstant(Idx));
02166 
02167   SDValue UndefVal = DAG.getUNDEF(EltVT);
02168   for ( ; Idx < WidenNumElts; ++Idx)
02169     Ops[Idx] = UndefVal;
02170   return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts);
02171 }