LLVM API Documentation
00001 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 00002 // 00003 // The LLVM Compiler Infrastructure 00004 // 00005 // This file is distributed under the University of Illinois Open Source 00006 // License. See LICENSE.TXT for details. 00007 // 00008 //===----------------------------------------------------------------------===// 00009 // 00010 // This file performs vector type splitting and scalarization for LegalizeTypes. 00011 // Scalarization is the act of changing a computation in an illegal one-element 00012 // vector type to be a computation in its scalar element type. For example, 00013 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 00014 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 00015 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 00016 // types. 00017 // Splitting is the act of changing a computation in an invalid vector type to 00018 // be a computation in two vectors of half the size. For example, implementing 00019 // <128 x f32> operations in terms of two <64 x f32> operations. 00020 // 00021 //===----------------------------------------------------------------------===// 00022 00023 #include "LegalizeTypes.h" 00024 #include "llvm/CodeGen/PseudoSourceValue.h" 00025 #include "llvm/Target/TargetData.h" 00026 #include "llvm/Support/ErrorHandling.h" 00027 using namespace llvm; 00028 00029 //===----------------------------------------------------------------------===// 00030 // Result Vector Scalarization: <1 x ty> -> ty. 00031 //===----------------------------------------------------------------------===// 00032 00033 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 00034 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 00035 cerr << "\n"); 00036 SDValue R = SDValue(); 00037 00038 switch (N->getOpcode()) { 00039 default: 00040 #ifndef NDEBUG 00041 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 00042 N->dump(&DAG); cerr << "\n"; 00043 #endif 00044 llvm_unreachable("Do not know how to scalarize the result of this operator!"); 00045 00046 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 00047 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 00048 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 00049 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 00050 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 00051 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 00052 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 00053 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 00054 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 00055 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 00056 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 00057 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 00058 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 00059 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 00060 00061 case ISD::CTLZ: 00062 case ISD::CTPOP: 00063 case ISD::CTTZ: 00064 case ISD::FABS: 00065 case ISD::FCOS: 00066 case ISD::FNEG: 00067 case ISD::FP_TO_SINT: 00068 case ISD::FP_TO_UINT: 00069 case ISD::FSIN: 00070 case ISD::FSQRT: 00071 case ISD::FTRUNC: 00072 case ISD::FFLOOR: 00073 case ISD::FCEIL: 00074 case ISD::FRINT: 00075 case ISD::FNEARBYINT: 00076 case ISD::UINT_TO_FP: 00077 case ISD::SINT_TO_FP: 00078 case ISD::TRUNCATE: 00079 case ISD::SIGN_EXTEND: 00080 case ISD::ZERO_EXTEND: 00081 case ISD::ANY_EXTEND: 00082 R = ScalarizeVecRes_UnaryOp(N); 00083 break; 00084 00085 case ISD::ADD: 00086 case ISD::AND: 00087 case ISD::FADD: 00088 case ISD::FDIV: 00089 case ISD::FMUL: 00090 case ISD::FPOW: 00091 case ISD::FREM: 00092 case ISD::FSUB: 00093 case ISD::MUL: 00094 case ISD::OR: 00095 case ISD::SDIV: 00096 case ISD::SREM: 00097 case ISD::SUB: 00098 case ISD::UDIV: 00099 case ISD::UREM: 00100 case ISD::XOR: 00101 case ISD::SHL: 00102 case ISD::SRA: 00103 case ISD::SRL: 00104 R = ScalarizeVecRes_BinOp(N); 00105 break; 00106 } 00107 00108 // If R is null, the sub-method took care of registering the result. 00109 if (R.getNode()) 00110 SetScalarizedVector(SDValue(N, ResNo), R); 00111 } 00112 00113 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 00114 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 00115 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 00116 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 00117 LHS.getValueType(), LHS, RHS); 00118 } 00119 00120 SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 00121 EVT NewVT = N->getValueType(0).getVectorElementType(); 00122 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 00123 NewVT, N->getOperand(0)); 00124 } 00125 00126 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 00127 EVT NewVT = N->getValueType(0).getVectorElementType(); 00128 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 00129 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 00130 Op0, DAG.getValueType(NewVT), 00131 DAG.getValueType(Op0.getValueType()), 00132 N->getOperand(3), 00133 N->getOperand(4), 00134 cast<CvtRndSatSDNode>(N)->getCvtCode()); 00135 } 00136 00137 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 00138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 00139 N->getValueType(0).getVectorElementType(), 00140 N->getOperand(0), N->getOperand(1)); 00141 } 00142 00143 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 00144 SDValue Op = GetScalarizedVector(N->getOperand(0)); 00145 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 00146 Op.getValueType(), Op, N->getOperand(1)); 00147 } 00148 00149 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 00150 // The value to insert may have a wider type than the vector element type, 00151 // so be sure to truncate it to the element type if necessary. 00152 SDValue Op = N->getOperand(1); 00153 EVT EltVT = N->getValueType(0).getVectorElementType(); 00154 if (Op.getValueType() != EltVT) 00155 // FIXME: Can this happen for floating point types? 00156 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 00157 return Op; 00158 } 00159 00160 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 00161 assert(N->isUnindexed() && "Indexed vector load?"); 00162 00163 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 00164 N->getExtensionType(), 00165 N->getValueType(0).getVectorElementType(), 00166 N->getChain(), N->getBasePtr(), 00167 DAG.getUNDEF(N->getBasePtr().getValueType()), 00168 N->getSrcValue(), N->getSrcValueOffset(), 00169 N->getMemoryVT().getVectorElementType(), 00170 N->isVolatile(), N->getAlignment()); 00171 00172 // Legalized the chain result - switch anything that used the old chain to 00173 // use the new one. 00174 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 00175 return Result; 00176 } 00177 00178 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 00179 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 00180 EVT DestVT = N->getValueType(0).getVectorElementType(); 00181 SDValue Op = GetScalarizedVector(N->getOperand(0)); 00182 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 00183 } 00184 00185 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 00186 // If the operand is wider than the vector element type then it is implicitly 00187 // truncated. Make that explicit here. 00188 EVT EltVT = N->getValueType(0).getVectorElementType(); 00189 SDValue InOp = N->getOperand(0); 00190 if (InOp.getValueType() != EltVT) 00191 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 00192 return InOp; 00193 } 00194 00195 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 00196 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 00197 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 00198 LHS.getValueType(), N->getOperand(0), LHS, 00199 GetScalarizedVector(N->getOperand(2))); 00200 } 00201 00202 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 00203 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 00204 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 00205 N->getOperand(0), N->getOperand(1), 00206 LHS, GetScalarizedVector(N->getOperand(3)), 00207 N->getOperand(4)); 00208 } 00209 00210 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 00211 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 00212 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 00213 DebugLoc DL = N->getDebugLoc(); 00214 00215 // Turn it into a scalar SETCC. 00216 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 00217 } 00218 00219 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 00220 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 00221 } 00222 00223 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 00224 // Figure out if the scalar is the LHS or RHS and return it. 00225 SDValue Arg = N->getOperand(2).getOperand(0); 00226 if (Arg.getOpcode() == ISD::UNDEF) 00227 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 00228 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 00229 return GetScalarizedVector(N->getOperand(Op)); 00230 } 00231 00232 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 00233 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 00234 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 00235 EVT NVT = N->getValueType(0).getVectorElementType(); 00236 EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 00237 DebugLoc DL = N->getDebugLoc(); 00238 00239 // Turn it into a scalar SETCC. 00240 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 00241 00242 // VSETCC always returns a sign-extended value, while SETCC may not. The 00243 // SETCC result type may not match the vector element type. Correct these. 00244 if (NVT.bitsLE(SVT)) { 00245 // The SETCC result type is bigger than the vector element type. 00246 // Ensure the SETCC result is sign-extended. 00247 if (TLI.getBooleanContents() != 00248 TargetLowering::ZeroOrNegativeOneBooleanContent) 00249 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 00250 DAG.getValueType(MVT::i1)); 00251 // Truncate to the final type. 00252 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 00253 } 00254 00255 // The SETCC result type is smaller than the vector element type. 00256 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 00257 if (TLI.getBooleanContents() != 00258 TargetLowering::ZeroOrNegativeOneBooleanContent) 00259 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 00260 // Sign extend to the final type. 00261 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 00262 } 00263 00264 00265 //===----------------------------------------------------------------------===// 00266 // Operand Vector Scalarization <1 x ty> -> ty. 00267 //===----------------------------------------------------------------------===// 00268 00269 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 00270 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 00271 cerr << "\n"); 00272 SDValue Res = SDValue(); 00273 00274 if (Res.getNode() == 0) { 00275 switch (N->getOpcode()) { 00276 default: 00277 #ifndef NDEBUG 00278 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 00279 N->dump(&DAG); cerr << "\n"; 00280 #endif 00281 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 00282 case ISD::BIT_CONVERT: 00283 Res = ScalarizeVecOp_BIT_CONVERT(N); 00284 break; 00285 case ISD::CONCAT_VECTORS: 00286 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 00287 break; 00288 case ISD::EXTRACT_VECTOR_ELT: 00289 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 00290 break; 00291 case ISD::STORE: 00292 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 00293 break; 00294 } 00295 } 00296 00297 // If the result is null, the sub-method took care of registering results etc. 00298 if (!Res.getNode()) return false; 00299 00300 // If the result is N, the sub-method updated N in place. Tell the legalizer 00301 // core about this. 00302 if (Res.getNode() == N) 00303 return true; 00304 00305 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 00306 "Invalid operand expansion"); 00307 00308 ReplaceValueWith(SDValue(N, 0), Res); 00309 return false; 00310 } 00311 00312 /// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 00313 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 00314 SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 00315 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 00316 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 00317 N->getValueType(0), Elt); 00318 } 00319 00320 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 00321 /// use a BUILD_VECTOR instead. 00322 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 00323 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 00324 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 00325 Ops[i] = GetScalarizedVector(N->getOperand(i)); 00326 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 00327 &Ops[0], Ops.size()); 00328 } 00329 00330 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 00331 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 00332 /// index. 00333 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 00334 SDValue Res = GetScalarizedVector(N->getOperand(0)); 00335 if (Res.getValueType() != N->getValueType(0)) 00336 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 00337 Res); 00338 return Res; 00339 } 00340 00341 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 00342 /// scalarized, it must be <1 x ty>. Just store the element. 00343 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 00344 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 00345 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 00346 DebugLoc dl = N->getDebugLoc(); 00347 00348 if (N->isTruncatingStore()) 00349 return DAG.getTruncStore(N->getChain(), dl, 00350 GetScalarizedVector(N->getOperand(1)), 00351 N->getBasePtr(), 00352 N->getSrcValue(), N->getSrcValueOffset(), 00353 N->getMemoryVT().getVectorElementType(), 00354 N->isVolatile(), N->getAlignment()); 00355 00356 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 00357 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 00358 N->isVolatile(), N->getAlignment()); 00359 } 00360 00361 00362 //===----------------------------------------------------------------------===// 00363 // Result Vector Splitting 00364 //===----------------------------------------------------------------------===// 00365 00366 /// SplitVectorResult - This method is called when the specified result of the 00367 /// specified node is found to need vector splitting. At this point, the node 00368 /// may also have invalid operands or may have other results that need 00369 /// legalization, we just know that (at least) one result needs vector 00370 /// splitting. 00371 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 00372 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 00373 SDValue Lo, Hi; 00374 00375 switch (N->getOpcode()) { 00376 default: 00377 #ifndef NDEBUG 00378 cerr << "SplitVectorResult #" << ResNo << ": "; 00379 N->dump(&DAG); cerr << "\n"; 00380 #endif 00381 llvm_unreachable("Do not know how to split the result of this operator!"); 00382 00383 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 00384 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 00385 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 00386 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 00387 00388 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 00389 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 00390 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 00391 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 00392 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 00393 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 00394 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 00395 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 00396 case ISD::LOAD: 00397 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 00398 break; 00399 case ISD::SETCC: 00400 case ISD::VSETCC: 00401 SplitVecRes_SETCC(N, Lo, Hi); 00402 break; 00403 case ISD::VECTOR_SHUFFLE: 00404 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 00405 break; 00406 00407 case ISD::CTTZ: 00408 case ISD::CTLZ: 00409 case ISD::CTPOP: 00410 case ISD::FNEG: 00411 case ISD::FABS: 00412 case ISD::FSQRT: 00413 case ISD::FSIN: 00414 case ISD::FCOS: 00415 case ISD::FTRUNC: 00416 case ISD::FFLOOR: 00417 case ISD::FCEIL: 00418 case ISD::FRINT: 00419 case ISD::FNEARBYINT: 00420 case ISD::FP_TO_SINT: 00421 case ISD::FP_TO_UINT: 00422 case ISD::SINT_TO_FP: 00423 case ISD::UINT_TO_FP: 00424 case ISD::TRUNCATE: 00425 case ISD::SIGN_EXTEND: 00426 case ISD::ZERO_EXTEND: 00427 case ISD::ANY_EXTEND: 00428 SplitVecRes_UnaryOp(N, Lo, Hi); 00429 break; 00430 00431 case ISD::ADD: 00432 case ISD::SUB: 00433 case ISD::MUL: 00434 case ISD::FADD: 00435 case ISD::FSUB: 00436 case ISD::FMUL: 00437 case ISD::SDIV: 00438 case ISD::UDIV: 00439 case ISD::FDIV: 00440 case ISD::FPOW: 00441 case ISD::AND: 00442 case ISD::OR: 00443 case ISD::XOR: 00444 case ISD::SHL: 00445 case ISD::SRA: 00446 case ISD::SRL: 00447 case ISD::UREM: 00448 case ISD::SREM: 00449 case ISD::FREM: 00450 SplitVecRes_BinOp(N, Lo, Hi); 00451 break; 00452 } 00453 00454 // If Lo/Hi is null, the sub-method took care of registering results etc. 00455 if (Lo.getNode()) 00456 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 00457 } 00458 00459 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 00460 SDValue &Hi) { 00461 SDValue LHSLo, LHSHi; 00462 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 00463 SDValue RHSLo, RHSHi; 00464 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 00465 DebugLoc dl = N->getDebugLoc(); 00466 00467 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 00468 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 00469 } 00470 00471 void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 00472 SDValue &Hi) { 00473 // We know the result is a vector. The input may be either a vector or a 00474 // scalar value. 00475 EVT LoVT, HiVT; 00476 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00477 DebugLoc dl = N->getDebugLoc(); 00478 00479 SDValue InOp = N->getOperand(0); 00480 EVT InVT = InOp.getValueType(); 00481 00482 // Handle some special cases efficiently. 00483 switch (getTypeAction(InVT)) { 00484 default: 00485 assert(false && "Unknown type action!"); 00486 case Legal: 00487 case PromoteInteger: 00488 case SoftenFloat: 00489 case ScalarizeVector: 00490 break; 00491 case ExpandInteger: 00492 case ExpandFloat: 00493 // A scalar to vector conversion, where the scalar needs expansion. 00494 // If the vector is being split in two then we can just convert the 00495 // expanded pieces. 00496 if (LoVT == HiVT) { 00497 GetExpandedOp(InOp, Lo, Hi); 00498 if (TLI.isBigEndian()) 00499 std::swap(Lo, Hi); 00500 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 00501 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 00502 return; 00503 } 00504 break; 00505 case SplitVector: 00506 // If the input is a vector that needs to be split, convert each split 00507 // piece of the input now. 00508 GetSplitVector(InOp, Lo, Hi); 00509 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 00510 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 00511 return; 00512 } 00513 00514 // In the general case, convert the input to an integer and split it by hand. 00515 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 00516 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 00517 if (TLI.isBigEndian()) 00518 std::swap(LoIntVT, HiIntVT); 00519 00520 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 00521 00522 if (TLI.isBigEndian()) 00523 std::swap(Lo, Hi); 00524 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 00525 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 00526 } 00527 00528 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 00529 SDValue &Hi) { 00530 EVT LoVT, HiVT; 00531 DebugLoc dl = N->getDebugLoc(); 00532 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00533 unsigned LoNumElts = LoVT.getVectorNumElements(); 00534 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 00535 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 00536 00537 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 00538 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 00539 } 00540 00541 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 00542 SDValue &Hi) { 00543 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 00544 DebugLoc dl = N->getDebugLoc(); 00545 unsigned NumSubvectors = N->getNumOperands() / 2; 00546 if (NumSubvectors == 1) { 00547 Lo = N->getOperand(0); 00548 Hi = N->getOperand(1); 00549 return; 00550 } 00551 00552 EVT LoVT, HiVT; 00553 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00554 00555 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 00556 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 00557 00558 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 00559 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 00560 } 00561 00562 void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 00563 SDValue &Hi) { 00564 EVT LoVT, HiVT; 00565 DebugLoc dl = N->getDebugLoc(); 00566 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00567 00568 SDValue DTyOpLo = DAG.getValueType(LoVT); 00569 SDValue DTyOpHi = DAG.getValueType(HiVT); 00570 00571 SDValue RndOp = N->getOperand(3); 00572 SDValue SatOp = N->getOperand(4); 00573 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 00574 00575 // Split the input. 00576 SDValue VLo, VHi; 00577 EVT InVT = N->getOperand(0).getValueType(); 00578 switch (getTypeAction(InVT)) { 00579 default: llvm_unreachable("Unexpected type action!"); 00580 case Legal: { 00581 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 00582 LoVT.getVectorNumElements()); 00583 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 00584 DAG.getIntPtrConstant(0)); 00585 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 00586 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 00587 break; 00588 } 00589 case SplitVector: 00590 GetSplitVector(N->getOperand(0), VLo, VHi); 00591 break; 00592 case WidenVector: { 00593 // If the result needs to be split and the input needs to be widened, 00594 // the two types must have different lengths. Use the widened result 00595 // and extract from it to do the split. 00596 SDValue InOp = GetWidenedVector(N->getOperand(0)); 00597 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 00598 LoVT.getVectorNumElements()); 00599 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 00600 DAG.getIntPtrConstant(0)); 00601 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 00602 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 00603 break; 00604 } 00605 } 00606 00607 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 00608 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 00609 00610 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 00611 CvtCode); 00612 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 00613 CvtCode); 00614 } 00615 00616 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 00617 SDValue &Hi) { 00618 SDValue Vec = N->getOperand(0); 00619 SDValue Idx = N->getOperand(1); 00620 EVT IdxVT = Idx.getValueType(); 00621 DebugLoc dl = N->getDebugLoc(); 00622 00623 EVT LoVT, HiVT; 00624 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00625 00626 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 00627 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 00628 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 00629 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 00630 } 00631 00632 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 00633 SDValue &Hi) { 00634 DebugLoc dl = N->getDebugLoc(); 00635 GetSplitVector(N->getOperand(0), Lo, Hi); 00636 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 00637 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 00638 } 00639 00640 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 00641 SDValue &Hi) { 00642 SDValue Vec = N->getOperand(0); 00643 SDValue Elt = N->getOperand(1); 00644 SDValue Idx = N->getOperand(2); 00645 DebugLoc dl = N->getDebugLoc(); 00646 GetSplitVector(Vec, Lo, Hi); 00647 00648 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 00649 unsigned IdxVal = CIdx->getZExtValue(); 00650 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 00651 if (IdxVal < LoNumElts) 00652 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 00653 Lo.getValueType(), Lo, Elt, Idx); 00654 else 00655 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 00656 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 00657 return; 00658 } 00659 00660 // Spill the vector to the stack. 00661 EVT VecVT = Vec.getValueType(); 00662 EVT EltVT = VecVT.getVectorElementType(); 00663 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 00664 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 00665 00666 // Store the new element. This may be larger than the vector element type, 00667 // so use a truncating store. 00668 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 00669 unsigned Alignment = 00670 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); 00671 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 00672 00673 // Load the Lo part from the stack slot. 00674 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 00675 00676 // Increment the pointer to the other part. 00677 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 00678 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 00679 DAG.getIntPtrConstant(IncrementSize)); 00680 00681 // Load the Hi part from the stack slot. 00682 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 00683 MinAlign(Alignment, IncrementSize)); 00684 } 00685 00686 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 00687 SDValue &Hi) { 00688 EVT LoVT, HiVT; 00689 DebugLoc dl = N->getDebugLoc(); 00690 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00691 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 00692 Hi = DAG.getUNDEF(HiVT); 00693 } 00694 00695 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 00696 SDValue &Hi) { 00697 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 00698 EVT LoVT, HiVT; 00699 DebugLoc dl = LD->getDebugLoc(); 00700 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 00701 00702 ISD::LoadExtType ExtType = LD->getExtensionType(); 00703 SDValue Ch = LD->getChain(); 00704 SDValue Ptr = LD->getBasePtr(); 00705 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 00706 const Value *SV = LD->getSrcValue(); 00707 int SVOffset = LD->getSrcValueOffset(); 00708 EVT MemoryVT = LD->getMemoryVT(); 00709 unsigned Alignment = LD->getAlignment(); 00710 bool isVolatile = LD->isVolatile(); 00711 00712 EVT LoMemVT, HiMemVT; 00713 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 00714 00715 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 00716 SV, SVOffset, LoMemVT, isVolatile, Alignment); 00717 00718 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 00719 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 00720 DAG.getIntPtrConstant(IncrementSize)); 00721 SVOffset += IncrementSize; 00722 Alignment = MinAlign(Alignment, IncrementSize); 00723 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 00724 SV, SVOffset, HiMemVT, isVolatile, Alignment); 00725 00726 // Build a factor node to remember that this load is independent of the 00727 // other one. 00728 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 00729 Hi.getValue(1)); 00730 00731 // Legalized the chain result - switch anything that used the old chain to 00732 // use the new one. 00733 ReplaceValueWith(SDValue(LD, 1), Ch); 00734 } 00735 00736 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 00737 EVT LoVT, HiVT; 00738 DebugLoc DL = N->getDebugLoc(); 00739 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00740 00741 // Split the input. 00742 EVT InVT = N->getOperand(0).getValueType(); 00743 SDValue LL, LH, RL, RH; 00744 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 00745 LoVT.getVectorNumElements()); 00746 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 00747 DAG.getIntPtrConstant(0)); 00748 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 00749 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 00750 00751 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 00752 DAG.getIntPtrConstant(0)); 00753 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 00754 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 00755 00756 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 00757 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 00758 } 00759 00760 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 00761 SDValue &Hi) { 00762 // Get the dest types - they may not match the input types, e.g. int_to_fp. 00763 EVT LoVT, HiVT; 00764 DebugLoc dl = N->getDebugLoc(); 00765 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 00766 00767 // Split the input. 00768 EVT InVT = N->getOperand(0).getValueType(); 00769 switch (getTypeAction(InVT)) { 00770 default: llvm_unreachable("Unexpected type action!"); 00771 case Legal: { 00772 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 00773 LoVT.getVectorNumElements()); 00774 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 00775 DAG.getIntPtrConstant(0)); 00776 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 00777 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 00778 break; 00779 } 00780 case SplitVector: 00781 GetSplitVector(N->getOperand(0), Lo, Hi); 00782 break; 00783 case WidenVector: { 00784 // If the result needs to be split and the input needs to be widened, 00785 // the two types must have different lengths. Use the widened result 00786 // and extract from it to do the split. 00787 SDValue InOp = GetWidenedVector(N->getOperand(0)); 00788 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 00789 LoVT.getVectorNumElements()); 00790 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 00791 DAG.getIntPtrConstant(0)); 00792 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 00793 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 00794 break; 00795 } 00796 } 00797 00798 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 00799 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 00800 } 00801 00802 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 00803 SDValue &Lo, SDValue &Hi) { 00804 // The low and high parts of the original input give four input vectors. 00805 SDValue Inputs[4]; 00806 DebugLoc dl = N->getDebugLoc(); 00807 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 00808 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 00809 EVT NewVT = Inputs[0].getValueType(); 00810 unsigned NewElts = NewVT.getVectorNumElements(); 00811 00812 // If Lo or Hi uses elements from at most two of the four input vectors, then 00813 // express it as a vector shuffle of those two inputs. Otherwise extract the 00814 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 00815 SmallVector<int, 16> Ops; 00816 for (unsigned High = 0; High < 2; ++High) { 00817 SDValue &Output = High ? Hi : Lo; 00818 00819 // Build a shuffle mask for the output, discovering on the fly which 00820 // input vectors to use as shuffle operands (recorded in InputUsed). 00821 // If building a suitable shuffle vector proves too hard, then bail 00822 // out with useBuildVector set. 00823 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 00824 unsigned FirstMaskIdx = High * NewElts; 00825 bool useBuildVector = false; 00826 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 00827 // The mask element. This indexes into the input. 00828 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 00829 00830 // The input vector this mask element indexes into. 00831 unsigned Input = (unsigned)Idx / NewElts; 00832 00833 if (Input >= array_lengthof(Inputs)) { 00834 // The mask element does not index into any input vector. 00835 Ops.push_back(-1); 00836 continue; 00837 } 00838 00839 // Turn the index into an offset from the start of the input vector. 00840 Idx -= Input * NewElts; 00841 00842 // Find or create a shuffle vector operand to hold this input. 00843 unsigned OpNo; 00844 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 00845 if (InputUsed[OpNo] == Input) { 00846 // This input vector is already an operand. 00847 break; 00848 } else if (InputUsed[OpNo] == -1U) { 00849 // Create a new operand for this input vector. 00850 InputUsed[OpNo] = Input; 00851 break; 00852 } 00853 } 00854 00855 if (OpNo >= array_lengthof(InputUsed)) { 00856 // More than two input vectors used! Give up on trying to create a 00857 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 00858 useBuildVector = true; 00859 break; 00860 } 00861 00862 // Add the mask index for the new shuffle vector. 00863 Ops.push_back(Idx + OpNo * NewElts); 00864 } 00865 00866 if (useBuildVector) { 00867 EVT EltVT = NewVT.getVectorElementType(); 00868 SmallVector<SDValue, 16> SVOps; 00869 00870 // Extract the input elements by hand. 00871 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 00872 // The mask element. This indexes into the input. 00873 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 00874 00875 // The input vector this mask element indexes into. 00876 unsigned Input = (unsigned)Idx / NewElts; 00877 00878 if (Input >= array_lengthof(Inputs)) { 00879 // The mask element is "undef" or indexes off the end of the input. 00880 SVOps.push_back(DAG.getUNDEF(EltVT)); 00881 continue; 00882 } 00883 00884 // Turn the index into an offset from the start of the input vector. 00885 Idx -= Input * NewElts; 00886 00887 // Extract the vector element by hand. 00888 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 00889 Inputs[Input], DAG.getIntPtrConstant(Idx))); 00890 } 00891 00892 // Construct the Lo/Hi output using a BUILD_VECTOR. 00893 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 00894 } else if (InputUsed[0] == -1U) { 00895 // No input vectors were used! The result is undefined. 00896 Output = DAG.getUNDEF(NewVT); 00897 } else { 00898 SDValue Op0 = Inputs[InputUsed[0]]; 00899 // If only one input was used, use an undefined vector for the other. 00900 SDValue Op1 = InputUsed[1] == -1U ? 00901 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 00902 // At least one input vector was used. Create a new shuffle vector. 00903 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 00904 } 00905 00906 Ops.clear(); 00907 } 00908 } 00909 00910 00911 //===----------------------------------------------------------------------===// 00912 // Operand Vector Splitting 00913 //===----------------------------------------------------------------------===// 00914 00915 /// SplitVectorOperand - This method is called when the specified operand of the 00916 /// specified node is found to need vector splitting. At this point, all of the 00917 /// result types of the node are known to be legal, but other operands of the 00918 /// node may need legalization as well as the specified one. 00919 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 00920 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 00921 SDValue Res = SDValue(); 00922 00923 if (Res.getNode() == 0) { 00924 switch (N->getOpcode()) { 00925 default: 00926 #ifndef NDEBUG 00927 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 00928 N->dump(&DAG); cerr << "\n"; 00929 #endif 00930 llvm_unreachable("Do not know how to split this operator's operand!"); 00931 00932 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 00933 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 00934 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 00935 case ISD::STORE: 00936 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 00937 break; 00938 00939 case ISD::CTTZ: 00940 case ISD::CTLZ: 00941 case ISD::CTPOP: 00942 case ISD::FP_TO_SINT: 00943 case ISD::FP_TO_UINT: 00944 case ISD::SINT_TO_FP: 00945 case ISD::UINT_TO_FP: 00946 case ISD::TRUNCATE: 00947 case ISD::SIGN_EXTEND: 00948 case ISD::ZERO_EXTEND: 00949 case ISD::ANY_EXTEND: 00950 Res = SplitVecOp_UnaryOp(N); 00951 break; 00952 } 00953 } 00954 00955 // If the result is null, the sub-method took care of registering results etc. 00956 if (!Res.getNode()) return false; 00957 00958 // If the result is N, the sub-method updated N in place. Tell the legalizer 00959 // core about this. 00960 if (Res.getNode() == N) 00961 return true; 00962 00963 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 00964 "Invalid operand expansion"); 00965 00966 ReplaceValueWith(SDValue(N, 0), Res); 00967 return false; 00968 } 00969 00970 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 00971 // The result has a legal vector type, but the input needs splitting. 00972 EVT ResVT = N->getValueType(0); 00973 SDValue Lo, Hi; 00974 DebugLoc dl = N->getDebugLoc(); 00975 GetSplitVector(N->getOperand(0), Lo, Hi); 00976 EVT InVT = Lo.getValueType(); 00977 00978 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 00979 InVT.getVectorNumElements()); 00980 00981 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 00982 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 00983 00984 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 00985 } 00986 00987 SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 00988 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 00989 // end up being split all the way down to individual components. Convert the 00990 // split pieces into integers and reassemble. 00991 SDValue Lo, Hi; 00992 GetSplitVector(N->getOperand(0), Lo, Hi); 00993 Lo = BitConvertToInteger(Lo); 00994 Hi = BitConvertToInteger(Hi); 00995 00996 if (TLI.isBigEndian()) 00997 std::swap(Lo, Hi); 00998 00999 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 01000 JoinIntegers(Lo, Hi)); 01001 } 01002 01003 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 01004 // We know that the extracted result type is legal. For now, assume the index 01005 // is a constant. 01006 EVT SubVT = N->getValueType(0); 01007 SDValue Idx = N->getOperand(1); 01008 DebugLoc dl = N->getDebugLoc(); 01009 SDValue Lo, Hi; 01010 GetSplitVector(N->getOperand(0), Lo, Hi); 01011 01012 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 01013 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 01014 01015 if (IdxVal < LoElts) { 01016 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 01017 "Extracted subvector crosses vector split!"); 01018 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 01019 } else { 01020 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 01021 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 01022 } 01023 } 01024 01025 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 01026 SDValue Vec = N->getOperand(0); 01027 SDValue Idx = N->getOperand(1); 01028 EVT VecVT = Vec.getValueType(); 01029 01030 if (isa<ConstantSDNode>(Idx)) { 01031 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 01032 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 01033 01034 SDValue Lo, Hi; 01035 GetSplitVector(Vec, Lo, Hi); 01036 01037 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 01038 01039 if (IdxVal < LoElts) 01040 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 01041 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 01042 DAG.getConstant(IdxVal - LoElts, 01043 Idx.getValueType())); 01044 } 01045 01046 // Store the vector to the stack. 01047 EVT EltVT = VecVT.getVectorElementType(); 01048 DebugLoc dl = N->getDebugLoc(); 01049 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 01050 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 01051 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 01052 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 01053 01054 // Load back the required element. 01055 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 01056 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 01057 SV, 0, EltVT); 01058 } 01059 01060 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 01061 assert(N->isUnindexed() && "Indexed store of vector?"); 01062 assert(OpNo == 1 && "Can only split the stored value"); 01063 DebugLoc dl = N->getDebugLoc(); 01064 01065 bool isTruncating = N->isTruncatingStore(); 01066 SDValue Ch = N->getChain(); 01067 SDValue Ptr = N->getBasePtr(); 01068 int SVOffset = N->getSrcValueOffset(); 01069 EVT MemoryVT = N->getMemoryVT(); 01070 unsigned Alignment = N->getAlignment(); 01071 bool isVol = N->isVolatile(); 01072 SDValue Lo, Hi; 01073 GetSplitVector(N->getOperand(1), Lo, Hi); 01074 01075 EVT LoMemVT, HiMemVT; 01076 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 01077 01078 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 01079 01080 if (isTruncating) 01081 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 01082 LoMemVT, isVol, Alignment); 01083 else 01084 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 01085 isVol, Alignment); 01086 01087 // Increment the pointer to the other half. 01088 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 01089 DAG.getIntPtrConstant(IncrementSize)); 01090 01091 if (isTruncating) 01092 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 01093 N->getSrcValue(), SVOffset+IncrementSize, 01094 HiMemVT, 01095 isVol, MinAlign(Alignment, IncrementSize)); 01096 else 01097 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 01098 isVol, MinAlign(Alignment, IncrementSize)); 01099 01100 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 01101 } 01102 01103 01104 //===----------------------------------------------------------------------===// 01105 // Result Vector Widening 01106 //===----------------------------------------------------------------------===// 01107 01108 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 01109 DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG); 01110 cerr << "\n"); 01111 SDValue Res = SDValue(); 01112 01113 switch (N->getOpcode()) { 01114 default: 01115 #ifndef NDEBUG 01116 cerr << "WidenVectorResult #" << ResNo << ": "; 01117 N->dump(&DAG); cerr << "\n"; 01118 #endif 01119 llvm_unreachable("Do not know how to widen the result of this operator!"); 01120 01121 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 01122 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 01123 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 01124 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 01125 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 01126 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 01127 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 01128 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 01129 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 01130 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 01131 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 01132 case ISD::VECTOR_SHUFFLE: 01133 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 01134 break; 01135 case ISD::VSETCC: 01136 Res = WidenVecRes_VSETCC(N); 01137 break; 01138 01139 case ISD::ADD: 01140 case ISD::AND: 01141 case ISD::BSWAP: 01142 case ISD::FADD: 01143 case ISD::FCOPYSIGN: 01144 case ISD::FDIV: 01145 case ISD::FMUL: 01146 case ISD::FPOW: 01147 case ISD::FPOWI: 01148 case ISD::FREM: 01149 case ISD::FSUB: 01150 case ISD::MUL: 01151 case ISD::MULHS: 01152 case ISD::MULHU: 01153 case ISD::OR: 01154 case ISD::SDIV: 01155 case ISD::SREM: 01156 case ISD::UDIV: 01157 case ISD::UREM: 01158 case ISD::SUB: 01159 case ISD::XOR: 01160 Res = WidenVecRes_Binary(N); 01161 break; 01162 01163 case ISD::SHL: 01164 case ISD::SRA: 01165 case ISD::SRL: 01166 Res = WidenVecRes_Shift(N); 01167 break; 01168 01169 case ISD::FP_ROUND: 01170 case ISD::FP_TO_SINT: 01171 case ISD::FP_TO_UINT: 01172 case ISD::SINT_TO_FP: 01173 case ISD::UINT_TO_FP: 01174 case ISD::TRUNCATE: 01175 case ISD::SIGN_EXTEND: 01176 case ISD::ZERO_EXTEND: 01177 case ISD::ANY_EXTEND: 01178 Res = WidenVecRes_Convert(N); 01179 break; 01180 01181 case ISD::CTLZ: 01182 case ISD::CTPOP: 01183 case ISD::CTTZ: 01184 case ISD::FABS: 01185 case ISD::FCOS: 01186 case ISD::FNEG: 01187 case ISD::FSIN: 01188 case ISD::FSQRT: 01189 Res = WidenVecRes_Unary(N); 01190 break; 01191 } 01192 01193 // If Res is null, the sub-method took care of registering the result. 01194 if (Res.getNode()) 01195 SetWidenedVector(SDValue(N, ResNo), Res); 01196 } 01197 01198 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 01199 // Binary op widening. 01200 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01201 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 01202 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 01203 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 01204 } 01205 01206 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 01207 SDValue InOp = N->getOperand(0); 01208 DebugLoc dl = N->getDebugLoc(); 01209 01210 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01211 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01212 01213 EVT InVT = InOp.getValueType(); 01214 EVT InEltVT = InVT.getVectorElementType(); 01215 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 01216 01217 unsigned Opcode = N->getOpcode(); 01218 unsigned InVTNumElts = InVT.getVectorNumElements(); 01219 01220 if (getTypeAction(InVT) == WidenVector) { 01221 InOp = GetWidenedVector(N->getOperand(0)); 01222 InVT = InOp.getValueType(); 01223 InVTNumElts = InVT.getVectorNumElements(); 01224 if (InVTNumElts == WidenNumElts) 01225 return DAG.getNode(Opcode, dl, WidenVT, InOp); 01226 } 01227 01228 if (TLI.isTypeLegal(InWidenVT)) { 01229 // Because the result and the input are different vector types, widening 01230 // the result could create a legal type but widening the input might make 01231 // it an illegal type that might lead to repeatedly splitting the input 01232 // and then widening it. To avoid this, we widen the input only if 01233 // it results in a legal type. 01234 if (WidenNumElts % InVTNumElts == 0) { 01235 // Widen the input and call convert on the widened input vector. 01236 unsigned NumConcat = WidenNumElts/InVTNumElts; 01237 SmallVector<SDValue, 16> Ops(NumConcat); 01238 Ops[0] = InOp; 01239 SDValue UndefVal = DAG.getUNDEF(InVT); 01240 for (unsigned i = 1; i != NumConcat; ++i) 01241 Ops[i] = UndefVal; 01242 return DAG.getNode(Opcode, dl, WidenVT, 01243 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 01244 &Ops[0], NumConcat)); 01245 } 01246 01247 if (InVTNumElts % WidenNumElts == 0) { 01248 // Extract the input and convert the shorten input vector. 01249 return DAG.getNode(Opcode, dl, WidenVT, 01250 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 01251 InOp, DAG.getIntPtrConstant(0))); 01252 } 01253 } 01254 01255 // Otherwise unroll into some nasty scalar code and rebuild the vector. 01256 SmallVector<SDValue, 16> Ops(WidenNumElts); 01257 EVT EltVT = WidenVT.getVectorElementType(); 01258 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 01259 unsigned i; 01260 for (i=0; i < MinElts; ++i) 01261 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 01262 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 01263 DAG.getIntPtrConstant(i))); 01264 01265 SDValue UndefVal = DAG.getUNDEF(EltVT); 01266 for (; i < WidenNumElts; ++i) 01267 Ops[i] = UndefVal; 01268 01269 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 01270 } 01271 01272 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 01273 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01274 SDValue InOp = GetWidenedVector(N->getOperand(0)); 01275 SDValue ShOp = N->getOperand(1); 01276 01277 EVT ShVT = ShOp.getValueType(); 01278 if (getTypeAction(ShVT) == WidenVector) { 01279 ShOp = GetWidenedVector(ShOp); 01280 ShVT = ShOp.getValueType(); 01281 } 01282 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), 01283 WidenVT.getVectorNumElements()); 01284 if (ShVT != ShWidenVT) 01285 ShOp = ModifyToType(ShOp, ShWidenVT); 01286 01287 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 01288 } 01289 01290 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 01291 // Unary op widening. 01292 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01293 SDValue InOp = GetWidenedVector(N->getOperand(0)); 01294 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 01295 } 01296 01297 SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 01298 SDValue InOp = N->getOperand(0); 01299 EVT InVT = InOp.getValueType(); 01300 EVT VT = N->getValueType(0); 01301 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 01302 DebugLoc dl = N->getDebugLoc(); 01303 01304 switch (getTypeAction(InVT)) { 01305 default: 01306 assert(false && "Unknown type action!"); 01307 break; 01308 case Legal: 01309 break; 01310 case PromoteInteger: 01311 // If the InOp is promoted to the same size, convert it. Otherwise, 01312 // fall out of the switch and widen the promoted input. 01313 InOp = GetPromotedInteger(InOp); 01314 InVT = InOp.getValueType(); 01315 if (WidenVT.bitsEq(InVT)) 01316 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 01317 break; 01318 case SoftenFloat: 01319 case ExpandInteger: 01320 case ExpandFloat: 01321 case ScalarizeVector: 01322 case SplitVector: 01323 break; 01324 case WidenVector: 01325 // If the InOp is widened to the same size, convert it. Otherwise, fall 01326 // out of the switch and widen the widened input. 01327 InOp = GetWidenedVector(InOp); 01328 InVT = InOp.getValueType(); 01329 if (WidenVT.bitsEq(InVT)) 01330 // The input widens to the same size. Convert to the widen value. 01331 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 01332 break; 01333 } 01334 01335 unsigned WidenSize = WidenVT.getSizeInBits(); 01336 unsigned InSize = InVT.getSizeInBits(); 01337 if (WidenSize % InSize == 0) { 01338 // Determine new input vector type. The new input vector type will use 01339 // the same element type (if its a vector) or use the input type as a 01340 // vector. It is the same size as the type to widen to. 01341 EVT NewInVT; 01342 unsigned NewNumElts = WidenSize / InSize; 01343 if (InVT.isVector()) { 01344 EVT InEltVT = InVT.getVectorElementType(); 01345 NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); 01346 } else { 01347 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 01348 } 01349 01350 if (TLI.isTypeLegal(NewInVT)) { 01351 // Because the result and the input are different vector types, widening 01352 // the result could create a legal type but widening the input might make 01353 // it an illegal type that might lead to repeatedly splitting the input 01354 // and then widening it. To avoid this, we widen the input only if 01355 // it results in a legal type. 01356 SmallVector<SDValue, 16> Ops(NewNumElts); 01357 SDValue UndefVal = DAG.getUNDEF(InVT); 01358 Ops[0] = InOp; 01359 for (unsigned i = 1; i < NewNumElts; ++i) 01360 Ops[i] = UndefVal; 01361 01362 SDValue NewVec; 01363 if (InVT.isVector()) 01364 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 01365 NewInVT, &Ops[0], NewNumElts); 01366 else 01367 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 01368 NewInVT, &Ops[0], NewNumElts); 01369 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 01370 } 01371 } 01372 01373 return CreateStackStoreLoad(InOp, WidenVT); 01374 } 01375 01376 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 01377 DebugLoc dl = N->getDebugLoc(); 01378 // Build a vector with undefined for the new nodes. 01379 EVT VT = N->getValueType(0); 01380 EVT EltVT = VT.getVectorElementType(); 01381 unsigned NumElts = VT.getVectorNumElements(); 01382 01383 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 01384 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01385 01386 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 01387 NewOps.reserve(WidenNumElts); 01388 for (unsigned i = NumElts; i < WidenNumElts; ++i) 01389 NewOps.push_back(DAG.getUNDEF(EltVT)); 01390 01391 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 01392 } 01393 01394 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 01395 EVT InVT = N->getOperand(0).getValueType(); 01396 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01397 DebugLoc dl = N->getDebugLoc(); 01398 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01399 unsigned NumOperands = N->getNumOperands(); 01400 01401 bool InputWidened = false; // Indicates we need to widen the input. 01402 if (getTypeAction(InVT) != WidenVector) { 01403 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 01404 // Add undef vectors to widen to correct length. 01405 unsigned NumConcat = WidenVT.getVectorNumElements() / 01406 InVT.getVectorNumElements(); 01407 SDValue UndefVal = DAG.getUNDEF(InVT); 01408 SmallVector<SDValue, 16> Ops(NumConcat); 01409 for (unsigned i=0; i < NumOperands; ++i) 01410 Ops[i] = N->getOperand(i); 01411 for (unsigned i = NumOperands; i != NumConcat; ++i) 01412 Ops[i] = UndefVal; 01413 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 01414 } 01415 } else { 01416 InputWidened = true; 01417 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 01418 // The inputs and the result are widen to the same value. 01419 unsigned i; 01420 for (i=1; i < NumOperands; ++i) 01421 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 01422 break; 01423 01424 if (i > NumOperands) 01425 // Everything but the first operand is an UNDEF so just return the 01426 // widened first operand. 01427 return GetWidenedVector(N->getOperand(0)); 01428 01429 if (NumOperands == 2) { 01430 // Replace concat of two operands with a shuffle. 01431 SmallVector<int, 16> MaskOps(WidenNumElts); 01432 for (unsigned i=0; i < WidenNumElts/2; ++i) { 01433 MaskOps[i] = i; 01434 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 01435 } 01436 return DAG.getVectorShuffle(WidenVT, dl, 01437 GetWidenedVector(N->getOperand(0)), 01438 GetWidenedVector(N->getOperand(1)), 01439 &MaskOps[0]); 01440 } 01441 } 01442 } 01443 01444 // Fall back to use extracts and build vector. 01445 EVT EltVT = WidenVT.getVectorElementType(); 01446 unsigned NumInElts = InVT.getVectorNumElements(); 01447 SmallVector<SDValue, 16> Ops(WidenNumElts); 01448 unsigned Idx = 0; 01449 for (unsigned i=0; i < NumOperands; ++i) { 01450 SDValue InOp = N->getOperand(i); 01451 if (InputWidened) 01452 InOp = GetWidenedVector(InOp); 01453 for (unsigned j=0; j < NumInElts; ++j) 01454 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 01455 DAG.getIntPtrConstant(j)); 01456 } 01457 SDValue UndefVal = DAG.getUNDEF(EltVT); 01458 for (; Idx < WidenNumElts; ++Idx) 01459 Ops[Idx] = UndefVal; 01460 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 01461 } 01462 01463 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 01464 DebugLoc dl = N->getDebugLoc(); 01465 SDValue InOp = N->getOperand(0); 01466 SDValue RndOp = N->getOperand(3); 01467 SDValue SatOp = N->getOperand(4); 01468 01469 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01470 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01471 01472 EVT InVT = InOp.getValueType(); 01473 EVT InEltVT = InVT.getVectorElementType(); 01474 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 01475 01476 SDValue DTyOp = DAG.getValueType(WidenVT); 01477 SDValue STyOp = DAG.getValueType(InWidenVT); 01478 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 01479 01480 unsigned InVTNumElts = InVT.getVectorNumElements(); 01481 if (getTypeAction(InVT) == WidenVector) { 01482 InOp = GetWidenedVector(InOp); 01483 InVT = InOp.getValueType(); 01484 InVTNumElts = InVT.getVectorNumElements(); 01485 if (InVTNumElts == WidenNumElts) 01486 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 01487 SatOp, CvtCode); 01488 } 01489 01490 if (TLI.isTypeLegal(InWidenVT)) { 01491 // Because the result and the input are different vector types, widening 01492 // the result could create a legal type but widening the input might make 01493 // it an illegal type that might lead to repeatedly splitting the input 01494 // and then widening it. To avoid this, we widen the input only if 01495 // it results in a legal type. 01496 if (WidenNumElts % InVTNumElts == 0) { 01497 // Widen the input and call convert on the widened input vector. 01498 unsigned NumConcat = WidenNumElts/InVTNumElts; 01499 SmallVector<SDValue, 16> Ops(NumConcat); 01500 Ops[0] = InOp; 01501 SDValue UndefVal = DAG.getUNDEF(InVT); 01502 for (unsigned i = 1; i != NumConcat; ++i) { 01503 Ops[i] = UndefVal; 01504 } 01505 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 01506 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 01507 SatOp, CvtCode); 01508 } 01509 01510 if (InVTNumElts % WidenNumElts == 0) { 01511 // Extract the input and convert the shorten input vector. 01512 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 01513 DAG.getIntPtrConstant(0)); 01514 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 01515 SatOp, CvtCode); 01516 } 01517 } 01518 01519 // Otherwise unroll into some nasty scalar code and rebuild the vector. 01520 SmallVector<SDValue, 16> Ops(WidenNumElts); 01521 EVT EltVT = WidenVT.getVectorElementType(); 01522 DTyOp = DAG.getValueType(EltVT); 01523 STyOp = DAG.getValueType(InEltVT); 01524 01525 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 01526 unsigned i; 01527 for (i=0; i < MinElts; ++i) { 01528 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 01529 DAG.getIntPtrConstant(i)); 01530 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 01531 SatOp, CvtCode); 01532 } 01533 01534 SDValue UndefVal = DAG.getUNDEF(EltVT); 01535 for (; i < WidenNumElts; ++i) 01536 Ops[i] = UndefVal; 01537 01538 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 01539 } 01540 01541 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 01542 EVT VT = N->getValueType(0); 01543 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 01544 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01545 SDValue InOp = N->getOperand(0); 01546 SDValue Idx = N->getOperand(1); 01547 DebugLoc dl = N->getDebugLoc(); 01548 01549 if (getTypeAction(InOp.getValueType()) == WidenVector) 01550 InOp = GetWidenedVector(InOp); 01551 01552 EVT InVT = InOp.getValueType(); 01553 01554 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 01555 if (CIdx) { 01556 unsigned IdxVal = CIdx->getZExtValue(); 01557 // Check if we can just return the input vector after widening. 01558 if (IdxVal == 0 && InVT == WidenVT) 01559 return InOp; 01560 01561 // Check if we can extract from the vector. 01562 unsigned InNumElts = InVT.getVectorNumElements(); 01563 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 01564 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 01565 } 01566 01567 // We could try widening the input to the right length but for now, extract 01568 // the original elements, fill the rest with undefs and build a vector. 01569 SmallVector<SDValue, 16> Ops(WidenNumElts); 01570 EVT EltVT = VT.getVectorElementType(); 01571 EVT IdxVT = Idx.getValueType(); 01572 unsigned NumElts = VT.getVectorNumElements(); 01573 unsigned i; 01574 if (CIdx) { 01575 unsigned IdxVal = CIdx->getZExtValue(); 01576 for (i=0; i < NumElts; ++i) 01577 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 01578 DAG.getConstant(IdxVal+i, IdxVT)); 01579 } else { 01580 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 01581 for (i=1; i < NumElts; ++i) { 01582 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 01583 DAG.getConstant(i, IdxVT)); 01584 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 01585 } 01586 } 01587 01588 SDValue UndefVal = DAG.getUNDEF(EltVT); 01589 for (; i < WidenNumElts; ++i) 01590 Ops[i] = UndefVal; 01591 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 01592 } 01593 01594 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 01595 SDValue InOp = GetWidenedVector(N->getOperand(0)); 01596 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 01597 InOp.getValueType(), InOp, 01598 N->getOperand(1), N->getOperand(2)); 01599 } 01600 01601 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 01602 LoadSDNode *LD = cast<LoadSDNode>(N); 01603 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 01604 EVT LdVT = LD->getMemoryVT(); 01605 DebugLoc dl = N->getDebugLoc(); 01606 assert(LdVT.isVector() && WidenVT.isVector()); 01607 01608 // Load information 01609 SDValue Chain = LD->getChain(); 01610 SDValue BasePtr = LD->getBasePtr(); 01611 int SVOffset = LD->getSrcValueOffset(); 01612 unsigned Align = LD->getAlignment(); 01613 bool isVolatile = LD->isVolatile(); 01614 const Value *SV = LD->getSrcValue(); 01615 ISD::LoadExtType ExtType = LD->getExtensionType(); 01616 01617 SDValue Result; 01618 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 01619 if (ExtType != ISD::NON_EXTLOAD) { 01620 // For extension loads, we can not play the tricks of chopping legal 01621 // vector types and bit cast it to the right type. Instead, we unroll 01622 // the load and build a vector. 01623 EVT EltVT = WidenVT.getVectorElementType(); 01624 EVT LdEltVT = LdVT.getVectorElementType(); 01625 unsigned NumElts = LdVT.getVectorNumElements(); 01626 01627 // Load each element and widen 01628 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01629 SmallVector<SDValue, 16> Ops(WidenNumElts); 01630 unsigned Increment = LdEltVT.getSizeInBits() / 8; 01631 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 01632 LdEltVT, isVolatile, Align); 01633 LdChain.push_back(Ops[0].getValue(1)); 01634 unsigned i = 0, Offset = Increment; 01635 for (i=1; i < NumElts; ++i, Offset += Increment) { 01636 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 01637 BasePtr, DAG.getIntPtrConstant(Offset)); 01638 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 01639 SVOffset + Offset, LdEltVT, isVolatile, Align); 01640 LdChain.push_back(Ops[i].getValue(1)); 01641 } 01642 01643 // Fill the rest with undefs 01644 SDValue UndefVal = DAG.getUNDEF(EltVT); 01645 for (; i != WidenNumElts; ++i) 01646 Ops[i] = UndefVal; 01647 01648 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 01649 } else { 01650 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 01651 unsigned int LdWidth = LdVT.getSizeInBits(); 01652 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 01653 Align, isVolatile, LdWidth, WidenVT, dl); 01654 } 01655 01656 // If we generate a single load, we can use that for the chain. Otherwise, 01657 // build a factor node to remember the multiple loads are independent and 01658 // chain to that. 01659 SDValue NewChain; 01660 if (LdChain.size() == 1) 01661 NewChain = LdChain[0]; 01662 else 01663 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 01664 LdChain.size()); 01665 01666 // Modified the chain - switch anything that used the old chain to use 01667 // the new one. 01668 ReplaceValueWith(SDValue(N, 1), NewChain); 01669 01670 return Result; 01671 } 01672 01673 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 01674 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01675 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 01676 WidenVT, N->getOperand(0)); 01677 } 01678 01679 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 01680 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01681 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01682 01683 SDValue Cond1 = N->getOperand(0); 01684 EVT CondVT = Cond1.getValueType(); 01685 if (CondVT.isVector()) { 01686 EVT CondEltVT = CondVT.getVectorElementType(); 01687 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); 01688 if (getTypeAction(CondVT) == WidenVector) 01689 Cond1 = GetWidenedVector(Cond1); 01690 01691 if (Cond1.getValueType() != CondWidenVT) 01692 Cond1 = ModifyToType(Cond1, CondWidenVT); 01693 } 01694 01695 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 01696 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 01697 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 01698 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 01699 WidenVT, Cond1, InOp1, InOp2); 01700 } 01701 01702 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 01703 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 01704 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 01705 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 01706 InOp1.getValueType(), N->getOperand(0), 01707 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 01708 } 01709 01710 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 01711 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01712 return DAG.getUNDEF(WidenVT); 01713 } 01714 01715 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 01716 EVT VT = N->getValueType(0); 01717 DebugLoc dl = N->getDebugLoc(); 01718 01719 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 01720 unsigned NumElts = VT.getVectorNumElements(); 01721 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01722 01723 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 01724 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 01725 01726 // Adjust mask based on new input vector length. 01727 SmallVector<int, 16> NewMask; 01728 for (unsigned i = 0; i != NumElts; ++i) { 01729 int Idx = N->getMaskElt(i); 01730 if (Idx < (int)NumElts) 01731 NewMask.push_back(Idx); 01732 else 01733 NewMask.push_back(Idx - NumElts + WidenNumElts); 01734 } 01735 for (unsigned i = NumElts; i != WidenNumElts; ++i) 01736 NewMask.push_back(-1); 01737 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 01738 } 01739 01740 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 01741 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 01742 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 01743 01744 SDValue InOp1 = N->getOperand(0); 01745 EVT InVT = InOp1.getValueType(); 01746 assert(InVT.isVector() && "can not widen non vector type"); 01747 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); 01748 InOp1 = GetWidenedVector(InOp1); 01749 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 01750 01751 // Assume that the input and output will be widen appropriately. If not, 01752 // we will have to unroll it at some point. 01753 assert(InOp1.getValueType() == WidenInVT && 01754 InOp2.getValueType() == WidenInVT && 01755 "Input not widened to expected type!"); 01756 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 01757 WidenVT, InOp1, InOp2, N->getOperand(2)); 01758 } 01759 01760 01761 //===----------------------------------------------------------------------===// 01762 // Widen Vector Operand 01763 //===----------------------------------------------------------------------===// 01764 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 01765 DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG); 01766 cerr << "\n"); 01767 SDValue Res = SDValue(); 01768 01769 switch (N->getOpcode()) { 01770 default: 01771 #ifndef NDEBUG 01772 cerr << "WidenVectorOperand op #" << ResNo << ": "; 01773 N->dump(&DAG); cerr << "\n"; 01774 #endif 01775 llvm_unreachable("Do not know how to widen this operator's operand!"); 01776 01777 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 01778 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 01779 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 01780 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 01781 01782 case ISD::FP_ROUND: 01783 case ISD::FP_TO_SINT: 01784 case ISD::FP_TO_UINT: 01785 case ISD::SINT_TO_FP: 01786 case ISD::UINT_TO_FP: 01787 case ISD::TRUNCATE: 01788 case ISD::SIGN_EXTEND: 01789 case ISD::ZERO_EXTEND: 01790 case ISD::ANY_EXTEND: 01791 Res = WidenVecOp_Convert(N); 01792 break; 01793 } 01794 01795 // If Res is null, the sub-method took care of registering the result. 01796 if (!Res.getNode()) return false; 01797 01798 // If the result is N, the sub-method updated N in place. Tell the legalizer 01799 // core about this. 01800 if (Res.getNode() == N) 01801 return true; 01802 01803 01804 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 01805 "Invalid operand expansion"); 01806 01807 ReplaceValueWith(SDValue(N, 0), Res); 01808 return false; 01809 } 01810 01811 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 01812 // Since the result is legal and the input is illegal, it is unlikely 01813 // that we can fix the input to a legal type so unroll the convert 01814 // into some scalar code and create a nasty build vector. 01815 EVT VT = N->getValueType(0); 01816 EVT EltVT = VT.getVectorElementType(); 01817 DebugLoc dl = N->getDebugLoc(); 01818 unsigned NumElts = VT.getVectorNumElements(); 01819 SDValue InOp = N->getOperand(0); 01820 if (getTypeAction(InOp.getValueType()) == WidenVector) 01821 InOp = GetWidenedVector(InOp); 01822 EVT InVT = InOp.getValueType(); 01823 EVT InEltVT = InVT.getVectorElementType(); 01824 01825 unsigned Opcode = N->getOpcode(); 01826 SmallVector<SDValue, 16> Ops(NumElts); 01827 for (unsigned i=0; i < NumElts; ++i) 01828 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 01829 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 01830 DAG.getIntPtrConstant(i))); 01831 01832 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 01833 } 01834 01835 SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 01836 EVT VT = N->getValueType(0); 01837 SDValue InOp = GetWidenedVector(N->getOperand(0)); 01838 EVT InWidenVT = InOp.getValueType(); 01839 DebugLoc dl = N->getDebugLoc(); 01840 01841 // Check if we can convert between two legal vector types and extract. 01842 unsigned InWidenSize = InWidenVT.getSizeInBits(); 01843 unsigned Size = VT.getSizeInBits(); 01844 if (InWidenSize % Size == 0 && !VT.isVector()) { 01845 unsigned NewNumElts = InWidenSize / Size; 01846 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 01847 if (TLI.isTypeLegal(NewVT)) { 01848 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 01849 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 01850 DAG.getIntPtrConstant(0)); 01851 } 01852 } 01853 01854 return CreateStackStoreLoad(InOp, VT); 01855 } 01856 01857 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 01858 // If the input vector is not legal, it is likely that we will not find a 01859 // legal vector of the same size. Replace the concatenate vector with a 01860 // nasty build vector. 01861 EVT VT = N->getValueType(0); 01862 EVT EltVT = VT.getVectorElementType(); 01863 DebugLoc dl = N->getDebugLoc(); 01864 unsigned NumElts = VT.getVectorNumElements(); 01865 SmallVector<SDValue, 16> Ops(NumElts); 01866 01867 EVT InVT = N->getOperand(0).getValueType(); 01868 unsigned NumInElts = InVT.getVectorNumElements(); 01869 01870 unsigned Idx = 0; 01871 unsigned NumOperands = N->getNumOperands(); 01872 for (unsigned i=0; i < NumOperands; ++i) { 01873 SDValue InOp = N->getOperand(i); 01874 if (getTypeAction(InOp.getValueType()) == WidenVector) 01875 InOp = GetWidenedVector(InOp); 01876 for (unsigned j=0; j < NumInElts; ++j) 01877 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 01878 DAG.getIntPtrConstant(j)); 01879 } 01880 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 01881 } 01882 01883 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 01884 SDValue InOp = GetWidenedVector(N->getOperand(0)); 01885 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 01886 N->getValueType(0), InOp, N->getOperand(1)); 01887 } 01888 01889 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 01890 // We have to widen the value but we want only to store the original 01891 // vector type. 01892 StoreSDNode *ST = cast<StoreSDNode>(N); 01893 SDValue Chain = ST->getChain(); 01894 SDValue BasePtr = ST->getBasePtr(); 01895 const Value *SV = ST->getSrcValue(); 01896 int SVOffset = ST->getSrcValueOffset(); 01897 unsigned Align = ST->getAlignment(); 01898 bool isVolatile = ST->isVolatile(); 01899 SDValue ValOp = GetWidenedVector(ST->getValue()); 01900 DebugLoc dl = N->getDebugLoc(); 01901 01902 EVT StVT = ST->getMemoryVT(); 01903 EVT ValVT = ValOp.getValueType(); 01904 // It must be true that we the widen vector type is bigger than where 01905 // we need to store. 01906 assert(StVT.isVector() && ValOp.getValueType().isVector()); 01907 assert(StVT.bitsLT(ValOp.getValueType())); 01908 01909 SmallVector<SDValue, 16> StChain; 01910 if (ST->isTruncatingStore()) { 01911 // For truncating stores, we can not play the tricks of chopping legal 01912 // vector types and bit cast it to the right type. Instead, we unroll 01913 // the store. 01914 EVT StEltVT = StVT.getVectorElementType(); 01915 EVT ValEltVT = ValVT.getVectorElementType(); 01916 unsigned Increment = ValEltVT.getSizeInBits() / 8; 01917 unsigned NumElts = StVT.getVectorNumElements(); 01918 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 01919 DAG.getIntPtrConstant(0)); 01920 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 01921 SVOffset, StEltVT, 01922 isVolatile, Align)); 01923 unsigned Offset = Increment; 01924 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 01925 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 01926 BasePtr, DAG.getIntPtrConstant(Offset)); 01927 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 01928 DAG.getIntPtrConstant(0)); 01929 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 01930 SVOffset + Offset, StEltVT, 01931 isVolatile, MinAlign(Align, Offset))); 01932 } 01933 } 01934 else { 01935 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 01936 // Store value 01937 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 01938 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 01939 } 01940 if (StChain.size() == 1) 01941 return StChain[0]; 01942 else 01943 return DAG.getNode(ISD::TokenFactor, dl, 01944 MVT::Other,&StChain[0],StChain.size()); 01945 } 01946 01947 //===----------------------------------------------------------------------===// 01948 // Vector Widening Utilities 01949 //===----------------------------------------------------------------------===// 01950 01951 01952 // Utility function to find a vector type and its associated element 01953 // type from a preferred width and whose vector type must be the same size 01954 // as the VecVT. 01955 // TLI: Target lowering used to determine legal types. 01956 // Width: Preferred width to store. 01957 // VecVT: Vector value type whose size we must match. 01958 // Returns NewVecVT and NewEltVT - the vector type and its associated 01959 // element type. 01960 static void FindAssocWidenVecType(SelectionDAG& DAG, 01961 const TargetLowering &TLI, unsigned Width, 01962 EVT VecVT, 01963 EVT& NewEltVT, EVT& NewVecVT) { 01964 unsigned EltWidth = Width + 1; 01965 if (TLI.isTypeLegal(VecVT)) { 01966 // We start with the preferred with, making it a power of 2 and find a 01967 // legal vector type of that width. If not, we reduce it by another of 2. 01968 // For incoming type is legal, this process will end as a vector of the 01969 // smallest loadable type should always be legal. 01970 do { 01971 assert(EltWidth > 0); 01972 EltWidth = 1 << Log2_32(EltWidth - 1); 01973 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 01974 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 01975 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 01976 } while (!TLI.isTypeLegal(NewVecVT) || 01977 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 01978 } else { 01979 // The incoming vector type is illegal and is the result of widening 01980 // a vector to a power of 2. In this case, we will use the preferred 01981 // with as long as it is a multiple of the incoming vector length. 01982 // The legalization process will eventually make this into a legal type 01983 // and remove the illegal bit converts (which would turn to stack converts 01984 // if they are allow to exist). 01985 do { 01986 assert(EltWidth > 0); 01987 EltWidth = 1 << Log2_32(EltWidth - 1); 01988 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 01989 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 01990 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 01991 } while (!TLI.isTypeLegal(NewEltVT) || 01992 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 01993 } 01994 } 01995 01996 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 01997 SDValue Chain, 01998 SDValue BasePtr, 01999 const Value *SV, 02000 int SVOffset, 02001 unsigned Alignment, 02002 bool isVolatile, 02003 unsigned LdWidth, 02004 EVT ResType, 02005 DebugLoc dl) { 02006 // The strategy assumes that we can efficiently load powers of two widths. 02007 // The routines chops the vector into the largest power of 2 load and 02008 // can be inserted into a legal vector and then cast the result into the 02009 // vector type we want. This avoids unnecessary stack converts. 02010 02011 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 02012 // the load is nonvolatile, we an use a wider load for the value. 02013 02014 // Find the vector type that can load from. 02015 EVT NewEltVT, NewVecVT; 02016 unsigned NewEltVTWidth; 02017 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 02018 NewEltVTWidth = NewEltVT.getSizeInBits(); 02019 02020 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 02021 isVolatile, Alignment); 02022 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 02023 LdChain.push_back(LdOp.getValue(1)); 02024 02025 // Check if we can load the element with one instruction 02026 if (LdWidth == NewEltVTWidth) { 02027 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 02028 } 02029 02030 unsigned Idx = 1; 02031 LdWidth -= NewEltVTWidth; 02032 unsigned Offset = 0; 02033 02034 while (LdWidth > 0) { 02035 unsigned Increment = NewEltVTWidth / 8; 02036 Offset += Increment; 02037 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 02038 DAG.getIntPtrConstant(Increment)); 02039 02040 if (LdWidth < NewEltVTWidth) { 02041 // Our current type we are using is too large, use a smaller size by 02042 // using a smaller power of 2 02043 unsigned oNewEltVTWidth = NewEltVTWidth; 02044 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 02045 NewEltVTWidth = NewEltVT.getSizeInBits(); 02046 // Readjust position and vector position based on new load type 02047 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 02048 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 02049 } 02050 02051 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 02052 SVOffset+Offset, isVolatile, 02053 MinAlign(Alignment, Offset)); 02054 LdChain.push_back(LdOp.getValue(1)); 02055 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 02056 DAG.getIntPtrConstant(Idx++)); 02057 02058 LdWidth -= NewEltVTWidth; 02059 } 02060 02061 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 02062 } 02063 02064 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 02065 SDValue Chain, 02066 SDValue BasePtr, 02067 const Value *SV, 02068 int SVOffset, 02069 unsigned Alignment, 02070 bool isVolatile, 02071 SDValue ValOp, 02072 unsigned StWidth, 02073 DebugLoc dl) { 02074 // Breaks the stores into a series of power of 2 width stores. For any 02075 // width, we convert the vector to the vector of element size that we 02076 // want to store. This avoids requiring a stack convert. 02077 02078 // Find a width of the element type we can store with 02079 EVT WidenVT = ValOp.getValueType(); 02080 EVT NewEltVT, NewVecVT; 02081 02082 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 02083 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 02084 02085 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 02086 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 02087 DAG.getIntPtrConstant(0)); 02088 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 02089 isVolatile, Alignment); 02090 StChain.push_back(StOp); 02091 02092 // Check if we are done 02093 if (StWidth == NewEltVTWidth) { 02094 return; 02095 } 02096 02097 unsigned Idx = 1; 02098 StWidth -= NewEltVTWidth; 02099 unsigned Offset = 0; 02100 02101 while (StWidth > 0) { 02102 unsigned Increment = NewEltVTWidth / 8; 02103 Offset += Increment; 02104 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 02105 DAG.getIntPtrConstant(Increment)); 02106 02107 if (StWidth < NewEltVTWidth) { 02108 // Our current type we are using is too large, use a smaller size by 02109 // using a smaller power of 2 02110 unsigned oNewEltVTWidth = NewEltVTWidth; 02111 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 02112 NewEltVTWidth = NewEltVT.getSizeInBits(); 02113 // Readjust position and vector position based on new load type 02114 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 02115 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 02116 } 02117 02118 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 02119 DAG.getIntPtrConstant(Idx++)); 02120 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 02121 SVOffset + Offset, isVolatile, 02122 MinAlign(Alignment, Offset))); 02123 StWidth -= NewEltVTWidth; 02124 } 02125 } 02126 02127 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 02128 /// input vector must have the same element type as NVT. 02129 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 02130 // Note that InOp might have been widened so it might already have 02131 // the right width or it might need be narrowed. 02132 EVT InVT = InOp.getValueType(); 02133 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 02134 "input and widen element type must match"); 02135 DebugLoc dl = InOp.getDebugLoc(); 02136 02137 // Check if InOp already has the right width. 02138 if (InVT == NVT) 02139 return InOp; 02140 02141 unsigned InNumElts = InVT.getVectorNumElements(); 02142 unsigned WidenNumElts = NVT.getVectorNumElements(); 02143 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 02144 unsigned NumConcat = WidenNumElts / InNumElts; 02145 SmallVector<SDValue, 16> Ops(NumConcat); 02146 SDValue UndefVal = DAG.getUNDEF(InVT); 02147 Ops[0] = InOp; 02148 for (unsigned i = 1; i != NumConcat; ++i) 02149 Ops[i] = UndefVal; 02150 02151 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 02152 } 02153 02154 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 02155 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 02156 DAG.getIntPtrConstant(0)); 02157 02158 // Fall back to extract and build. 02159 SmallVector<SDValue, 16> Ops(WidenNumElts); 02160 EVT EltVT = NVT.getVectorElementType(); 02161 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 02162 unsigned Idx; 02163 for (Idx = 0; Idx < MinNumElts; ++Idx) 02164 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 02165 DAG.getIntPtrConstant(Idx)); 02166 02167 SDValue UndefVal = DAG.getUNDEF(EltVT); 02168 for ( ; Idx < WidenNumElts; ++Idx) 02169 Ops[Idx] = UndefVal; 02170 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 02171 }