diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 727526055e592..191f4cc78fcc5 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4187,6 +4187,16 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { /// More limited version of SimplifyDemandedBits that can be used to "look /// through" ops that don't contribute to the DemandedBits/DemandedElts - /// bitwise ops etc. + /// Vector elements that aren't demanded can be turned into poison unless the + /// corresponding bit in the \p DoNotPoisonEltMask is set. + SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts, + const APInt &DoNotPoisonEltMask, + SelectionDAG &DAG, + unsigned Depth = 0) const; + + /// Helper wrapper around SimplifyMultipleUseDemandedBits, with + /// DoNotPoisonEltMask being set to zero. SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, @@ -4202,6 +4212,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { /// bits from only some vector elements. SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op, const APInt &DemandedElts, + const APInt &DoNotPoisonEltMask, SelectionDAG &DAG, unsigned Depth = 0) const; @@ -4219,6 +4230,15 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { /// results of this function, because simply replacing TLO.Old /// with TLO.New will be incorrect when this parameter is true and TLO.Old /// has multiple uses. + /// Vector elements that aren't demanded can be turned into poison unless the + /// corresponding bit in \p DoNotPoisonEltMask is set. + bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, + const APInt &DoNotPoisonEltMask, + APInt &KnownUndef, APInt &KnownZero, + TargetLoweringOpt &TLO, unsigned Depth = 0, + bool AssumeSingleUse = false) const; + /// Version of SimplifyDemandedVectorElts without the DoNotPoisonEltMask + /// argument. All undemanded elements can be turned into poison. bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth = 0, @@ -4303,8 +4323,9 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { /// (used to simplify the caller). The KnownUndef/Zero elements may only be /// accurate for those bits in the DemandedMask. virtual bool SimplifyDemandedVectorEltsForTargetNode( - SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, - APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth = 0) const; + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonEltMask, + APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, + unsigned Depth = 0) const; /// Attempt to simplify any target nodes based on the demanded bits/elts, /// returning true on success. Otherwise, analyze the @@ -4323,6 +4344,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { /// bitwise ops etc. virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, + const APInt &DoNotPoisonEltMask, SelectionDAG &DAG, unsigned Depth) const; /// Return true if this function can prove that \p Op is never poison diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c8088d68b6f1b..1f53742b2c6f7 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1457,8 +1457,10 @@ bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op, bool AssumeSingleUse) { TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations); APInt KnownUndef, KnownZero; - if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, - TLO, 0, AssumeSingleUse)) + APInt DoNotPoisonElts = APInt::getZero(DemandedElts.getBitWidth()); + if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, DoNotPoisonElts, + KnownUndef, KnownZero, TLO, 0, + AssumeSingleUse)) return false; // Revisit the node. diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 524c97ab3eab8..931154b922924 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -679,7 +679,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, // TODO: Under what circumstances can we create nodes? Constant folding? SDValue TargetLowering::SimplifyMultipleUseDemandedBits( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, - SelectionDAG &DAG, unsigned Depth) const { + const APInt &DoNotPoisonEltMask, SelectionDAG &DAG, unsigned Depth) const { EVT VT = Op.getValueType(); // Limit search depth. @@ -713,9 +713,12 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( unsigned NumDstEltBits = DstVT.getScalarSizeInBits(); if (NumSrcEltBits == NumDstEltBits) if (SDValue V = SimplifyMultipleUseDemandedBits( - Src, DemandedBits, DemandedElts, DAG, Depth + 1)) + Src, DemandedBits, DemandedElts, DoNotPoisonEltMask, DAG, + Depth + 1)) return DAG.getBitcast(DstVT, V); + // FIXME: Handle DoNotPoisonEltMask better? + if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) { unsigned Scale = NumDstEltBits / NumSrcEltBits; unsigned NumSrcElts = SrcVT.getVectorNumElements(); @@ -731,9 +734,12 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( // destination element, since recursive calls below may turn not demanded // elements into poison. APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); + APInt DoNotPoisonSrcElts = + APIntOps::ScaleBitMask(DoNotPoisonEltMask, NumSrcElts); if (SDValue V = SimplifyMultipleUseDemandedBits( - Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1)) + Src, DemandedSrcBits, DemandedSrcElts, DoNotPoisonSrcElts, DAG, + Depth + 1)) return DAG.getBitcast(DstVT, V); } @@ -743,15 +749,21 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1; APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits); APInt DemandedSrcElts = APInt::getZero(NumSrcElts); + APInt DoNotPoisonSrcElts = APInt::getZero(NumSrcElts); for (unsigned i = 0; i != NumElts; ++i) if (DemandedElts[i]) { unsigned Offset = (i % Scale) * NumDstEltBits; DemandedSrcBits.insertBits(DemandedBits, Offset); DemandedSrcElts.setBit(i / Scale); + } else if (DoNotPoisonEltMask[i]) { + DoNotPoisonSrcElts.setBit(i / Scale); } + // FIXME: Handle DoNotPoisonEltMask better? + if (SDValue V = SimplifyMultipleUseDemandedBits( - Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1)) + Src, DemandedSrcBits, DemandedSrcElts, DoNotPoisonSrcElts, DAG, + Depth + 1)) return DAG.getBitcast(DstVT, V); } @@ -759,7 +771,8 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( } case ISD::FREEZE: { SDValue N0 = Op.getOperand(0); - if (DAG.isGuaranteedNotToBeUndefOrPoison(N0, DemandedElts, + if (DAG.isGuaranteedNotToBeUndefOrPoison(N0, + DemandedElts | DoNotPoisonEltMask, /*PoisonOnly=*/false)) return N0; break; @@ -815,12 +828,12 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( case ISD::SHL: { // If we are only demanding sign bits then we can use the shift source // directly. - if (std::optional MaxSA = - DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) { + if (std::optional MaxSA = DAG.getValidMaximumShiftAmount( + Op, DemandedElts | DoNotPoisonEltMask, Depth + 1)) { SDValue Op0 = Op.getOperand(0); unsigned ShAmt = *MaxSA; - unsigned NumSignBits = - DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1); + unsigned NumSignBits = DAG.ComputeNumSignBits( + Op0, DemandedElts | DoNotPoisonEltMask, Depth + 1); unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero(); if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits)) return Op0; @@ -830,15 +843,15 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( case ISD::SRL: { // If we are only demanding sign bits then we can use the shift source // directly. - if (std::optional MaxSA = - DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) { + if (std::optional MaxSA = DAG.getValidMaximumShiftAmount( + Op, DemandedElts | DoNotPoisonEltMask, Depth + 1)) { SDValue Op0 = Op.getOperand(0); unsigned ShAmt = *MaxSA; // Must already be signbits in DemandedBits bounds, and can't demand any // shifted in zeroes. if (DemandedBits.countl_zero() >= ShAmt) { - unsigned NumSignBits = - DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1); + unsigned NumSignBits = DAG.ComputeNumSignBits( + Op0, DemandedElts | DoNotPoisonEltMask, Depth + 1); if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits)) return Op0; } @@ -875,7 +888,9 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( shouldRemoveRedundantExtend(Op)) return Op0; // If the input is already sign extended, just drop the extension. - unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1); + // FIXME: Can we skip DoNotPoisonEltMask here? + unsigned NumSignBits = DAG.ComputeNumSignBits( + Op0, DemandedElts | DoNotPoisonEltMask, Depth + 1); if (NumSignBits >= (BitWidth - ExBits + 1)) return Op0; break; @@ -891,7 +906,8 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( SDValue Src = Op.getOperand(0); EVT SrcVT = Src.getValueType(); EVT DstVT = Op.getValueType(); - if (IsLE && DemandedElts == 1 && + // FIXME: Can we skip DoNotPoisonEltMask here? + if (IsLE && (DemandedElts | DoNotPoisonEltMask) == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() && DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) { return DAG.getBitcast(DstVT, Src); @@ -906,8 +922,10 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( SDValue Vec = Op.getOperand(0); auto *CIdx = dyn_cast(Op.getOperand(2)); EVT VecVT = Vec.getValueType(); + // FIXME: Handle DoNotPoisonEltMask better. if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) && - !DemandedElts[CIdx->getZExtValue()]) + !DemandedElts[CIdx->getZExtValue()] && + !DoNotPoisonEltMask[CIdx->getZExtValue()]) return Vec; break; } @@ -920,8 +938,10 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( uint64_t Idx = Op.getConstantOperandVal(2); unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); + APInt DoNotPoisonSubElts = DoNotPoisonEltMask.extractBits(NumSubElts, Idx); // If we don't demand the inserted subvector, return the base vector. - if (DemandedSubElts == 0) + // FIXME: Handle DoNotPoisonEltMask better. + if (DemandedSubElts == 0 && DoNotPoisonSubElts == 0) return Vec; break; } @@ -934,9 +954,10 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( bool AllUndef = true, IdentityLHS = true, IdentityRHS = true; for (unsigned i = 0; i != NumElts; ++i) { int M = ShuffleMask[i]; - if (M < 0 || !DemandedElts[i]) + if (M < 0 || (!DemandedElts[i] && !DoNotPoisonEltMask[i])) continue; - AllUndef = false; + if (DemandedElts[i]) + AllUndef = false; IdentityLHS &= (M == (int)i); IdentityRHS &= ((M - NumElts) == i); } @@ -957,13 +978,21 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( if (Op.getOpcode() >= ISD::BUILTIN_OP_END) if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode( - Op, DemandedBits, DemandedElts, DAG, Depth)) + Op, DemandedBits, DemandedElts, DoNotPoisonEltMask, DAG, Depth)) return V; break; } return SDValue(); } +SDValue TargetLowering::SimplifyMultipleUseDemandedBits( + SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, + SelectionDAG &DAG, unsigned Depth) const { + APInt DoNotPoisonEltMask = APInt::getZero(DemandedElts.getBitWidth()); + return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, + DoNotPoisonEltMask, DAG, Depth); +} + SDValue TargetLowering::SimplifyMultipleUseDemandedBits( SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG, unsigned Depth) const { @@ -974,13 +1003,14 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( APInt DemandedElts = VT.isFixedLengthVector() ? APInt::getAllOnes(VT.getVectorNumElements()) : APInt(1, 1); - return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG, - Depth); + APInt DoNotPoisonEltMask = APInt::getZero(DemandedElts.getBitWidth()); + return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, + DoNotPoisonEltMask, DAG, Depth); } SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts( - SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, - unsigned Depth) const { + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonEltMask, + SelectionDAG &DAG, unsigned Depth) const { APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits()); return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG, Depth); @@ -2756,34 +2786,51 @@ bool TargetLowering::SimplifyDemandedBits( TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); } } - // Bitcast from a vector using SimplifyDemanded Bits/VectorElts. // Demand the elt/bit if any of the original elts/bits are demanded. if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) { unsigned Scale = BitWidth / NumSrcEltBits; unsigned NumSrcElts = SrcVT.getVectorNumElements(); APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits); + APInt DemandedSrcElts = APInt::getZero(NumSrcElts); for (unsigned i = 0; i != Scale; ++i) { unsigned EltOffset = IsLE ? i : (Scale - 1 - i); unsigned BitOffset = EltOffset * NumSrcEltBits; APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset); - if (!Sub.isZero()) + if (!Sub.isZero()) { DemandedSrcBits |= Sub; + for (unsigned j = 0; j != NumElts; ++j) + if (DemandedElts[j]) + DemandedSrcElts.setBit((j * Scale) + i); + } } - // Need to demand all smaller source elements that maps to a demanded - // destination element, since recursive calls below may turn not demanded - // elements into poison. - APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); + // Need to "semi demand" all smaller source elements that maps to a + // demanded destination element, since recursive calls below may turn not + // demanded elements into poison. Instead of demanding such elements we + // use a special bitmask to indicate that the recursive calls must not + // turn such elements into poison. + APInt NoPoisonSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); APInt KnownSrcUndef, KnownSrcZero; - if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef, - KnownSrcZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, NoPoisonSrcElts, + KnownSrcUndef, KnownSrcZero, TLO, + Depth + 1)) return true; KnownBits KnownSrcBits; - if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, - KnownSrcBits, TLO, Depth + 1)) + if (SimplifyDemandedBits(Src, DemandedSrcBits, + DemandedSrcElts | NoPoisonSrcElts, KnownSrcBits, + TLO, Depth + 1)) return true; + + if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) { + if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits( + Src, DemandedSrcBits, DemandedSrcElts, NoPoisonSrcElts, TLO.DAG, + Depth + 1)) { + SDValue NewOp = TLO.DAG.getBitcast(VT, DemandedSrc); + return TLO.CombineTo(Op, NewOp); + } + } } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) { // TODO - bigendian once we have test coverage. unsigned Scale = NumSrcEltBits / BitWidth; @@ -3090,8 +3137,9 @@ bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op, !DCI.isBeforeLegalizeOps()); APInt KnownUndef, KnownZero; - bool Simplified = - SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO); + APInt DoNotPoisonEltMask = APInt::getZero(DemandedElts.getBitWidth()); + bool Simplified = SimplifyDemandedVectorElts( + Op, DemandedElts, DoNotPoisonEltMask, KnownUndef, KnownZero, TLO); if (Simplified) { DCI.AddToWorklist(Op.getNode()); DCI.CommitTargetLoweringOpt(TLO); @@ -3152,6 +3200,16 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth, bool AssumeSingleUse) const { + APInt DoNotPoisonEltMask = APInt::getZero(OriginalDemandedElts.getBitWidth()); + return SimplifyDemandedVectorElts(Op, OriginalDemandedElts, + DoNotPoisonEltMask, KnownUndef, KnownZero, + TLO, Depth, AssumeSingleUse); +} + +bool TargetLowering::SimplifyDemandedVectorElts( + SDValue Op, const APInt &OriginalDemandedElts, + const APInt &DoNotPoisonEltMask, APInt &KnownUndef, APInt &KnownZero, + TargetLoweringOpt &TLO, unsigned Depth, bool AssumeSingleUse) const { EVT VT = Op.getValueType(); unsigned Opcode = Op.getOpcode(); APInt DemandedElts = OriginalDemandedElts; @@ -3190,6 +3248,7 @@ bool TargetLowering::SimplifyDemandedVectorElts( if (Depth >= SelectionDAG::MaxRecursionDepth) return false; + APInt DemandedEltsInclDoNotPoison = DemandedElts | DoNotPoisonEltMask; SDLoc DL(Op); unsigned EltSizeInBits = VT.getScalarSizeInBits(); bool IsLE = TLO.DAG.getDataLayout().isLittleEndian(); @@ -3197,10 +3256,10 @@ bool TargetLowering::SimplifyDemandedVectorElts( // Helper for demanding the specified elements and all the bits of both binary // operands. auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) { - SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts, - TLO.DAG, Depth + 1); - SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts, - TLO.DAG, Depth + 1); + SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts( + Op0, DemandedElts, DoNotPoisonEltMask, TLO.DAG, Depth + 1); + SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts( + Op1, DemandedElts, DoNotPoisonEltMask, TLO.DAG, Depth + 1); if (NewOp0 || NewOp1) { SDValue NewOp = TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, @@ -3230,6 +3289,7 @@ bool TargetLowering::SimplifyDemandedVectorElts( unsigned NumSrcElts = SrcEltCnt.getFixedValue(); if (isNullConstant(Idx)) { APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0); + APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts); APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts); if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero, @@ -3265,17 +3325,20 @@ bool TargetLowering::SimplifyDemandedVectorElts( // Fast handling of 'identity' bitcasts. unsigned NumSrcElts = SrcVT.getVectorNumElements(); if (NumSrcElts == NumElts) - return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, - KnownZero, TLO, Depth + 1); + return SimplifyDemandedVectorElts(Src, DemandedElts, DoNotPoisonEltMask, + KnownUndef, KnownZero, TLO, Depth + 1); - APInt SrcDemandedElts, SrcZero, SrcUndef; + APInt SrcDemandedElts, SrcDoNotPoisonEltMask, SrcZero, SrcUndef; // Bitcast from 'large element' src vector to 'small element' vector, we // must demand a source element if any DemandedElt maps to it. if ((NumElts % NumSrcElts) == 0) { unsigned Scale = NumElts / NumSrcElts; SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); - if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero, + SrcDoNotPoisonEltMask = + APIntOps::ScaleBitMask(DoNotPoisonEltMask, NumSrcElts); + if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, + SrcDoNotPoisonEltMask, SrcUndef, SrcZero, TLO, Depth + 1)) return true; @@ -3286,7 +3349,7 @@ bool TargetLowering::SimplifyDemandedVectorElts( unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits(); APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits); for (unsigned i = 0; i != NumElts; ++i) - if (DemandedElts[i]) { + if (DemandedEltsInclDoNotPoison[i]) { unsigned Ofs = (i % Scale) * EltSizeInBits; SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits); } @@ -3330,7 +3393,10 @@ bool TargetLowering::SimplifyDemandedVectorElts( if ((NumSrcElts % NumElts) == 0) { unsigned Scale = NumSrcElts / NumElts; SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); - if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero, + SrcDoNotPoisonEltMask = + APIntOps::ScaleBitMask(DoNotPoisonEltMask, NumSrcElts); + if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, + SrcDoNotPoisonEltMask, SrcUndef, SrcZero, TLO, Depth + 1)) return true; @@ -3349,7 +3415,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( } case ISD::FREEZE: { SDValue N0 = Op.getOperand(0); - if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(N0, DemandedElts, + if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(N0, + DemandedEltsInclDoNotPoison, /*PoisonOnly=*/false)) return TLO.CombineTo(Op, N0); @@ -3398,9 +3465,11 @@ bool TargetLowering::SimplifyDemandedVectorElts( for (unsigned i = 0; i != NumSubVecs; ++i) { SDValue SubOp = Op.getOperand(i); APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts); + APInt DoNotPoisonSubElts = + DoNotPoisonEltMask.extractBits(NumSubElts, i * NumSubElts); APInt SubUndef, SubZero; - if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(SubOp, SubElts, DoNotPoisonSubElts, + SubUndef, SubZero, TLO, Depth + 1)) return true; KnownUndef.insertBits(SubUndef, i * NumSubElts); KnownZero.insertBits(SubZero, i * NumSubElts); @@ -3413,10 +3482,18 @@ bool TargetLowering::SimplifyDemandedVectorElts( for (unsigned i = 0; i != NumSubVecs; ++i) { SDValue SubOp = Op.getOperand(i); APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts); - SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts( - SubOp, SubElts, TLO.DAG, Depth + 1); - DemandedSubOps.push_back(NewSubOp ? NewSubOp : SubOp); - FoundNewSub = NewSubOp ? true : FoundNewSub; + APInt DoNotPoisonSubElts = + DoNotPoisonEltMask.extractBits(NumSubElts, i * NumSubElts); + if (SubElts != 0) { + SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts( + SubOp, SubElts, DoNotPoisonSubElts, TLO.DAG, Depth + 1); + DemandedSubOps.push_back(NewSubOp ? NewSubOp : SubOp); + FoundNewSub = NewSubOp ? true : FoundNewSub; + } else if (!SubOp.isUndef()) { + DemandedSubOps.push_back(TLO.DAG.getUNDEF(SubOp.getValueType())); + FoundNewSub = true; + } else + DemandedSubOps.push_back(SubOp); } if (FoundNewSub) { SDValue NewOp = @@ -3432,18 +3509,28 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Src = Op.getOperand(0); SDValue Sub = Op.getOperand(1); uint64_t Idx = Op.getConstantOperandVal(2); - unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); + EVT SubVT = Sub.getValueType(); + unsigned NumSubElts = SubVT.getVectorNumElements(); APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); + APInt DoNoPoisonSubElts = DoNotPoisonEltMask.extractBits(NumSubElts, Idx); APInt DemandedSrcElts = DemandedElts; DemandedSrcElts.clearBits(Idx, Idx + NumSubElts); + APInt DoNoPoisonSrcElts = DoNotPoisonEltMask; + DoNoPoisonSrcElts.clearBits(Idx, Idx + NumSubElts); - // If none of the sub operand elements are demanded, bypass the insert. - if (!DemandedSubElts) + // If none of the sub operand elements are demanded and may be poisoned, + // bypass the insert. + if (!DemandedSubElts && !DoNoPoisonSubElts) return TLO.CombineTo(Op, Src); + // If none of the sub operand elements are demanded, replace it with undef. + if (!DemandedSubElts && !Sub.isUndef()) + return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, + Src, TLO.DAG.getUNDEF(SubVT), + Op.getOperand(2))); APInt SubUndef, SubZero; - if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, DoNoPoisonSubElts, + SubUndef, SubZero, TLO, Depth + 1)) return true; // If none of the src operand elements are demanded, replace it with undef. @@ -3452,8 +3539,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( TLO.DAG.getUNDEF(VT), Sub, Op.getOperand(2))); - if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero, - TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, DoNoPoisonSrcElts, + KnownUndef, KnownZero, TLO, Depth + 1)) return true; KnownUndef.insertBits(SubUndef, Idx); KnownZero.insertBits(SubZero, Idx); @@ -3461,9 +3548,9 @@ bool TargetLowering::SimplifyDemandedVectorElts( // Attempt to avoid multi-use ops if we don't need anything from them. if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) { SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts( - Src, DemandedSrcElts, TLO.DAG, Depth + 1); + Src, DemandedSrcElts, DoNoPoisonSrcElts, TLO.DAG, Depth + 1); SDValue NewSub = SimplifyMultipleUseDemandedVectorElts( - Sub, DemandedSubElts, TLO.DAG, Depth + 1); + Sub, DemandedSubElts, DoNoPoisonSubElts, TLO.DAG, Depth + 1); if (NewSrc || NewSub) { NewSrc = NewSrc ? NewSrc : Src; NewSub = NewSub ? NewSub : Sub; @@ -3482,10 +3569,11 @@ bool TargetLowering::SimplifyDemandedVectorElts( uint64_t Idx = Op.getConstantOperandVal(1); unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); + APInt DoNotPoisonSrcElts = DoNotPoisonEltMask.zext(NumSrcElts).shl(Idx); APInt SrcUndef, SrcZero; - if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, DoNotPoisonSrcElts, + SrcUndef, SrcZero, TLO, Depth + 1)) return true; KnownUndef = SrcUndef.extractBits(NumElts, Idx); KnownZero = SrcZero.extractBits(NumElts, Idx); @@ -3493,7 +3581,7 @@ bool TargetLowering::SimplifyDemandedVectorElts( // Attempt to avoid multi-use ops if we don't need anything from them. if (!DemandedElts.isAllOnes()) { SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts( - Src, DemandedSrcElts, TLO.DAG, Depth + 1); + Src, DemandedSrcElts, DoNotPoisonSrcElts, TLO.DAG, Depth + 1); if (NewSrc) { SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, Op.getOperand(1)); @@ -3505,18 +3593,26 @@ bool TargetLowering::SimplifyDemandedVectorElts( case ISD::INSERT_VECTOR_ELT: { SDValue Vec = Op.getOperand(0); SDValue Scl = Op.getOperand(1); + EVT SclVT = Scl.getValueType(); auto *CIdx = dyn_cast(Op.getOperand(2)); // For a legal, constant insertion index, if we don't need this insertion // then strip it, else remove it from the demanded elts. if (CIdx && CIdx->getAPIntValue().ult(NumElts)) { unsigned Idx = CIdx->getZExtValue(); - if (!DemandedElts[Idx]) + if (!DemandedEltsInclDoNotPoison[Idx]) return TLO.CombineTo(Op, Vec); + if (!DemandedElts[Idx]) + return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, + Vec, TLO.DAG.getUNDEF(SclVT), + Op.getOperand(2))); APInt DemandedVecElts(DemandedElts); DemandedVecElts.clearBit(Idx); - if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef, + APInt SrcDoNotPoisonEltMask(DoNotPoisonEltMask); + SrcDoNotPoisonEltMask.clearBit(Idx); + if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, + SrcDoNotPoisonEltMask, KnownUndef, KnownZero, TLO, Depth + 1)) return true; @@ -3527,8 +3623,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( } APInt VecUndef, VecZero; - if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Vec, DemandedElts, DoNotPoisonEltMask, + VecUndef, VecZero, TLO, Depth + 1)) return true; // Without knowing the insertion index we can't set KnownUndef/KnownZero. break; @@ -3541,20 +3637,18 @@ bool TargetLowering::SimplifyDemandedVectorElts( // Try to transform the select condition based on the current demanded // elements. APInt UndefSel, ZeroSel; - if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, ZeroSel, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Sel, DemandedElts, DoNotPoisonEltMask, + UndefSel, ZeroSel, TLO, Depth + 1)) return true; // See if we can simplify either vselect operand. - APInt DemandedLHS(DemandedElts); - APInt DemandedRHS(DemandedElts); APInt UndefLHS, ZeroLHS; APInt UndefRHS, ZeroRHS; - if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedElts, DoNotPoisonEltMask, + UndefLHS, ZeroLHS, TLO, Depth + 1)) return true; - if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedElts, DoNotPoisonEltMask, + UndefRHS, ZeroRHS, TLO, Depth + 1)) return true; KnownUndef = UndefLHS & UndefRHS; @@ -3564,8 +3658,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( // select value element. APInt DemandedSel = DemandedElts & ~KnownZero; if (DemandedSel != DemandedElts) - if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, ZeroSel, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Sel, DemandedSel, DoNotPoisonEltMask, + UndefSel, ZeroSel, TLO, Depth + 1)) return true; break; @@ -3575,19 +3669,16 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue RHS = Op.getOperand(1); ArrayRef ShuffleMask = cast(Op)->getMask(); - // Collect demanded elements from shuffle operands.. - APInt DemandedLHS(NumElts, 0); - APInt DemandedRHS(NumElts, 0); - for (unsigned i = 0; i != NumElts; ++i) { - int M = ShuffleMask[i]; - if (M < 0 || !DemandedElts[i]) - continue; - assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"); - if (M < (int)NumElts) - DemandedLHS.setBit(M); - else - DemandedRHS.setBit(M - NumElts); - } + // Collect demanded elements from shuffle operands. + APInt DemandedLHS, DemandedRHS; + APInt DoNotPoisonLHS, DoNotPoisonRHS; + if (!getShuffleDemandedElts(NumElts, ShuffleMask, DemandedElts, DemandedLHS, + DemandedRHS, + /*AllowUndefElts=*/true) || + !getShuffleDemandedElts(NumElts, ShuffleMask, DoNotPoisonEltMask, + DoNotPoisonLHS, DoNotPoisonRHS, + /*AllowUndefElts=*/true)) + break; // If either side isn't demanded, replace it by UNDEF. We handle this // explicitly here to also simplify in case of multiple uses (on the @@ -3605,11 +3696,11 @@ bool TargetLowering::SimplifyDemandedVectorElts( // See if we can simplify either shuffle operand. APInt UndefLHS, ZeroLHS; APInt UndefRHS, ZeroRHS; - if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedLHS, DoNotPoisonLHS, UndefLHS, + ZeroLHS, TLO, Depth + 1)) return true; - if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedRHS, DoNotPoisonRHS, UndefRHS, + ZeroRHS, TLO, Depth + 1)) return true; // Simplify mask using undef elements from LHS/RHS. @@ -3628,7 +3719,6 @@ bool TargetLowering::SimplifyDemandedVectorElts( IdentityLHS &= (M < 0) || (M == (int)i); IdentityRHS &= (M < 0) || ((M - NumElts) == i); } - // Update legal shuffle masks based on demanded elements if it won't reduce // to Identity which can cause premature removal of the shuffle mask. if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) { @@ -3663,9 +3753,10 @@ bool TargetLowering::SimplifyDemandedVectorElts( APInt SrcUndef, SrcZero; SDValue Src = Op.getOperand(0); unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); - APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts); - if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO, - Depth + 1)) + APInt DemandedSrcElts = DemandedEltsInclDoNotPoison.zext(NumSrcElts); + APInt SrcDoNotPoisonEltMask = APInt::getZero(NumSrcElts); + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcDoNotPoisonEltMask, + SrcUndef, SrcZero, TLO, Depth + 1)) return true; KnownZero = SrcZero.zextOrTrunc(NumElts); KnownUndef = SrcUndef.zextOrTrunc(NumElts); @@ -3712,8 +3803,9 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Op1 = Op.getOperand(1); if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) { APInt UndefLHS, ZeroLHS; - if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO, - Depth + 1, /*AssumeSingleUse*/ true)) + if (SimplifyDemandedVectorElts(Op0, DemandedElts, DoNotPoisonEltMask, + UndefLHS, ZeroLHS, TLO, Depth + 1, + /*AssumeSingleUse*/ true)) return true; } [[fallthrough]]; @@ -3734,12 +3826,12 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Op1 = Op.getOperand(1); APInt UndefRHS, ZeroRHS; - if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Op1, DemandedElts, DoNotPoisonEltMask, + UndefRHS, ZeroRHS, TLO, Depth + 1)) return true; APInt UndefLHS, ZeroLHS; - if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Op0, DemandedElts, DoNotPoisonEltMask, + UndefLHS, ZeroLHS, TLO, Depth + 1)) return true; KnownZero = ZeroLHS & ZeroRHS; @@ -3761,12 +3853,12 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Op1 = Op.getOperand(1); APInt UndefRHS, ZeroRHS; - if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Op1, DemandedElts, DoNotPoisonEltMask, + UndefRHS, ZeroRHS, TLO, Depth + 1)) return true; APInt UndefLHS, ZeroLHS; - if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Op0, DemandedElts, DoNotPoisonEltMask, + UndefLHS, ZeroLHS, TLO, Depth + 1)) return true; KnownZero = ZeroLHS; @@ -3787,14 +3879,14 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Op1 = Op.getOperand(1); APInt SrcUndef, SrcZero; - if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Op1, DemandedElts, DoNotPoisonEltMask, + SrcUndef, SrcZero, TLO, Depth + 1)) return true; // If we know that a demanded element was zero in Op1 we don't need to // demand it in Op0 - its guaranteed to be zero. APInt DemandedElts0 = DemandedElts & ~SrcZero; - if (SimplifyDemandedVectorElts(Op0, DemandedElts0, KnownUndef, KnownZero, - TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Op0, DemandedElts0, DoNotPoisonEltMask, + KnownUndef, KnownZero, TLO, Depth + 1)) return true; KnownUndef &= DemandedElts0; @@ -3823,13 +3915,15 @@ bool TargetLowering::SimplifyDemandedVectorElts( case ISD::TRUNCATE: case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: - if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef, - KnownZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, + DoNotPoisonEltMask, KnownUndef, KnownZero, + TLO, Depth + 1)) return true; if (!DemandedElts.isAllOnes()) if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts( - Op.getOperand(0), DemandedElts, TLO.DAG, Depth + 1)) + Op.getOperand(0), DemandedElts, DoNotPoisonEltMask, TLO.DAG, + Depth + 1)) return TLO.CombineTo(Op, TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp)); if (Op.getOpcode() == ISD::ZERO_EXTEND) { @@ -3843,20 +3937,23 @@ bool TargetLowering::SimplifyDemandedVectorElts( case ISD::UINT_TO_FP: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: - if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef, - KnownZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, + DoNotPoisonEltMask, KnownUndef, KnownZero, + TLO, Depth + 1)) return true; // Don't fall through to generic undef -> undef handling. return false; default: { if (Op.getOpcode() >= ISD::BUILTIN_OP_END) { - if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef, - KnownZero, TLO, Depth)) + if (SimplifyDemandedVectorEltsForTargetNode( + Op, DemandedElts, DoNotPoisonEltMask, KnownUndef, KnownZero, TLO, + Depth)) return true; } else { KnownBits Known; APInt DemandedBits = APInt::getAllOnes(EltSizeInBits); - if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known, + if (SimplifyDemandedBits(Op, DemandedBits, + OriginalDemandedElts | DoNotPoisonEltMask, Known, TLO, Depth, AssumeSingleUse)) return true; } @@ -3937,8 +4034,9 @@ unsigned TargetLowering::computeNumSignBitsForTargetInstr( } bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode( - SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, - TargetLoweringOpt &TLO, unsigned Depth) const { + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonEltMask, + APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, + unsigned Depth) const { assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || @@ -3957,12 +4055,14 @@ bool TargetLowering::SimplifyDemandedBitsForTargetNode( Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op" " is a target node!"); - computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth); + computeKnownBitsForTargetNode(Op, Known, DemandedElts, + TLO.DAG, Depth); return false; } SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, + const APInt &DoNotPoisonEltMask, SelectionDAG &DAG, unsigned Depth) const { assert( (Op.getOpcode() >= ISD::BUILTIN_OP_END || diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 7c26dd6e2dc2f..528e75246c40c 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -41511,10 +41511,11 @@ static SDValue combineX86ShufflesRecursively( // The Op itself may be of different VT, so we need to scale the mask. unsigned NumOpElts = Op.getValueType().getVectorNumElements(); APInt OpScaledDemandedElts = APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts); + APInt DoNotPoisonElts = APInt::getZero(NumOpElts); // Can this operand be simplified any further, given it's demanded elements? if (SDValue NewOp = TLI.SimplifyMultipleUseDemandedVectorElts( - Op, OpScaledDemandedElts, DAG)) + Op, OpScaledDemandedElts, DoNotPoisonElts, DAG)) Op = NewOp; } // FIXME: should we rerun resolveTargetShuffleInputsAndMask() now? @@ -43382,8 +43383,9 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG, // Simplify variable target shuffle masks based on the demanded elements. // TODO: Handle DemandedBits in mask indices as well? bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle( - SDValue Op, const APInt &DemandedElts, unsigned MaskIndex, - TargetLowering::TargetLoweringOpt &TLO, unsigned Depth) const { + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonEltMask, + unsigned MaskIndex, TargetLowering::TargetLoweringOpt &TLO, + unsigned Depth) const { // If we're demanding all elements don't bother trying to simplify the mask. unsigned NumElts = DemandedElts.getBitWidth(); if (DemandedElts.isAllOnes()) @@ -43395,8 +43397,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle( // Attempt to generically simplify the variable shuffle mask. APInt MaskUndef, MaskZero; - if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Mask, DemandedElts, DoNotPoisonEltMask, + MaskUndef, MaskZero, TLO, Depth + 1)) return true; // Attempt to extract+simplify a (constant pool load) shuffle mask. @@ -43422,7 +43424,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle( return false; unsigned Scale = NumCstElts / NumElts; - // Simplify mask if we have an undemanded element that is not undef. + // Simplify mask if we have an undemanded element that is not undef/posion. bool Simplified = false; SmallVector ConstVecOps; for (unsigned i = 0; i != NumCstElts; ++i) { @@ -43449,8 +43451,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle( } bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( - SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, - TargetLoweringOpt &TLO, unsigned Depth) const { + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonEltMask, + APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, + unsigned Depth) const { int NumElts = DemandedElts.getBitWidth(); unsigned Opc = Op.getOpcode(); EVT VT = Op.getValueType(); @@ -43463,11 +43466,11 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( APInt RHSUndef, RHSZero; SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); - if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedElts, DoNotPoisonEltMask, + LHSUndef, LHSZero, TLO, Depth + 1)) return true; - if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedElts, DoNotPoisonEltMask, + RHSUndef, RHSZero, TLO, Depth + 1)) return true; // Multiply by zero. KnownZero = LHSZero | RHSZero; @@ -43480,24 +43483,26 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, 2 * NumElts); + APInt DoNotPoisonSrcElts = + APIntOps::ScaleBitMask(DoNotPoisonEltMask, 2 * NumElts); - if (SimplifyDemandedVectorElts(LHS, DemandedSrcElts, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedSrcElts, DoNotPoisonSrcElts, + LHSUndef, LHSZero, TLO, Depth + 1)) return true; - if (SimplifyDemandedVectorElts(RHS, DemandedSrcElts, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedSrcElts, DoNotPoisonSrcElts, + RHSUndef, RHSZero, TLO, Depth + 1)) return true; // TODO: Multiply by zero. // If RHS/LHS elements are known zero then we don't need the LHS/RHS equivalent. APInt DemandedLHSElts = DemandedSrcElts & ~RHSZero; - if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, DoNotPoisonSrcElts, + LHSUndef, LHSZero, TLO, Depth + 1)) return true; APInt DemandedRHSElts = DemandedSrcElts & ~LHSZero; - if (SimplifyDemandedVectorElts(RHS, DemandedRHSElts, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedRHSElts, DoNotPoisonSrcElts, + RHSUndef, RHSZero, TLO, Depth + 1)) return true; break; } @@ -43513,10 +43518,12 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( if (!DemandedElts.isAllOnes()) { unsigned NumSrcElts = LHS.getValueType().getVectorNumElements(); APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); + APInt DoNotPoisonSrcElts = + APIntOps::ScaleBitMask(DoNotPoisonEltMask, NumSrcElts); SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts( - LHS, DemandedSrcElts, TLO.DAG, Depth + 1); + LHS, DemandedSrcElts, DoNotPoisonSrcElts, TLO.DAG, Depth + 1); SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts( - RHS, DemandedSrcElts, TLO.DAG, Depth + 1); + RHS, DemandedSrcElts, DoNotPoisonSrcElts, TLO.DAG, Depth + 1); if (NewLHS || NewRHS) { NewLHS = NewLHS ? NewLHS : LHS; NewRHS = NewRHS ? NewRHS : RHS; @@ -43556,8 +43563,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( case X86ISD::VSRAI: { SDValue Src = Op.getOperand(0); APInt SrcUndef; - if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Src, DemandedElts, DoNotPoisonEltMask, + SrcUndef, KnownZero, TLO, Depth + 1)) return true; // Fold shift(0,x) -> 0 @@ -43568,7 +43575,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( // Aggressively peek through ops to get at the demanded elts. if (!DemandedElts.isAllOnes()) if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts( - Src, DemandedElts, TLO.DAG, Depth + 1)) + Src, DemandedElts, DoNotPoisonEltMask, TLO.DAG, Depth + 1)) return TLO.CombineTo( Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); break; @@ -43582,8 +43589,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( APInt RHSUndef, RHSZero; SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); - if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedElts, DoNotPoisonEltMask, + LHSUndef, LHSZero, TLO, Depth + 1)) return true; // Fold shift(0,x) -> 0 @@ -43591,8 +43598,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( return TLO.CombineTo( Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); - if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedElts, DoNotPoisonEltMask, + RHSUndef, RHSZero, TLO, Depth + 1)) return true; KnownZero = LHSZero; @@ -43626,11 +43633,11 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( APInt RHSUndef, RHSZero; SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); - if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(LHS, DemandedElts, DoNotPoisonEltMask, + LHSUndef, LHSZero, TLO, Depth + 1)) return true; - if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(RHS, DemandedElts, DoNotPoisonEltMask, + RHSUndef, RHSZero, TLO, Depth + 1)) return true; break; } @@ -43664,8 +43671,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( } APInt DemandedSrc = DemandedElts.lshr(ShiftAmt); - if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO, - Depth + 1)) + APInt DoNotPoisonSrcElts = DoNotPoisonEltMask.lshr(ShiftAmt); + if (SimplifyDemandedVectorElts(Src, DemandedSrc, DoNotPoisonSrcElts, + KnownUndef, KnownZero, TLO, Depth + 1)) return true; KnownUndef <<= ShiftAmt; @@ -43703,8 +43711,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( } APInt DemandedSrc = DemandedElts.shl(ShiftAmt); - if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO, - Depth + 1)) + APInt DoNotPoisonSrcElts = DoNotPoisonEltMask.shl(ShiftAmt); + if (SimplifyDemandedVectorElts(Src, DemandedSrc, DoNotPoisonSrcElts, + KnownUndef, KnownZero, TLO, Depth + 1)) return true; KnownUndef.lshrInPlace(ShiftAmt); @@ -43717,19 +43726,23 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); + // For now, treat "DoNotPoison" elements as demanded. + // FIXME: Handle DoNotPoisonEltMask better. + APInt DemandedEltsInclDoNotPoison = DemandedElts | DoNotPoisonEltMask; + auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) { APInt UndefElts; SmallVector EltBits; int NumElts = VT.getVectorNumElements(); int EltSizeInBits = VT.getScalarSizeInBits(); APInt OpBits = APInt::getAllOnes(EltSizeInBits); - APInt OpElts = DemandedElts; + APInt OpElts = DemandedEltsInclDoNotPoison; if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits)) { OpBits.clearAllBits(); OpElts.clearAllBits(); for (int I = 0; I != NumElts; ++I) { - if (!DemandedElts[I]) + if (!DemandedEltsInclDoNotPoison[I]) continue; if (UndefElts[I]) { // We can't assume an undef src element gives an undef dst - the @@ -43781,8 +43794,10 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( EVT SrcVT = Src.getValueType(); APInt SrcUndef, SrcZero; APInt SrcElts = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements()); - if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO, - Depth + 1)) + APInt DoNotPoisonSrcElts = + DoNotPoisonEltMask.zextOrTrunc(SrcVT.getVectorNumElements()); + if (SimplifyDemandedVectorElts(Src, SrcElts, DoNotPoisonSrcElts, SrcUndef, + SrcZero, TLO, Depth + 1)) return true; break; } @@ -43793,14 +43808,16 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( APInt DemandedLHS, DemandedRHS; getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); + APInt DoNotPoisonLHS, DoNotPoisonRHS; + getPackDemandedElts(VT, DoNotPoisonEltMask, DoNotPoisonLHS, DoNotPoisonRHS); APInt LHSUndef, LHSZero; - if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(N0, DemandedLHS, DoNotPoisonLHS, LHSUndef, + LHSZero, TLO, Depth + 1)) return true; APInt RHSUndef, RHSZero; - if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(N1, DemandedRHS, DoNotPoisonRHS, RHSUndef, + RHSZero, TLO, Depth + 1)) return true; // TODO - pass on known zero/undef. @@ -43808,10 +43825,10 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( // Aggressively peek through ops to get at the demanded elts. // TODO - we should do this for all target/faux shuffles ops. if (!DemandedElts.isAllOnes()) { - SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS, - TLO.DAG, Depth + 1); - SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS, - TLO.DAG, Depth + 1); + SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts( + N0, DemandedLHS, DoNotPoisonLHS, TLO.DAG, Depth + 1); + SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts( + N1, DemandedRHS, DoNotPoisonRHS, TLO.DAG, Depth + 1); if (NewN0 || NewN1) { NewN0 = NewN0 ? NewN0 : N0; NewN1 = NewN1 ? NewN1 : N1; @@ -43830,14 +43847,17 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( APInt DemandedLHS, DemandedRHS; getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); + APInt DoNotPoisonLHS, DoNotPoisonRHS; + getHorizDemandedElts(VT, DoNotPoisonEltMask, DoNotPoisonLHS, + DoNotPoisonRHS); APInt LHSUndef, LHSZero; - if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(N0, DemandedLHS, DoNotPoisonLHS, LHSUndef, + LHSZero, TLO, Depth + 1)) return true; APInt RHSUndef, RHSZero; - if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(N1, DemandedRHS, DoNotPoisonRHS, RHSUndef, + RHSZero, TLO, Depth + 1)) return true; // TODO - pass on known zero/undef. @@ -43845,10 +43865,10 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( // Aggressively peek through ops to get at the demanded elts. // TODO: Handle repeated operands. if (N0 != N1 && !DemandedElts.isAllOnes()) { - SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS, - TLO.DAG, Depth + 1); - SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS, - TLO.DAG, Depth + 1); + SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts( + N0, DemandedLHS, DoNotPoisonLHS, TLO.DAG, Depth + 1); + SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts( + N1, DemandedRHS, DoNotPoisonRHS, TLO.DAG, Depth + 1); if (NewN0 || NewN1) { NewN0 = NewN0 ? NewN0 : N0; NewN1 = NewN1 ? NewN1 : N1; @@ -43864,9 +43884,11 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( SDValue Src = Op.getOperand(0); MVT SrcVT = Src.getSimpleValueType(); APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements()); + APInt DoNotPoisonSrcElts = + DoNotPoisonEltMask.zextOrTrunc(SrcVT.getVectorNumElements()); APInt SrcUndef, SrcZero; - if (SimplifyDemandedVectorElts(Src, DemandedSrc, SrcUndef, SrcZero, TLO, - Depth + 1)) + if (SimplifyDemandedVectorElts(Src, DemandedSrc, DoNotPoisonSrcElts, + SrcUndef, SrcZero, TLO, Depth + 1)) return true; KnownZero = SrcZero.zextOrTrunc(NumElts); KnownUndef = SrcUndef.zextOrTrunc(NumElts); @@ -43877,25 +43899,28 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( DecodeBLENDMask(NumElts, Op.getConstantOperandVal(2), BlendMask); if (SDValue R = combineBlendOfPermutes( VT.getSimpleVT(), Op.getOperand(0), Op.getOperand(1), BlendMask, - DemandedElts, TLO.DAG, Subtarget, SDLoc(Op))) + DemandedElts | DoNotPoisonEltMask, TLO.DAG, Subtarget, SDLoc(Op))) return TLO.CombineTo(Op, R); break; } case X86ISD::BLENDV: { APInt SelUndef, SelZero; - if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, SelUndef, - SelZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, + DoNotPoisonEltMask, SelUndef, SelZero, TLO, + Depth + 1)) return true; // TODO: Use SelZero to adjust LHS/RHS DemandedElts. APInt LHSUndef, LHSZero; - if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, LHSUndef, - LHSZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, + DoNotPoisonEltMask, LHSUndef, LHSZero, TLO, + Depth + 1)) return true; APInt RHSUndef, RHSZero; - if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, RHSUndef, - RHSZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, + DoNotPoisonEltMask, RHSUndef, RHSZero, TLO, + Depth + 1)) return true; KnownZero = LHSZero & RHSZero; @@ -43905,7 +43930,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( case X86ISD::VZEXT_MOVL: { // If upper demanded elements are already zero then we have nothing to do. SDValue Src = Op.getOperand(0); - APInt DemandedUpperElts = DemandedElts; + APInt DemandedUpperElts = DemandedElts | DoNotPoisonEltMask; DemandedUpperElts.clearLowBits(1); if (TLO.DAG.MaskedVectorIsZero(Src, DemandedUpperElts, Depth + 1)) return TLO.CombineTo(Op, Src); @@ -43941,32 +43966,34 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( break; APInt SrcUndef, SrcZero; APInt SrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(), 0); - if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO, - Depth + 1)) + // VBROADCAST only uses element zero. Allow poison in other elements. + APInt DoNotPoisonSrcElts = APInt::getZero(SrcVT.getVectorNumElements()); + if (SimplifyDemandedVectorElts(Src, SrcElts, DoNotPoisonSrcElts, SrcUndef, + SrcZero, TLO, Depth + 1)) return true; // Aggressively peek through src to get at the demanded elt. // TODO - we should do this for all target/faux shuffles ops. if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts( - Src, SrcElts, TLO.DAG, Depth + 1)) + Src, SrcElts, DoNotPoisonSrcElts, TLO.DAG, Depth + 1)) return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); break; } case X86ISD::VPERMV: - if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 0, TLO, - Depth)) + if (SimplifyDemandedVectorEltsForTargetShuffle( + Op, DemandedElts, DoNotPoisonEltMask, 0, TLO, Depth)) return true; break; case X86ISD::PSHUFB: case X86ISD::VPERMV3: case X86ISD::VPERMILPV: - if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 1, TLO, - Depth)) + if (SimplifyDemandedVectorEltsForTargetShuffle( + Op, DemandedElts, DoNotPoisonEltMask, 1, TLO, Depth)) return true; break; case X86ISD::VPPERM: case X86ISD::VPERMIL2: - if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 2, TLO, - Depth)) + if (SimplifyDemandedVectorEltsForTargetShuffle( + Op, DemandedElts, DoNotPoisonEltMask, 2, TLO, Depth)) return true; break; } @@ -44297,6 +44324,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( return TLO.CombineTo( Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); } + // FIXME: Do we need to consider DoNotPoisonEltMask? for (int Src = 0; Src != NumSrcs; ++Src) if (isSequentialOrUndefInRange(OpMask, 0, NumElts, Src * NumElts)) return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src])); @@ -44310,7 +44338,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( int Lo = Src * NumElts; APInt SrcElts = APInt::getZero(NumElts); for (int i = 0; i != NumElts; ++i) - if (DemandedElts[i]) { + if (DemandedElts[i] || DoNotPoisonEltMask[i]) { int M = OpMask[i] - Lo; if (0 <= M && M < NumElts) SrcElts.setBit(M); @@ -44330,6 +44358,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( // to match. This prevents combineX86ShuffleChain from returning a // combined shuffle that's the same as the original root, causing an // infinite loop. + // FIXME: Here we assume that combineX86ShufflesRecursively doesn't make + // undemanded vector elements more poisonous. No idea if that is true. if (!DemandedElts.isAllOnes()) { assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range"); @@ -44929,6 +44959,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, + const APInt &DoNotPoisonEltMask, SelectionDAG &DAG, unsigned Depth) const { int NumElts = DemandedElts.getBitWidth(); unsigned Opc = Op.getOpcode(); @@ -44942,7 +44973,8 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( auto *CIdx = dyn_cast(Op.getOperand(2)); MVT VecVT = Vec.getSimpleValueType(); if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) && - !DemandedElts[CIdx->getZExtValue()]) + !DemandedElts[CIdx->getZExtValue()] && + !DoNotPoisonEltMask[CIdx->getZExtValue()]) return Vec; break; } @@ -44977,7 +45009,7 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( SDValue LHS = Op.getOperand(1); SDValue RHS = Op.getOperand(2); - KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts, Depth + 1); + KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts | DoNotPoisonEltMask, Depth + 1); if (CondKnown.isNegative()) return LHS; if (CondKnown.isNonNegative()) @@ -44989,8 +45021,8 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); - KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts, Depth + 1); - KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts, Depth + 1); + KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts | DoNotPoisonEltMask, Depth + 1); + KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts | DoNotPoisonEltMask, Depth + 1); // If all of the demanded bits are known 0 on LHS and known 0 on RHS, then // the (inverted) LHS bits cannot contribute to the result of the 'andn' in @@ -45004,7 +45036,8 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( APInt ShuffleUndef, ShuffleZero; SmallVector ShuffleMask; SmallVector ShuffleOps; - if (getTargetShuffleInputs(Op, DemandedElts, ShuffleOps, ShuffleMask, + if (getTargetShuffleInputs(Op, DemandedElts | DoNotPoisonEltMask, + ShuffleOps, ShuffleMask, ShuffleUndef, ShuffleZero, DAG, Depth, false)) { // If all the demanded elts are from one operand and are inline, // then we can use the operand directly. @@ -45023,7 +45056,7 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( APInt IdentityOp = APInt::getAllOnes(NumOps); for (int i = 0; i != NumElts; ++i) { int M = ShuffleMask[i]; - if (!DemandedElts[i] || ShuffleUndef[i]) + if (!(DemandedElts[i] || DoNotPoisonEltMask[i]) || ShuffleUndef[i]) continue; int OpIdx = M / NumElts; int EltIdx = M % NumElts; @@ -45044,7 +45077,7 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( } return TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( - Op, DemandedBits, DemandedElts, DAG, Depth); + Op, DemandedBits, DemandedElts, DoNotPoisonEltMask, DAG, Depth); } bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 5cb6b3e493a32..81a6e480d3167 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1304,18 +1304,14 @@ namespace llvm { const SelectionDAG &DAG, unsigned Depth) const override; - bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, - const APInt &DemandedElts, - APInt &KnownUndef, - APInt &KnownZero, - TargetLoweringOpt &TLO, - unsigned Depth) const override; - - bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op, - const APInt &DemandedElts, - unsigned MaskIndex, - TargetLoweringOpt &TLO, - unsigned Depth) const; + bool SimplifyDemandedVectorEltsForTargetNode( + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonElts, + APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, + unsigned Depth) const override; + + bool SimplifyDemandedVectorEltsForTargetShuffle( + SDValue Op, const APInt &DemandedElts, const APInt &DoNotPoisonElts, + unsigned MaskIndex, TargetLoweringOpt &TLO, unsigned Depth) const; bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, @@ -1326,6 +1322,7 @@ namespace llvm { SDValue SimplifyMultipleUseDemandedBitsForTargetNode( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, + const APInt &DoNotPoisonEltMask, SelectionDAG &DAG, unsigned Depth) const override; bool isGuaranteedNotToBeUndefOrPoisonForTargetNode( diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll index 5aacd238b4211..67c2ee6403558 100644 --- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll +++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll @@ -8339,216 +8339,191 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 -; GFX6-NEXT: s_mov_b32 s7, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NEXT: s_mov_b32 s9, s7 -; GFX6-NEXT: s_mov_b32 s11, s7 -; GFX6-NEXT: s_mov_b32 s13, s7 -; GFX6-NEXT: s_mov_b32 s17, s7 -; GFX6-NEXT: s_mov_b32 s19, s7 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshr_b32 s6, s5, 30 -; GFX6-NEXT: s_lshr_b32 s8, s5, 28 -; GFX6-NEXT: s_lshr_b32 s10, s5, 29 -; GFX6-NEXT: s_lshr_b32 s12, s5, 26 -; GFX6-NEXT: s_lshr_b32 s16, s5, 27 -; GFX6-NEXT: s_mov_b32 s18, s5 -; GFX6-NEXT: s_bfe_i64 s[14:15], s[4:5], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[44:45], s[18:19], 0x10000 -; GFX6-NEXT: s_ashr_i32 s18, s5, 31 -; GFX6-NEXT: s_bfe_i64 s[28:29], s[16:17], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[36:37], s[12:13], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[38:39], s[10:11], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[40:41], s[8:9], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[42:43], s[6:7], 0x10000 ; GFX6-NEXT: s_mov_b32 s2, -1 -; GFX6-NEXT: s_mov_b32 s31, s7 -; GFX6-NEXT: s_mov_b32 s35, s7 -; GFX6-NEXT: s_mov_b32 s25, s7 -; GFX6-NEXT: s_mov_b32 s27, s7 -; GFX6-NEXT: s_mov_b32 s21, s7 -; GFX6-NEXT: s_mov_b32 s23, s7 -; GFX6-NEXT: v_mov_b32_e32 v4, s18 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: s_lshr_b32 s42, s5, 30 +; GFX6-NEXT: s_lshr_b32 s36, s5, 28 +; GFX6-NEXT: s_lshr_b32 s38, s5, 29 +; GFX6-NEXT: s_lshr_b32 s30, s5, 26 +; GFX6-NEXT: s_lshr_b32 s34, s5, 27 +; GFX6-NEXT: s_lshr_b32 s26, s5, 24 +; GFX6-NEXT: s_lshr_b32 s28, s5, 25 +; GFX6-NEXT: s_lshr_b32 s22, s5, 22 +; GFX6-NEXT: s_lshr_b32 s24, s5, 23 +; GFX6-NEXT: s_lshr_b32 s18, s5, 20 +; GFX6-NEXT: s_lshr_b32 s20, s5, 21 +; GFX6-NEXT: s_lshr_b32 s14, s5, 18 +; GFX6-NEXT: s_lshr_b32 s16, s5, 19 +; GFX6-NEXT: s_lshr_b32 s10, s5, 16 +; GFX6-NEXT: s_lshr_b32 s12, s5, 17 +; GFX6-NEXT: s_lshr_b32 s6, s5, 14 +; GFX6-NEXT: s_lshr_b32 s8, s5, 15 +; GFX6-NEXT: s_mov_b32 s40, s5 +; GFX6-NEXT: s_ashr_i32 s7, s5, 31 +; GFX6-NEXT: s_bfe_i64 s[44:45], s[40:41], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v4, s7 +; GFX6-NEXT: s_lshr_b32 s40, s5, 12 ; GFX6-NEXT: v_mov_b32_e32 v0, s44 ; GFX6-NEXT: v_mov_b32_e32 v1, s45 -; GFX6-NEXT: s_mov_b32 s45, s7 -; GFX6-NEXT: v_mov_b32_e32 v6, s14 -; GFX6-NEXT: v_mov_b32_e32 v7, s15 -; GFX6-NEXT: s_mov_b32 s47, s7 +; GFX6-NEXT: s_bfe_i64 s[44:45], s[4:5], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v6, s44 +; GFX6-NEXT: v_mov_b32_e32 v7, s45 +; GFX6-NEXT: s_lshr_b32 s44, s5, 13 ; GFX6-NEXT: v_mov_b32_e32 v2, s42 ; GFX6-NEXT: v_mov_b32_e32 v3, s43 -; GFX6-NEXT: s_mov_b32 s43, s7 -; GFX6-NEXT: v_mov_b32_e32 v8, s40 -; GFX6-NEXT: v_mov_b32_e32 v9, s41 -; GFX6-NEXT: s_mov_b32 s41, s7 +; GFX6-NEXT: s_lshr_b32 s42, s5, 10 +; GFX6-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v8, s36 +; GFX6-NEXT: v_mov_b32_e32 v9, s37 +; GFX6-NEXT: s_lshr_b32 s36, s5, 11 ; GFX6-NEXT: v_mov_b32_e32 v10, s38 ; GFX6-NEXT: v_mov_b32_e32 v11, s39 -; GFX6-NEXT: s_mov_b32 s39, s7 -; GFX6-NEXT: v_mov_b32_e32 v12, s36 -; GFX6-NEXT: v_mov_b32_e32 v13, s37 -; GFX6-NEXT: s_mov_b32 s15, s7 -; GFX6-NEXT: v_mov_b32_e32 v14, s28 -; GFX6-NEXT: v_mov_b32_e32 v15, s29 -; GFX6-NEXT: s_mov_b32 s37, s7 -; GFX6-NEXT: s_lshr_b32 s30, s5, 24 -; GFX6-NEXT: s_lshr_b32 s34, s5, 25 +; GFX6-NEXT: s_lshr_b32 s38, s5, 8 +; GFX6-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[28:29], s[30:31], 0x10000 -; GFX6-NEXT: v_mov_b32_e32 v5, s18 +; GFX6-NEXT: v_mov_b32_e32 v12, s30 +; GFX6-NEXT: v_mov_b32_e32 v13, s31 +; GFX6-NEXT: s_lshr_b32 s30, s5, 9 +; GFX6-NEXT: v_mov_b32_e32 v14, s34 +; GFX6-NEXT: v_mov_b32_e32 v15, s35 +; GFX6-NEXT: s_lshr_b32 s34, s5, 6 +; GFX6-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v5, s7 ; GFX6-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:496 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v2, s28 -; GFX6-NEXT: v_mov_b32_e32 v3, s29 -; GFX6-NEXT: s_mov_b32 s29, s7 -; GFX6-NEXT: v_mov_b32_e32 v4, s34 -; GFX6-NEXT: v_mov_b32_e32 v5, s35 -; GFX6-NEXT: s_lshr_b32 s24, s5, 22 -; GFX6-NEXT: s_lshr_b32 s26, s5, 23 -; GFX6-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v2, s26 +; GFX6-NEXT: v_mov_b32_e32 v3, s27 +; GFX6-NEXT: s_lshr_b32 s26, s5, 7 +; GFX6-NEXT: v_mov_b32_e32 v4, s28 +; GFX6-NEXT: v_mov_b32_e32 v5, s29 +; GFX6-NEXT: s_lshr_b32 s28, s5, 4 ; GFX6-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:480 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v8, s24 -; GFX6-NEXT: v_mov_b32_e32 v9, s25 -; GFX6-NEXT: s_mov_b32 s25, s7 -; GFX6-NEXT: v_mov_b32_e32 v10, s26 -; GFX6-NEXT: v_mov_b32_e32 v11, s27 -; GFX6-NEXT: s_mov_b32 s27, s7 -; GFX6-NEXT: s_lshr_b32 s20, s5, 20 -; GFX6-NEXT: s_lshr_b32 s22, s5, 21 -; GFX6-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v8, s22 +; GFX6-NEXT: v_mov_b32_e32 v9, s23 +; GFX6-NEXT: s_lshr_b32 s22, s5, 5 +; GFX6-NEXT: v_mov_b32_e32 v10, s24 +; GFX6-NEXT: v_mov_b32_e32 v11, s25 +; GFX6-NEXT: s_lshr_b32 s24, s5, 2 ; GFX6-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:464 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v12, s20 -; GFX6-NEXT: v_mov_b32_e32 v13, s21 -; GFX6-NEXT: s_mov_b32 s35, s7 -; GFX6-NEXT: v_mov_b32_e32 v14, s22 -; GFX6-NEXT: v_mov_b32_e32 v15, s23 -; GFX6-NEXT: s_mov_b32 s21, s7 -; GFX6-NEXT: s_mov_b32 s23, s7 -; GFX6-NEXT: s_lshr_b32 s16, s5, 18 -; GFX6-NEXT: s_lshr_b32 s18, s5, 19 -; GFX6-NEXT: s_lshr_b32 s10, s5, 16 -; GFX6-NEXT: s_lshr_b32 s12, s5, 17 -; GFX6-NEXT: s_lshr_b32 s8, s5, 14 -; GFX6-NEXT: s_lshr_b32 s44, s5, 15 -; GFX6-NEXT: s_lshr_b32 s46, s5, 12 -; GFX6-NEXT: s_lshr_b32 s42, s5, 13 -; GFX6-NEXT: s_lshr_b32 s40, s5, 10 -; GFX6-NEXT: s_lshr_b32 s38, s5, 11 -; GFX6-NEXT: s_lshr_b32 s14, s5, 8 -; GFX6-NEXT: s_lshr_b32 s36, s5, 9 -; GFX6-NEXT: s_lshr_b32 s28, s5, 6 -; GFX6-NEXT: s_lshr_b32 s30, s5, 7 -; GFX6-NEXT: s_lshr_b32 s24, s5, 4 -; GFX6-NEXT: s_lshr_b32 s26, s5, 5 -; GFX6-NEXT: s_lshr_b32 s34, s5, 2 -; GFX6-NEXT: s_lshr_b32 s20, s5, 3 -; GFX6-NEXT: s_lshr_b32 s22, s5, 1 -; GFX6-NEXT: s_bfe_i64 s[6:7], s[18:19], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v12, s18 +; GFX6-NEXT: v_mov_b32_e32 v13, s19 +; GFX6-NEXT: s_lshr_b32 s18, s5, 3 +; GFX6-NEXT: v_mov_b32_e32 v14, s20 +; GFX6-NEXT: v_mov_b32_e32 v15, s21 +; GFX6-NEXT: s_lshr_b32 s20, s5, 1 ; GFX6-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:448 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v2, s16 -; GFX6-NEXT: v_mov_b32_e32 v3, s17 -; GFX6-NEXT: s_lshr_b32 s16, s4, 30 -; GFX6-NEXT: v_mov_b32_e32 v4, s6 -; GFX6-NEXT: v_mov_b32_e32 v5, s7 -; GFX6-NEXT: s_lshr_b32 s18, s4, 31 -; GFX6-NEXT: s_bfe_i64 s[6:7], s[12:13], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v2, s14 +; GFX6-NEXT: v_mov_b32_e32 v3, s15 +; GFX6-NEXT: s_lshr_b32 s14, s4, 30 +; GFX6-NEXT: v_mov_b32_e32 v4, s16 +; GFX6-NEXT: v_mov_b32_e32 v5, s17 +; GFX6-NEXT: s_lshr_b32 s16, s4, 31 +; GFX6-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:432 ; GFX6-NEXT: s_waitcnt expcnt(0) ; GFX6-NEXT: v_mov_b32_e32 v8, s10 ; GFX6-NEXT: v_mov_b32_e32 v9, s11 ; GFX6-NEXT: s_lshr_b32 s10, s4, 28 -; GFX6-NEXT: v_mov_b32_e32 v10, s6 -; GFX6-NEXT: v_mov_b32_e32 v11, s7 +; GFX6-NEXT: v_mov_b32_e32 v10, s12 +; GFX6-NEXT: v_mov_b32_e32 v11, s13 ; GFX6-NEXT: s_lshr_b32 s12, s4, 29 -; GFX6-NEXT: s_bfe_i64 s[6:7], s[44:45], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:416 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v12, s8 -; GFX6-NEXT: v_mov_b32_e32 v13, s9 -; GFX6-NEXT: s_lshr_b32 s8, s4, 26 -; GFX6-NEXT: v_mov_b32_e32 v14, s6 -; GFX6-NEXT: v_mov_b32_e32 v15, s7 -; GFX6-NEXT: s_lshr_b32 s44, s4, 27 -; GFX6-NEXT: s_bfe_i64 s[6:7], s[42:43], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[42:43], s[46:47], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v12, s6 +; GFX6-NEXT: v_mov_b32_e32 v13, s7 +; GFX6-NEXT: s_lshr_b32 s46, s4, 26 +; GFX6-NEXT: v_mov_b32_e32 v14, s8 +; GFX6-NEXT: v_mov_b32_e32 v15, s9 +; GFX6-NEXT: s_lshr_b32 s8, s4, 27 +; GFX6-NEXT: s_bfe_i64 s[6:7], s[44:45], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:400 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v2, s42 -; GFX6-NEXT: v_mov_b32_e32 v3, s43 -; GFX6-NEXT: s_lshr_b32 s42, s4, 24 +; GFX6-NEXT: v_mov_b32_e32 v2, s40 +; GFX6-NEXT: v_mov_b32_e32 v3, s41 +; GFX6-NEXT: s_lshr_b32 s40, s4, 24 ; GFX6-NEXT: v_mov_b32_e32 v4, s6 ; GFX6-NEXT: v_mov_b32_e32 v5, s7 -; GFX6-NEXT: s_lshr_b32 s46, s4, 25 -; GFX6-NEXT: s_bfe_i64 s[6:7], s[38:39], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[38:39], s[40:41], 0x10000 +; GFX6-NEXT: s_lshr_b32 s44, s4, 25 +; GFX6-NEXT: s_bfe_i64 s[6:7], s[36:37], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[36:37], s[42:43], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:384 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v8, s38 -; GFX6-NEXT: v_mov_b32_e32 v9, s39 -; GFX6-NEXT: s_lshr_b32 s38, s4, 22 +; GFX6-NEXT: v_mov_b32_e32 v8, s36 +; GFX6-NEXT: v_mov_b32_e32 v9, s37 +; GFX6-NEXT: s_lshr_b32 s36, s4, 22 ; GFX6-NEXT: v_mov_b32_e32 v10, s6 ; GFX6-NEXT: v_mov_b32_e32 v11, s7 -; GFX6-NEXT: s_lshr_b32 s40, s4, 23 -; GFX6-NEXT: s_bfe_i64 s[6:7], s[36:37], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000 +; GFX6-NEXT: s_lshr_b32 s42, s4, 23 +; GFX6-NEXT: s_bfe_i64 s[6:7], s[30:31], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[30:31], s[38:39], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:368 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v12, s14 -; GFX6-NEXT: v_mov_b32_e32 v13, s15 -; GFX6-NEXT: s_lshr_b32 s14, s4, 20 +; GFX6-NEXT: v_mov_b32_e32 v12, s30 +; GFX6-NEXT: v_mov_b32_e32 v13, s31 +; GFX6-NEXT: s_lshr_b32 s30, s4, 20 ; GFX6-NEXT: v_mov_b32_e32 v14, s6 ; GFX6-NEXT: v_mov_b32_e32 v15, s7 ; GFX6-NEXT: s_lshr_b32 s6, s4, 21 -; GFX6-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000 -; GFX6-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:352 -; GFX6-NEXT: v_mov_b32_e32 v16, s28 -; GFX6-NEXT: v_mov_b32_e32 v17, s29 -; GFX6-NEXT: s_lshr_b32 s28, s4, 18 -; GFX6-NEXT: v_mov_b32_e32 v18, s30 -; GFX6-NEXT: v_mov_b32_e32 v19, s31 -; GFX6-NEXT: s_lshr_b32 s30, s4, 19 ; GFX6-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000 +; GFX6-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:352 +; GFX6-NEXT: v_mov_b32_e32 v16, s34 +; GFX6-NEXT: v_mov_b32_e32 v17, s35 +; GFX6-NEXT: s_lshr_b32 s34, s4, 18 +; GFX6-NEXT: v_mov_b32_e32 v18, s26 +; GFX6-NEXT: v_mov_b32_e32 v19, s27 +; GFX6-NEXT: s_lshr_b32 s26, s4, 19 +; GFX6-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:336 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v8, s24 -; GFX6-NEXT: v_mov_b32_e32 v9, s25 -; GFX6-NEXT: s_lshr_b32 s24, s4, 16 -; GFX6-NEXT: v_mov_b32_e32 v10, s26 -; GFX6-NEXT: v_mov_b32_e32 v11, s27 -; GFX6-NEXT: s_lshr_b32 s26, s4, 17 -; GFX6-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v8, s28 +; GFX6-NEXT: v_mov_b32_e32 v9, s29 +; GFX6-NEXT: s_lshr_b32 s28, s4, 16 +; GFX6-NEXT: v_mov_b32_e32 v10, s22 +; GFX6-NEXT: v_mov_b32_e32 v11, s23 +; GFX6-NEXT: s_lshr_b32 s22, s4, 17 +; GFX6-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:320 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v12, s34 -; GFX6-NEXT: v_mov_b32_e32 v13, s35 -; GFX6-NEXT: s_lshr_b32 s34, s4, 14 -; GFX6-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 -; GFX6-NEXT: v_mov_b32_e32 v14, s20 -; GFX6-NEXT: v_mov_b32_e32 v15, s21 -; GFX6-NEXT: s_lshr_b32 s20, s4, 15 -; GFX6-NEXT: v_mov_b32_e32 v2, s22 -; GFX6-NEXT: v_mov_b32_e32 v3, s23 -; GFX6-NEXT: s_lshr_b32 s22, s4, 12 +; GFX6-NEXT: v_mov_b32_e32 v12, s24 +; GFX6-NEXT: v_mov_b32_e32 v13, s25 +; GFX6-NEXT: s_lshr_b32 s24, s4, 14 ; GFX6-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v14, s18 +; GFX6-NEXT: v_mov_b32_e32 v15, s19 +; GFX6-NEXT: s_lshr_b32 s18, s4, 15 +; GFX6-NEXT: v_mov_b32_e32 v2, s20 +; GFX6-NEXT: v_mov_b32_e32 v3, s21 +; GFX6-NEXT: s_lshr_b32 s20, s4, 12 ; GFX6-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:304 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v16, s16 -; GFX6-NEXT: v_mov_b32_e32 v17, s17 -; GFX6-NEXT: s_lshr_b32 s16, s4, 13 -; GFX6-NEXT: v_mov_b32_e32 v18, s18 -; GFX6-NEXT: v_mov_b32_e32 v19, s19 -; GFX6-NEXT: s_lshr_b32 s18, s4, 10 +; GFX6-NEXT: v_mov_b32_e32 v16, s14 +; GFX6-NEXT: v_mov_b32_e32 v17, s15 +; GFX6-NEXT: s_lshr_b32 s14, s4, 13 +; GFX6-NEXT: v_mov_b32_e32 v18, s16 +; GFX6-NEXT: v_mov_b32_e32 v19, s17 +; GFX6-NEXT: s_lshr_b32 s16, s4, 10 ; GFX6-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:288 @@ -8559,62 +8534,62 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX6-NEXT: v_mov_b32_e32 v10, s12 ; GFX6-NEXT: v_mov_b32_e32 v11, s13 ; GFX6-NEXT: s_lshr_b32 s12, s4, 8 -; GFX6-NEXT: s_bfe_i64 s[36:37], s[44:45], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[38:39], s[46:47], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:272 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v12, s8 -; GFX6-NEXT: v_mov_b32_e32 v13, s9 -; GFX6-NEXT: s_lshr_b32 s8, s4, 9 -; GFX6-NEXT: v_mov_b32_e32 v14, s36 -; GFX6-NEXT: v_mov_b32_e32 v15, s37 -; GFX6-NEXT: s_lshr_b32 s36, s4, 6 -; GFX6-NEXT: s_bfe_i64 s[44:45], s[46:47], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v12, s38 +; GFX6-NEXT: v_mov_b32_e32 v13, s39 +; GFX6-NEXT: s_lshr_b32 s38, s4, 9 +; GFX6-NEXT: v_mov_b32_e32 v14, s8 +; GFX6-NEXT: v_mov_b32_e32 v15, s9 +; GFX6-NEXT: s_lshr_b32 s8, s4, 6 +; GFX6-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:256 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v0, s42 -; GFX6-NEXT: v_mov_b32_e32 v1, s43 -; GFX6-NEXT: s_lshr_b32 s42, s4, 7 +; GFX6-NEXT: v_mov_b32_e32 v0, s40 +; GFX6-NEXT: v_mov_b32_e32 v1, s41 +; GFX6-NEXT: s_lshr_b32 s40, s4, 7 ; GFX6-NEXT: v_mov_b32_e32 v2, s44 ; GFX6-NEXT: v_mov_b32_e32 v3, s45 ; GFX6-NEXT: s_lshr_b32 s44, s4, 4 -; GFX6-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:240 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v16, s38 -; GFX6-NEXT: v_mov_b32_e32 v17, s39 -; GFX6-NEXT: s_lshr_b32 s38, s4, 5 -; GFX6-NEXT: v_mov_b32_e32 v18, s40 -; GFX6-NEXT: v_mov_b32_e32 v19, s41 -; GFX6-NEXT: s_lshr_b32 s40, s4, 2 -; GFX6-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000 +; GFX6-NEXT: v_mov_b32_e32 v16, s36 +; GFX6-NEXT: v_mov_b32_e32 v17, s37 +; GFX6-NEXT: s_lshr_b32 s36, s4, 5 +; GFX6-NEXT: v_mov_b32_e32 v18, s42 +; GFX6-NEXT: v_mov_b32_e32 v19, s43 +; GFX6-NEXT: s_lshr_b32 s42, s4, 2 +; GFX6-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:224 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v8, s14 -; GFX6-NEXT: v_mov_b32_e32 v9, s15 -; GFX6-NEXT: s_lshr_b32 s14, s4, 3 +; GFX6-NEXT: v_mov_b32_e32 v8, s30 +; GFX6-NEXT: v_mov_b32_e32 v9, s31 +; GFX6-NEXT: s_lshr_b32 s30, s4, 3 ; GFX6-NEXT: s_lshr_b32 s4, s4, 1 ; GFX6-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000 -; GFX6-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000 +; GFX6-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000 ; GFX6-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000 ; GFX6-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:208 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:192 @@ -8623,58 +8598,58 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX6-NEXT: v_mov_b32_e32 v11, s7 ; GFX6-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:160 ; GFX6-NEXT: s_waitcnt expcnt(2) +; GFX6-NEXT: v_mov_b32_e32 v0, s34 +; GFX6-NEXT: v_mov_b32_e32 v1, s35 +; GFX6-NEXT: v_mov_b32_e32 v2, s26 +; GFX6-NEXT: v_mov_b32_e32 v3, s27 +; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:144 +; GFX6-NEXT: s_waitcnt expcnt(0) ; GFX6-NEXT: v_mov_b32_e32 v0, s28 ; GFX6-NEXT: v_mov_b32_e32 v1, s29 -; GFX6-NEXT: v_mov_b32_e32 v2, s30 -; GFX6-NEXT: v_mov_b32_e32 v3, s31 -; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:144 +; GFX6-NEXT: v_mov_b32_e32 v2, s22 +; GFX6-NEXT: v_mov_b32_e32 v3, s23 +; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:128 ; GFX6-NEXT: s_waitcnt expcnt(0) ; GFX6-NEXT: v_mov_b32_e32 v0, s24 ; GFX6-NEXT: v_mov_b32_e32 v1, s25 -; GFX6-NEXT: v_mov_b32_e32 v2, s26 -; GFX6-NEXT: v_mov_b32_e32 v3, s27 -; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:128 -; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v0, s34 -; GFX6-NEXT: v_mov_b32_e32 v1, s35 -; GFX6-NEXT: v_mov_b32_e32 v2, s20 -; GFX6-NEXT: v_mov_b32_e32 v3, s21 +; GFX6-NEXT: v_mov_b32_e32 v2, s18 +; GFX6-NEXT: v_mov_b32_e32 v3, s19 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v0, s22 -; GFX6-NEXT: v_mov_b32_e32 v1, s23 -; GFX6-NEXT: v_mov_b32_e32 v2, s16 -; GFX6-NEXT: v_mov_b32_e32 v3, s17 +; GFX6-NEXT: v_mov_b32_e32 v0, s20 +; GFX6-NEXT: v_mov_b32_e32 v1, s21 +; GFX6-NEXT: v_mov_b32_e32 v2, s14 +; GFX6-NEXT: v_mov_b32_e32 v3, s15 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v0, s18 -; GFX6-NEXT: v_mov_b32_e32 v1, s19 +; GFX6-NEXT: v_mov_b32_e32 v0, s16 +; GFX6-NEXT: v_mov_b32_e32 v1, s17 ; GFX6-NEXT: v_mov_b32_e32 v2, s10 ; GFX6-NEXT: v_mov_b32_e32 v3, s11 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80 ; GFX6-NEXT: s_waitcnt expcnt(0) ; GFX6-NEXT: v_mov_b32_e32 v0, s12 ; GFX6-NEXT: v_mov_b32_e32 v1, s13 -; GFX6-NEXT: v_mov_b32_e32 v2, s8 -; GFX6-NEXT: v_mov_b32_e32 v3, s9 +; GFX6-NEXT: v_mov_b32_e32 v2, s38 +; GFX6-NEXT: v_mov_b32_e32 v3, s39 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v0, s36 -; GFX6-NEXT: v_mov_b32_e32 v1, s37 -; GFX6-NEXT: v_mov_b32_e32 v2, s42 -; GFX6-NEXT: v_mov_b32_e32 v3, s43 +; GFX6-NEXT: v_mov_b32_e32 v0, s8 +; GFX6-NEXT: v_mov_b32_e32 v1, s9 +; GFX6-NEXT: v_mov_b32_e32 v2, s40 +; GFX6-NEXT: v_mov_b32_e32 v3, s41 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48 ; GFX6-NEXT: s_waitcnt expcnt(0) ; GFX6-NEXT: v_mov_b32_e32 v0, s44 ; GFX6-NEXT: v_mov_b32_e32 v1, s45 -; GFX6-NEXT: v_mov_b32_e32 v2, s38 -; GFX6-NEXT: v_mov_b32_e32 v3, s39 +; GFX6-NEXT: v_mov_b32_e32 v2, s36 +; GFX6-NEXT: v_mov_b32_e32 v3, s37 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32 ; GFX6-NEXT: s_waitcnt expcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v0, s40 -; GFX6-NEXT: v_mov_b32_e32 v1, s41 -; GFX6-NEXT: v_mov_b32_e32 v2, s14 -; GFX6-NEXT: v_mov_b32_e32 v3, s15 +; GFX6-NEXT: v_mov_b32_e32 v0, s42 +; GFX6-NEXT: v_mov_b32_e32 v1, s43 +; GFX6-NEXT: v_mov_b32_e32 v2, s30 +; GFX6-NEXT: v_mov_b32_e32 v3, s31 ; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16 ; GFX6-NEXT: v_mov_b32_e32 v8, s4 ; GFX6-NEXT: v_mov_b32_e32 v9, s5 @@ -8685,16 +8660,8 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX8: ; %bb.0: ; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 ; GFX8-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane -; GFX8-NEXT: s_mov_b32 s69, 0 -; GFX8-NEXT: s_mov_b32 s67, s69 -; GFX8-NEXT: s_mov_b32 s41, s69 -; GFX8-NEXT: s_mov_b32 s61, s69 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_load_dwordx2 s[2:3], s[10:11], 0x0 -; GFX8-NEXT: s_mov_b32 s43, s69 -; GFX8-NEXT: s_mov_b32 s65, s69 -; GFX8-NEXT: s_mov_b32 s45, s69 -; GFX8-NEXT: s_mov_b32 s57, s69 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_lshr_b32 s0, s3, 8 ; GFX8-NEXT: s_lshr_b32 s48, s3, 15 @@ -8707,15 +8674,14 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX8-NEXT: s_lshr_b32 s36, s3, 27 ; GFX8-NEXT: s_lshr_b32 s68, s3, 24 ; GFX8-NEXT: s_lshr_b32 s38, s3, 25 -; GFX8-NEXT: s_lshr_b32 s66, s3, 22 +; GFX8-NEXT: s_lshr_b32 s64, s3, 22 ; GFX8-NEXT: s_lshr_b32 s40, s3, 23 ; GFX8-NEXT: s_lshr_b32 s60, s3, 20 ; GFX8-NEXT: s_lshr_b32 s42, s3, 21 -; GFX8-NEXT: s_lshr_b32 s64, s3, 18 +; GFX8-NEXT: s_lshr_b32 s66, s3, 18 ; GFX8-NEXT: s_lshr_b32 s44, s3, 19 ; GFX8-NEXT: s_lshr_b32 s56, s3, 16 ; GFX8-NEXT: s_lshr_b32 s46, s3, 17 -; GFX8-NEXT: s_mov_b32 s47, s69 ; GFX8-NEXT: s_lshr_b32 s58, s3, 14 ; GFX8-NEXT: s_lshr_b32 s62, s3, 12 ; GFX8-NEXT: s_lshr_b32 s54, s3, 10 @@ -8724,14 +8690,13 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX8-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x10000 ; GFX8-NEXT: s_lshr_b32 s52, s3, 11 ; GFX8-NEXT: v_writelane_b32 v62, s0, 2 -; GFX8-NEXT: s_mov_b32 s23, s69 ; GFX8-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x10000 -; GFX8-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x10000 +; GFX8-NEXT: s_bfe_i64 s[64:65], s[64:65], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[68:69], s[68:69], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[70:71], s[70:71], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[72:73], s[72:73], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[74:75], s[74:75], 0x10000 -; GFX8-NEXT: s_bfe_i64 s[64:65], s[64:65], 0x10000 +; GFX8-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x10000 ; GFX8-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x10000 @@ -8763,9 +8728,9 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX8-NEXT: v_mov_b32_e32 v12, s72 ; GFX8-NEXT: v_mov_b32_e32 v0, s70 ; GFX8-NEXT: v_mov_b32_e32 v8, s68 -; GFX8-NEXT: v_mov_b32_e32 v16, s66 +; GFX8-NEXT: v_mov_b32_e32 v16, s64 ; GFX8-NEXT: v_mov_b32_e32 v20, s60 -; GFX8-NEXT: v_mov_b32_e32 v24, s64 +; GFX8-NEXT: v_mov_b32_e32 v24, s66 ; GFX8-NEXT: v_mov_b32_e32 v28, s56 ; GFX8-NEXT: v_mov_b32_e32 v32, s58 ; GFX8-NEXT: v_mov_b32_e32 v36, s62 @@ -8828,11 +8793,11 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX8-NEXT: v_mov_b32_e32 v3, s37 ; GFX8-NEXT: v_mov_b32_e32 v9, s69 ; GFX8-NEXT: v_mov_b32_e32 v11, s39 -; GFX8-NEXT: v_mov_b32_e32 v17, s67 +; GFX8-NEXT: v_mov_b32_e32 v17, s65 ; GFX8-NEXT: v_mov_b32_e32 v19, s41 ; GFX8-NEXT: v_mov_b32_e32 v21, s61 ; GFX8-NEXT: v_mov_b32_e32 v23, s43 -; GFX8-NEXT: v_mov_b32_e32 v25, s65 +; GFX8-NEXT: v_mov_b32_e32 v25, s67 ; GFX8-NEXT: v_mov_b32_e32 v27, s45 ; GFX8-NEXT: v_mov_b32_e32 v29, s57 ; GFX8-NEXT: v_mov_b32_e32 v31, s47 @@ -9496,59 +9461,48 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX12-LABEL: constant_sextload_v64i1_to_v64i64: ; GFX12: ; %bb.0: ; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX12-NEXT: s_mov_b32 s67, 0 -; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_mov_b32 s57, s67 -; GFX12-NEXT: s_mov_b32 s63, s67 -; GFX12-NEXT: s_mov_b32 s45, s67 -; GFX12-NEXT: s_mov_b32 s53, s67 -; GFX12-NEXT: s_mov_b32 s31, s67 -; GFX12-NEXT: s_mov_b32 s41, s67 -; GFX12-NEXT: s_mov_b32 s19, s67 -; GFX12-NEXT: s_mov_b32 s27, s67 -; GFX12-NEXT: s_mov_b32 s47, s67 ; GFX12-NEXT: s_wait_kmcnt 0x0 -; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x0 +; GFX12-NEXT: s_load_b64 s[10:11], s[2:3], 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 -; GFX12-NEXT: s_lshr_b32 s96, s3, 30 -; GFX12-NEXT: s_lshr_b32 s98, s3, 31 -; GFX12-NEXT: s_lshr_b32 s92, s3, 28 -; GFX12-NEXT: s_lshr_b32 s94, s3, 29 -; GFX12-NEXT: s_lshr_b32 s78, s3, 26 -; GFX12-NEXT: s_lshr_b32 s88, s3, 27 +; GFX12-NEXT: s_lshr_b32 s96, s11, 30 +; GFX12-NEXT: s_lshr_b32 s98, s11, 31 +; GFX12-NEXT: s_lshr_b32 s92, s11, 28 +; GFX12-NEXT: s_lshr_b32 s94, s11, 29 +; GFX12-NEXT: s_lshr_b32 s78, s11, 26 +; GFX12-NEXT: s_lshr_b32 s88, s11, 27 ; GFX12-NEXT: s_bfe_i64 s[96:97], s[96:97], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[100:101], s[98:99], 0x10000 -; GFX12-NEXT: s_lshr_b32 s66, s3, 24 -; GFX12-NEXT: s_lshr_b32 s74, s3, 25 +; GFX12-NEXT: s_lshr_b32 s66, s11, 24 +; GFX12-NEXT: s_lshr_b32 s74, s11, 25 ; GFX12-NEXT: s_bfe_i64 s[92:93], s[92:93], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[94:95], s[94:95], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s96 -; GFX12-NEXT: s_lshr_b32 s56, s3, 22 -; GFX12-NEXT: s_lshr_b32 s62, s3, 23 +; GFX12-NEXT: s_lshr_b32 s56, s11, 22 +; GFX12-NEXT: s_lshr_b32 s62, s11, 23 ; GFX12-NEXT: v_dual_mov_b32 v2, s97 :: v_dual_mov_b32 v3, s100 ; GFX12-NEXT: v_dual_mov_b32 v4, s101 :: v_dual_mov_b32 v5, s92 ; GFX12-NEXT: s_bfe_i64 s[78:79], s[78:79], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[88:89], s[88:89], 0x10000 -; GFX12-NEXT: s_lshr_b32 s44, s3, 20 -; GFX12-NEXT: s_lshr_b32 s52, s3, 21 -; GFX12-NEXT: s_lshr_b32 s30, s3, 18 -; GFX12-NEXT: s_lshr_b32 s40, s3, 19 -; GFX12-NEXT: s_lshr_b32 s18, s3, 16 -; GFX12-NEXT: s_lshr_b32 s26, s3, 17 -; GFX12-NEXT: s_lshr_b32 s4, s3, 14 -; GFX12-NEXT: s_lshr_b32 s6, s3, 15 +; GFX12-NEXT: s_lshr_b32 s44, s11, 20 +; GFX12-NEXT: s_lshr_b32 s52, s11, 21 +; GFX12-NEXT: s_lshr_b32 s30, s11, 18 +; GFX12-NEXT: s_lshr_b32 s40, s11, 19 +; GFX12-NEXT: s_lshr_b32 s18, s11, 16 +; GFX12-NEXT: s_lshr_b32 s26, s11, 17 +; GFX12-NEXT: s_lshr_b32 s2, s11, 14 +; GFX12-NEXT: s_lshr_b32 s4, s11, 15 ; GFX12-NEXT: v_dual_mov_b32 v6, s93 :: v_dual_mov_b32 v7, s94 ; GFX12-NEXT: v_dual_mov_b32 v8, s95 :: v_dual_mov_b32 v9, s78 ; GFX12-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[74:75], s[74:75], 0x10000 -; GFX12-NEXT: s_lshr_b32 s8, s3, 12 -; GFX12-NEXT: s_lshr_b32 s10, s3, 13 +; GFX12-NEXT: s_lshr_b32 s6, s11, 12 +; GFX12-NEXT: s_lshr_b32 s8, s11, 13 ; GFX12-NEXT: v_dual_mov_b32 v10, s79 :: v_dual_mov_b32 v11, s88 ; GFX12-NEXT: v_dual_mov_b32 v12, s89 :: v_dual_mov_b32 v13, s66 ; GFX12-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x10000 -; GFX12-NEXT: s_lshr_b32 s12, s3, 10 -; GFX12-NEXT: s_lshr_b32 s14, s3, 11 +; GFX12-NEXT: s_lshr_b32 s12, s11, 10 +; GFX12-NEXT: s_lshr_b32 s14, s11, 11 ; GFX12-NEXT: v_dual_mov_b32 v14, s67 :: v_dual_mov_b32 v15, s74 ; GFX12-NEXT: v_dual_mov_b32 v16, s75 :: v_dual_mov_b32 v17, s56 ; GFX12-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000 @@ -9557,16 +9511,16 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX12-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000 -; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000 -; GFX12-NEXT: s_lshr_b32 s16, s3, 8 -; GFX12-NEXT: s_lshr_b32 s20, s3, 9 +; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000 +; GFX12-NEXT: s_lshr_b32 s16, s11, 8 +; GFX12-NEXT: s_lshr_b32 s20, s11, 9 ; GFX12-NEXT: v_dual_mov_b32 v18, s57 :: v_dual_mov_b32 v19, s62 ; GFX12-NEXT: v_dual_mov_b32 v20, s63 :: v_dual_mov_b32 v21, s44 -; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000 -; GFX12-NEXT: s_lshr_b32 s22, s3, 6 -; GFX12-NEXT: s_lshr_b32 s24, s3, 7 +; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000 +; GFX12-NEXT: s_lshr_b32 s22, s11, 6 +; GFX12-NEXT: s_lshr_b32 s24, s11, 7 ; GFX12-NEXT: v_dual_mov_b32 v22, s45 :: v_dual_mov_b32 v23, s52 ; GFX12-NEXT: v_dual_mov_b32 v24, s53 :: v_dual_mov_b32 v25, s30 ; GFX12-NEXT: v_dual_mov_b32 v26, s31 :: v_dual_mov_b32 v27, s40 @@ -9584,39 +9538,39 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX12-NEXT: global_store_b128 v0, v[21:24], s[0:1] offset:416 ; GFX12-NEXT: global_store_b128 v0, v[25:28], s[0:1] offset:400 ; GFX12-NEXT: global_store_b128 v0, v[29:32], s[0:1] offset:384 -; GFX12-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s5 -; GFX12-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v4, s7 -; GFX12-NEXT: v_mov_b32_e32 v5, s8 -; GFX12-NEXT: s_lshr_b32 s28, s3, 4 -; GFX12-NEXT: s_lshr_b32 s34, s3, 5 -; GFX12-NEXT: s_lshr_b32 s36, s3, 2 -; GFX12-NEXT: s_lshr_b32 s38, s3, 3 +; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3 +; GFX12-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s5 +; GFX12-NEXT: v_mov_b32_e32 v5, s6 +; GFX12-NEXT: s_lshr_b32 s28, s11, 4 +; GFX12-NEXT: s_lshr_b32 s34, s11, 5 +; GFX12-NEXT: s_lshr_b32 s36, s11, 2 +; GFX12-NEXT: s_lshr_b32 s38, s11, 3 ; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000 -; GFX12-NEXT: v_dual_mov_b32 v6, s9 :: v_dual_mov_b32 v7, s10 -; GFX12-NEXT: v_dual_mov_b32 v8, s11 :: v_dual_mov_b32 v9, s12 -; GFX12-NEXT: s_lshr_b32 s42, s3, 1 -; GFX12-NEXT: s_mov_b32 s46, s3 +; GFX12-NEXT: v_dual_mov_b32 v6, s7 :: v_dual_mov_b32 v7, s8 +; GFX12-NEXT: v_dual_mov_b32 v8, s9 :: v_dual_mov_b32 v9, s12 +; GFX12-NEXT: s_lshr_b32 s42, s11, 1 +; GFX12-NEXT: s_mov_b32 s46, s11 ; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v10, s13 :: v_dual_mov_b32 v11, s14 ; GFX12-NEXT: v_dual_mov_b32 v12, s15 :: v_dual_mov_b32 v13, s16 -; GFX12-NEXT: s_lshr_b32 s48, s2, 30 -; GFX12-NEXT: s_lshr_b32 s50, s2, 31 +; GFX12-NEXT: s_lshr_b32 s48, s10, 30 +; GFX12-NEXT: s_lshr_b32 s50, s10, 31 ; GFX12-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s20 ; GFX12-NEXT: v_dual_mov_b32 v16, s21 :: v_dual_mov_b32 v17, s22 -; GFX12-NEXT: s_lshr_b32 s54, s2, 28 -; GFX12-NEXT: s_lshr_b32 s58, s2, 29 +; GFX12-NEXT: s_lshr_b32 s54, s10, 28 +; GFX12-NEXT: s_lshr_b32 s58, s10, 29 ; GFX12-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v18, s23 :: v_dual_mov_b32 v19, s24 ; GFX12-NEXT: v_dual_mov_b32 v20, s25 :: v_dual_mov_b32 v21, s28 -; GFX12-NEXT: s_lshr_b32 s60, s2, 26 -; GFX12-NEXT: s_lshr_b32 s64, s2, 27 +; GFX12-NEXT: s_lshr_b32 s60, s10, 26 +; GFX12-NEXT: s_lshr_b32 s64, s10, 27 ; GFX12-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v22, s29 :: v_dual_mov_b32 v23, s34 @@ -9631,43 +9585,43 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX12-NEXT: v_dual_mov_b32 v1, s36 :: v_dual_mov_b32 v2, s37 ; GFX12-NEXT: v_dual_mov_b32 v3, s38 :: v_dual_mov_b32 v4, s39 ; GFX12-NEXT: v_mov_b32_e32 v5, s46 -; GFX12-NEXT: s_lshr_b32 s68, s2, 24 -; GFX12-NEXT: s_lshr_b32 s70, s2, 25 -; GFX12-NEXT: s_lshr_b32 s72, s2, 22 -; GFX12-NEXT: s_lshr_b32 s76, s2, 23 +; GFX12-NEXT: s_lshr_b32 s68, s10, 24 +; GFX12-NEXT: s_lshr_b32 s70, s10, 25 +; GFX12-NEXT: s_lshr_b32 s72, s10, 22 +; GFX12-NEXT: s_lshr_b32 s76, s10, 23 ; GFX12-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v6, s47 :: v_dual_mov_b32 v7, s42 ; GFX12-NEXT: v_dual_mov_b32 v8, s43 :: v_dual_mov_b32 v9, s48 -; GFX12-NEXT: s_lshr_b32 s80, s2, 20 -; GFX12-NEXT: s_lshr_b32 s82, s2, 21 +; GFX12-NEXT: s_lshr_b32 s80, s10, 20 +; GFX12-NEXT: s_lshr_b32 s82, s10, 21 ; GFX12-NEXT: s_bfe_i64 s[64:65], s[64:65], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v10, s49 :: v_dual_mov_b32 v11, s50 ; GFX12-NEXT: v_dual_mov_b32 v12, s51 :: v_dual_mov_b32 v13, s54 -; GFX12-NEXT: s_lshr_b32 s84, s2, 18 -; GFX12-NEXT: s_lshr_b32 s86, s2, 19 +; GFX12-NEXT: s_lshr_b32 s84, s10, 18 +; GFX12-NEXT: s_lshr_b32 s86, s10, 19 ; GFX12-NEXT: s_bfe_i64 s[76:77], s[76:77], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[72:73], s[72:73], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[70:71], s[70:71], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[68:69], s[68:69], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v14, s55 :: v_dual_mov_b32 v15, s58 ; GFX12-NEXT: v_dual_mov_b32 v16, s59 :: v_dual_mov_b32 v17, s60 -; GFX12-NEXT: s_lshr_b32 s90, s2, 16 -; GFX12-NEXT: s_lshr_b32 s98, s2, 17 +; GFX12-NEXT: s_lshr_b32 s90, s10, 16 +; GFX12-NEXT: s_lshr_b32 s98, s10, 17 ; GFX12-NEXT: s_bfe_i64 s[82:83], s[82:83], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[80:81], s[80:81], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v18, s61 :: v_dual_mov_b32 v19, s64 ; GFX12-NEXT: v_dual_mov_b32 v20, s65 :: v_dual_mov_b32 v21, s68 -; GFX12-NEXT: s_lshr_b32 s96, s2, 14 -; GFX12-NEXT: s_lshr_b32 s100, s2, 15 -; GFX12-NEXT: s_lshr_b32 s94, s2, 13 -; GFX12-NEXT: s_lshr_b32 s88, s2, 11 -; GFX12-NEXT: s_lshr_b32 s74, s2, 9 -; GFX12-NEXT: s_lshr_b32 s62, s2, 7 -; GFX12-NEXT: s_lshr_b32 s52, s2, 5 -; GFX12-NEXT: s_lshr_b32 s40, s2, 3 -; GFX12-NEXT: s_lshr_b32 s26, s2, 1 +; GFX12-NEXT: s_lshr_b32 s96, s10, 14 +; GFX12-NEXT: s_lshr_b32 s100, s10, 15 +; GFX12-NEXT: s_lshr_b32 s94, s10, 13 +; GFX12-NEXT: s_lshr_b32 s88, s10, 11 +; GFX12-NEXT: s_lshr_b32 s74, s10, 9 +; GFX12-NEXT: s_lshr_b32 s62, s10, 7 +; GFX12-NEXT: s_lshr_b32 s52, s10, 5 +; GFX12-NEXT: s_lshr_b32 s40, s10, 3 +; GFX12-NEXT: s_lshr_b32 s26, s10, 1 ; GFX12-NEXT: s_bfe_i64 s[86:87], s[86:87], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[84:85], s[84:85], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v22, s69 :: v_dual_mov_b32 v23, s70 @@ -9682,19 +9636,19 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX12-NEXT: v_dual_mov_b32 v1, s72 :: v_dual_mov_b32 v2, s73 ; GFX12-NEXT: v_dual_mov_b32 v3, s76 :: v_dual_mov_b32 v4, s77 ; GFX12-NEXT: v_mov_b32_e32 v5, s80 -; GFX12-NEXT: s_lshr_b32 s92, s2, 12 -; GFX12-NEXT: s_lshr_b32 s78, s2, 10 +; GFX12-NEXT: s_lshr_b32 s92, s10, 12 +; GFX12-NEXT: s_lshr_b32 s78, s10, 10 ; GFX12-NEXT: s_bfe_i64 s[98:99], s[98:99], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[90:91], s[90:91], 0x10000 ; GFX12-NEXT: v_dual_mov_b32 v6, s81 :: v_dual_mov_b32 v7, s82 ; GFX12-NEXT: v_dual_mov_b32 v8, s83 :: v_dual_mov_b32 v9, s84 -; GFX12-NEXT: s_lshr_b32 s66, s2, 8 -; GFX12-NEXT: s_lshr_b32 s56, s2, 6 -; GFX12-NEXT: s_lshr_b32 s44, s2, 4 -; GFX12-NEXT: s_lshr_b32 s30, s2, 2 -; GFX12-NEXT: s_bfe_i64 s[18:19], s[2:3], 0x10000 +; GFX12-NEXT: s_lshr_b32 s66, s10, 8 +; GFX12-NEXT: s_lshr_b32 s56, s10, 6 +; GFX12-NEXT: s_lshr_b32 s44, s10, 4 +; GFX12-NEXT: s_lshr_b32 s30, s10, 2 +; GFX12-NEXT: s_bfe_i64 s[18:19], s[10:11], 0x10000 ; GFX12-NEXT: s_wait_alu 0xfffe -; GFX12-NEXT: s_bfe_i64 s[2:3], s[26:27], 0x10000 +; GFX12-NEXT: s_bfe_i64 s[10:11], s[26:27], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[26:27], s[40:41], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[40:41], s[52:53], 0x10000 ; GFX12-NEXT: s_bfe_i64 s[52:53], s[62:63], 0x10000 @@ -9738,8 +9692,8 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o ; GFX12-NEXT: v_dual_mov_b32 v16, s41 :: v_dual_mov_b32 v17, s30 ; GFX12-NEXT: v_dual_mov_b32 v18, s31 :: v_dual_mov_b32 v19, s26 ; GFX12-NEXT: v_dual_mov_b32 v20, s27 :: v_dual_mov_b32 v21, s18 -; GFX12-NEXT: v_dual_mov_b32 v22, s19 :: v_dual_mov_b32 v23, s2 -; GFX12-NEXT: v_mov_b32_e32 v24, s3 +; GFX12-NEXT: v_dual_mov_b32 v22, s19 :: v_dual_mov_b32 v23, s10 +; GFX12-NEXT: v_mov_b32_e32 v24, s11 ; GFX12-NEXT: s_clause 0x5 ; GFX12-NEXT: global_store_b128 v0, v[1:4], s[0:1] offset:80 ; GFX12-NEXT: global_store_b128 v0, v[5:8], s[0:1] offset:64 diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll index df8d09b7932e6..8e312a0e195ff 100644 --- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll +++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll @@ -6209,7 +6209,6 @@ define amdgpu_kernel void @constant_sextload_v4i16_to_v4i64(ptr addrspace(1) %ou ; GCN-NOHSA-SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s3, 0xf000 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s2, -1 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s7, 0 ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NOHSA-SI-NEXT: s_mov_b32 s6, s5 ; GCN-NOHSA-SI-NEXT: s_lshr_b32 s8, s4, 16 @@ -6235,7 +6234,6 @@ define amdgpu_kernel void @constant_sextload_v4i16_to_v4i64(ptr addrspace(1) %ou ; GCN-HSA: ; %bb.0: ; GCN-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GCN-HSA-NEXT: s_add_i32 s12, s12, s17 -; GCN-HSA-NEXT: s_mov_b32 s5, 0 ; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 ; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) @@ -6575,9 +6573,7 @@ define amdgpu_kernel void @constant_sextload_v8i16_to_v8i64(ptr addrspace(1) %ou ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NOHSA-SI-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s3, 0xf000 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s9, 0 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s2, -1 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s11, s9 ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NOHSA-SI-NEXT: s_mov_b32 s8, s7 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s10, s5 @@ -6621,25 +6617,23 @@ define amdgpu_kernel void @constant_sextload_v8i16_to_v8i64(ptr addrspace(1) %ou ; GCN-HSA: ; %bb.0: ; GCN-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GCN-HSA-NEXT: s_add_i32 s12, s12, s17 -; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 ; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 +; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GCN-HSA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 -; GCN-HSA-NEXT: s_mov_b32 s3, 0 -; GCN-HSA-NEXT: s_mov_b32 s9, s3 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GCN-HSA-NEXT: s_mov_b32 s2, s7 ; GCN-HSA-NEXT: s_mov_b32 s8, s5 ; GCN-HSA-NEXT: s_lshr_b32 s10, s6, 16 ; GCN-HSA-NEXT: s_lshr_b32 s12, s4, 16 +; GCN-HSA-NEXT: s_ashr_i32 s13, s5, 16 ; GCN-HSA-NEXT: s_bfe_i64 s[14:15], s[4:5], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[16:17], s[6:7], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x100000 ; GCN-HSA-NEXT: s_ashr_i32 s6, s5, 31 -; GCN-HSA-NEXT: s_ashr_i32 s18, s5, 16 -; GCN-HSA-NEXT: s_bfe_i64 s[4:5], s[8:9], 0x100000 -; GCN-HSA-NEXT: s_ashr_i32 s8, s7, 31 +; GCN-HSA-NEXT: s_bfe_i64 s[4:5], s[12:13], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x100000 +; GCN-HSA-NEXT: s_ashr_i32 s12, s7, 31 ; GCN-HSA-NEXT: s_ashr_i32 s7, s7, 16 ; GCN-HSA-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000 ; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2 @@ -6650,15 +6644,15 @@ define amdgpu_kernel void @constant_sextload_v8i16_to_v8i64(ptr addrspace(1) %ou ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2 ; GCN-HSA-NEXT: s_add_u32 s2, s0, 16 ; GCN-HSA-NEXT: v_mov_b32_e32 v2, s7 -; GCN-HSA-NEXT: v_mov_b32_e32 v3, s8 +; GCN-HSA-NEXT: v_mov_b32_e32 v3, s12 ; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3 ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2 ; GCN-HSA-NEXT: s_add_u32 s2, s0, 32 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s4 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s5 -; GCN-HSA-NEXT: v_mov_b32_e32 v2, s18 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s8 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s9 +; GCN-HSA-NEXT: v_mov_b32_e32 v2, s13 ; GCN-HSA-NEXT: v_mov_b32_e32 v3, s6 ; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] @@ -6672,8 +6666,8 @@ define amdgpu_kernel void @constant_sextload_v8i16_to_v8i64(ptr addrspace(1) %ou ; GCN-HSA-NEXT: v_mov_b32_e32 v5, s1 ; GCN-HSA-NEXT: v_mov_b32_e32 v0, s14 ; GCN-HSA-NEXT: v_mov_b32_e32 v1, s15 -; GCN-HSA-NEXT: v_mov_b32_e32 v2, s12 -; GCN-HSA-NEXT: v_mov_b32_e32 v3, s13 +; GCN-HSA-NEXT: v_mov_b32_e32 v2, s4 +; GCN-HSA-NEXT: v_mov_b32_e32 v3, s5 ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s0 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: s_endpgm @@ -7201,11 +7195,7 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) % ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NOHSA-SI-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s11, 0xf000 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s13, 0 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s10, -1 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s15, s13 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s17, s13 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s19, s13 ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NOHSA-SI-NEXT: s_mov_b32 s12, s7 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s14, s5 @@ -7285,14 +7275,10 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) % ; GCN-HSA: ; %bb.0: ; GCN-HSA-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0 ; GCN-HSA-NEXT: s_add_i32 s12, s12, s17 -; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 -; GCN-HSA-NEXT: s_mov_b32 s13, 0 +; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GCN-HSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 -; GCN-HSA-NEXT: s_mov_b32 s15, s13 -; GCN-HSA-NEXT: s_mov_b32 s17, s13 -; GCN-HSA-NEXT: s_mov_b32 s19, s13 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GCN-HSA-NEXT: s_mov_b32 s12, s7 ; GCN-HSA-NEXT: s_mov_b32 s14, s5 @@ -8355,42 +8341,39 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-NOHSA-SI-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x9 ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NOHSA-SI-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s45, 0 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s19, s45 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s21, s45 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s25, s45 ; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NOHSA-SI-NEXT: s_mov_b32 s44, s15 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s18, s13 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s20, s11 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s24, s9 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s18, s15 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s20, s13 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s24, s11 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s26, s9 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s22, s7 ; GCN-NOHSA-SI-NEXT: s_ashr_i32 s33, s1, 31 ; GCN-NOHSA-SI-NEXT: s_ashr_i32 s39, s1, 16 ; GCN-NOHSA-SI-NEXT: s_ashr_i32 s41, s3, 31 ; GCN-NOHSA-SI-NEXT: s_ashr_i32 s43, s3, 16 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s27, s5, 31 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s29, s5, 16 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s31, s7, 31 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s35, s7, 16 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s37, s9, 31 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s58, s9, 16 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[50:51], s[24:25], 0x100000 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s59, s11, 31 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s60, s11, 16 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[52:53], s[20:21], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[54:55], s[18:19], 0x100000 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s61, s13, 31 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s62, s13, 16 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s63, s15, 31 -; GCN-NOHSA-SI-NEXT: s_ashr_i32 s64, s15, 16 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s56, s5 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s46, s3 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s48, s1 -; GCN-NOHSA-SI-NEXT: s_lshr_b32 s28, s14, 16 -; GCN-NOHSA-SI-NEXT: s_lshr_b32 s34, s12, 16 -; GCN-NOHSA-SI-NEXT: s_lshr_b32 s36, s10, 16 -; GCN-NOHSA-SI-NEXT: s_lshr_b32 s30, s8, 16 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s23, s5, 31 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s27, s5, 16 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s29, s7, 31 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s31, s7, 16 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s35, s9, 31 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s37, s9, 16 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[46:47], s[26:27], 0x100000 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s55, s11, 31 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s57, s11, 16 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[48:49], s[24:25], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[50:51], s[20:21], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[52:53], s[18:19], 0x100000 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s58, s13, 31 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s59, s13, 16 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s60, s15, 31 +; GCN-NOHSA-SI-NEXT: s_ashr_i32 s61, s15, 16 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s54, s5 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s44, s3 +; GCN-NOHSA-SI-NEXT: s_mov_b32 s56, s1 +; GCN-NOHSA-SI-NEXT: s_lshr_b32 s30, s14, 16 +; GCN-NOHSA-SI-NEXT: s_lshr_b32 s36, s12, 16 +; GCN-NOHSA-SI-NEXT: s_lshr_b32 s34, s10, 16 +; GCN-NOHSA-SI-NEXT: s_lshr_b32 s28, s8, 16 ; GCN-NOHSA-SI-NEXT: s_lshr_b32 s26, s6, 16 ; GCN-NOHSA-SI-NEXT: s_lshr_b32 s38, s4, 16 ; GCN-NOHSA-SI-NEXT: s_lshr_b32 s40, s2, 16 @@ -8402,67 +8385,62 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[2:3], s[44:45], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[24:25], s[14:15], 0x100000 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s0, s16 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s1, s17 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s23, s45 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s57, s45 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s47, s45 -; GCN-NOHSA-SI-NEXT: s_mov_b32 s49, s45 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s2 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s3 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v4, s54 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, s55 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v8, s52 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v9, s53 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v12, s50 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v13, s51 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s52 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s53 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v4, s50 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, s51 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v8, s48 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v9, s49 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v12, s46 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v13, s47 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s3, 0xf000 ; GCN-NOHSA-SI-NEXT: s_mov_b32 s2, -1 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[14:15], s[22:23], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[16:17], s[56:57], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[44:45], s[48:49], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[16:17], s[54:55], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[46:47], s[56:57], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x100000 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v16, s14 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v17, s15 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s64 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s63 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v6, s62 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, s61 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v10, s60 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v11, s59 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v14, s58 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, s37 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v18, s35 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v19, s31 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s61 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s60 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v6, s59 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, s58 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v10, s57 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v11, s55 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v14, s37 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, s35 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v18, s31 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v19, s29 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v20, s16 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:240 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v21, s17 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v22, s29 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v22, s27 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:208 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v23, s27 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v23, s23 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[14:15], s[42:43], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[16:17], s[40:41], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[22:23], s[38:39], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x100000 -; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x100000 ; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x100000 +; GCN-NOHSA-SI-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x100000 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:176 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:112 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:80 ; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(5) -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s46 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s47 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s44 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s45 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s43 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s41 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48 ; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0) -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s44 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s45 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s46 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s47 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s39 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s33 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16 @@ -8481,20 +8459,20 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v21, s5 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v24, s20 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v25, s21 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s28 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s29 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s30 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, s31 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224 ; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0) ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s18 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, s19 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v6, s34 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, s35 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v6, s36 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, s37 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:192 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v10, s36 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v11, s37 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v10, s34 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v11, s35 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:160 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v14, s30 -; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, s31 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v14, s28 +; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, s29 ; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:128 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v18, s26 ; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v19, s27 @@ -8516,23 +8494,19 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-HSA-NEXT: s_load_dwordx4 s[16:19], s[8:9], 0x0 ; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 -; GCN-HSA-NEXT: s_mov_b32 s31, 0 -; GCN-HSA-NEXT: s_mov_b32 s45, s31 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GCN-HSA-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0 -; GCN-HSA-NEXT: s_mov_b32 s47, s31 -; GCN-HSA-NEXT: s_mov_b32 s49, s31 -; GCN-HSA-NEXT: s_mov_b32 s51, s31 -; GCN-HSA-NEXT: s_mov_b32 s53, s31 ; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0) -; GCN-HSA-NEXT: s_mov_b32 s30, s15 -; GCN-HSA-NEXT: s_ashr_i32 s43, s5, 31 +; GCN-HSA-NEXT: s_mov_b32 s34, s15 +; GCN-HSA-NEXT: s_ashr_i32 s41, s3, 31 +; GCN-HSA-NEXT: s_ashr_i32 s42, s3, 16 ; GCN-HSA-NEXT: s_ashr_i32 s57, s5, 16 ; GCN-HSA-NEXT: s_ashr_i32 s59, s7, 31 ; GCN-HSA-NEXT: s_ashr_i32 s61, s7, 16 ; GCN-HSA-NEXT: s_ashr_i32 s63, s9, 31 ; GCN-HSA-NEXT: s_ashr_i32 s65, s9, 16 ; GCN-HSA-NEXT: s_ashr_i32 s67, s11, 31 +; GCN-HSA-NEXT: s_ashr_i32 s69, s11, 16 ; GCN-HSA-NEXT: s_mov_b32 s44, s13 ; GCN-HSA-NEXT: s_mov_b32 s46, s11 ; GCN-HSA-NEXT: s_mov_b32 s48, s9 @@ -8540,44 +8514,40 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-HSA-NEXT: s_mov_b32 s52, s5 ; GCN-HSA-NEXT: s_mov_b32 s38, s3 ; GCN-HSA-NEXT: s_mov_b32 s36, s1 -; GCN-HSA-NEXT: s_lshr_b32 s34, s14, 16 -; GCN-HSA-NEXT: s_lshr_b32 s54, s12, 16 -; GCN-HSA-NEXT: s_lshr_b32 s56, s10, 16 -; GCN-HSA-NEXT: s_lshr_b32 s58, s8, 16 -; GCN-HSA-NEXT: s_lshr_b32 s60, s6, 16 -; GCN-HSA-NEXT: s_lshr_b32 s62, s4, 16 -; GCN-HSA-NEXT: s_lshr_b32 s64, s2, 16 -; GCN-HSA-NEXT: s_lshr_b32 s66, s0, 16 -; GCN-HSA-NEXT: s_bfe_i64 s[18:19], s[4:5], 0x100000 -; GCN-HSA-NEXT: s_mov_b32 s39, s31 -; GCN-HSA-NEXT: s_mov_b32 s37, s31 -; GCN-HSA-NEXT: s_bfe_i64 s[4:5], s[30:31], 0x100000 +; GCN-HSA-NEXT: s_lshr_b32 s54, s14, 16 +; GCN-HSA-NEXT: s_lshr_b32 s56, s12, 16 +; GCN-HSA-NEXT: s_lshr_b32 s58, s10, 16 +; GCN-HSA-NEXT: s_lshr_b32 s60, s8, 16 +; GCN-HSA-NEXT: s_lshr_b32 s62, s6, 16 +; GCN-HSA-NEXT: s_lshr_b32 s64, s4, 16 +; GCN-HSA-NEXT: s_lshr_b32 s66, s2, 16 +; GCN-HSA-NEXT: s_lshr_b32 s68, s0, 16 +; GCN-HSA-NEXT: s_bfe_i64 s[18:19], s[2:3], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[2:3], s[34:35], 0x100000 ; GCN-HSA-NEXT: s_ashr_i32 s33, s1, 31 ; GCN-HSA-NEXT: s_ashr_i32 s40, s1, 16 -; GCN-HSA-NEXT: s_ashr_i32 s41, s3, 31 -; GCN-HSA-NEXT: s_ashr_i32 s42, s3, 16 -; GCN-HSA-NEXT: s_ashr_i32 s68, s11, 16 -; GCN-HSA-NEXT: s_ashr_i32 s69, s13, 31 -; GCN-HSA-NEXT: s_ashr_i32 s70, s13, 16 -; GCN-HSA-NEXT: s_ashr_i32 s71, s15, 31 -; GCN-HSA-NEXT: s_ashr_i32 s72, s15, 16 +; GCN-HSA-NEXT: s_ashr_i32 s43, s5, 31 +; GCN-HSA-NEXT: s_ashr_i32 s70, s13, 31 +; GCN-HSA-NEXT: s_ashr_i32 s71, s13, 16 +; GCN-HSA-NEXT: s_ashr_i32 s72, s15, 31 +; GCN-HSA-NEXT: s_ashr_i32 s73, s15, 16 ; GCN-HSA-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[20:21], s[6:7], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[22:23], s[8:9], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[24:25], s[10:11], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[26:27], s[12:13], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[28:29], s[14:15], 0x100000 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s4 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s5 +; GCN-HSA-NEXT: s_bfe_i64 s[20:21], s[4:5], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[22:23], s[6:7], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[24:25], s[8:9], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[26:27], s[10:11], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[28:29], s[12:13], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[30:31], s[14:15], 0x100000 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3 +; GCN-HSA-NEXT: s_bfe_i64 s[2:3], s[68:69], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[4:5], s[66:67], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[6:7], s[64:65], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[8:9], s[62:63], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[10:11], s[60:61], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[12:13], s[58:59], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[14:15], s[56:57], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[30:31], s[54:55], 0x100000 -; GCN-HSA-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x100000 +; GCN-HSA-NEXT: s_bfe_i64 s[34:35], s[54:55], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x100000 ; GCN-HSA-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x100000 @@ -8604,19 +8574,19 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-HSA-NEXT: v_mov_b32_e32 v28, s45 ; GCN-HSA-NEXT: s_add_u32 s44, s16, 0x70 ; GCN-HSA-NEXT: v_mov_b32_e32 v19, s55 -; GCN-HSA-NEXT: v_mov_b32_e32 v2, s72 -; GCN-HSA-NEXT: v_mov_b32_e32 v3, s71 +; GCN-HSA-NEXT: v_mov_b32_e32 v2, s73 +; GCN-HSA-NEXT: v_mov_b32_e32 v3, s72 ; GCN-HSA-NEXT: s_addc_u32 s45, s17, 0 ; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[0:3] -; GCN-HSA-NEXT: v_mov_b32_e32 v6, s70 +; GCN-HSA-NEXT: v_mov_b32_e32 v6, s71 ; GCN-HSA-NEXT: v_mov_b32_e32 v0, s38 ; GCN-HSA-NEXT: s_add_u32 s38, s16, 0x50 -; GCN-HSA-NEXT: v_mov_b32_e32 v7, s69 +; GCN-HSA-NEXT: v_mov_b32_e32 v7, s70 ; GCN-HSA-NEXT: v_mov_b32_e32 v1, s39 ; GCN-HSA-NEXT: s_addc_u32 s39, s17, 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v8, s46 ; GCN-HSA-NEXT: v_mov_b32_e32 v9, s47 -; GCN-HSA-NEXT: v_mov_b32_e32 v10, s68 +; GCN-HSA-NEXT: v_mov_b32_e32 v10, s69 ; GCN-HSA-NEXT: v_mov_b32_e32 v11, s67 ; GCN-HSA-NEXT: flat_store_dwordx4 v[23:24], v[4:7] ; GCN-HSA-NEXT: flat_store_dwordx4 v[25:26], v[8:11] @@ -8628,25 +8598,27 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-HSA-NEXT: v_mov_b32_e32 v25, s37 ; GCN-HSA-NEXT: s_add_u32 s36, s16, 16 ; GCN-HSA-NEXT: s_addc_u32 s37, s17, 0 -; GCN-HSA-NEXT: v_mov_b32_e32 v8, s28 -; GCN-HSA-NEXT: s_add_u32 s28, s16, 0xe0 +; GCN-HSA-NEXT: v_mov_b32_e32 v8, s30 +; GCN-HSA-NEXT: s_add_u32 s30, s16, 0xe0 ; GCN-HSA-NEXT: v_mov_b32_e32 v12, s48 ; GCN-HSA-NEXT: v_mov_b32_e32 v13, s49 ; GCN-HSA-NEXT: v_mov_b32_e32 v14, s65 ; GCN-HSA-NEXT: v_mov_b32_e32 v15, s63 +; GCN-HSA-NEXT: v_mov_b32_e32 v9, s31 +; GCN-HSA-NEXT: s_addc_u32 s31, s17, 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v29, s44 -; GCN-HSA-NEXT: v_mov_b32_e32 v9, s29 -; GCN-HSA-NEXT: s_addc_u32 s29, s17, 0 +; GCN-HSA-NEXT: flat_store_dwordx4 v[27:28], v[12:15] ; GCN-HSA-NEXT: v_mov_b32_e32 v16, s50 +; GCN-HSA-NEXT: v_mov_b32_e32 v14, s14 +; GCN-HSA-NEXT: s_add_u32 s14, s16, 0xc0 ; GCN-HSA-NEXT: v_mov_b32_e32 v17, s51 ; GCN-HSA-NEXT: v_mov_b32_e32 v30, s45 ; GCN-HSA-NEXT: v_mov_b32_e32 v18, s61 ; GCN-HSA-NEXT: v_mov_b32_e32 v19, s59 ; GCN-HSA-NEXT: v_mov_b32_e32 v10, s38 -; GCN-HSA-NEXT: flat_store_dwordx4 v[27:28], v[12:15] +; GCN-HSA-NEXT: v_mov_b32_e32 v15, s15 +; GCN-HSA-NEXT: s_addc_u32 s15, s17, 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v20, s52 -; GCN-HSA-NEXT: v_mov_b32_e32 v12, s26 -; GCN-HSA-NEXT: s_add_u32 s26, s16, 0xc0 ; GCN-HSA-NEXT: v_mov_b32_e32 v21, s53 ; GCN-HSA-NEXT: v_mov_b32_e32 v22, s57 ; GCN-HSA-NEXT: v_mov_b32_e32 v23, s43 @@ -8655,78 +8627,76 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) % ; GCN-HSA-NEXT: v_mov_b32_e32 v3, s41 ; GCN-HSA-NEXT: v_mov_b32_e32 v26, s36 ; GCN-HSA-NEXT: flat_store_dwordx4 v[29:30], v[16:19] -; GCN-HSA-NEXT: v_mov_b32_e32 v13, s27 -; GCN-HSA-NEXT: v_mov_b32_e32 v16, s28 -; GCN-HSA-NEXT: s_addc_u32 s27, s17, 0 -; GCN-HSA-NEXT: v_mov_b32_e32 v18, s26 ; GCN-HSA-NEXT: v_mov_b32_e32 v27, s37 +; GCN-HSA-NEXT: v_mov_b32_e32 v16, s30 +; GCN-HSA-NEXT: v_mov_b32_e32 v19, s15 ; GCN-HSA-NEXT: v_mov_b32_e32 v6, s40 ; GCN-HSA-NEXT: v_mov_b32_e32 v7, s33 ; GCN-HSA-NEXT: flat_store_dwordx4 v[10:11], v[20:23] +; GCN-HSA-NEXT: v_mov_b32_e32 v12, s28 ; GCN-HSA-NEXT: v_mov_b32_e32 v10, s34 ; GCN-HSA-NEXT: v_mov_b32_e32 v11, s35 -; GCN-HSA-NEXT: v_mov_b32_e32 v17, s29 -; GCN-HSA-NEXT: v_mov_b32_e32 v14, s30 -; GCN-HSA-NEXT: v_mov_b32_e32 v15, s31 -; GCN-HSA-NEXT: v_mov_b32_e32 v19, s27 +; GCN-HSA-NEXT: v_mov_b32_e32 v13, s29 +; GCN-HSA-NEXT: v_mov_b32_e32 v17, s31 +; GCN-HSA-NEXT: v_mov_b32_e32 v18, s14 ; GCN-HSA-NEXT: flat_store_dwordx4 v[24:25], v[0:3] ; GCN-HSA-NEXT: flat_store_dwordx4 v[26:27], v[4:7] ; GCN-HSA-NEXT: flat_store_dwordx4 v[16:17], v[8:11] ; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[12:15] -; GCN-HSA-NEXT: v_mov_b32_e32 v2, s14 -; GCN-HSA-NEXT: s_add_u32 s14, s16, 0xa0 -; GCN-HSA-NEXT: v_mov_b32_e32 v3, s15 -; GCN-HSA-NEXT: s_addc_u32 s15, s17, 0 -; GCN-HSA-NEXT: v_mov_b32_e32 v4, s14 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s24 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s25 -; GCN-HSA-NEXT: v_mov_b32_e32 v5, s15 -; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] -; GCN-HSA-NEXT: s_nop 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v2, s12 -; GCN-HSA-NEXT: s_add_u32 s12, s16, 0x80 +; GCN-HSA-NEXT: s_add_u32 s12, s16, 0xa0 ; GCN-HSA-NEXT: v_mov_b32_e32 v3, s13 ; GCN-HSA-NEXT: s_addc_u32 s13, s17, 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s12 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s22 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s23 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s26 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s27 ; GCN-HSA-NEXT: v_mov_b32_e32 v5, s13 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: s_nop 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v2, s10 -; GCN-HSA-NEXT: s_add_u32 s10, s16, 0x60 +; GCN-HSA-NEXT: s_add_u32 s10, s16, 0x80 ; GCN-HSA-NEXT: v_mov_b32_e32 v3, s11 ; GCN-HSA-NEXT: s_addc_u32 s11, s17, 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s10 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s20 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s21 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s24 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s25 ; GCN-HSA-NEXT: v_mov_b32_e32 v5, s11 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: s_nop 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v2, s8 -; GCN-HSA-NEXT: s_add_u32 s8, s16, 64 +; GCN-HSA-NEXT: s_add_u32 s8, s16, 0x60 ; GCN-HSA-NEXT: v_mov_b32_e32 v3, s9 ; GCN-HSA-NEXT: s_addc_u32 s9, s17, 0 ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s8 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s18 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s19 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s22 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s23 ; GCN-HSA-NEXT: v_mov_b32_e32 v5, s9 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: s_nop 0 -; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2 -; GCN-HSA-NEXT: s_add_u32 s2, s16, 32 -; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3 -; GCN-HSA-NEXT: s_addc_u32 s3, s17, 0 -; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3 ; GCN-HSA-NEXT: v_mov_b32_e32 v2, s6 +; GCN-HSA-NEXT: s_add_u32 s6, s16, 64 ; GCN-HSA-NEXT: v_mov_b32_e32 v3, s7 -; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2 +; GCN-HSA-NEXT: s_addc_u32 s7, s17, 0 +; GCN-HSA-NEXT: v_mov_b32_e32 v4, s6 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s20 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s21 +; GCN-HSA-NEXT: v_mov_b32_e32 v5, s7 +; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; GCN-HSA-NEXT: s_nop 0 +; GCN-HSA-NEXT: v_mov_b32_e32 v2, s4 +; GCN-HSA-NEXT: s_add_u32 s4, s16, 32 +; GCN-HSA-NEXT: v_mov_b32_e32 v3, s5 +; GCN-HSA-NEXT: s_addc_u32 s5, s17, 0 +; GCN-HSA-NEXT: v_mov_b32_e32 v4, s4 +; GCN-HSA-NEXT: v_mov_b32_e32 v0, s18 +; GCN-HSA-NEXT: v_mov_b32_e32 v1, s19 +; GCN-HSA-NEXT: v_mov_b32_e32 v5, s5 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: v_mov_b32_e32 v4, s16 ; GCN-HSA-NEXT: v_mov_b32_e32 v0, s0 ; GCN-HSA-NEXT: v_mov_b32_e32 v1, s1 -; GCN-HSA-NEXT: v_mov_b32_e32 v2, s4 -; GCN-HSA-NEXT: v_mov_b32_e32 v3, s5 +; GCN-HSA-NEXT: v_mov_b32_e32 v2, s2 +; GCN-HSA-NEXT: v_mov_b32_e32 v3, s3 ; GCN-HSA-NEXT: v_mov_b32_e32 v5, s17 ; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GCN-HSA-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll index a4aa49e53ec5f..70bdebedf85bc 100644 --- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll +++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll @@ -6395,10 +6395,7 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NOHSA-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 ; GFX6-NOHSA-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NOHSA-NEXT: s_mov_b32 s7, 0 ; GFX6-NOHSA-NEXT: s_mov_b32 s2, -1 -; GFX6-NOHSA-NEXT: s_mov_b32 s9, s7 -; GFX6-NOHSA-NEXT: s_mov_b32 s11, s7 ; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NOHSA-NEXT: s_lshr_b32 s6, s5, 16 ; GFX6-NOHSA-NEXT: s_lshr_b32 s8, s5, 8 @@ -6442,13 +6439,10 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX7-HSA: ; %bb.0: ; GFX7-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX7-HSA-NEXT: s_add_i32 s12, s12, s17 -; GFX7-HSA-NEXT: s_mov_b32 s5, 0 ; GFX7-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 -; GFX7-HSA-NEXT: s_mov_b32 s7, s5 +; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-HSA-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 -; GFX7-HSA-NEXT: s_mov_b32 s9, s5 -; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-HSA-NEXT: s_lshr_b32 s4, s3, 16 ; GFX7-HSA-NEXT: s_lshr_b32 s6, s3, 8 @@ -6456,14 +6450,14 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX7-HSA-NEXT: s_lshr_b32 s10, s2, 16 ; GFX7-HSA-NEXT: s_lshr_b32 s12, s2, 24 ; GFX7-HSA-NEXT: s_lshr_b32 s14, s2, 8 +; GFX7-HSA-NEXT: s_ashr_i32 s5, s3, 31 ; GFX7-HSA-NEXT: s_bfe_i64 s[16:17], s[2:3], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x80000 -; GFX7-HSA-NEXT: s_ashr_i32 s18, s3, 31 -; GFX7-HSA-NEXT: s_ashr_i32 s19, s3, 24 +; GFX7-HSA-NEXT: s_ashr_i32 s18, s3, 24 ; GFX7-HSA-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x80000 ; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s2 ; GFX7-HSA-NEXT: s_add_u32 s2, s0, 48 @@ -6472,8 +6466,8 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3 ; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2 ; GFX7-HSA-NEXT: s_add_u32 s2, s0, 32 -; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s19 -; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s18 +; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s18 +; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s5 ; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3 @@ -6504,25 +6498,23 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX8-NOHSA-LABEL: constant_sextload_v8i8_to_v8i64: ; GFX8-NOHSA: ; %bb.0: ; GFX8-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX8-NOHSA-NEXT: s_mov_b32 s5, 0 -; GFX8-NOHSA-NEXT: s_mov_b32 s7, s5 ; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NOHSA-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0 ; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NOHSA-NEXT: s_lshr_b32 s4, s3, 16 -; GFX8-NOHSA-NEXT: s_lshr_b32 s8, s3, 8 -; GFX8-NOHSA-NEXT: s_mov_b32 s6, s3 +; GFX8-NOHSA-NEXT: s_lshr_b32 s6, s3, 8 +; GFX8-NOHSA-NEXT: s_mov_b32 s8, s3 ; GFX8-NOHSA-NEXT: s_lshr_b32 s10, s2, 16 ; GFX8-NOHSA-NEXT: s_lshr_b32 s12, s2, 24 ; GFX8-NOHSA-NEXT: s_lshr_b32 s14, s2, 8 +; GFX8-NOHSA-NEXT: s_ashr_i32 s5, s3, 31 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[16:17], s[2:3], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x80000 -; GFX8-NOHSA-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x80000 -; GFX8-NOHSA-NEXT: s_ashr_i32 s18, s3, 31 -; GFX8-NOHSA-NEXT: s_ashr_i32 s19, s3, 24 +; GFX8-NOHSA-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x80000 +; GFX8-NOHSA-NEXT: s_ashr_i32 s18, s3, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x80000 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s2 ; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 48 @@ -6531,17 +6523,17 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s2 ; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 32 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s19 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s18 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s18 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s5 ; GFX8-NOHSA-NEXT: s_addc_u32 s3, s1, 0 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s2 ; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 16 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s7 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s8 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s9 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s8 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s9 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s6 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s7 ; GFX8-NOHSA-NEXT: s_addc_u32 s3, s1, 0 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3 @@ -6619,15 +6611,12 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX12-LABEL: constant_sextload_v8i8_to_v8i64: ; GFX12: ; %bb.0: ; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX12-NEXT: s_mov_b32 s5, 0 -; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_mov_b32 s7, s5 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: s_lshr_b32 s4, s3, 16 -; GFX12-NEXT: s_lshr_b32 s8, s3, 8 -; GFX12-NEXT: s_mov_b32 s6, s3 +; GFX12-NEXT: s_lshr_b32 s6, s3, 8 +; GFX12-NEXT: s_mov_b32 s8, s3 ; GFX12-NEXT: s_lshr_b32 s10, s2, 16 ; GFX12-NEXT: s_lshr_b32 s12, s2, 24 ; GFX12-NEXT: s_bfe_i64 s[16:17], s[2:3], 0x80000 @@ -6635,17 +6624,17 @@ define amdgpu_kernel void @constant_sextload_v8i8_to_v8i64(ptr addrspace(1) %out ; GFX12-NEXT: s_ashr_i32 s18, s3, 24 ; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x80000 ; GFX12-NEXT: s_lshr_b32 s14, s2, 8 -; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x80000 ; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v3, s15 ; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x80000 ; GFX12-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v5, s17 ; GFX12-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v1, s5 -; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v9, s7 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v9, s9 ; GFX12-NEXT: s_bfe_i64 s[2:3], s[14:15], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v11, s9 -; GFX12-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v13, s11 +; GFX12-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v11, s7 +; GFX12-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v13, s11 ; GFX12-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v15, s13 ; GFX12-NEXT: v_dual_mov_b32 v14, s12 :: v_dual_mov_b32 v7, s3 ; GFX12-NEXT: v_mov_b32_e32 v6, s2 @@ -7042,29 +7031,23 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GFX6-NOHSA-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NOHSA-NEXT: s_mov_b32 s11, 0 ; GFX6-NOHSA-NEXT: s_mov_b32 s2, -1 -; GFX6-NOHSA-NEXT: s_mov_b32 s13, s11 -; GFX6-NOHSA-NEXT: s_mov_b32 s15, s11 -; GFX6-NOHSA-NEXT: s_mov_b32 s17, s11 -; GFX6-NOHSA-NEXT: s_mov_b32 s19, s11 -; GFX6-NOHSA-NEXT: s_mov_b32 s21, s11 ; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NOHSA-NEXT: s_lshr_b32 s10, s7, 16 ; GFX6-NOHSA-NEXT: s_lshr_b32 s12, s7, 8 ; GFX6-NOHSA-NEXT: s_mov_b32 s14, s7 -; GFX6-NOHSA-NEXT: s_lshr_b32 s22, s6, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s24, s6, 24 -; GFX6-NOHSA-NEXT: s_lshr_b32 s26, s6, 8 -; GFX6-NOHSA-NEXT: s_lshr_b32 s16, s5, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s18, s5, 8 -; GFX6-NOHSA-NEXT: s_mov_b32 s20, s5 +; GFX6-NOHSA-NEXT: s_lshr_b32 s16, s6, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s18, s6, 24 +; GFX6-NOHSA-NEXT: s_lshr_b32 s20, s6, 8 +; GFX6-NOHSA-NEXT: s_lshr_b32 s22, s5, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s24, s5, 8 +; GFX6-NOHSA-NEXT: s_mov_b32 s26, s5 ; GFX6-NOHSA-NEXT: s_lshr_b32 s28, s4, 16 ; GFX6-NOHSA-NEXT: s_lshr_b32 s30, s4, 24 ; GFX6-NOHSA-NEXT: s_lshr_b32 s34, s4, 8 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[8:9], s[4:5], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[36:37], s[6:7], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 ; GFX6-NOHSA-NEXT: s_ashr_i32 s31, s5, 31 ; GFX6-NOHSA-NEXT: s_ashr_i32 s33, s5, 24 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 @@ -7073,11 +7056,11 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX6-NOHSA-NEXT: s_bfe_i64 s[4:5], s[34:35], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[6:7], s[30:31], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s38 @@ -7088,8 +7071,8 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s37 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v12, s33 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v13, s31 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v14, s20 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v15, s21 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v14, s26 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v15, s27 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s10 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s11 ; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112 @@ -7099,21 +7082,21 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s12 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s13 ; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s22 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s23 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s16 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s17 ; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s24 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s25 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s18 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s19 ; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:80 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s26 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s27 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s20 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s21 ; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:64 ; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s16 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s17 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s22 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s23 ; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:48 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s18 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s19 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s24 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s25 ; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:32 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s28 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s29 @@ -7130,15 +7113,10 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX7-HSA: ; %bb.0: ; GFX7-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX7-HSA-NEXT: s_add_i32 s12, s12, s17 -; GFX7-HSA-NEXT: s_mov_b32 s9, 0 -; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GFX7-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 +; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-HSA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 -; GFX7-HSA-NEXT: s_mov_b32 s11, s9 -; GFX7-HSA-NEXT: s_mov_b32 s13, s9 -; GFX7-HSA-NEXT: s_mov_b32 s21, s9 -; GFX7-HSA-NEXT: s_mov_b32 s23, s9 ; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-HSA-NEXT: s_lshr_b32 s8, s7, 16 ; GFX7-HSA-NEXT: s_lshr_b32 s10, s7, 8 @@ -7155,7 +7133,6 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX7-HSA-NEXT: s_lshr_b32 s28, s4, 24 ; GFX7-HSA-NEXT: s_lshr_b32 s30, s4, 8 ; GFX7-HSA-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x80000 -; GFX7-HSA-NEXT: s_mov_b32 s25, s9 ; GFX7-HSA-NEXT: s_bfe_i64 s[4:5], s[8:9], 0x80000 ; GFX7-HSA-NEXT: s_ashr_i32 s33, s7, 31 ; GFX7-HSA-NEXT: s_ashr_i32 s36, s7, 24 @@ -7247,16 +7224,12 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX8-NOHSA-LABEL: constant_sextload_v16i8_to_v16i64: ; GFX8-NOHSA: ; %bb.0: ; GFX8-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 -; GFX8-NOHSA-NEXT: s_mov_b32 s19, 0 -; GFX8-NOHSA-NEXT: s_mov_b32 s21, s19 -; GFX8-NOHSA-NEXT: s_mov_b32 s31, s19 -; GFX8-NOHSA-NEXT: s_mov_b32 s15, s19 ; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NOHSA-NEXT: s_lshr_b32 s18, s7, 16 -; GFX8-NOHSA-NEXT: s_lshr_b32 s22, s7, 8 -; GFX8-NOHSA-NEXT: s_mov_b32 s20, s7 +; GFX8-NOHSA-NEXT: s_lshr_b32 s20, s7, 8 +; GFX8-NOHSA-NEXT: s_mov_b32 s22, s7 ; GFX8-NOHSA-NEXT: s_lshr_b32 s24, s6, 16 ; GFX8-NOHSA-NEXT: s_lshr_b32 s26, s6, 24 ; GFX8-NOHSA-NEXT: s_lshr_b32 s28, s6, 8 @@ -7266,6 +7239,8 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: s_lshr_b32 s12, s4, 16 ; GFX8-NOHSA-NEXT: s_lshr_b32 s10, s4, 24 ; GFX8-NOHSA-NEXT: s_lshr_b32 s8, s4, 8 +; GFX8-NOHSA-NEXT: s_ashr_i32 s19, s5, 31 +; GFX8-NOHSA-NEXT: s_ashr_i32 s31, s5, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[34:35], s[6:7], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x80000 @@ -7273,33 +7248,31 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 -; GFX8-NOHSA-NEXT: s_ashr_i32 s33, s5, 31 -; GFX8-NOHSA-NEXT: s_ashr_i32 s36, s5, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[4:5], s[30:31], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x80000 -; GFX8-NOHSA-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 +; GFX8-NOHSA-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 ; GFX8-NOHSA-NEXT: s_ashr_i32 s30, s7, 31 -; GFX8-NOHSA-NEXT: s_ashr_i32 s31, s7, 24 +; GFX8-NOHSA-NEXT: s_ashr_i32 s33, s7, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[6:7], s[18:19], 0x80000 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NOHSA-NEXT: s_add_u32 s6, s0, 0x70 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NOHSA-NEXT: s_addc_u32 s7, s1, 0 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s6 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s31 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s33 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s30 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s7 ; GFX8-NOHSA-NEXT: s_add_u32 s6, s0, 0x60 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NOHSA-NEXT: s_addc_u32 s7, s1, 0 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s6 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s20 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s21 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s22 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s23 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s22 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s23 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s20 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s21 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s7 ; GFX8-NOHSA-NEXT: s_add_u32 s6, s0, 0x50 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] @@ -7326,8 +7299,8 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s5 ; GFX8-NOHSA-NEXT: s_addc_u32 s5, s1, 0 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s4 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s36 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s33 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s31 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s19 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s5 ; GFX8-NOHSA-NEXT: s_add_u32 s4, s0, 32 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] @@ -7459,70 +7432,65 @@ define amdgpu_kernel void @constant_sextload_v16i8_to_v16i64(ptr addrspace(1) %o ; GFX12-LABEL: constant_sextload_v16i8_to_v16i64: ; GFX12: ; %bb.0: ; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX12-NEXT: s_mov_b32 s9, 0 -; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_mov_b32 s11, s9 -; GFX12-NEXT: s_mov_b32 s13, s9 -; GFX12-NEXT: s_mov_b32 s15, s9 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: s_load_b128 s[4:7], s[2:3], 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: s_lshr_b32 s8, s7, 16 -; GFX12-NEXT: s_lshr_b32 s16, s7, 8 -; GFX12-NEXT: s_mov_b32 s10, s7 +; GFX12-NEXT: s_lshr_b32 s10, s7, 8 +; GFX12-NEXT: s_mov_b32 s12, s7 +; GFX12-NEXT: s_bfe_i64 s[34:35], s[6:7], 0x80000 ; GFX12-NEXT: s_ashr_i32 s33, s7, 31 ; GFX12-NEXT: s_ashr_i32 s36, s7, 24 ; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x80000 -; GFX12-NEXT: s_lshr_b32 s18, s6, 16 +; GFX12-NEXT: s_lshr_b32 s14, s6, 16 +; GFX12-NEXT: s_lshr_b32 s16, s6, 24 +; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 ; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v3, s33 -; GFX12-NEXT: s_lshr_b32 s20, s6, 24 -; GFX12-NEXT: v_dual_mov_b32 v2, s36 :: v_dual_mov_b32 v1, s9 -; GFX12-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v5, s11 -; GFX12-NEXT: s_lshr_b32 s22, s6, 8 +; GFX12-NEXT: s_lshr_b32 s18, s6, 8 +; GFX12-NEXT: v_dual_mov_b32 v2, s36 :: v_dual_mov_b32 v5, s35 +; GFX12-NEXT: v_dual_mov_b32 v4, s34 :: v_dual_mov_b32 v1, s9 +; GFX12-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v9, s13 +; GFX12-NEXT: s_lshr_b32 s20, s5, 16 +; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s11 +; GFX12-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v13, s15 +; GFX12-NEXT: s_lshr_b32 s22, s5, 8 +; GFX12-NEXT: s_mov_b32 s24, s5 ; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v4, s10 :: v_dual_mov_b32 v7, s17 -; GFX12-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v9, s19 -; GFX12-NEXT: s_lshr_b32 s12, s5, 16 -; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 -; GFX12-NEXT: s_lshr_b32 s24, s5, 8 -; GFX12-NEXT: s_mov_b32 s14, s5 -; GFX12-NEXT: s_bfe_i64 s[34:35], s[6:7], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 ; GFX12-NEXT: s_lshr_b32 s26, s4, 16 ; GFX12-NEXT: s_lshr_b32 s28, s4, 24 ; GFX12-NEXT: s_ashr_i32 s29, s5, 31 ; GFX12-NEXT: s_ashr_i32 s31, s5, 24 -; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v8, s18 :: v_dual_mov_b32 v11, s21 -; GFX12-NEXT: v_mov_b32_e32 v10, s20 +; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17 +; GFX12-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v7, s19 ; GFX12-NEXT: s_lshr_b32 s30, s4, 8 -; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x80000 -; GFX12-NEXT: s_clause 0x1 -; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112 -; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96 -; GFX12-NEXT: v_dual_mov_b32 v0, s34 :: v_dual_mov_b32 v3, s23 -; GFX12-NEXT: v_dual_mov_b32 v1, s35 :: v_dual_mov_b32 v2, s22 -; GFX12-NEXT: v_mov_b32_e32 v5, s13 +; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 +; GFX12-NEXT: v_mov_b32_e32 v6, s18 ; GFX12-NEXT: s_bfe_i64 s[6:7], s[28:29], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s29 -; GFX12-NEXT: v_dual_mov_b32 v6, s31 :: v_dual_mov_b32 v13, s15 +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112 +; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:96 +; GFX12-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v3, s29 +; GFX12-NEXT: v_dual_mov_b32 v1, s21 :: v_dual_mov_b32 v2, s31 +; GFX12-NEXT: v_mov_b32_e32 v9, s25 ; GFX12-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[4:5], s[30:31], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s25 -; GFX12-NEXT: v_dual_mov_b32 v14, s24 :: v_dual_mov_b32 v17, s27 +; GFX12-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v11, s23 +; GFX12-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v17, s27 ; GFX12-NEXT: v_dual_mov_b32 v16, s26 :: v_dual_mov_b32 v19, s7 ; GFX12-NEXT: v_dual_mov_b32 v18, s6 :: v_dual_mov_b32 v21, s3 ; GFX12-NEXT: v_dual_mov_b32 v20, s2 :: v_dual_mov_b32 v23, s5 ; GFX12-NEXT: v_mov_b32_e32 v22, s4 ; GFX12-NEXT: s_clause 0x5 -; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80 -; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:64 -; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:48 -; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:32 +; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:80 +; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:64 +; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48 +; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:32 ; GFX12-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16 ; GFX12-NEXT: global_store_b128 v24, v[20:23], s[0:1] ; GFX12-NEXT: s_endpgm @@ -8235,337 +8203,312 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX6-NOHSA-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9 ; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NOHSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 -; GFX6-NOHSA-NEXT: s_mov_b32 s13, 0 -; GFX6-NOHSA-NEXT: s_mov_b32 s27, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s11, s13 ; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NOHSA-NEXT: s_lshr_b32 s12, s7, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s14, s7, 8 -; GFX6-NOHSA-NEXT: s_mov_b32 s26, s7 -; GFX6-NOHSA-NEXT: s_lshr_b32 s24, s6, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s18, s6, 24 -; GFX6-NOHSA-NEXT: s_lshr_b32 s22, s6, 8 -; GFX6-NOHSA-NEXT: s_lshr_b32 s20, s5, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s16, s5, 8 -; GFX6-NOHSA-NEXT: s_mov_b32 s10, s5 -; GFX6-NOHSA-NEXT: s_ashr_i32 s25, s1, 31 -; GFX6-NOHSA-NEXT: s_ashr_i32 s33, s1, 24 -; GFX6-NOHSA-NEXT: s_ashr_i32 s19, s3, 31 -; GFX6-NOHSA-NEXT: s_ashr_i32 s15, s3, 24 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x80000 -; GFX6-NOHSA-NEXT: s_ashr_i32 s23, s5, 31 -; GFX6-NOHSA-NEXT: s_ashr_i32 s17, s5, 24 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[58:59], s[26:27], 0x80000 -; GFX6-NOHSA-NEXT: s_ashr_i32 s21, s7, 31 +; GFX6-NOHSA-NEXT: s_lshr_b32 s14, s7, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s22, s7, 8 +; GFX6-NOHSA-NEXT: s_mov_b32 s30, s7 +; GFX6-NOHSA-NEXT: s_lshr_b32 s28, s6, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s24, s6, 24 +; GFX6-NOHSA-NEXT: s_lshr_b32 s26, s6, 8 +; GFX6-NOHSA-NEXT: s_lshr_b32 s10, s5, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s12, s5, 8 +; GFX6-NOHSA-NEXT: s_mov_b32 s34, s5 +; GFX6-NOHSA-NEXT: s_lshr_b32 s16, s4, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s18, s4, 24 +; GFX6-NOHSA-NEXT: s_lshr_b32 s20, s4, 8 +; GFX6-NOHSA-NEXT: s_ashr_i32 s11, s1, 31 +; GFX6-NOHSA-NEXT: s_ashr_i32 s13, s1, 24 +; GFX6-NOHSA-NEXT: s_ashr_i32 s17, s3, 31 +; GFX6-NOHSA-NEXT: s_ashr_i32 s19, s3, 24 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[56:57], s[34:35], 0x80000 +; GFX6-NOHSA-NEXT: s_ashr_i32 s21, s5, 31 +; GFX6-NOHSA-NEXT: s_ashr_i32 s23, s5, 24 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[58:59], s[30:31], 0x80000 +; GFX6-NOHSA-NEXT: s_ashr_i32 s25, s7, 31 ; GFX6-NOHSA-NEXT: s_ashr_i32 s27, s7, 24 -; GFX6-NOHSA-NEXT: s_lshr_b32 s42, s4, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s38, s4, 24 -; GFX6-NOHSA-NEXT: s_lshr_b32 s26, s4, 8 -; GFX6-NOHSA-NEXT: s_lshr_b32 s28, s3, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s30, s3, 8 -; GFX6-NOHSA-NEXT: s_mov_b32 s52, s3 -; GFX6-NOHSA-NEXT: s_lshr_b32 s34, s2, 16 -; GFX6-NOHSA-NEXT: s_lshr_b32 s36, s2, 24 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[60:61], s[14:15], 0x80000 +; GFX6-NOHSA-NEXT: s_lshr_b32 s30, s3, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s34, s3, 8 +; GFX6-NOHSA-NEXT: s_mov_b32 s44, s3 +; GFX6-NOHSA-NEXT: s_lshr_b32 s36, s2, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s38, s2, 24 ; GFX6-NOHSA-NEXT: s_lshr_b32 s40, s2, 8 -; GFX6-NOHSA-NEXT: s_lshr_b32 s44, s1, 16 +; GFX6-NOHSA-NEXT: s_lshr_b32 s42, s1, 16 ; GFX6-NOHSA-NEXT: s_lshr_b32 s46, s1, 8 -; GFX6-NOHSA-NEXT: s_mov_b32 s56, s1 +; GFX6-NOHSA-NEXT: s_mov_b32 s52, s1 ; GFX6-NOHSA-NEXT: s_lshr_b32 s48, s0, 16 ; GFX6-NOHSA-NEXT: s_lshr_b32 s50, s0, 24 ; GFX6-NOHSA-NEXT: s_lshr_b32 s54, s0, 8 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[14:15], s[0:1], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[62:63], s[4:5], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x80000 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s58 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s59 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v8, s6 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s7 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s10 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s11 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s4 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s5 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v18, s27 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x80000 +; GFX6-NOHSA-NEXT: s_mov_b32 s0, s8 +; GFX6-NOHSA-NEXT: s_mov_b32 s1, s9 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s58 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s59 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s6 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s7 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v8, s56 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s57 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v12, s62 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v13, s63 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v14, s60 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v15, s61 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s27 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s25 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v18, s23 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v19, s21 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s17 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v14, s15 -; GFX6-NOHSA-NEXT: s_mov_b32 s11, 0xf000 -; GFX6-NOHSA-NEXT: s_mov_b32 s15, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s21, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s17, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s29, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s31, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s53, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s45, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s47, s13 -; GFX6-NOHSA-NEXT: s_mov_b32 s57, s13 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[4:5], s[12:13], 0x80000 -; GFX6-NOHSA-NEXT: s_mov_b32 s10, -1 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s23 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v15, s19 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s4 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s5 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[8:11], 0 offset:240 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x80000 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v20, s19 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v21, s17 +; GFX6-NOHSA-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[6:7], s[28:29], 0x80000 +; GFX6-NOHSA-NEXT: s_mov_b32 s2, -1 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v22, s6 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:240 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[8:9], s[26:27], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x80000 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s22 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s23 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v23, s7 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v24, s24 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v25, s25 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s8 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s9 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:208 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[58:59], s[24:25], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[60:61], s[14:15], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[62:63], s[22:23], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[64:65], s[18:19], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[66:67], s[20:21], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[68:69], s[16:17], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[6:7], s[54:55], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[8:9], s[50:51], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[22:23], s[48:49], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[24:25], s[46:47], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[26:27], s[42:43], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[28:29], s[40:41], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[4:5], s[54:55], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[6:7], s[50:51], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[12:13], s[48:49], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[14:15], s[46:47], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[16:17], s[44:45], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[18:19], s[40:41], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[20:21], s[36:37], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[22:23], s[34:35], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x80000 ; GFX6-NOHSA-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x80000 -; GFX6-NOHSA-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v12, s60 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v13, s61 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[10:13], off, s[8:11], 0 offset:224 -; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s58 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s59 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v12, s64 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v13, s65 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[10:13], off, s[8:11], 0 offset:208 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s52 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s53 -; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s62 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s63 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:192 -; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v8, s2 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s3 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s66 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s67 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:176 -; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s33 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s25 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s68 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s69 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[2:5], off, s[8:11], 0 offset:160 -; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s42 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s43 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s38 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s39 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[2:5], off, s[8:11], 0 offset:144 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v20, s56 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v21, s57 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[40:41], s[12:13], 0x80000 +; GFX6-NOHSA-NEXT: s_bfe_i64 s[42:43], s[10:11], 0x80000 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:192 +; GFX6-NOHSA-NEXT: s_waitcnt expcnt(2) +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s44 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s45 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s42 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s43 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176 +; GFX6-NOHSA-NEXT: s_waitcnt expcnt(1) +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s4 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s5 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s40 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s41 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:160 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s16 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s17 ; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s26 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s27 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 offset:128 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v8, s18 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s19 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:144 ; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s0 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s1 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v12, s28 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v13, s29 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[8:11], 0 offset:112 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v8, s13 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v9, s11 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v14, s20 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v15, s21 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:128 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s52 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s53 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v18, s30 ; GFX6-NOHSA-NEXT: v_mov_b32_e32 v19, s31 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[8:11], 0 offset:96 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s22 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s23 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s20 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s21 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[2:5], off, s[8:11], 0 offset:80 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v10, s18 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v11, s19 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:64 -; GFX6-NOHSA-NEXT: s_waitcnt expcnt(1) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s16 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s17 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:48 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v22, s14 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v23, s15 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[20:23], off, s[8:11], 0 offset:32 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s12 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s13 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:112 ; GFX6-NOHSA-NEXT: s_waitcnt expcnt(1) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v4, s6 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v5, s7 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[2:5], off, s[8:11], 0 offset:16 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v14, s14 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v15, s15 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s34 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s35 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96 ; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s4 -; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s5 -; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s36 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s37 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s38 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s39 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s28 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s29 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64 +; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0) +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v6, s26 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v7, s27 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:48 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v12, s24 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v13, s25 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:32 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s22 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v1, s23 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s8 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v3, s9 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v16, s6 +; GFX6-NOHSA-NEXT: v_mov_b32_e32 v17, s7 +; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 ; GFX6-NOHSA-NEXT: s_endpgm ; ; GFX7-HSA-LABEL: constant_sextload_v32i8_to_v32i64: ; GFX7-HSA: ; %bb.0: ; GFX7-HSA-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0 ; GFX7-HSA-NEXT: s_add_i32 s12, s12, s17 -; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GFX7-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8 +; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13 ; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-HSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 -; GFX7-HSA-NEXT: s_mov_b32 s11, 0 -; GFX7-HSA-NEXT: s_mov_b32 s37, s11 -; GFX7-HSA-NEXT: s_mov_b32 s41, s11 -; GFX7-HSA-NEXT: s_mov_b32 s55, s11 ; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-HSA-NEXT: s_lshr_b32 s10, s7, 16 -; GFX7-HSA-NEXT: s_lshr_b32 s36, s7, 8 -; GFX7-HSA-NEXT: s_mov_b32 s40, s7 +; GFX7-HSA-NEXT: s_lshr_b32 s40, s7, 8 +; GFX7-HSA-NEXT: s_mov_b32 s42, s7 ; GFX7-HSA-NEXT: s_lshr_b32 s44, s6, 16 -; GFX7-HSA-NEXT: s_ashr_i32 s45, s1, 24 -; GFX7-HSA-NEXT: s_ashr_i32 s48, s3, 31 -; GFX7-HSA-NEXT: s_ashr_i32 s49, s3, 24 -; GFX7-HSA-NEXT: s_ashr_i32 s63, s5, 31 -; GFX7-HSA-NEXT: s_ashr_i32 s65, s5, 24 -; GFX7-HSA-NEXT: s_lshr_b32 s50, s6, 24 -; GFX7-HSA-NEXT: s_lshr_b32 s52, s6, 8 -; GFX7-HSA-NEXT: s_lshr_b32 s54, s5, 16 +; GFX7-HSA-NEXT: s_ashr_i32 s41, s1, 24 +; GFX7-HSA-NEXT: s_ashr_i32 s43, s3, 31 +; GFX7-HSA-NEXT: s_ashr_i32 s45, s3, 24 +; GFX7-HSA-NEXT: s_lshr_b32 s48, s6, 24 +; GFX7-HSA-NEXT: s_lshr_b32 s50, s6, 8 +; GFX7-HSA-NEXT: s_lshr_b32 s52, s5, 16 ; GFX7-HSA-NEXT: s_lshr_b32 s46, s5, 8 -; GFX7-HSA-NEXT: s_mov_b32 s56, s5 -; GFX7-HSA-NEXT: s_lshr_b32 s42, s4, 16 -; GFX7-HSA-NEXT: s_lshr_b32 s38, s4, 24 +; GFX7-HSA-NEXT: s_mov_b32 s54, s5 +; GFX7-HSA-NEXT: s_lshr_b32 s38, s4, 16 +; GFX7-HSA-NEXT: s_lshr_b32 s36, s4, 24 ; GFX7-HSA-NEXT: s_lshr_b32 s34, s4, 8 -; GFX7-HSA-NEXT: s_lshr_b32 s30, s3, 16 +; GFX7-HSA-NEXT: s_lshr_b32 s28, s3, 16 ; GFX7-HSA-NEXT: s_lshr_b32 s24, s3, 8 ; GFX7-HSA-NEXT: s_mov_b32 s26, s3 ; GFX7-HSA-NEXT: s_lshr_b32 s22, s2, 16 ; GFX7-HSA-NEXT: s_lshr_b32 s20, s2, 24 ; GFX7-HSA-NEXT: s_lshr_b32 s18, s2, 8 -; GFX7-HSA-NEXT: s_lshr_b32 s16, s1, 16 -; GFX7-HSA-NEXT: s_lshr_b32 s58, s1, 8 +; GFX7-HSA-NEXT: s_lshr_b32 s14, s1, 16 +; GFX7-HSA-NEXT: s_lshr_b32 s56, s1, 8 ; GFX7-HSA-NEXT: s_mov_b32 s12, s1 -; GFX7-HSA-NEXT: s_lshr_b32 s60, s0, 16 -; GFX7-HSA-NEXT: s_lshr_b32 s62, s0, 24 -; GFX7-HSA-NEXT: s_lshr_b32 s64, s0, 8 -; GFX7-HSA-NEXT: s_bfe_i64 s[14:15], s[2:3], 0x80000 -; GFX7-HSA-NEXT: s_mov_b32 s47, s11 -; GFX7-HSA-NEXT: s_mov_b32 s57, s11 -; GFX7-HSA-NEXT: s_mov_b32 s31, s11 -; GFX7-HSA-NEXT: s_mov_b32 s25, s11 -; GFX7-HSA-NEXT: s_mov_b32 s27, s11 -; GFX7-HSA-NEXT: s_mov_b32 s17, s11 -; GFX7-HSA-NEXT: s_mov_b32 s59, s11 -; GFX7-HSA-NEXT: s_mov_b32 s13, s11 +; GFX7-HSA-NEXT: s_lshr_b32 s58, s0, 16 +; GFX7-HSA-NEXT: s_lshr_b32 s60, s0, 24 +; GFX7-HSA-NEXT: s_lshr_b32 s62, s0, 8 +; GFX7-HSA-NEXT: s_bfe_i64 s[16:17], s[2:3], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[2:3], s[10:11], 0x80000 ; GFX7-HSA-NEXT: s_ashr_i32 s33, s1, 31 +; GFX7-HSA-NEXT: s_ashr_i32 s66, s5, 31 +; GFX7-HSA-NEXT: s_ashr_i32 s67, s5, 24 ; GFX7-HSA-NEXT: s_ashr_i32 s68, s7, 31 ; GFX7-HSA-NEXT: s_ashr_i32 s69, s7, 24 ; GFX7-HSA-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[28:29], s[4:5], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[66:67], s[6:7], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[30:31], s[4:5], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[64:65], s[6:7], 0x80000 ; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s2 ; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s3 -; GFX7-HSA-NEXT: s_bfe_i64 s[2:3], s[64:65], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[4:5], s[62:63], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[6:7], s[60:61], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[2:3], s[62:63], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[4:5], s[60:61], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[6:7], s[58:59], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[10:11], s[58:59], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[10:11], s[56:57], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x80000 ; GFX7-HSA-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[58:59], s[44:45], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x80000 -; GFX7-HSA-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x80000 -; GFX7-HSA-NEXT: s_add_u32 s60, s8, 0xf0 -; GFX7-HSA-NEXT: s_addc_u32 s61, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s36 -; GFX7-HSA-NEXT: s_add_u32 s36, s8, 0xe0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s37 -; GFX7-HSA-NEXT: s_addc_u32 s37, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v24, s36 -; GFX7-HSA-NEXT: v_mov_b32_e32 v25, s37 -; GFX7-HSA-NEXT: s_add_u32 s36, s8, 0xd0 -; GFX7-HSA-NEXT: s_addc_u32 s37, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v26, s36 -; GFX7-HSA-NEXT: v_mov_b32_e32 v27, s37 -; GFX7-HSA-NEXT: s_add_u32 s36, s8, 0xc0 -; GFX7-HSA-NEXT: s_addc_u32 s37, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v28, s36 -; GFX7-HSA-NEXT: v_mov_b32_e32 v29, s37 -; GFX7-HSA-NEXT: s_add_u32 s36, s8, 0xb0 -; GFX7-HSA-NEXT: s_addc_u32 s37, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v30, s36 -; GFX7-HSA-NEXT: v_mov_b32_e32 v31, s37 -; GFX7-HSA-NEXT: s_add_u32 s36, s8, 0xa0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s40 -; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s41 -; GFX7-HSA-NEXT: s_addc_u32 s37, s9, 0 +; GFX7-HSA-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[56:57], s[44:45], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[58:59], s[42:43], 0x80000 +; GFX7-HSA-NEXT: s_bfe_i64 s[60:61], s[40:41], 0x80000 +; GFX7-HSA-NEXT: s_add_u32 s62, s8, 0xf0 +; GFX7-HSA-NEXT: s_addc_u32 s63, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s58 +; GFX7-HSA-NEXT: s_add_u32 s58, s8, 0xe0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s59 +; GFX7-HSA-NEXT: s_addc_u32 s59, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s48 +; GFX7-HSA-NEXT: s_add_u32 s48, s8, 0xd0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s49 +; GFX7-HSA-NEXT: s_addc_u32 s49, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v26, s48 +; GFX7-HSA-NEXT: v_mov_b32_e32 v27, s49 +; GFX7-HSA-NEXT: s_add_u32 s48, s8, 0xc0 +; GFX7-HSA-NEXT: s_addc_u32 s49, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v28, s48 +; GFX7-HSA-NEXT: v_mov_b32_e32 v18, s62 +; GFX7-HSA-NEXT: v_mov_b32_e32 v29, s49 +; GFX7-HSA-NEXT: s_add_u32 s48, s8, 0xb0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v19, s63 +; GFX7-HSA-NEXT: s_addc_u32 s49, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s69 +; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s68 +; GFX7-HSA-NEXT: v_mov_b32_e32 v24, s58 +; GFX7-HSA-NEXT: flat_store_dwordx4 v[18:19], v[0:3] +; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s60 +; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s38 +; GFX7-HSA-NEXT: s_add_u32 s38, s8, 0xa0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s61 +; GFX7-HSA-NEXT: v_mov_b32_e32 v25, s59 +; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s39 +; GFX7-HSA-NEXT: s_addc_u32 s39, s9, 0 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[24:25], v[4:7] -; GFX7-HSA-NEXT: v_mov_b32_e32 v8, s58 -; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s28 -; GFX7-HSA-NEXT: s_add_u32 s28, s8, 0x90 -; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s29 -; GFX7-HSA-NEXT: s_addc_u32 s29, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v24, s28 -; GFX7-HSA-NEXT: v_mov_b32_e32 v9, s59 -; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s50 -; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s51 -; GFX7-HSA-NEXT: v_mov_b32_e32 v25, s29 -; GFX7-HSA-NEXT: s_add_u32 s28, s8, 0x80 +; GFX7-HSA-NEXT: v_mov_b32_e32 v8, s56 +; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s30 +; GFX7-HSA-NEXT: s_add_u32 s30, s8, 0x90 +; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s31 +; GFX7-HSA-NEXT: s_addc_u32 s31, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v24, s30 +; GFX7-HSA-NEXT: v_mov_b32_e32 v25, s31 +; GFX7-HSA-NEXT: s_add_u32 s30, s8, 0x80 +; GFX7-HSA-NEXT: v_mov_b32_e32 v9, s57 +; GFX7-HSA-NEXT: s_addc_u32 s31, s9, 0 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[26:27], v[8:11] -; GFX7-HSA-NEXT: s_addc_u32 s29, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v26, s28 -; GFX7-HSA-NEXT: v_mov_b32_e32 v18, s60 -; GFX7-HSA-NEXT: v_mov_b32_e32 v27, s29 +; GFX7-HSA-NEXT: v_mov_b32_e32 v12, s64 +; GFX7-HSA-NEXT: v_mov_b32_e32 v8, s28 ; GFX7-HSA-NEXT: s_add_u32 s28, s8, 0x70 -; GFX7-HSA-NEXT: v_mov_b32_e32 v19, s61 -; GFX7-HSA-NEXT: v_mov_b32_e32 v12, s66 -; GFX7-HSA-NEXT: v_mov_b32_e32 v13, s67 -; GFX7-HSA-NEXT: v_mov_b32_e32 v14, s52 -; GFX7-HSA-NEXT: v_mov_b32_e32 v15, s53 -; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s69 -; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s68 +; GFX7-HSA-NEXT: v_mov_b32_e32 v13, s65 +; GFX7-HSA-NEXT: v_mov_b32_e32 v14, s50 +; GFX7-HSA-NEXT: v_mov_b32_e32 v15, s51 +; GFX7-HSA-NEXT: v_mov_b32_e32 v30, s48 +; GFX7-HSA-NEXT: v_mov_b32_e32 v9, s29 ; GFX7-HSA-NEXT: s_addc_u32 s29, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v16, s54 -; GFX7-HSA-NEXT: v_mov_b32_e32 v17, s55 -; GFX7-HSA-NEXT: flat_store_dwordx4 v[18:19], v[0:3] -; GFX7-HSA-NEXT: v_mov_b32_e32 v18, s65 -; GFX7-HSA-NEXT: v_mov_b32_e32 v19, s63 -; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s36 +; GFX7-HSA-NEXT: v_mov_b32_e32 v16, s52 +; GFX7-HSA-NEXT: v_mov_b32_e32 v17, s53 +; GFX7-HSA-NEXT: v_mov_b32_e32 v31, s49 +; GFX7-HSA-NEXT: v_mov_b32_e32 v18, s67 +; GFX7-HSA-NEXT: v_mov_b32_e32 v19, s66 +; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s38 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[28:29], v[12:15] -; GFX7-HSA-NEXT: v_mov_b32_e32 v20, s56 +; GFX7-HSA-NEXT: v_mov_b32_e32 v20, s54 ; GFX7-HSA-NEXT: v_mov_b32_e32 v14, s24 ; GFX7-HSA-NEXT: s_add_u32 s24, s8, 0x60 -; GFX7-HSA-NEXT: v_mov_b32_e32 v21, s57 +; GFX7-HSA-NEXT: v_mov_b32_e32 v21, s55 ; GFX7-HSA-NEXT: v_mov_b32_e32 v22, s46 ; GFX7-HSA-NEXT: v_mov_b32_e32 v23, s47 -; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s42 -; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s43 -; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s37 -; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s38 -; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s39 +; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s39 +; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s36 +; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s37 +; GFX7-HSA-NEXT: v_mov_b32_e32 v26, s30 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[30:31], v[16:19] ; GFX7-HSA-NEXT: v_mov_b32_e32 v15, s25 ; GFX7-HSA-NEXT: v_mov_b32_e32 v16, s28 ; GFX7-HSA-NEXT: s_addc_u32 s25, s9, 0 ; GFX7-HSA-NEXT: v_mov_b32_e32 v18, s24 +; GFX7-HSA-NEXT: v_mov_b32_e32 v27, s31 ; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s34 ; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s35 -; GFX7-HSA-NEXT: v_mov_b32_e32 v8, s30 -; GFX7-HSA-NEXT: v_mov_b32_e32 v9, s31 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[10:11], v[20:23] ; GFX7-HSA-NEXT: v_mov_b32_e32 v12, s26 -; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s49 -; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s48 +; GFX7-HSA-NEXT: v_mov_b32_e32 v10, s45 +; GFX7-HSA-NEXT: v_mov_b32_e32 v11, s43 ; GFX7-HSA-NEXT: v_mov_b32_e32 v13, s27 ; GFX7-HSA-NEXT: v_mov_b32_e32 v17, s29 ; GFX7-HSA-NEXT: v_mov_b32_e32 v19, s25 @@ -8583,21 +8526,22 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s21 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX7-HSA-NEXT: s_nop 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s14 -; GFX7-HSA-NEXT: s_add_u32 s14, s8, 64 -; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s15 -; GFX7-HSA-NEXT: s_addc_u32 s15, s9, 0 -; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s14 +; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s16 +; GFX7-HSA-NEXT: s_add_u32 s16, s8, 64 +; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s17 +; GFX7-HSA-NEXT: s_addc_u32 s17, s9, 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s16 ; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s18 ; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s19 -; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s15 -; GFX7-HSA-NEXT: s_add_u32 s14, s8, 48 +; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s17 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; GFX7-HSA-NEXT: s_nop 0 +; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s14 +; GFX7-HSA-NEXT: s_add_u32 s14, s8, 48 +; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s15 ; GFX7-HSA-NEXT: s_addc_u32 s15, s9, 0 ; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s14 -; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s16 -; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s17 -; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s45 +; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s41 ; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s33 ; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s15 ; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] @@ -8633,26 +8577,18 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX8-NOHSA-LABEL: constant_sextload_v32i8_to_v32i64: ; GFX8-NOHSA: ; %bb.0: ; GFX8-NOHSA-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 -; GFX8-NOHSA-NEXT: s_mov_b32 s47, 0 -; GFX8-NOHSA-NEXT: s_mov_b32 s53, s47 -; GFX8-NOHSA-NEXT: s_mov_b32 s63, s47 -; GFX8-NOHSA-NEXT: s_mov_b32 s49, s47 ; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NOHSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 -; GFX8-NOHSA-NEXT: s_mov_b32 s65, s47 -; GFX8-NOHSA-NEXT: s_mov_b32 s31, s47 -; GFX8-NOHSA-NEXT: s_mov_b32 s67, s47 -; GFX8-NOHSA-NEXT: s_mov_b32 s19, s47 ; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NOHSA-NEXT: s_lshr_b32 s46, s7, 16 -; GFX8-NOHSA-NEXT: s_lshr_b32 s54, s7, 8 -; GFX8-NOHSA-NEXT: s_mov_b32 s52, s7 +; GFX8-NOHSA-NEXT: s_lshr_b32 s50, s7, 16 +; GFX8-NOHSA-NEXT: s_lshr_b32 s52, s7, 8 +; GFX8-NOHSA-NEXT: s_mov_b32 s54, s7 ; GFX8-NOHSA-NEXT: s_lshr_b32 s56, s6, 16 ; GFX8-NOHSA-NEXT: s_lshr_b32 s58, s6, 24 ; GFX8-NOHSA-NEXT: s_lshr_b32 s60, s6, 8 ; GFX8-NOHSA-NEXT: s_lshr_b32 s62, s5, 16 -; GFX8-NOHSA-NEXT: s_lshr_b32 s50, s5, 8 -; GFX8-NOHSA-NEXT: s_mov_b32 s48, s5 +; GFX8-NOHSA-NEXT: s_lshr_b32 s48, s5, 8 +; GFX8-NOHSA-NEXT: s_mov_b32 s46, s5 ; GFX8-NOHSA-NEXT: s_lshr_b32 s42, s4, 16 ; GFX8-NOHSA-NEXT: s_lshr_b32 s40, s4, 24 ; GFX8-NOHSA-NEXT: s_lshr_b32 s38, s4, 8 @@ -8668,6 +8604,7 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: s_lshr_b32 s16, s0, 16 ; GFX8-NOHSA-NEXT: s_lshr_b32 s14, s0, 24 ; GFX8-NOHSA-NEXT: s_lshr_b32 s12, s0, 8 +; GFX8-NOHSA-NEXT: s_ashr_i32 s65, s3, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[10:11], s[0:1], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[22:23], s[2:3], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[36:37], s[4:5], 0x80000 @@ -8686,50 +8623,49 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x80000 ; GFX8-NOHSA-NEXT: s_ashr_i32 s33, s3, 31 -; GFX8-NOHSA-NEXT: s_ashr_i32 s66, s3, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[2:3], s[64:65], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x80000 +; GFX8-NOHSA-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x80000 -; GFX8-NOHSA-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x80000 ; GFX8-NOHSA-NEXT: s_ashr_i32 s64, s5, 31 ; GFX8-NOHSA-NEXT: s_ashr_i32 s5, s5, 24 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x80000 -; GFX8-NOHSA-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x80000 ; GFX8-NOHSA-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x80000 -; GFX8-NOHSA-NEXT: s_ashr_i32 s65, s7, 31 +; GFX8-NOHSA-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x80000 +; GFX8-NOHSA-NEXT: s_ashr_i32 s66, s7, 31 ; GFX8-NOHSA-NEXT: s_ashr_i32 s7, s7, 24 -; GFX8-NOHSA-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x80000 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s46 -; GFX8-NOHSA-NEXT: s_add_u32 s46, s8, 0xf0 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s47 -; GFX8-NOHSA-NEXT: s_addc_u32 s47, s9, 0 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s46 +; GFX8-NOHSA-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x80000 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s50 +; GFX8-NOHSA-NEXT: s_add_u32 s50, s8, 0xf0 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s51 +; GFX8-NOHSA-NEXT: s_addc_u32 s51, s9, 0 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s50 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s7 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s65 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s47 -; GFX8-NOHSA-NEXT: s_add_u32 s46, s8, 0xe0 -; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] -; GFX8-NOHSA-NEXT: s_addc_u32 s47, s9, 0 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s46 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s52 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s53 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s54 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s55 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s47 -; GFX8-NOHSA-NEXT: s_add_u32 s46, s8, 0xd0 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s66 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s51 +; GFX8-NOHSA-NEXT: s_add_u32 s50, s8, 0xe0 +; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; GFX8-NOHSA-NEXT: s_addc_u32 s51, s9, 0 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s50 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s54 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s55 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s52 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s53 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s51 +; GFX8-NOHSA-NEXT: s_add_u32 s50, s8, 0xd0 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] -; GFX8-NOHSA-NEXT: s_addc_u32 s47, s9, 0 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s46 +; GFX8-NOHSA-NEXT: s_addc_u32 s51, s9, 0 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s50 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s56 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s57 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s58 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s59 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s47 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s51 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NOHSA-NEXT: s_nop 0 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s44 @@ -8753,10 +8689,10 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NOHSA-NEXT: s_addc_u32 s45, s9, 0 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s44 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s48 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s49 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s50 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s51 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s46 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v1, s47 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s48 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s49 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s45 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; GFX8-NOHSA-NEXT: s_nop 0 @@ -8787,7 +8723,7 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s2 ; GFX8-NOHSA-NEXT: s_add_u32 s2, s8, 0x60 -; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s66 +; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s65 ; GFX8-NOHSA-NEXT: v_mov_b32_e32 v3, s33 ; GFX8-NOHSA-NEXT: s_addc_u32 s3, s9, 0 ; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] @@ -9045,78 +8981,57 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX12-LABEL: constant_sextload_v32i8_to_v32i64: ; GFX12: ; %bb.0: ; GFX12-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 -; GFX12-NEXT: s_mov_b32 s41, 0 -; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_mov_b32 s47, s41 -; GFX12-NEXT: s_mov_b32 s25, s41 -; GFX12-NEXT: s_mov_b32 s43, s41 -; GFX12-NEXT: s_mov_b32 s31, s41 -; GFX12-NEXT: s_mov_b32 s45, s41 -; GFX12-NEXT: s_mov_b32 s37, s41 -; GFX12-NEXT: s_mov_b32 s19, s41 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: s_load_b256 s[0:7], s[10:11], 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: s_lshr_b32 s40, s7, 16 -; GFX12-NEXT: s_lshr_b32 s54, s6, 8 -; GFX12-NEXT: s_lshr_b32 s62, s4, 8 -; GFX12-NEXT: s_lshr_b32 s46, s3, 16 -; GFX12-NEXT: s_lshr_b32 s64, s2, 8 -; GFX12-NEXT: s_lshr_b32 s24, s1, 16 -; GFX12-NEXT: s_ashr_i32 s55, s3, 31 -; GFX12-NEXT: s_ashr_i32 s63, s3, 24 -; GFX12-NEXT: s_lshr_b32 s48, s7, 8 -; GFX12-NEXT: s_mov_b32 s42, s7 -; GFX12-NEXT: s_lshr_b32 s50, s6, 16 -; GFX12-NEXT: s_lshr_b32 s52, s6, 24 -; GFX12-NEXT: s_lshr_b32 s30, s5, 16 -; GFX12-NEXT: s_lshr_b32 s56, s5, 8 -; GFX12-NEXT: s_mov_b32 s44, s5 -; GFX12-NEXT: s_lshr_b32 s58, s4, 16 -; GFX12-NEXT: s_lshr_b32 s60, s4, 24 -; GFX12-NEXT: s_lshr_b32 s38, s3, 8 -; GFX12-NEXT: s_mov_b32 s36, s3 +; GFX12-NEXT: s_lshr_b32 s50, s6, 8 +; GFX12-NEXT: s_lshr_b32 s62, s3, 16 +; GFX12-NEXT: s_ashr_i32 s51, s3, 24 +; GFX12-NEXT: s_lshr_b32 s42, s7, 8 +; GFX12-NEXT: s_mov_b32 s44, s7 +; GFX12-NEXT: s_lshr_b32 s46, s6, 16 +; GFX12-NEXT: s_lshr_b32 s48, s6, 24 +; GFX12-NEXT: s_lshr_b32 s38, s5, 16 +; GFX12-NEXT: s_lshr_b32 s52, s5, 8 +; GFX12-NEXT: s_mov_b32 s54, s5 +; GFX12-NEXT: s_lshr_b32 s56, s4, 16 +; GFX12-NEXT: s_lshr_b32 s58, s4, 24 +; GFX12-NEXT: s_lshr_b32 s60, s4, 8 +; GFX12-NEXT: s_lshr_b32 s36, s3, 8 +; GFX12-NEXT: s_mov_b32 s34, s3 ; GFX12-NEXT: s_lshr_b32 s28, s2, 16 ; GFX12-NEXT: s_lshr_b32 s26, s2, 24 -; GFX12-NEXT: s_lshr_b32 s20, s1, 8 -; GFX12-NEXT: s_mov_b32 s18, s1 -; GFX12-NEXT: s_lshr_b32 s16, s0, 16 -; GFX12-NEXT: s_lshr_b32 s14, s0, 24 -; GFX12-NEXT: s_lshr_b32 s12, s0, 8 -; GFX12-NEXT: s_bfe_i64 s[10:11], s[0:1], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[22:23], s[2:3], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[34:35], s[4:5], 0x80000 +; GFX12-NEXT: s_lshr_b32 s24, s2, 8 +; GFX12-NEXT: s_bfe_i64 s[20:21], s[2:3], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[30:31], s[4:5], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[66:67], s[6:7], 0x80000 -; GFX12-NEXT: s_ashr_i32 s6, s1, 31 -; GFX12-NEXT: s_ashr_i32 s33, s1, 24 -; GFX12-NEXT: s_bfe_i64 s[0:1], s[24:25], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[24:25], s[64:65], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[2:3], s[46:47], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[46:47], s[62:63], 0x80000 +; GFX12-NEXT: s_ashr_i32 s39, s3, 31 +; GFX12-NEXT: s_bfe_i64 s[2:3], s[62:63], 0x80000 ; GFX12-NEXT: s_ashr_i32 s62, s5, 31 -; GFX12-NEXT: s_ashr_i32 s64, s5, 24 -; GFX12-NEXT: s_bfe_i64 s[4:5], s[54:55], 0x80000 -; GFX12-NEXT: s_ashr_i32 s54, s7, 31 +; GFX12-NEXT: s_ashr_i32 s63, s5, 24 +; GFX12-NEXT: s_bfe_i64 s[4:5], s[50:51], 0x80000 +; GFX12-NEXT: s_ashr_i32 s50, s7, 31 ; GFX12-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x80000 ; GFX12-NEXT: s_ashr_i32 s7, s7, 24 +; GFX12-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x80000 ; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s41 -; GFX12-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v0, s40 :: v_dual_mov_b32 v3, s54 -; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v5, s43 -; GFX12-NEXT: v_dual_mov_b32 v4, s42 :: v_dual_mov_b32 v7, s49 -; GFX12-NEXT: v_dual_mov_b32 v6, s48 :: v_dual_mov_b32 v9, s51 -; GFX12-NEXT: v_dual_mov_b32 v8, s50 :: v_dual_mov_b32 v11, s53 -; GFX12-NEXT: v_dual_mov_b32 v10, s52 :: v_dual_mov_b32 v13, s67 +; GFX12-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v0, s40 :: v_dual_mov_b32 v3, s50 +; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v5, s45 +; GFX12-NEXT: v_dual_mov_b32 v4, s44 :: v_dual_mov_b32 v7, s43 +; GFX12-NEXT: v_dual_mov_b32 v6, s42 :: v_dual_mov_b32 v9, s47 +; GFX12-NEXT: v_dual_mov_b32 v8, s46 :: v_dual_mov_b32 v11, s49 +; GFX12-NEXT: v_dual_mov_b32 v10, s48 :: v_dual_mov_b32 v13, s67 ; GFX12-NEXT: v_dual_mov_b32 v12, s66 :: v_dual_mov_b32 v15, s5 ; GFX12-NEXT: v_mov_b32_e32 v14, s4 -; GFX12-NEXT: s_bfe_i64 s[4:5], s[30:31], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[4:5], s[38:39], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x80000 ; GFX12-NEXT: s_clause 0x3 ; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:240 ; GFX12-NEXT: global_store_b128 v24, v[4:7], s[8:9] offset:224 @@ -9124,22 +9039,32 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX12-NEXT: global_store_b128 v24, v[12:15], s[8:9] offset:192 ; GFX12-NEXT: s_wait_alu 0xfffe ; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s62 -; GFX12-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s64 -; GFX12-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v4, s44 -; GFX12-NEXT: v_dual_mov_b32 v7, s57 :: v_dual_mov_b32 v6, s56 -; GFX12-NEXT: v_dual_mov_b32 v9, s59 :: v_dual_mov_b32 v8, s58 -; GFX12-NEXT: v_dual_mov_b32 v11, s61 :: v_dual_mov_b32 v10, s60 -; GFX12-NEXT: v_mov_b32_e32 v13, s35 +; GFX12-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s63 +; GFX12-NEXT: v_mov_b32_e32 v5, s55 +; GFX12-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v4, s54 :: v_dual_mov_b32 v7, s53 +; GFX12-NEXT: v_dual_mov_b32 v6, s52 :: v_dual_mov_b32 v9, s57 +; GFX12-NEXT: v_dual_mov_b32 v8, s56 :: v_dual_mov_b32 v11, s59 +; GFX12-NEXT: v_dual_mov_b32 v10, s58 :: v_dual_mov_b32 v13, s31 +; GFX12-NEXT: s_lshr_b32 s22, s1, 16 ; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v12, s34 :: v_dual_mov_b32 v15, s47 -; GFX12-NEXT: v_dual_mov_b32 v14, s46 :: v_dual_mov_b32 v17, s3 -; GFX12-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v19, s55 -; GFX12-NEXT: v_dual_mov_b32 v18, s63 :: v_dual_mov_b32 v21, s37 -; GFX12-NEXT: v_dual_mov_b32 v20, s36 :: v_dual_mov_b32 v23, s39 -; GFX12-NEXT: v_mov_b32_e32 v22, s38 +; GFX12-NEXT: v_dual_mov_b32 v12, s30 :: v_dual_mov_b32 v15, s61 +; GFX12-NEXT: v_dual_mov_b32 v14, s60 :: v_dual_mov_b32 v17, s3 +; GFX12-NEXT: s_lshr_b32 s16, s1, 8 +; GFX12-NEXT: s_mov_b32 s18, s1 +; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v19, s39 +; GFX12-NEXT: v_dual_mov_b32 v18, s51 :: v_dual_mov_b32 v21, s35 +; GFX12-NEXT: s_lshr_b32 s14, s0, 16 +; GFX12-NEXT: s_lshr_b32 s12, s0, 24 +; GFX12-NEXT: s_ashr_i32 s6, s1, 31 +; GFX12-NEXT: s_ashr_i32 s33, s1, 24 +; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v20, s34 :: v_dual_mov_b32 v23, s37 +; GFX12-NEXT: v_mov_b32_e32 v22, s36 ; GFX12-NEXT: s_clause 0x5 ; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:176 ; GFX12-NEXT: global_store_b128 v24, v[4:7], s[8:9] offset:160 @@ -9149,22 +9074,25 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o ; GFX12-NEXT: global_store_b128 v24, v[20:23], s[8:9] offset:96 ; GFX12-NEXT: v_dual_mov_b32 v0, s28 :: v_dual_mov_b32 v3, s27 ; GFX12-NEXT: v_dual_mov_b32 v1, s29 :: v_dual_mov_b32 v2, s26 -; GFX12-NEXT: v_mov_b32_e32 v5, s23 +; GFX12-NEXT: v_mov_b32_e32 v5, s21 +; GFX12-NEXT: s_lshr_b32 s64, s0, 8 ; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x80000 -; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v4, s22 :: v_dual_mov_b32 v7, s25 -; GFX12-NEXT: v_dual_mov_b32 v6, s24 :: v_dual_mov_b32 v9, s1 -; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 ; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v11, s6 -; GFX12-NEXT: v_dual_mov_b32 v10, s33 :: v_dual_mov_b32 v13, s19 +; GFX12-NEXT: v_dual_mov_b32 v4, s20 :: v_dual_mov_b32 v7, s25 +; GFX12-NEXT: v_dual_mov_b32 v6, s24 :: v_dual_mov_b32 v9, s23 ; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x80000 -; GFX12-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v15, s21 -; GFX12-NEXT: v_dual_mov_b32 v14, s20 :: v_dual_mov_b32 v17, s17 -; GFX12-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v19, s15 -; GFX12-NEXT: v_dual_mov_b32 v18, s14 :: v_dual_mov_b32 v21, s11 -; GFX12-NEXT: v_dual_mov_b32 v20, s10 :: v_dual_mov_b32 v23, s13 -; GFX12-NEXT: v_mov_b32_e32 v22, s12 +; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x80000 +; GFX12-NEXT: s_wait_alu 0xfffe +; GFX12-NEXT: v_dual_mov_b32 v8, s22 :: v_dual_mov_b32 v11, s6 +; GFX12-NEXT: v_dual_mov_b32 v10, s33 :: v_dual_mov_b32 v13, s19 +; GFX12-NEXT: s_bfe_i64 s[10:11], s[0:1], 0x80000 +; GFX12-NEXT: s_bfe_i64 s[0:1], s[64:65], 0x80000 +; GFX12-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v15, s17 +; GFX12-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s15 +; GFX12-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v19, s13 +; GFX12-NEXT: v_dual_mov_b32 v18, s12 :: v_dual_mov_b32 v21, s11 +; GFX12-NEXT: v_dual_mov_b32 v20, s10 :: v_dual_mov_b32 v23, s1 +; GFX12-NEXT: v_mov_b32_e32 v22, s0 ; GFX12-NEXT: s_clause 0x5 ; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:80 ; GFX12-NEXT: global_store_b128 v24, v[4:7], s[8:9] offset:64 diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll index 5bc9357190c88..5734c81c043fd 100644 --- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll +++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll @@ -150,12 +150,11 @@ define i128 @v_lshr_i128_kv(i128 %rhs) { ; GCN-NEXT: s_mov_b64 s[4:5], 0x41 ; GCN-NEXT: v_lshr_b64 v[1:2], s[4:5], v0 ; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0 ; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-NEXT: v_mov_b32_e32 v3, 0x41 -; GCN-NEXT: s_and_b64 vcc, s[4:5], vcc -; GCN-NEXT: v_cndmask_b32_e64 v0, v3, v1, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc +; GCN-NEXT: v_mov_b32_e32 v2, 0x41 +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: v_mov_b32_e32 v2, 0 ; GCN-NEXT: v_mov_b32_e32 v3, 0 ; GCN-NEXT: s_setpc_b64 s[30:31] @@ -169,11 +168,10 @@ define i128 @v_ashr_i128_kv(i128 %rhs) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_lshr_b64 v[1:2], 33, v0 ; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0 -; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0 ; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; GCN-NEXT: s_and_b64 vcc, s[4:5], vcc -; GCN-NEXT: v_cndmask_b32_e64 v0, 33, v1, s[4:5] -; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc +; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, 33, v1, vcc +; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: v_mov_b32_e32 v2, 0 ; GCN-NEXT: v_mov_b32_e32 v3, 0 ; GCN-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll index 5914253b5f58e..8e24e581c7ea4 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll @@ -985,9 +985,8 @@ define void @v_shuffle_v3bf16_v2bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -998,9 +997,8 @@ define void @v_shuffle_v3bf16_v2bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1011,9 +1009,8 @@ define void @v_shuffle_v3bf16_v2bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x bfloat> asm "; def $0", "=v"() @@ -1030,9 +1027,8 @@ define void @v_shuffle_v3bf16_v2bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1043,9 +1039,8 @@ define void @v_shuffle_v3bf16_v2bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1056,9 +1051,8 @@ define void @v_shuffle_v3bf16_v2bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x bfloat> asm "; def $0", "=v"() @@ -1126,9 +1120,8 @@ define void @v_shuffle_v3bf16_v2bf16__2_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1139,9 +1132,8 @@ define void @v_shuffle_v3bf16_v2bf16__2_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1152,9 +1144,8 @@ define void @v_shuffle_v3bf16_v2bf16__2_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x bfloat> asm "; def $0", "=v"() @@ -1713,9 +1704,8 @@ define void @v_shuffle_v3bf16_v2bf16__u_3_3(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1726,9 +1716,8 @@ define void @v_shuffle_v3bf16_v2bf16__u_3_3(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1739,9 +1728,8 @@ define void @v_shuffle_v3bf16_v2bf16__u_3_3(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x bfloat> asm "; def $0", "=v"() @@ -1882,9 +1870,8 @@ define void @v_shuffle_v3bf16_v2bf16__2_3_3(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1895,9 +1882,8 @@ define void @v_shuffle_v3bf16_v2bf16__2_3_3(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1908,9 +1894,8 @@ define void @v_shuffle_v3bf16_v2bf16__2_3_3(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x bfloat> asm "; def $0", "=v"() diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll index cd4dbe93e8a11..771c233df116a 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll @@ -1562,9 +1562,8 @@ define void @v_shuffle_v3bf16_v3bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1575,9 +1574,8 @@ define void @v_shuffle_v3bf16_v3bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1588,9 +1586,8 @@ define void @v_shuffle_v3bf16_v3bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -1608,9 +1605,8 @@ define void @v_shuffle_v3bf16_v3bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1621,9 +1617,8 @@ define void @v_shuffle_v3bf16_v3bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1634,9 +1629,8 @@ define void @v_shuffle_v3bf16_v3bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -1758,9 +1752,8 @@ define void @v_shuffle_v3bf16_v3bf16__3_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1771,9 +1764,8 @@ define void @v_shuffle_v3bf16_v3bf16__3_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1784,9 +1776,8 @@ define void @v_shuffle_v3bf16_v3bf16__3_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -3418,9 +3409,8 @@ define void @v_shuffle_v3bf16_v3bf16__u_4_4(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -3431,9 +3421,8 @@ define void @v_shuffle_v3bf16_v3bf16__u_4_4(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -3444,9 +3433,8 @@ define void @v_shuffle_v3bf16_v3bf16__u_4_4(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -3658,9 +3646,8 @@ define void @v_shuffle_v3bf16_v3bf16__3_4_4(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -3671,9 +3658,8 @@ define void @v_shuffle_v3bf16_v3bf16__3_4_4(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -3684,9 +3670,8 @@ define void @v_shuffle_v3bf16_v3bf16__3_4_4(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v4bf16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v4bf16.ll index 311ca98227da3..e9859ec1e5119 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v4bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v4bf16.ll @@ -2008,9 +2008,8 @@ define void @v_shuffle_v3bf16_v4bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -2021,9 +2020,8 @@ define void @v_shuffle_v3bf16_v4bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -2034,9 +2032,8 @@ define void @v_shuffle_v3bf16_v4bf16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -2053,9 +2050,8 @@ define void @v_shuffle_v3bf16_v4bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -2066,9 +2062,8 @@ define void @v_shuffle_v3bf16_v4bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -2079,9 +2074,8 @@ define void @v_shuffle_v3bf16_v4bf16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -2251,9 +2245,8 @@ define void @v_shuffle_v3bf16_v4bf16__4_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -2264,9 +2257,8 @@ define void @v_shuffle_v3bf16_v4bf16__4_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -2277,9 +2269,8 @@ define void @v_shuffle_v3bf16_v4bf16__4_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -5314,9 +5305,8 @@ define void @v_shuffle_v3bf16_v4bf16__u_5_5(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -5327,9 +5317,8 @@ define void @v_shuffle_v3bf16_v4bf16__u_5_5(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -5340,9 +5329,8 @@ define void @v_shuffle_v3bf16_v4bf16__u_5_5(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() @@ -5608,9 +5596,8 @@ define void @v_shuffle_v3bf16_v4bf16__4_5_5(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -5621,9 +5608,8 @@ define void @v_shuffle_v3bf16_v4bf16__4_5_5(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -5634,9 +5620,8 @@ define void @v_shuffle_v3bf16_v4bf16__4_5_5(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x bfloat> asm "; def $0", "=v"() diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll index 99cb8a38f57c3..3398894e478dd 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll @@ -985,9 +985,8 @@ define void @v_shuffle_v3f16_v2f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -998,9 +997,8 @@ define void @v_shuffle_v3f16_v2f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1011,9 +1009,8 @@ define void @v_shuffle_v3f16_v2f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x half> asm "; def $0", "=v"() @@ -1030,9 +1027,8 @@ define void @v_shuffle_v3f16_v2f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1043,9 +1039,8 @@ define void @v_shuffle_v3f16_v2f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1056,9 +1051,8 @@ define void @v_shuffle_v3f16_v2f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x half> asm "; def $0", "=v"() @@ -1126,9 +1120,8 @@ define void @v_shuffle_v3f16_v2f16__2_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1139,9 +1132,8 @@ define void @v_shuffle_v3f16_v2f16__2_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1152,9 +1144,8 @@ define void @v_shuffle_v3f16_v2f16__2_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x half> asm "; def $0", "=v"() @@ -1713,9 +1704,8 @@ define void @v_shuffle_v3f16_v2f16__u_3_3(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1726,9 +1716,8 @@ define void @v_shuffle_v3f16_v2f16__u_3_3(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1739,9 +1728,8 @@ define void @v_shuffle_v3f16_v2f16__u_3_3(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x half> asm "; def $0", "=v"() @@ -1882,9 +1870,8 @@ define void @v_shuffle_v3f16_v2f16__2_3_3(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v1 ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v0, v1, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX900-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1895,9 +1882,8 @@ define void @v_shuffle_v3f16_v2f16__2_3_3(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v1 ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v0, v1, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX90A-NEXT: global_store_short v0, v1, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1908,9 +1894,8 @@ define void @v_shuffle_v3f16_v2f16__2_3_3(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v1 ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v0, v1, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v0, v1, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX942-NEXT: global_store_short v0, v1, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <2 x half> asm "; def $0", "=v"() diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll index 0854ff2ebfc5d..b8115490be536 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll @@ -1562,9 +1562,8 @@ define void @v_shuffle_v3f16_v3f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1575,9 +1574,8 @@ define void @v_shuffle_v3f16_v3f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1588,9 +1586,8 @@ define void @v_shuffle_v3f16_v3f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -1608,9 +1605,8 @@ define void @v_shuffle_v3f16_v3f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1621,9 +1617,8 @@ define void @v_shuffle_v3f16_v3f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1634,9 +1629,8 @@ define void @v_shuffle_v3f16_v3f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -1758,9 +1752,8 @@ define void @v_shuffle_v3f16_v3f16__3_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -1771,9 +1764,8 @@ define void @v_shuffle_v3f16_v3f16__3_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -1784,9 +1776,8 @@ define void @v_shuffle_v3f16_v3f16__3_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -3418,9 +3409,8 @@ define void @v_shuffle_v3f16_v3f16__u_4_4(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -3431,9 +3421,8 @@ define void @v_shuffle_v3f16_v3f16__u_4_4(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -3444,9 +3433,8 @@ define void @v_shuffle_v3f16_v3f16__u_4_4(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -3658,9 +3646,8 @@ define void @v_shuffle_v3f16_v3f16__3_4_4(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -3671,9 +3658,8 @@ define void @v_shuffle_v3f16_v3f16__3_4_4(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -3684,9 +3670,8 @@ define void @v_shuffle_v3f16_v3f16__3_4_4(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v4f16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v4f16.ll index ecc7ff618932b..3b262a65b9f82 100644 --- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v4f16.ll +++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v4f16.ll @@ -2008,9 +2008,8 @@ define void @v_shuffle_v3f16_v4f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -2021,9 +2020,8 @@ define void @v_shuffle_v3f16_v4f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -2034,9 +2032,8 @@ define void @v_shuffle_v3f16_v4f16__u_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -2053,9 +2050,8 @@ define void @v_shuffle_v3f16_v4f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -2066,9 +2062,8 @@ define void @v_shuffle_v3f16_v4f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -2079,9 +2074,8 @@ define void @v_shuffle_v3f16_v4f16__0_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -2251,9 +2245,8 @@ define void @v_shuffle_v3f16_v4f16__4_1_1(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -2264,9 +2257,8 @@ define void @v_shuffle_v3f16_v4f16__4_1_1(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -2277,9 +2269,8 @@ define void @v_shuffle_v3f16_v4f16__4_1_1(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -5314,9 +5305,8 @@ define void @v_shuffle_v3f16_v4f16__u_5_5(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -5327,9 +5317,8 @@ define void @v_shuffle_v3f16_v4f16__u_5_5(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -5340,9 +5329,8 @@ define void @v_shuffle_v3f16_v4f16__u_5_5(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() @@ -5608,9 +5596,8 @@ define void @v_shuffle_v3f16_v4f16__4_5_5(ptr addrspace(1) inreg %ptr) { ; GFX900-NEXT: ;;#ASMSTART ; GFX900-NEXT: ; def v[0:1] ; GFX900-NEXT: ;;#ASMEND +; GFX900-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX900-NEXT: global_store_dword v2, v0, s[16:17] -; GFX900-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX900-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_setpc_b64 s[30:31] ; @@ -5621,9 +5608,8 @@ define void @v_shuffle_v3f16_v4f16__4_5_5(ptr addrspace(1) inreg %ptr) { ; GFX90A-NEXT: ;;#ASMSTART ; GFX90A-NEXT: ; def v[0:1] ; GFX90A-NEXT: ;;#ASMEND +; GFX90A-NEXT: global_store_short_d16_hi v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: global_store_dword v2, v0, s[16:17] -; GFX90A-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX90A-NEXT: global_store_short v2, v0, s[16:17] offset:4 ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_setpc_b64 s[30:31] ; @@ -5634,9 +5620,8 @@ define void @v_shuffle_v3f16_v4f16__4_5_5(ptr addrspace(1) inreg %ptr) { ; GFX942-NEXT: ;;#ASMSTART ; GFX942-NEXT: ; def v[0:1] ; GFX942-NEXT: ;;#ASMEND +; GFX942-NEXT: global_store_short_d16_hi v2, v0, s[0:1] offset:4 ; GFX942-NEXT: global_store_dword v2, v0, s[0:1] -; GFX942-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX942-NEXT: global_store_short v2, v0, s[0:1] offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] %vec0 = call <4 x half> asm "; def $0", "=v"() diff --git a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll index 8c3f2880f22a2..4d091c2302658 100644 --- a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll @@ -26,7 +26,7 @@ define <2 x i32> @stest_f64i32(<2 x double> %x) { ; CHECK-NEXT: vmov.32 d9[1], r5 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: mov r5, #0 -; CHECK-NEXT: mvn r12, #0 +; CHECK-NEXT: mvn r4, #0 ; CHECK-NEXT: movwlt r5, #1 ; CHECK-NEXT: cmp r5, #0 ; CHECK-NEXT: mvnne r5, #0 @@ -34,6 +34,7 @@ define <2 x i32> @stest_f64i32(<2 x double> %x) { ; CHECK-NEXT: sbcs r0, r1, #0 ; CHECK-NEXT: vmov.32 d8[1], r1 ; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: vmov.i32 q10, #0x80000000 ; CHECK-NEXT: movwlt r0, #1 ; CHECK-NEXT: cmp r0, #0 ; CHECK-NEXT: vdup.32 d19, r5 @@ -42,24 +43,22 @@ define <2 x i32> @stest_f64i32(<2 x double> %x) { ; CHECK-NEXT: mov r2, #0 ; CHECK-NEXT: vdup.32 d18, r0 ; CHECK-NEXT: vbit q8, q4, q9 -; CHECK-NEXT: adr r4, .LCPI0_1 -; CHECK-NEXT: vld1.64 {d18, d19}, [r4:128] ; CHECK-NEXT: vmov r0, r1, d17 ; CHECK-NEXT: vmov r3, r5, d16 ; CHECK-NEXT: rsbs r0, r0, #-2147483648 -; CHECK-NEXT: sbcs r0, r12, r1 +; CHECK-NEXT: sbcs r0, r4, r1 ; CHECK-NEXT: mov r0, #0 ; CHECK-NEXT: movwlt r0, #1 ; CHECK-NEXT: cmp r0, #0 ; CHECK-NEXT: mvnne r0, #0 ; CHECK-NEXT: rsbs r1, r3, #-2147483648 -; CHECK-NEXT: sbcs r1, r12, r5 -; CHECK-NEXT: vdup.32 d21, r0 +; CHECK-NEXT: sbcs r1, r4, r5 +; CHECK-NEXT: vdup.32 d19, r0 ; CHECK-NEXT: movwlt r2, #1 ; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: mvnne r2, #0 -; CHECK-NEXT: vdup.32 d20, r2 -; CHECK-NEXT: vbif q8, q9, q10 +; CHECK-NEXT: vdup.32 d18, r2 +; CHECK-NEXT: vbif q8, q10, q9 ; CHECK-NEXT: vmovn.i64 d0, q8 ; CHECK-NEXT: vpop {d8, d9} ; CHECK-NEXT: pop {r4, r5, r11, pc} @@ -70,11 +69,6 @@ define <2 x i32> @stest_f64i32(<2 x double> %x) { ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 2147483647 @ 0x7fffffff ; CHECK-NEXT: .long 0 @ 0x0 -; CHECK-NEXT: .LCPI0_1: -; CHECK-NEXT: .long 2147483648 @ 0x80000000 -; CHECK-NEXT: .long 4294967295 @ 0xffffffff -; CHECK-NEXT: .long 2147483648 @ 0x80000000 -; CHECK-NEXT: .long 4294967295 @ 0xffffffff entry: %conv = fptosi <2 x double> %x to <2 x i64> %0 = icmp slt <2 x i64> %conv, @@ -100,22 +94,20 @@ define <2 x i32> @utest_f64i32(<2 x double> %x) { ; CHECK-NEXT: vmov r0, r1, d8 ; CHECK-NEXT: vmov.32 d9[0], r4 ; CHECK-NEXT: bl __aeabi_d2ulz -; CHECK-NEXT: vmov.32 d8[0], r0 ; CHECK-NEXT: mvn r3, #0 ; CHECK-NEXT: subs r4, r4, r3 -; CHECK-NEXT: mov r2, #0 -; CHECK-NEXT: vmov.32 d9[1], r5 ; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: vmov.32 d8[0], r0 ; CHECK-NEXT: mov r5, #0 +; CHECK-NEXT: mov r2, #0 ; CHECK-NEXT: movwlo r5, #1 ; CHECK-NEXT: cmp r5, #0 ; CHECK-NEXT: mvnne r5, #0 ; CHECK-NEXT: subs r0, r0, r3 ; CHECK-NEXT: sbcs r0, r1, #0 -; CHECK-NEXT: vmov.32 d8[1], r1 +; CHECK-NEXT: vdup.32 d17, r5 ; CHECK-NEXT: movwlo r2, #1 ; CHECK-NEXT: cmp r2, #0 -; CHECK-NEXT: vdup.32 d17, r5 ; CHECK-NEXT: mvnne r2, #0 ; CHECK-NEXT: vdup.32 d16, r2 ; CHECK-NEXT: vand q9, q4, q8 @@ -177,11 +169,11 @@ define <2 x i32> @ustest_f64i32(<2 x double> %x) { ; CHECK-NEXT: mvnne r0, #0 ; CHECK-NEXT: rsbs r1, r3, #0 ; CHECK-NEXT: rscs r1, r5, #0 -; CHECK-NEXT: vdup.32 d19, r0 +; CHECK-NEXT: vmov.32 d19[0], r0 ; CHECK-NEXT: movwlt r2, #1 ; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: mvnne r2, #0 -; CHECK-NEXT: vdup.32 d18, r2 +; CHECK-NEXT: vmov.32 d18[0], r2 ; CHECK-NEXT: vand q8, q8, q9 ; CHECK-NEXT: vmovn.i64 d0, q8 ; CHECK-NEXT: vpop {d8, d9} @@ -332,61 +324,57 @@ define <4 x i32> @utest_f32i32(<4 x float> %x) { ; CHECK-NEXT: .vsave {d8, d9, d10, d11} ; CHECK-NEXT: vpush {d8, d9, d10, d11} ; CHECK-NEXT: vorr q4, q0, q0 -; CHECK-NEXT: vmov r0, s19 +; CHECK-NEXT: vmov r0, s17 ; CHECK-NEXT: bl __aeabi_f2ulz -; CHECK-NEXT: mov r8, r0 +; CHECK-NEXT: mov r10, r0 ; CHECK-NEXT: vmov r0, s16 -; CHECK-NEXT: mov r9, r1 -; CHECK-NEXT: vmov r4, s17 -; CHECK-NEXT: vmov r6, s18 -; CHECK-NEXT: vmov.32 d9[0], r8 +; CHECK-NEXT: mov r8, r1 +; CHECK-NEXT: vmov r5, s19 +; CHECK-NEXT: vmov r7, s18 +; CHECK-NEXT: vmov.32 d9[0], r10 ; CHECK-NEXT: bl __aeabi_f2ulz -; CHECK-NEXT: mov r10, r0 -; CHECK-NEXT: vmov.32 d10[0], r0 -; CHECK-NEXT: mov r0, r4 -; CHECK-NEXT: mov r7, r1 +; CHECK-NEXT: mov r6, r0 +; CHECK-NEXT: vmov.32 d8[0], r0 +; CHECK-NEXT: mov r0, r5 +; CHECK-NEXT: mov r9, r1 ; CHECK-NEXT: bl __aeabi_f2ulz -; CHECK-NEXT: mov r4, r0 +; CHECK-NEXT: mov r5, r0 ; CHECK-NEXT: vmov.32 d11[0], r0 -; CHECK-NEXT: mov r0, r6 -; CHECK-NEXT: mov r5, r1 +; CHECK-NEXT: mov r0, r7 +; CHECK-NEXT: mov r4, r1 ; CHECK-NEXT: bl __aeabi_f2ulz -; CHECK-NEXT: vmov.32 d8[0], r0 ; CHECK-NEXT: mvn r3, #0 +; CHECK-NEXT: vmov.32 d10[0], r0 ; CHECK-NEXT: subs r0, r0, r3 ; CHECK-NEXT: mov r2, #0 ; CHECK-NEXT: sbcs r0, r1, #0 -; CHECK-NEXT: vmov.32 d9[1], r9 ; CHECK-NEXT: mov r0, #0 ; CHECK-NEXT: movwlo r0, #1 ; CHECK-NEXT: cmp r0, #0 -; CHECK-NEXT: vmov.32 d8[1], r1 ; CHECK-NEXT: mvnne r0, #0 -; CHECK-NEXT: subs r1, r8, r3 -; CHECK-NEXT: sbcs r1, r9, #0 -; CHECK-NEXT: vmov.32 d11[1], r5 +; CHECK-NEXT: subs r1, r5, r3 +; CHECK-NEXT: sbcs r1, r4, #0 ; CHECK-NEXT: mov r1, #0 ; CHECK-NEXT: movwlo r1, #1 ; CHECK-NEXT: cmp r1, #0 ; CHECK-NEXT: mvnne r1, #0 -; CHECK-NEXT: subs r6, r4, r3 -; CHECK-NEXT: sbcs r6, r5, #0 +; CHECK-NEXT: subs r7, r10, r3 +; CHECK-NEXT: sbcs r7, r8, #0 ; CHECK-NEXT: vdup.32 d19, r1 -; CHECK-NEXT: mov r6, #0 +; CHECK-NEXT: mov r7, #0 ; CHECK-NEXT: vdup.32 d18, r0 -; CHECK-NEXT: movwlo r6, #1 -; CHECK-NEXT: cmp r6, #0 -; CHECK-NEXT: mvnne r6, #0 -; CHECK-NEXT: subs r3, r10, r3 -; CHECK-NEXT: sbcs r3, r7, #0 -; CHECK-NEXT: vmov.32 d10[1], r7 +; CHECK-NEXT: movwlo r7, #1 +; CHECK-NEXT: cmp r7, #0 +; CHECK-NEXT: mvnne r7, #0 +; CHECK-NEXT: subs r3, r6, r3 +; CHECK-NEXT: sbcs r3, r9, #0 +; CHECK-NEXT: vdup.32 d17, r7 ; CHECK-NEXT: movwlo r2, #1 ; CHECK-NEXT: cmp r2, #0 -; CHECK-NEXT: vdup.32 d17, r6 ; CHECK-NEXT: mvnne r2, #0 -; CHECK-NEXT: vand q10, q4, q9 +; CHECK-NEXT: vand q10, q5, q9 ; CHECK-NEXT: vdup.32 d16, r2 -; CHECK-NEXT: vand q11, q5, q8 +; CHECK-NEXT: vand q11, q4, q8 ; CHECK-NEXT: vorn q9, q10, q9 ; CHECK-NEXT: vorn q8, q11, q8 ; CHECK-NEXT: vmovn.i64 d1, q9 @@ -409,46 +397,45 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) { ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: vorr q4, q0, q0 -; CHECK-NEXT: vmov r0, s18 -; CHECK-NEXT: bl __aeabi_f2lz -; CHECK-NEXT: mov r5, r0 ; CHECK-NEXT: vmov r0, s19 -; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: bl __aeabi_f2lz -; CHECK-NEXT: mov r2, r0 -; CHECK-NEXT: vmov r0, s17 -; CHECK-NEXT: vmov.32 d17[0], r2 -; CHECK-NEXT: mvn r4, #0 -; CHECK-NEXT: subs r2, r2, r4 +; CHECK-NEXT: mov r6, r0 +; CHECK-NEXT: vmov r0, s18 +; CHECK-NEXT: mov r7, r1 +; CHECK-NEXT: vmov r5, s17 ; CHECK-NEXT: vmov r8, s16 -; CHECK-NEXT: vmov.32 d16[0], r5 -; CHECK-NEXT: vmov.i64 q5, #0xffffffff -; CHECK-NEXT: mov r7, #0 -; CHECK-NEXT: vmov.32 d17[1], r1 -; CHECK-NEXT: sbcs r1, r1, #0 -; CHECK-NEXT: mov r1, #0 -; CHECK-NEXT: movwlt r1, #1 -; CHECK-NEXT: cmp r1, #0 -; CHECK-NEXT: mvnne r1, #0 -; CHECK-NEXT: subs r2, r5, r4 -; CHECK-NEXT: sbcs r2, r6, #0 -; CHECK-NEXT: vdup.32 d19, r1 +; CHECK-NEXT: vmov.32 d9[0], r6 +; CHECK-NEXT: bl __aeabi_f2lz +; CHECK-NEXT: mvn r9, #0 +; CHECK-NEXT: subs r2, r6, r9 +; CHECK-NEXT: sbcs r2, r7, #0 +; CHECK-NEXT: vmov.32 d8[0], r0 ; CHECK-NEXT: mov r2, #0 -; CHECK-NEXT: vmov.32 d16[1], r6 +; CHECK-NEXT: vmov.i64 q5, #0xffffffff ; CHECK-NEXT: movwlt r2, #1 ; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: mvnne r2, #0 -; CHECK-NEXT: vdup.32 d18, r2 -; CHECK-NEXT: vorr q4, q9, q9 -; CHECK-NEXT: vbsl q4, q8, q5 -; CHECK-NEXT: vmov r10, r9, d8 +; CHECK-NEXT: subs r0, r0, r9 +; CHECK-NEXT: sbcs r0, r1, #0 +; CHECK-NEXT: vmov.32 d9[1], r7 +; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: mov r4, #0 +; CHECK-NEXT: movwlt r0, #1 +; CHECK-NEXT: cmp r0, #0 +; CHECK-NEXT: vmov.32 d8[1], r1 +; CHECK-NEXT: mvnne r0, #0 +; CHECK-NEXT: vdup.32 d17, r2 +; CHECK-NEXT: vdup.32 d16, r0 +; CHECK-NEXT: mov r0, r5 +; CHECK-NEXT: vbif q4, q5, q8 ; CHECK-NEXT: bl __aeabi_f2lz ; CHECK-NEXT: mov r5, r0 ; CHECK-NEXT: vmov.32 d13[0], r0 ; CHECK-NEXT: mov r0, r8 ; CHECK-NEXT: mov r6, r1 +; CHECK-NEXT: vmov r7, r10, d8 ; CHECK-NEXT: bl __aeabi_f2lz -; CHECK-NEXT: subs r2, r5, r4 +; CHECK-NEXT: subs r2, r5, r9 ; CHECK-NEXT: vmov.32 d12[0], r0 ; CHECK-NEXT: sbcs r2, r6, #0 ; CHECK-NEXT: mov r2, #0 @@ -456,25 +443,25 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) { ; CHECK-NEXT: movwlt r2, #1 ; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: mvnne r2, #0 -; CHECK-NEXT: subs r0, r0, r4 +; CHECK-NEXT: subs r0, r0, r9 ; CHECK-NEXT: sbcs r0, r1, #0 -; CHECK-NEXT: vmov.32 d12[1], r1 +; CHECK-NEXT: vdup.32 d17, r2 ; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: vmov.32 d12[1], r1 ; CHECK-NEXT: movwlt r0, #1 ; CHECK-NEXT: cmp r0, #0 ; CHECK-NEXT: mvnne r0, #0 -; CHECK-NEXT: vdup.32 d17, r2 +; CHECK-NEXT: vmov r2, r3, d9 ; CHECK-NEXT: vdup.32 d16, r0 -; CHECK-NEXT: vmov r0, r1, d9 +; CHECK-NEXT: rsbs r7, r7, #0 ; CHECK-NEXT: vbsl q8, q6, q5 -; CHECK-NEXT: rsbs r6, r10, #0 -; CHECK-NEXT: rscs r6, r9, #0 -; CHECK-NEXT: mov r6, #0 -; CHECK-NEXT: vmov r2, r3, d17 -; CHECK-NEXT: movwlt r6, #1 -; CHECK-NEXT: vmov r5, r4, d16 -; CHECK-NEXT: cmp r6, #0 -; CHECK-NEXT: mvnne r6, #0 +; CHECK-NEXT: rscs r7, r10, #0 +; CHECK-NEXT: mov r7, #0 +; CHECK-NEXT: movwlt r7, #1 +; CHECK-NEXT: cmp r7, #0 +; CHECK-NEXT: vmov r0, r1, d17 +; CHECK-NEXT: mvnne r7, #0 +; CHECK-NEXT: vmov r6, r5, d16 ; CHECK-NEXT: rsbs r0, r0, #0 ; CHECK-NEXT: rscs r0, r1, #0 ; CHECK-NEXT: mov r0, #0 @@ -483,20 +470,20 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) { ; CHECK-NEXT: mvnne r0, #0 ; CHECK-NEXT: rsbs r1, r2, #0 ; CHECK-NEXT: rscs r1, r3, #0 -; CHECK-NEXT: vdup.32 d21, r0 +; CHECK-NEXT: vmov.32 d19[0], r0 ; CHECK-NEXT: mov r1, #0 -; CHECK-NEXT: vdup.32 d20, r6 ; CHECK-NEXT: movwlt r1, #1 ; CHECK-NEXT: cmp r1, #0 ; CHECK-NEXT: mvnne r1, #0 -; CHECK-NEXT: rsbs r2, r5, #0 -; CHECK-NEXT: rscs r2, r4, #0 -; CHECK-NEXT: vdup.32 d19, r1 -; CHECK-NEXT: movwlt r7, #1 -; CHECK-NEXT: cmp r7, #0 -; CHECK-NEXT: mvnne r7, #0 +; CHECK-NEXT: rsbs r0, r6, #0 +; CHECK-NEXT: rscs r0, r5, #0 +; CHECK-NEXT: vmov.32 d21[0], r1 +; CHECK-NEXT: movwlt r4, #1 +; CHECK-NEXT: cmp r4, #0 +; CHECK-NEXT: vmov.32 d20[0], r7 +; CHECK-NEXT: mvnne r4, #0 +; CHECK-NEXT: vmov.32 d18[0], r4 ; CHECK-NEXT: vand q10, q4, q10 -; CHECK-NEXT: vdup.32 d18, r7 ; CHECK-NEXT: vand q8, q8, q9 ; CHECK-NEXT: vmovn.i64 d1, q10 ; CHECK-NEXT: vmovn.i64 d0, q8 @@ -764,151 +751,141 @@ entry: define <4 x i32> @utesth_f16i32(<4 x half> %x) { ; CHECK-NEON-LABEL: utesth_f16i32: ; CHECK-NEON: @ %bb.0: @ %entry -; CHECK-NEON-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr} -; CHECK-NEON-NEXT: push {r4, r5, r6, r7, r8, r9, r10, lr} -; CHECK-NEON-NEXT: .vsave {d8, d9, d10, d11} -; CHECK-NEON-NEXT: vpush {d8, d9, d10, d11} +; CHECK-NEON-NEXT: .save {r4, r5, r6, r7, r8, r9, r11, lr} +; CHECK-NEON-NEXT: push {r4, r5, r6, r7, r8, r9, r11, lr} +; CHECK-NEON-NEXT: .vsave {d12, d13} +; CHECK-NEON-NEXT: vpush {d12, d13} +; CHECK-NEON-NEXT: .vsave {d8, d9, d10} +; CHECK-NEON-NEXT: vpush {d8, d9, d10} ; CHECK-NEON-NEXT: vmov r0, s3 ; CHECK-NEON-NEXT: vmov.f32 s16, s2 ; CHECK-NEON-NEXT: vmov.f32 s18, s1 ; CHECK-NEON-NEXT: vmov.f32 s20, s0 ; CHECK-NEON-NEXT: bl __aeabi_h2f ; CHECK-NEON-NEXT: bl __aeabi_f2ulz -; CHECK-NEON-NEXT: mov r8, r0 +; CHECK-NEON-NEXT: mov r4, r0 +; CHECK-NEON-NEXT: vmov r0, s18 +; CHECK-NEON-NEXT: mov r8, r1 +; CHECK-NEON-NEXT: bl __aeabi_h2f +; CHECK-NEON-NEXT: bl __aeabi_f2ulz +; CHECK-NEON-NEXT: mov r6, r0 +; CHECK-NEON-NEXT: vmov.32 d13[0], r0 ; CHECK-NEON-NEXT: vmov r0, s20 ; CHECK-NEON-NEXT: mov r9, r1 -; CHECK-NEON-NEXT: vmov r4, s18 -; CHECK-NEON-NEXT: vmov r6, s16 -; CHECK-NEON-NEXT: vmov.32 d9[0], r8 ; CHECK-NEON-NEXT: bl __aeabi_h2f ; CHECK-NEON-NEXT: bl __aeabi_f2ulz -; CHECK-NEON-NEXT: mov r10, r0 -; CHECK-NEON-NEXT: vmov.32 d10[0], r0 -; CHECK-NEON-NEXT: mov r0, r4 +; CHECK-NEON-NEXT: mov r5, r0 +; CHECK-NEON-NEXT: vmov.32 d12[0], r0 +; CHECK-NEON-NEXT: vmov r0, s16 ; CHECK-NEON-NEXT: mov r7, r1 ; CHECK-NEON-NEXT: bl __aeabi_h2f +; CHECK-NEON-NEXT: vmov.32 d9[0], r4 ; CHECK-NEON-NEXT: bl __aeabi_f2ulz -; CHECK-NEON-NEXT: mov r4, r0 -; CHECK-NEON-NEXT: vmov.32 d11[0], r0 -; CHECK-NEON-NEXT: mov r0, r6 -; CHECK-NEON-NEXT: mov r5, r1 -; CHECK-NEON-NEXT: bl __aeabi_h2f -; CHECK-NEON-NEXT: bl __aeabi_f2ulz -; CHECK-NEON-NEXT: vmov.32 d8[0], r0 ; CHECK-NEON-NEXT: mvn r3, #0 +; CHECK-NEON-NEXT: vmov.32 d8[0], r0 ; CHECK-NEON-NEXT: subs r0, r0, r3 ; CHECK-NEON-NEXT: mov r2, #0 ; CHECK-NEON-NEXT: sbcs r0, r1, #0 -; CHECK-NEON-NEXT: vmov.32 d9[1], r9 ; CHECK-NEON-NEXT: mov r0, #0 ; CHECK-NEON-NEXT: movwlo r0, #1 ; CHECK-NEON-NEXT: cmp r0, #0 -; CHECK-NEON-NEXT: vmov.32 d8[1], r1 ; CHECK-NEON-NEXT: mvnne r0, #0 -; CHECK-NEON-NEXT: subs r1, r8, r3 -; CHECK-NEON-NEXT: sbcs r1, r9, #0 -; CHECK-NEON-NEXT: vmov.32 d11[1], r5 +; CHECK-NEON-NEXT: subs r1, r4, r3 +; CHECK-NEON-NEXT: sbcs r1, r8, #0 ; CHECK-NEON-NEXT: mov r1, #0 ; CHECK-NEON-NEXT: movwlo r1, #1 ; CHECK-NEON-NEXT: cmp r1, #0 ; CHECK-NEON-NEXT: mvnne r1, #0 -; CHECK-NEON-NEXT: subs r6, r4, r3 -; CHECK-NEON-NEXT: sbcs r6, r5, #0 +; CHECK-NEON-NEXT: subs r6, r6, r3 +; CHECK-NEON-NEXT: sbcs r6, r9, #0 ; CHECK-NEON-NEXT: vdup.32 d19, r1 ; CHECK-NEON-NEXT: mov r6, #0 ; CHECK-NEON-NEXT: vdup.32 d18, r0 ; CHECK-NEON-NEXT: movwlo r6, #1 ; CHECK-NEON-NEXT: cmp r6, #0 ; CHECK-NEON-NEXT: mvnne r6, #0 -; CHECK-NEON-NEXT: subs r3, r10, r3 +; CHECK-NEON-NEXT: subs r3, r5, r3 ; CHECK-NEON-NEXT: sbcs r3, r7, #0 -; CHECK-NEON-NEXT: vmov.32 d10[1], r7 +; CHECK-NEON-NEXT: vdup.32 d17, r6 ; CHECK-NEON-NEXT: movwlo r2, #1 ; CHECK-NEON-NEXT: cmp r2, #0 -; CHECK-NEON-NEXT: vdup.32 d17, r6 ; CHECK-NEON-NEXT: mvnne r2, #0 ; CHECK-NEON-NEXT: vand q10, q4, q9 ; CHECK-NEON-NEXT: vdup.32 d16, r2 -; CHECK-NEON-NEXT: vand q11, q5, q8 +; CHECK-NEON-NEXT: vand q11, q6, q8 ; CHECK-NEON-NEXT: vorn q9, q10, q9 ; CHECK-NEON-NEXT: vorn q8, q11, q8 ; CHECK-NEON-NEXT: vmovn.i64 d1, q9 ; CHECK-NEON-NEXT: vmovn.i64 d0, q8 -; CHECK-NEON-NEXT: vpop {d8, d9, d10, d11} -; CHECK-NEON-NEXT: pop {r4, r5, r6, r7, r8, r9, r10, pc} +; CHECK-NEON-NEXT: vpop {d8, d9, d10} +; CHECK-NEON-NEXT: vpop {d12, d13} +; CHECK-NEON-NEXT: pop {r4, r5, r6, r7, r8, r9, r11, pc} ; ; CHECK-FP16-LABEL: utesth_f16i32: ; CHECK-FP16: @ %bb.0: @ %entry ; CHECK-FP16-NEXT: .save {r4, r5, r6, r7, r8, r9, r11, lr} ; CHECK-FP16-NEXT: push {r4, r5, r6, r7, r8, r9, r11, lr} -; CHECK-FP16-NEXT: .vsave {d10, d11, d12, d13} -; CHECK-FP16-NEXT: vpush {d10, d11, d12, d13} -; CHECK-FP16-NEXT: .vsave {d8} -; CHECK-FP16-NEXT: vpush {d8} -; CHECK-FP16-NEXT: vmov.u16 r0, d0[3] +; CHECK-FP16-NEXT: .vsave {d8, d9, d10, d11} +; CHECK-FP16-NEXT: vpush {d8, d9, d10, d11} +; CHECK-FP16-NEXT: vmov.u16 r0, d0[1] ; CHECK-FP16-NEXT: vorr d8, d0, d0 -; CHECK-FP16-NEXT: vmov.u16 r4, d0[1] +; CHECK-FP16-NEXT: vmov.u16 r5, d0[3] ; CHECK-FP16-NEXT: vmov s0, r0 ; CHECK-FP16-NEXT: bl __fixunshfdi -; CHECK-FP16-NEXT: mov r6, r0 +; CHECK-FP16-NEXT: mov r4, r0 ; CHECK-FP16-NEXT: vmov.u16 r0, d8[0] -; CHECK-FP16-NEXT: mov r7, r1 -; CHECK-FP16-NEXT: vmov.32 d11[0], r6 +; CHECK-FP16-NEXT: mov r8, r1 +; CHECK-FP16-NEXT: vmov.32 d11[0], r4 ; CHECK-FP16-NEXT: vmov s0, r0 ; CHECK-FP16-NEXT: bl __fixunshfdi -; CHECK-FP16-NEXT: vmov s0, r4 -; CHECK-FP16-NEXT: mov r8, r0 +; CHECK-FP16-NEXT: vmov s0, r5 +; CHECK-FP16-NEXT: mov r6, r0 ; CHECK-FP16-NEXT: mov r9, r1 -; CHECK-FP16-NEXT: vmov.32 d12[0], r0 +; CHECK-FP16-NEXT: vmov.32 d10[0], r0 ; CHECK-FP16-NEXT: bl __fixunshfdi -; CHECK-FP16-NEXT: mov r4, r0 +; CHECK-FP16-NEXT: mov r5, r0 ; CHECK-FP16-NEXT: vmov.u16 r0, d8[2] -; CHECK-FP16-NEXT: mov r5, r1 -; CHECK-FP16-NEXT: vmov.32 d13[0], r4 +; CHECK-FP16-NEXT: mov r7, r1 +; CHECK-FP16-NEXT: vmov.32 d9[0], r5 ; CHECK-FP16-NEXT: vmov s0, r0 ; CHECK-FP16-NEXT: bl __fixunshfdi -; CHECK-FP16-NEXT: vmov.32 d10[0], r0 ; CHECK-FP16-NEXT: mvn r3, #0 +; CHECK-FP16-NEXT: vmov.32 d8[0], r0 ; CHECK-FP16-NEXT: subs r0, r0, r3 ; CHECK-FP16-NEXT: mov r2, #0 ; CHECK-FP16-NEXT: sbcs r0, r1, #0 -; CHECK-FP16-NEXT: vmov.32 d11[1], r7 ; CHECK-FP16-NEXT: mov r0, #0 ; CHECK-FP16-NEXT: movwlo r0, #1 ; CHECK-FP16-NEXT: cmp r0, #0 -; CHECK-FP16-NEXT: vmov.32 d10[1], r1 ; CHECK-FP16-NEXT: mvnne r0, #0 -; CHECK-FP16-NEXT: subs r1, r6, r3 +; CHECK-FP16-NEXT: subs r1, r5, r3 ; CHECK-FP16-NEXT: sbcs r1, r7, #0 -; CHECK-FP16-NEXT: vmov.32 d13[1], r5 ; CHECK-FP16-NEXT: mov r1, #0 ; CHECK-FP16-NEXT: movwlo r1, #1 ; CHECK-FP16-NEXT: cmp r1, #0 ; CHECK-FP16-NEXT: mvnne r1, #0 ; CHECK-FP16-NEXT: subs r7, r4, r3 -; CHECK-FP16-NEXT: sbcs r7, r5, #0 +; CHECK-FP16-NEXT: sbcs r7, r8, #0 ; CHECK-FP16-NEXT: vdup.32 d19, r1 ; CHECK-FP16-NEXT: mov r7, #0 ; CHECK-FP16-NEXT: vdup.32 d18, r0 ; CHECK-FP16-NEXT: movwlo r7, #1 ; CHECK-FP16-NEXT: cmp r7, #0 ; CHECK-FP16-NEXT: mvnne r7, #0 -; CHECK-FP16-NEXT: subs r3, r8, r3 +; CHECK-FP16-NEXT: subs r3, r6, r3 ; CHECK-FP16-NEXT: sbcs r3, r9, #0 -; CHECK-FP16-NEXT: vmov.32 d12[1], r9 +; CHECK-FP16-NEXT: vdup.32 d17, r7 ; CHECK-FP16-NEXT: movwlo r2, #1 ; CHECK-FP16-NEXT: cmp r2, #0 -; CHECK-FP16-NEXT: vdup.32 d17, r7 ; CHECK-FP16-NEXT: mvnne r2, #0 -; CHECK-FP16-NEXT: vand q10, q5, q9 +; CHECK-FP16-NEXT: vand q10, q4, q9 ; CHECK-FP16-NEXT: vdup.32 d16, r2 -; CHECK-FP16-NEXT: vand q11, q6, q8 +; CHECK-FP16-NEXT: vand q11, q5, q8 ; CHECK-FP16-NEXT: vorn q9, q10, q9 ; CHECK-FP16-NEXT: vorn q8, q11, q8 ; CHECK-FP16-NEXT: vmovn.i64 d1, q9 ; CHECK-FP16-NEXT: vmovn.i64 d0, q8 -; CHECK-FP16-NEXT: vpop {d8} -; CHECK-FP16-NEXT: vpop {d10, d11, d12, d13} +; CHECK-FP16-NEXT: vpop {d8, d9, d10, d11} ; CHECK-FP16-NEXT: pop {r4, r5, r6, r7, r8, r9, r11, pc} entry: %conv = fptoui <4 x half> %x to <4 x i64> @@ -925,8 +902,8 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NEON-NEXT: push {r4, r5, r6, r7, r8, r9, r10, lr} ; CHECK-NEON-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-NEON-NEXT: vpush {d8, d9, d10, d11, d12, d13} -; CHECK-NEON-NEXT: vmov r0, s2 -; CHECK-NEON-NEXT: vmov.f32 s16, s3 +; CHECK-NEON-NEXT: vmov r0, s3 +; CHECK-NEON-NEXT: vmov.f32 s16, s2 ; CHECK-NEON-NEXT: vmov.f32 s18, s1 ; CHECK-NEON-NEXT: vmov.f32 s20, s0 ; CHECK-NEON-NEXT: bl __aeabi_h2f @@ -935,43 +912,42 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NEON-NEXT: vmov r0, s16 ; CHECK-NEON-NEXT: mov r6, r1 ; CHECK-NEON-NEXT: bl __aeabi_h2f +; CHECK-NEON-NEXT: vmov r8, s20 +; CHECK-NEON-NEXT: vmov.32 d13[0], r5 ; CHECK-NEON-NEXT: bl __aeabi_f2lz -; CHECK-NEON-NEXT: mov r2, r0 -; CHECK-NEON-NEXT: vmov r0, s18 -; CHECK-NEON-NEXT: vmov.32 d17[0], r2 -; CHECK-NEON-NEXT: mvn r8, #0 -; CHECK-NEON-NEXT: subs r2, r2, r8 -; CHECK-NEON-NEXT: vmov r4, s20 -; CHECK-NEON-NEXT: vmov.32 d16[0], r5 -; CHECK-NEON-NEXT: vmov.i64 q5, #0xffffffff +; CHECK-NEON-NEXT: vmov r2, s18 +; CHECK-NEON-NEXT: vmov.32 d12[0], r0 +; CHECK-NEON-NEXT: mvn r9, #0 +; CHECK-NEON-NEXT: subs r0, r0, r9 +; CHECK-NEON-NEXT: sbcs r0, r1, #0 +; CHECK-NEON-NEXT: vmov.32 d13[1], r6 +; CHECK-NEON-NEXT: mov r0, #0 ; CHECK-NEON-NEXT: mov r7, #0 -; CHECK-NEON-NEXT: vmov.32 d17[1], r1 -; CHECK-NEON-NEXT: sbcs r1, r1, #0 +; CHECK-NEON-NEXT: movwlt r0, #1 +; CHECK-NEON-NEXT: cmp r0, #0 +; CHECK-NEON-NEXT: vmov.32 d12[1], r1 +; CHECK-NEON-NEXT: mvnne r0, #0 +; CHECK-NEON-NEXT: subs r1, r5, r9 +; CHECK-NEON-NEXT: sbcs r1, r6, #0 ; CHECK-NEON-NEXT: mov r1, #0 ; CHECK-NEON-NEXT: movwlt r1, #1 ; CHECK-NEON-NEXT: cmp r1, #0 ; CHECK-NEON-NEXT: mvnne r1, #0 -; CHECK-NEON-NEXT: subs r2, r5, r8 -; CHECK-NEON-NEXT: sbcs r2, r6, #0 -; CHECK-NEON-NEXT: vdup.32 d19, r1 -; CHECK-NEON-NEXT: mov r2, #0 -; CHECK-NEON-NEXT: vmov.32 d16[1], r6 -; CHECK-NEON-NEXT: movwlt r2, #1 -; CHECK-NEON-NEXT: cmp r2, #0 -; CHECK-NEON-NEXT: mvnne r2, #0 -; CHECK-NEON-NEXT: vdup.32 d18, r2 -; CHECK-NEON-NEXT: vorr q4, q9, q9 -; CHECK-NEON-NEXT: vbsl q4, q8, q5 -; CHECK-NEON-NEXT: vmov r10, r9, d8 +; CHECK-NEON-NEXT: vdup.32 d9, r1 +; CHECK-NEON-NEXT: vdup.32 d8, r0 +; CHECK-NEON-NEXT: mov r0, r2 ; CHECK-NEON-NEXT: bl __aeabi_h2f +; CHECK-NEON-NEXT: vmov.i64 q5, #0xffffffff +; CHECK-NEON-NEXT: vbsl q4, q6, q5 ; CHECK-NEON-NEXT: bl __aeabi_f2lz ; CHECK-NEON-NEXT: mov r5, r0 ; CHECK-NEON-NEXT: vmov.32 d13[0], r0 -; CHECK-NEON-NEXT: mov r0, r4 +; CHECK-NEON-NEXT: mov r0, r8 ; CHECK-NEON-NEXT: mov r6, r1 +; CHECK-NEON-NEXT: vmov r4, r10, d8 ; CHECK-NEON-NEXT: bl __aeabi_h2f ; CHECK-NEON-NEXT: bl __aeabi_f2lz -; CHECK-NEON-NEXT: subs r2, r5, r8 +; CHECK-NEON-NEXT: subs r2, r5, r9 ; CHECK-NEON-NEXT: vmov.32 d12[0], r0 ; CHECK-NEON-NEXT: sbcs r2, r6, #0 ; CHECK-NEON-NEXT: mov r2, #0 @@ -979,25 +955,25 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NEON-NEXT: movwlt r2, #1 ; CHECK-NEON-NEXT: cmp r2, #0 ; CHECK-NEON-NEXT: mvnne r2, #0 -; CHECK-NEON-NEXT: subs r0, r0, r8 +; CHECK-NEON-NEXT: subs r0, r0, r9 ; CHECK-NEON-NEXT: sbcs r0, r1, #0 -; CHECK-NEON-NEXT: vmov.32 d12[1], r1 +; CHECK-NEON-NEXT: vdup.32 d17, r2 ; CHECK-NEON-NEXT: mov r0, #0 +; CHECK-NEON-NEXT: vmov.32 d12[1], r1 ; CHECK-NEON-NEXT: movwlt r0, #1 ; CHECK-NEON-NEXT: cmp r0, #0 ; CHECK-NEON-NEXT: mvnne r0, #0 -; CHECK-NEON-NEXT: vdup.32 d17, r2 +; CHECK-NEON-NEXT: vmov r2, r3, d9 ; CHECK-NEON-NEXT: vdup.32 d16, r0 -; CHECK-NEON-NEXT: vmov r0, r1, d9 +; CHECK-NEON-NEXT: rsbs r6, r4, #0 ; CHECK-NEON-NEXT: vbsl q8, q6, q5 -; CHECK-NEON-NEXT: rsbs r6, r10, #0 -; CHECK-NEON-NEXT: rscs r6, r9, #0 +; CHECK-NEON-NEXT: rscs r6, r10, #0 ; CHECK-NEON-NEXT: mov r6, #0 -; CHECK-NEON-NEXT: vmov r2, r3, d17 ; CHECK-NEON-NEXT: movwlt r6, #1 -; CHECK-NEON-NEXT: vmov r5, r4, d16 ; CHECK-NEON-NEXT: cmp r6, #0 +; CHECK-NEON-NEXT: vmov r0, r1, d17 ; CHECK-NEON-NEXT: mvnne r6, #0 +; CHECK-NEON-NEXT: vmov r5, r4, d16 ; CHECK-NEON-NEXT: rsbs r0, r0, #0 ; CHECK-NEON-NEXT: rscs r0, r1, #0 ; CHECK-NEON-NEXT: mov r0, #0 @@ -1006,20 +982,20 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NEON-NEXT: mvnne r0, #0 ; CHECK-NEON-NEXT: rsbs r1, r2, #0 ; CHECK-NEON-NEXT: rscs r1, r3, #0 -; CHECK-NEON-NEXT: vdup.32 d21, r0 +; CHECK-NEON-NEXT: vmov.32 d19[0], r0 ; CHECK-NEON-NEXT: mov r1, #0 -; CHECK-NEON-NEXT: vdup.32 d20, r6 ; CHECK-NEON-NEXT: movwlt r1, #1 ; CHECK-NEON-NEXT: cmp r1, #0 ; CHECK-NEON-NEXT: mvnne r1, #0 -; CHECK-NEON-NEXT: rsbs r2, r5, #0 -; CHECK-NEON-NEXT: rscs r2, r4, #0 -; CHECK-NEON-NEXT: vdup.32 d19, r1 +; CHECK-NEON-NEXT: rsbs r0, r5, #0 +; CHECK-NEON-NEXT: rscs r0, r4, #0 +; CHECK-NEON-NEXT: vmov.32 d21[0], r1 ; CHECK-NEON-NEXT: movwlt r7, #1 ; CHECK-NEON-NEXT: cmp r7, #0 +; CHECK-NEON-NEXT: vmov.32 d20[0], r6 ; CHECK-NEON-NEXT: mvnne r7, #0 +; CHECK-NEON-NEXT: vmov.32 d18[0], r7 ; CHECK-NEON-NEXT: vand q10, q4, q10 -; CHECK-NEON-NEXT: vdup.32 d18, r7 ; CHECK-NEON-NEXT: vand q8, q8, q9 ; CHECK-NEON-NEXT: vmovn.i64 d1, q10 ; CHECK-NEON-NEXT: vmovn.i64 d0, q8 @@ -1028,78 +1004,78 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; ; CHECK-FP16-LABEL: ustest_f16i32: ; CHECK-FP16: @ %bb.0: @ %entry -; CHECK-FP16-NEXT: .save {r4, r5, r6, r7, r8, r9, r11, lr} -; CHECK-FP16-NEXT: push {r4, r5, r6, r7, r8, r9, r11, lr} +; CHECK-FP16-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr} +; CHECK-FP16-NEXT: push {r4, r5, r6, r7, r8, r9, r10, lr} ; CHECK-FP16-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-FP16-NEXT: vpush {d8, d9, d10, d11, d12, d13} ; CHECK-FP16-NEXT: vmov.u16 r0, d0[3] ; CHECK-FP16-NEXT: vorr d8, d0, d0 -; CHECK-FP16-NEXT: vmov.u16 r8, d0[1] +; CHECK-FP16-NEXT: vmov.u16 r8, d0[0] +; CHECK-FP16-NEXT: vmov.u16 r9, d0[1] ; CHECK-FP16-NEXT: vmov s0, r0 ; CHECK-FP16-NEXT: bl __fixhfdi ; CHECK-FP16-NEXT: mov r4, r0 ; CHECK-FP16-NEXT: vmov.u16 r0, d8[2] ; CHECK-FP16-NEXT: mov r5, r1 -; CHECK-FP16-NEXT: vmov.32 d11[0], r4 +; CHECK-FP16-NEXT: vmov.32 d9[0], r4 ; CHECK-FP16-NEXT: vmov s0, r0 ; CHECK-FP16-NEXT: bl __fixhfdi -; CHECK-FP16-NEXT: mvn r7, #0 -; CHECK-FP16-NEXT: subs r2, r4, r7 +; CHECK-FP16-NEXT: mvn r10, #0 +; CHECK-FP16-NEXT: subs r2, r4, r10 ; CHECK-FP16-NEXT: sbcs r2, r5, #0 -; CHECK-FP16-NEXT: vmov.32 d10[0], r0 +; CHECK-FP16-NEXT: vmov.32 d8[0], r0 ; CHECK-FP16-NEXT: mov r2, #0 -; CHECK-FP16-NEXT: vmov.i64 q6, #0xffffffff +; CHECK-FP16-NEXT: vmov s0, r9 ; CHECK-FP16-NEXT: movwlt r2, #1 ; CHECK-FP16-NEXT: cmp r2, #0 ; CHECK-FP16-NEXT: mvnne r2, #0 -; CHECK-FP16-NEXT: subs r0, r0, r7 +; CHECK-FP16-NEXT: subs r0, r0, r10 ; CHECK-FP16-NEXT: sbcs r0, r1, #0 -; CHECK-FP16-NEXT: vmov.32 d11[1], r5 +; CHECK-FP16-NEXT: vmov.32 d9[1], r5 ; CHECK-FP16-NEXT: mov r0, #0 -; CHECK-FP16-NEXT: vmov s0, r8 +; CHECK-FP16-NEXT: vmov.i64 q5, #0xffffffff ; CHECK-FP16-NEXT: movwlt r0, #1 ; CHECK-FP16-NEXT: cmp r0, #0 -; CHECK-FP16-NEXT: vmov.32 d10[1], r1 +; CHECK-FP16-NEXT: vmov.32 d8[1], r1 ; CHECK-FP16-NEXT: mvnne r0, #0 ; CHECK-FP16-NEXT: mov r6, #0 ; CHECK-FP16-NEXT: vdup.32 d17, r2 ; CHECK-FP16-NEXT: vdup.32 d16, r0 -; CHECK-FP16-NEXT: vbif q5, q6, q8 -; CHECK-FP16-NEXT: vmov r9, r8, d10 +; CHECK-FP16-NEXT: vbif q4, q5, q8 ; CHECK-FP16-NEXT: bl __fixhfdi +; CHECK-FP16-NEXT: vmov s0, r8 ; CHECK-FP16-NEXT: mov r4, r0 -; CHECK-FP16-NEXT: vmov.u16 r0, d8[0] ; CHECK-FP16-NEXT: mov r5, r1 -; CHECK-FP16-NEXT: vmov.32 d9[0], r4 -; CHECK-FP16-NEXT: vmov s0, r0 +; CHECK-FP16-NEXT: vmov.32 d13[0], r0 +; CHECK-FP16-NEXT: vmov r7, r8, d8 ; CHECK-FP16-NEXT: bl __fixhfdi -; CHECK-FP16-NEXT: subs r2, r4, r7 -; CHECK-FP16-NEXT: vmov.32 d8[0], r0 +; CHECK-FP16-NEXT: subs r2, r4, r10 +; CHECK-FP16-NEXT: vmov.32 d12[0], r0 ; CHECK-FP16-NEXT: sbcs r2, r5, #0 ; CHECK-FP16-NEXT: mov r2, #0 -; CHECK-FP16-NEXT: vmov.32 d9[1], r5 +; CHECK-FP16-NEXT: vmov.32 d13[1], r5 ; CHECK-FP16-NEXT: movwlt r2, #1 ; CHECK-FP16-NEXT: cmp r2, #0 ; CHECK-FP16-NEXT: mvnne r2, #0 -; CHECK-FP16-NEXT: subs r0, r0, r7 +; CHECK-FP16-NEXT: subs r0, r0, r10 ; CHECK-FP16-NEXT: sbcs r0, r1, #0 -; CHECK-FP16-NEXT: vmov.32 d8[1], r1 +; CHECK-FP16-NEXT: vdup.32 d17, r2 ; CHECK-FP16-NEXT: mov r0, #0 +; CHECK-FP16-NEXT: vmov.32 d12[1], r1 ; CHECK-FP16-NEXT: movwlt r0, #1 ; CHECK-FP16-NEXT: cmp r0, #0 ; CHECK-FP16-NEXT: mvnne r0, #0 -; CHECK-FP16-NEXT: vdup.32 d17, r2 +; CHECK-FP16-NEXT: vmov r2, r3, d9 ; CHECK-FP16-NEXT: vdup.32 d16, r0 -; CHECK-FP16-NEXT: vmov r0, r1, d11 -; CHECK-FP16-NEXT: vbsl q8, q4, q6 -; CHECK-FP16-NEXT: rsbs r7, r9, #0 +; CHECK-FP16-NEXT: rsbs r7, r7, #0 +; CHECK-FP16-NEXT: vbsl q8, q6, q5 ; CHECK-FP16-NEXT: rscs r7, r8, #0 ; CHECK-FP16-NEXT: mov r7, #0 -; CHECK-FP16-NEXT: vmov r2, r3, d17 ; CHECK-FP16-NEXT: movwlt r7, #1 -; CHECK-FP16-NEXT: vmov r5, r4, d16 ; CHECK-FP16-NEXT: cmp r7, #0 +; CHECK-FP16-NEXT: vmov r0, r1, d17 ; CHECK-FP16-NEXT: mvnne r7, #0 +; CHECK-FP16-NEXT: vmov r5, r4, d16 ; CHECK-FP16-NEXT: rsbs r0, r0, #0 ; CHECK-FP16-NEXT: rscs r0, r1, #0 ; CHECK-FP16-NEXT: mov r0, #0 @@ -1108,25 +1084,25 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-FP16-NEXT: mvnne r0, #0 ; CHECK-FP16-NEXT: rsbs r1, r2, #0 ; CHECK-FP16-NEXT: rscs r1, r3, #0 -; CHECK-FP16-NEXT: vdup.32 d21, r0 +; CHECK-FP16-NEXT: vmov.32 d19[0], r0 ; CHECK-FP16-NEXT: mov r1, #0 -; CHECK-FP16-NEXT: vdup.32 d20, r7 ; CHECK-FP16-NEXT: movwlt r1, #1 ; CHECK-FP16-NEXT: cmp r1, #0 ; CHECK-FP16-NEXT: mvnne r1, #0 -; CHECK-FP16-NEXT: rsbs r2, r5, #0 -; CHECK-FP16-NEXT: rscs r2, r4, #0 -; CHECK-FP16-NEXT: vdup.32 d19, r1 +; CHECK-FP16-NEXT: rsbs r0, r5, #0 +; CHECK-FP16-NEXT: rscs r0, r4, #0 +; CHECK-FP16-NEXT: vmov.32 d21[0], r1 ; CHECK-FP16-NEXT: movwlt r6, #1 ; CHECK-FP16-NEXT: cmp r6, #0 +; CHECK-FP16-NEXT: vmov.32 d20[0], r7 ; CHECK-FP16-NEXT: mvnne r6, #0 -; CHECK-FP16-NEXT: vand q10, q5, q10 -; CHECK-FP16-NEXT: vdup.32 d18, r6 +; CHECK-FP16-NEXT: vmov.32 d18[0], r6 +; CHECK-FP16-NEXT: vand q10, q4, q10 ; CHECK-FP16-NEXT: vand q8, q8, q9 ; CHECK-FP16-NEXT: vmovn.i64 d1, q10 ; CHECK-FP16-NEXT: vmovn.i64 d0, q8 ; CHECK-FP16-NEXT: vpop {d8, d9, d10, d11, d12, d13} -; CHECK-FP16-NEXT: pop {r4, r5, r6, r7, r8, r9, r11, pc} +; CHECK-FP16-NEXT: pop {r4, r5, r6, r7, r8, r9, r10, pc} entry: %conv = fptosi <4 x half> %x to <4 x i64> %0 = icmp slt <4 x i64> %conv, diff --git a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll index 0134ee48ad421..742f2a75a1aa8 100644 --- a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll @@ -279,42 +279,40 @@ define arm_aapcs_vfpcc <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13} -; CHECK-NEXT: vmov.u16 r0, q0[2] +; CHECK-NEXT: vmov.u16 r0, q0[3] ; CHECK-NEXT: vmov q4, q0 ; CHECK-NEXT: bl __fixhfdi ; CHECK-NEXT: mov r4, r0 -; CHECK-NEXT: vmov.u16 r0, q4[3] +; CHECK-NEXT: vmov.u16 r0, q4[2] ; CHECK-NEXT: mov r5, r1 ; CHECK-NEXT: bl __fixhfdi -; CHECK-NEXT: rsbs r2, r4, #0 +; CHECK-NEXT: rsbs r2, r0, #0 ; CHECK-NEXT: mov.w r6, #0 -; CHECK-NEXT: sbcs.w r2, r6, r5 -; CHECK-NEXT: vmov q0[2], q0[0], r4, r0 -; CHECK-NEXT: csetm r2, lt -; CHECK-NEXT: rsbs r0, r0, #0 -; CHECK-NEXT: mov.w r3, #0 +; CHECK-NEXT: vmov q0[2], q0[0], r0, r4 ; CHECK-NEXT: sbcs.w r0, r6, r1 -; CHECK-NEXT: bfi r3, r2, #0, #8 ; CHECK-NEXT: csetm r0, lt -; CHECK-NEXT: bfi r3, r0, #8, #8 -; CHECK-NEXT: vmov.u16 r0, q4[0] +; CHECK-NEXT: movs r1, #0 +; CHECK-NEXT: bfi r1, r0, #0, #8 +; CHECK-NEXT: rsbs r0, r4, #0 +; CHECK-NEXT: sbcs.w r0, r6, r5 ; CHECK-NEXT: vmov.i32 q5, #0x0 -; CHECK-NEXT: vmov q0[3], q0[1], r5, r1 -; CHECK-NEXT: vmsr p0, r3 +; CHECK-NEXT: csetm r0, lt +; CHECK-NEXT: bfi r1, r0, #8, #8 +; CHECK-NEXT: vmov.u16 r0, q4[1] +; CHECK-NEXT: vmsr p0, r1 ; CHECK-NEXT: vpsel q6, q0, q5 ; CHECK-NEXT: bl __fixhfdi ; CHECK-NEXT: mov r4, r0 -; CHECK-NEXT: vmov.u16 r0, q4[1] +; CHECK-NEXT: vmov.u16 r0, q4[0] ; CHECK-NEXT: mov r5, r1 ; CHECK-NEXT: bl __fixhfdi -; CHECK-NEXT: rsbs r2, r4, #0 -; CHECK-NEXT: vmov q0[2], q0[0], r4, r0 -; CHECK-NEXT: sbcs.w r2, r6, r5 -; CHECK-NEXT: vmov q0[3], q0[1], r5, r1 -; CHECK-NEXT: csetm r2, lt -; CHECK-NEXT: rsbs r0, r0, #0 +; CHECK-NEXT: rsbs r2, r0, #0 +; CHECK-NEXT: vmov q0[2], q0[0], r0, r4 ; CHECK-NEXT: sbcs.w r0, r6, r1 -; CHECK-NEXT: bfi r6, r2, #0, #8 +; CHECK-NEXT: csetm r0, lt +; CHECK-NEXT: rsbs r1, r4, #0 +; CHECK-NEXT: sbcs.w r1, r6, r5 +; CHECK-NEXT: bfi r6, r0, #0, #8 ; CHECK-NEXT: csetm r0, lt ; CHECK-NEXT: bfi r6, r0, #8, #8 ; CHECK-NEXT: vmsr p0, r6 @@ -1353,42 +1351,40 @@ define arm_aapcs_vfpcc <4 x i32> @ustest_f16i32_mm(<4 x half> %x) { ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13} -; CHECK-NEXT: vmov.u16 r0, q0[2] +; CHECK-NEXT: vmov.u16 r0, q0[3] ; CHECK-NEXT: vmov q4, q0 ; CHECK-NEXT: bl __fixhfdi ; CHECK-NEXT: mov r4, r0 -; CHECK-NEXT: vmov.u16 r0, q4[3] +; CHECK-NEXT: vmov.u16 r0, q4[2] ; CHECK-NEXT: mov r5, r1 ; CHECK-NEXT: bl __fixhfdi -; CHECK-NEXT: rsbs r2, r4, #0 +; CHECK-NEXT: rsbs r2, r0, #0 ; CHECK-NEXT: mov.w r6, #0 -; CHECK-NEXT: sbcs.w r2, r6, r5 -; CHECK-NEXT: vmov q0[2], q0[0], r4, r0 -; CHECK-NEXT: csetm r2, lt -; CHECK-NEXT: rsbs r0, r0, #0 -; CHECK-NEXT: mov.w r3, #0 +; CHECK-NEXT: vmov q0[2], q0[0], r0, r4 ; CHECK-NEXT: sbcs.w r0, r6, r1 -; CHECK-NEXT: bfi r3, r2, #0, #8 ; CHECK-NEXT: csetm r0, lt -; CHECK-NEXT: bfi r3, r0, #8, #8 -; CHECK-NEXT: vmov.u16 r0, q4[0] +; CHECK-NEXT: movs r1, #0 +; CHECK-NEXT: bfi r1, r0, #0, #8 +; CHECK-NEXT: rsbs r0, r4, #0 +; CHECK-NEXT: sbcs.w r0, r6, r5 ; CHECK-NEXT: vmov.i32 q5, #0x0 -; CHECK-NEXT: vmov q0[3], q0[1], r5, r1 -; CHECK-NEXT: vmsr p0, r3 +; CHECK-NEXT: csetm r0, lt +; CHECK-NEXT: bfi r1, r0, #8, #8 +; CHECK-NEXT: vmov.u16 r0, q4[1] +; CHECK-NEXT: vmsr p0, r1 ; CHECK-NEXT: vpsel q6, q0, q5 ; CHECK-NEXT: bl __fixhfdi ; CHECK-NEXT: mov r4, r0 -; CHECK-NEXT: vmov.u16 r0, q4[1] +; CHECK-NEXT: vmov.u16 r0, q4[0] ; CHECK-NEXT: mov r5, r1 ; CHECK-NEXT: bl __fixhfdi -; CHECK-NEXT: rsbs r2, r4, #0 -; CHECK-NEXT: vmov q0[2], q0[0], r4, r0 -; CHECK-NEXT: sbcs.w r2, r6, r5 -; CHECK-NEXT: vmov q0[3], q0[1], r5, r1 -; CHECK-NEXT: csetm r2, lt -; CHECK-NEXT: rsbs r0, r0, #0 +; CHECK-NEXT: rsbs r2, r0, #0 +; CHECK-NEXT: vmov q0[2], q0[0], r0, r4 ; CHECK-NEXT: sbcs.w r0, r6, r1 -; CHECK-NEXT: bfi r6, r2, #0, #8 +; CHECK-NEXT: csetm r0, lt +; CHECK-NEXT: rsbs r1, r4, #0 +; CHECK-NEXT: sbcs.w r1, r6, r5 +; CHECK-NEXT: bfi r6, r0, #0, #8 ; CHECK-NEXT: csetm r0, lt ; CHECK-NEXT: bfi r6, r0, #8, #8 ; CHECK-NEXT: vmsr p0, r6 diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll index 5ac88c581f33c..b4a2aa7a1ed1b 100644 --- a/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll +++ b/llvm/test/CodeGen/Thumb2/mve-gather-ind8-unscaled.ll @@ -33,7 +33,12 @@ define arm_aapcs_vfpcc <2 x i8> @unscaled_v2i8_i8(ptr %base, ptr %offptr) { ; CHECK-LABEL: unscaled_v2i8_i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: ldrb r2, [r1] +; CHECK-NEXT: vmov.i32 q0, #0xff ; CHECK-NEXT: ldrb r1, [r1, #1] +; CHECK-NEXT: vmov q1[2], q1[0], r2, r1 +; CHECK-NEXT: vand q0, q1, q0 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s0 ; CHECK-NEXT: ldrb r1, [r0, r1] ; CHECK-NEXT: ldrb r0, [r0, r2] ; CHECK-NEXT: vmov q0[2], q0[0], r0, r1 diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll index fe5d7f29f01ff..acbe48f9e5927 100644 --- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll +++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll @@ -302,37 +302,35 @@ define arm_aapcs_vfpcc <4 x i32> @ext_ops_trunc_i32(<4 x i32> %a, <4 x i32> %b) ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, lr} -; CHECK-NEXT: vmov.f32 s12, s2 +; CHECK-NEXT: vmov.f32 s8, s2 ; CHECK-NEXT: vmov.f32 s2, s3 ; CHECK-NEXT: vmov.f32 s10, s7 +; CHECK-NEXT: vmov r10, s8 ; CHECK-NEXT: vmov.f32 s8, s6 ; CHECK-NEXT: vmov.f32 s6, s5 -; CHECK-NEXT: vmov r10, s2 -; CHECK-NEXT: vmov r2, s10 -; CHECK-NEXT: vmov.f32 s2, s1 -; CHECK-NEXT: adds.w r6, r10, r2 +; CHECK-NEXT: vmov r2, s8 ; CHECK-NEXT: asr.w r0, r10, #31 +; CHECK-NEXT: adds.w r6, r10, r2 +; CHECK-NEXT: eor.w r7, r10, r2 ; CHECK-NEXT: adc r3, r0, #0 -; CHECK-NEXT: eor.w r1, r10, r2 ; CHECK-NEXT: asrl r6, r3, r2 ; CHECK-NEXT: subs r0, r6, r2 -; CHECK-NEXT: vmov r6, s12 +; CHECK-NEXT: vmov r6, s2 ; CHECK-NEXT: sbc lr, r3, #0 -; CHECK-NEXT: vmov r3, s8 +; CHECK-NEXT: vmov r3, s10 +; CHECK-NEXT: vmov.f32 s2, s1 ; CHECK-NEXT: umull r0, r8, r0, r2 -; CHECK-NEXT: vmov.i64 q3, #0xffffffff -; CHECK-NEXT: vand q2, q2, q3 +; CHECK-NEXT: asrs r5, r6, #31 ; CHECK-NEXT: adds r4, r6, r3 -; CHECK-NEXT: asr.w r7, r6, #31 -; CHECK-NEXT: adc r5, r7, #0 -; CHECK-NEXT: eor.w r7, r6, r3 +; CHECK-NEXT: adc r5, r5, #0 +; CHECK-NEXT: eor.w r1, r6, r3 ; CHECK-NEXT: asrl r4, r5, r3 ; CHECK-NEXT: subs r4, r4, r3 ; CHECK-NEXT: sbc r5, r5, #0 -; CHECK-NEXT: orrs.w r7, r7, r6, asr #31 +; CHECK-NEXT: orrs.w r7, r7, r10, asr #31 ; CHECK-NEXT: umull r4, r12, r4, r3 ; CHECK-NEXT: csetm r9, eq -; CHECK-NEXT: orrs.w r1, r1, r10, asr #31 +; CHECK-NEXT: orrs.w r1, r1, r6, asr #31 ; CHECK-NEXT: mov.w r7, #0 ; CHECK-NEXT: csetm r1, eq ; CHECK-NEXT: bfi r7, r9, #0, #8 @@ -345,49 +343,47 @@ define arm_aapcs_vfpcc <4 x i32> @ext_ops_trunc_i32(<4 x i32> %a, <4 x i32> %b) ; CHECK-NEXT: rsb.w r1, r10, #0 ; CHECK-NEXT: lsll r4, r5, r3 ; CHECK-NEXT: lsll r0, r7, r1 +; CHECK-NEXT: vmov r3, s2 ; CHECK-NEXT: vmov r1, s6 ; CHECK-NEXT: lsll r0, r7, r2 +; CHECK-NEXT: vmov q3[2], q3[0], r0, r4 ; CHECK-NEXT: mov.w r12, #0 -; CHECK-NEXT: vmov q3[2], q3[0], r4, r0 -; CHECK-NEXT: vmov r0, s2 -; CHECK-NEXT: vmov q3[3], q3[1], r5, r7 ; CHECK-NEXT: vpsel q2, q3, q2 -; CHECK-NEXT: adds r4, r0, r1 -; CHECK-NEXT: asr.w r2, r0, #31 -; CHECK-NEXT: adc r3, r2, #0 -; CHECK-NEXT: asrl r4, r3, r1 -; CHECK-NEXT: subs r2, r4, r1 -; CHECK-NEXT: vmov r4, s0 -; CHECK-NEXT: sbc r8, r3, #0 -; CHECK-NEXT: vmov r3, s4 -; CHECK-NEXT: umull r2, lr, r2, r1 -; CHECK-NEXT: adds r6, r4, r3 -; CHECK-NEXT: asr.w r5, r4, #31 +; CHECK-NEXT: adds r2, r3, r1 +; CHECK-NEXT: asr.w r0, r3, #31 +; CHECK-NEXT: adc r5, r0, #0 +; CHECK-NEXT: asrl r2, r5, r1 +; CHECK-NEXT: subs r0, r2, r1 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: sbc r8, r5, #0 +; CHECK-NEXT: umull r4, lr, r0, r1 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: asrs r5, r2, #31 +; CHECK-NEXT: adds r6, r2, r0 ; CHECK-NEXT: adc r7, r5, #0 ; CHECK-NEXT: mla r5, r8, r1, lr -; CHECK-NEXT: asrl r6, r7, r3 -; CHECK-NEXT: subs.w r8, r6, r3 -; CHECK-NEXT: eor.w r6, r4, r3 +; CHECK-NEXT: asrl r6, r7, r0 +; CHECK-NEXT: subs.w r8, r6, r0 +; CHECK-NEXT: eor.w r6, r2, r0 ; CHECK-NEXT: sbc lr, r7, #0 -; CHECK-NEXT: eor.w r7, r0, r1 -; CHECK-NEXT: orrs.w r6, r6, r4, asr #31 -; CHECK-NEXT: orr.w r7, r7, r0, asr #31 +; CHECK-NEXT: eor.w r7, r3, r1 +; CHECK-NEXT: orrs.w r6, r6, r2, asr #31 +; CHECK-NEXT: orr.w r7, r7, r3, asr #31 ; CHECK-NEXT: csetm r6, eq ; CHECK-NEXT: cmp r7, #0 ; CHECK-NEXT: bfi r12, r6, #0, #8 ; CHECK-NEXT: csetm r6, eq ; CHECK-NEXT: bfi r12, r6, #8, #8 -; CHECK-NEXT: umull r6, r7, r8, r3 -; CHECK-NEXT: rsbs r0, r0, #0 -; CHECK-NEXT: lsll r2, r5, r0 -; CHECK-NEXT: rsbs r0, r4, #0 -; CHECK-NEXT: mla r7, lr, r3, r7 -; CHECK-NEXT: lsll r2, r5, r1 +; CHECK-NEXT: umull r6, r7, r8, r0 +; CHECK-NEXT: rsb.w r8, r3, #0 +; CHECK-NEXT: lsll r4, r5, r8 ; CHECK-NEXT: vmsr p0, r12 -; CHECK-NEXT: lsll r6, r7, r0 -; CHECK-NEXT: lsll r6, r7, r3 -; CHECK-NEXT: vmov q0[2], q0[0], r6, r2 -; CHECK-NEXT: vmov q0[3], q0[1], r7, r5 +; CHECK-NEXT: mla r3, lr, r0, r7 +; CHECK-NEXT: lsll r4, r5, r1 +; CHECK-NEXT: rsbs r1, r2, #0 +; CHECK-NEXT: lsll r6, r3, r1 +; CHECK-NEXT: lsll r6, r3, r0 +; CHECK-NEXT: vmov q0[2], q0[0], r6, r4 ; CHECK-NEXT: vpsel q0, q0, q1 ; CHECK-NEXT: vmov.f32 s1, s2 ; CHECK-NEXT: vmov.f32 s2, s8 diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll index b8d9670710a00..0bec2b100911c 100644 --- a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll +++ b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll @@ -420,7 +420,6 @@ define arm_aapcs_vfpcc <2 x i64> @zext_v2i1_v2f64(<2 x double> %src) { ; CHECK-MVE-NEXT: cmp r0, #0 ; CHECK-MVE-NEXT: csetm r0, eq ; CHECK-MVE-NEXT: vmov q0[2], q0[0], r0, r6 -; CHECK-MVE-NEXT: vmov q0[3], q0[1], r0, r6 ; CHECK-MVE-NEXT: vand q0, q0, q4 ; CHECK-MVE-NEXT: vpop {d8, d9} ; CHECK-MVE-NEXT: pop {r4, r5, r6, pc} diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll index 5972a9a7cf934..29b56639bd769 100644 --- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll +++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll @@ -11,63 +11,59 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq.w .LBB0_8 ; CHECK-NEXT: @ %bb.1: @ %entry -; CHECK-NEXT: mov r5, r2 +; CHECK-NEXT: mov r11, r2 ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: bne .LBB0_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r2, #0 ; CHECK-NEXT: mov r12, r0 -; CHECK-NEXT: mov r11, r1 -; CHECK-NEXT: mov r10, r5 +; CHECK-NEXT: mov r8, r1 +; CHECK-NEXT: mov r10, r11 ; CHECK-NEXT: b .LBB0_6 ; CHECK-NEXT: .LBB0_3: @ %vector.ph ; CHECK-NEXT: bic r2, r3, #1 ; CHECK-NEXT: adr r4, .LCPI0_0 ; CHECK-NEXT: subs r7, r2, #2 ; CHECK-NEXT: movs r6, #1 -; CHECK-NEXT: vldrw.u32 q0, [r4] -; CHECK-NEXT: adr r4, .LCPI0_1 ; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill +; CHECK-NEXT: add.w r10, r11, r2, lsl #2 ; CHECK-NEXT: add.w lr, r6, r7, lsr #1 ; CHECK-NEXT: str r2, [sp] @ 4-byte Spill -; CHECK-NEXT: add.w r10, r5, r2, lsl #2 -; CHECK-NEXT: add.w r11, r1, r2, lsl #2 +; CHECK-NEXT: add.w r8, r1, r2, lsl #2 ; CHECK-NEXT: add.w r12, r0, r2, lsl #2 -; CHECK-NEXT: vldrw.u32 q1, [r4] +; CHECK-NEXT: vldrw.u32 q0, [r4] +; CHECK-NEXT: vmvn.i32 q1, #0x80000000 ; CHECK-NEXT: .LBB0_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrd r4, r2, [r0], #8 -; CHECK-NEXT: mov.w r3, #-1 -; CHECK-NEXT: ldrd r7, r8, [r1], #8 +; CHECK-NEXT: movs r5, #0 +; CHECK-NEXT: ldrd r7, r6, [r1], #8 ; CHECK-NEXT: smull r4, r7, r7, r4 ; CHECK-NEXT: asrl r4, r7, #31 ; CHECK-NEXT: rsbs.w r9, r4, #-2147483648 -; CHECK-NEXT: sbcs r3, r7 -; CHECK-NEXT: mov r9, r5 +; CHECK-NEXT: mov.w r9, #-1 +; CHECK-NEXT: sbcs.w r3, r9, r7 ; CHECK-NEXT: csetm r3, lt -; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: bfi r5, r3, #0, #8 -; CHECK-NEXT: smull r2, r3, r8, r2 +; CHECK-NEXT: smull r2, r3, r6, r2 ; CHECK-NEXT: asrl r2, r3, #31 ; CHECK-NEXT: rsbs.w r6, r2, #-2147483648 ; CHECK-NEXT: vmov q2[2], q2[0], r4, r2 -; CHECK-NEXT: mov.w r6, #-1 +; CHECK-NEXT: sbcs.w r6, r9, r3 ; CHECK-NEXT: vmov q2[3], q2[1], r7, r3 -; CHECK-NEXT: sbcs r6, r3 ; CHECK-NEXT: csetm r6, lt ; CHECK-NEXT: bfi r5, r6, #8, #8 -; CHECK-NEXT: mvn r6, #-2147483648 ; CHECK-NEXT: vmsr p0, r5 -; CHECK-NEXT: mov r5, r9 +; CHECK-NEXT: mvn r5, #-2147483648 ; CHECK-NEXT: vpsel q2, q2, q0 ; CHECK-NEXT: vmov r2, r3, d4 -; CHECK-NEXT: subs r2, r2, r6 +; CHECK-NEXT: subs r2, r2, r5 ; CHECK-NEXT: sbcs r2, r3, #0 ; CHECK-NEXT: mov.w r3, #0 ; CHECK-NEXT: csetm r2, lt ; CHECK-NEXT: bfi r3, r2, #0, #8 ; CHECK-NEXT: vmov r2, r4, d5 -; CHECK-NEXT: subs r2, r2, r6 +; CHECK-NEXT: subs r2, r2, r5 ; CHECK-NEXT: sbcs r2, r4, #0 ; CHECK-NEXT: csetm r2, lt ; CHECK-NEXT: bfi r3, r2, #8, #8 @@ -75,8 +71,7 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: vpsel q2, q2, q1 ; CHECK-NEXT: vmov r2, s10 ; CHECK-NEXT: vmov r3, s8 -; CHECK-NEXT: strd r3, r2, [r5] -; CHECK-NEXT: add.w r5, r9, #8 +; CHECK-NEXT: strd r3, r2, [r11], #8 ; CHECK-NEXT: le lr, .LBB0_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: ldrd r2, r3, [sp] @ 8-byte Folded Reload @@ -90,7 +85,7 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: .LBB0_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r2, [r12], #4 -; CHECK-NEXT: ldr r4, [r11], #4 +; CHECK-NEXT: ldr r4, [r8], #4 ; CHECK-NEXT: smull r2, r5, r4, r2 ; CHECK-NEXT: asrl r2, r5, #31 ; CHECK-NEXT: subs r4, r1, r2 @@ -112,11 +107,6 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: .long 4294967295 @ 0xffffffff ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff -; CHECK-NEXT: .LCPI0_1: -; CHECK-NEXT: .long 2147483647 @ 0x7fffffff -; CHECK-NEXT: .long 0 @ 0x0 -; CHECK-NEXT: .long 2147483647 @ 0x7fffffff -; CHECK-NEXT: .long 0 @ 0x0 entry: switch i32 %N, label %vector.ph [ i32 0, label %for.cond.cleanup @@ -613,57 +603,56 @@ define arm_aapcs_vfpcc void @usatmul_2_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} -; CHECK-NEXT: .pad #8 -; CHECK-NEXT: sub sp, #8 +; CHECK-NEXT: .pad #4 +; CHECK-NEXT: sub sp, #4 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB3_8 ; CHECK-NEXT: @ %bb.1: @ %entry +; CHECK-NEXT: mov r8, r2 ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: bne .LBB3_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r7, #0 ; CHECK-NEXT: mov r12, r0 -; CHECK-NEXT: mov r10, r1 -; CHECK-NEXT: mov r11, r2 +; CHECK-NEXT: mov r11, r1 +; CHECK-NEXT: mov r2, r8 ; CHECK-NEXT: b .LBB3_6 ; CHECK-NEXT: .LBB3_3: @ %vector.ph -; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill -; CHECK-NEXT: bic r3, r3, #1 -; CHECK-NEXT: subs r7, r3, #2 +; CHECK-NEXT: bic r5, r3, #1 ; CHECK-NEXT: movs r6, #1 -; CHECK-NEXT: str r3, [sp] @ 4-byte Spill -; CHECK-NEXT: add.w r11, r2, r3, lsl #2 +; CHECK-NEXT: subs r7, r5, #2 +; CHECK-NEXT: str r5, [sp] @ 4-byte Spill +; CHECK-NEXT: add.w r2, r8, r5, lsl #2 +; CHECK-NEXT: add.w r11, r1, r5, lsl #2 ; CHECK-NEXT: add.w lr, r6, r7, lsr #1 -; CHECK-NEXT: add.w r10, r1, r3, lsl #2 -; CHECK-NEXT: add.w r12, r0, r3, lsl #2 -; CHECK-NEXT: vmov.i64 q0, #0xffffffff +; CHECK-NEXT: add.w r12, r0, r5, lsl #2 +; CHECK-NEXT: vmov.i8 q0, #0xff ; CHECK-NEXT: .LBB3_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldrd r4, r6, [r0], #8 -; CHECK-NEXT: mov.w r8, #0 -; CHECK-NEXT: ldrd r7, r3, [r1], #8 -; CHECK-NEXT: umull r4, r9, r7, r4 -; CHECK-NEXT: lsrl r4, r9, #31 -; CHECK-NEXT: subs.w r5, r4, #-1 -; CHECK-NEXT: sbcs r5, r9, #0 +; CHECK-NEXT: ldrd r4, r9, [r0], #8 +; CHECK-NEXT: ldrd r5, r10, [r1], #8 +; CHECK-NEXT: umull r4, r5, r5, r4 +; CHECK-NEXT: lsrl r4, r5, #31 +; CHECK-NEXT: subs.w r6, r4, #-1 +; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: mov.w r6, #0 ; CHECK-NEXT: csetm r5, lo -; CHECK-NEXT: bfi r8, r5, #0, #8 -; CHECK-NEXT: umull r6, r5, r3, r6 -; CHECK-NEXT: lsrl r6, r5, #31 -; CHECK-NEXT: subs.w r7, r6, #-1 -; CHECK-NEXT: vmov q1[2], q1[0], r4, r6 -; CHECK-NEXT: sbcs r3, r5, #0 -; CHECK-NEXT: vmov q1[3], q1[1], r9, r5 -; CHECK-NEXT: csetm r3, lo -; CHECK-NEXT: bfi r8, r3, #8, #8 -; CHECK-NEXT: vmsr p0, r8 +; CHECK-NEXT: bfi r6, r5, #0, #8 +; CHECK-NEXT: umull r10, r5, r10, r9 +; CHECK-NEXT: lsrl r10, r5, #31 +; CHECK-NEXT: subs.w r7, r10, #-1 +; CHECK-NEXT: vmov q1[2], q1[0], r4, r10 +; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: csetm r5, lo +; CHECK-NEXT: bfi r6, r5, #8, #8 +; CHECK-NEXT: vmsr p0, r6 ; CHECK-NEXT: vpsel q1, q1, q0 -; CHECK-NEXT: vmov r3, s6 -; CHECK-NEXT: vmov r4, s4 -; CHECK-NEXT: strd r4, r3, [r2], #8 +; CHECK-NEXT: vmov r4, s6 +; CHECK-NEXT: vmov r5, s4 +; CHECK-NEXT: strd r5, r4, [r8], #8 ; CHECK-NEXT: le lr, .LBB3_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block -; CHECK-NEXT: ldrd r7, r3, [sp] @ 8-byte Folded Reload +; CHECK-NEXT: ldr r7, [sp] @ 4-byte Reload ; CHECK-NEXT: cmp r7, r3 ; CHECK-NEXT: beq .LBB3_8 ; CHECK-NEXT: .LBB3_6: @ %for.body.preheader @@ -671,17 +660,17 @@ define arm_aapcs_vfpcc void @usatmul_2_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: .LBB3_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r0, [r12], #4 -; CHECK-NEXT: ldr r1, [r10], #4 +; CHECK-NEXT: ldr r1, [r11], #4 ; CHECK-NEXT: umull r0, r1, r1, r0 ; CHECK-NEXT: lsrl r0, r1, #31 -; CHECK-NEXT: subs.w r2, r0, #-1 +; CHECK-NEXT: subs.w r3, r0, #-1 ; CHECK-NEXT: sbcs r1, r1, #0 ; CHECK-NEXT: it hs ; CHECK-NEXT: movhs.w r0, #-1 -; CHECK-NEXT: str r0, [r11], #4 +; CHECK-NEXT: str r0, [r2], #4 ; CHECK-NEXT: le lr, .LBB3_7 ; CHECK-NEXT: .LBB3_8: @ %for.cond.cleanup -; CHECK-NEXT: add sp, #8 +; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} entry: switch i32 %N, label %vector.ph [ @@ -761,78 +750,69 @@ define arm_aapcs_vfpcc void @usatmul_4_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: sub sp, #4 ; CHECK-NEXT: .vsave {d8, d9, d10, d11} ; CHECK-NEXT: vpush {d8, d9, d10, d11} -; CHECK-NEXT: .pad #16 -; CHECK-NEXT: sub sp, #16 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq.w .LBB4_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader -; CHECK-NEXT: movs r7, #0 +; CHECK-NEXT: mov.w r8, #0 ; CHECK-NEXT: cmp r3, #3 ; CHECK-NEXT: bhi .LBB4_3 ; CHECK-NEXT: @ %bb.2: -; CHECK-NEXT: mov r10, r1 ; CHECK-NEXT: mov r12, r0 -; CHECK-NEXT: mov r1, r2 +; CHECK-NEXT: mov r9, r1 +; CHECK-NEXT: mov r11, r2 ; CHECK-NEXT: b .LBB4_6 ; CHECK-NEXT: .LBB4_3: @ %vector.ph -; CHECK-NEXT: str r3, [sp, #8] @ 4-byte Spill -; CHECK-NEXT: bic r3, r3, #3 -; CHECK-NEXT: subs r7, r3, #4 +; CHECK-NEXT: bic r8, r3, #3 ; CHECK-NEXT: movs r6, #1 -; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill -; CHECK-NEXT: add.w r10, r1, r3, lsl #2 -; CHECK-NEXT: add.w lr, r6, r7, lsr #2 -; CHECK-NEXT: add.w r7, r2, r3, lsl #2 -; CHECK-NEXT: str r7, [sp] @ 4-byte Spill -; CHECK-NEXT: add.w r12, r0, r3, lsl #2 +; CHECK-NEXT: sub.w r7, r8, #4 ; CHECK-NEXT: vmov.i64 q0, #0xffffffff +; CHECK-NEXT: add.w r11, r2, r8, lsl #2 +; CHECK-NEXT: add.w r9, r1, r8, lsl #2 +; CHECK-NEXT: add.w lr, r6, r7, lsr #2 +; CHECK-NEXT: add.w r12, r0, r8, lsl #2 ; CHECK-NEXT: .LBB4_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q1, [r0], #16 ; CHECK-NEXT: vldrw.u32 q2, [r1], #16 -; CHECK-NEXT: movs r6, #0 -; CHECK-NEXT: str r2, [sp, #12] @ 4-byte Spill ; CHECK-NEXT: vmov.f32 s12, s6 ; CHECK-NEXT: vmov.f32 s14, s7 ; CHECK-NEXT: vmov.f32 s16, s10 ; CHECK-NEXT: vmov.f32 s18, s11 ; CHECK-NEXT: vmullb.u32 q5, q4, q3 ; CHECK-NEXT: vmov.f32 s6, s5 -; CHECK-NEXT: vmov r4, r9, d10 -; CHECK-NEXT: lsrl r4, r9, #31 +; CHECK-NEXT: vmov r10, r5, d10 +; CHECK-NEXT: lsrl r10, r5, #31 ; CHECK-NEXT: vmov.f32 s10, s9 -; CHECK-NEXT: subs.w r5, r4, #-1 -; CHECK-NEXT: sbcs r5, r9, #0 -; CHECK-NEXT: vmullb.u32 q4, q2, q1 +; CHECK-NEXT: subs.w r6, r10, #-1 +; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: mov.w r6, #0 ; CHECK-NEXT: csetm r5, lo +; CHECK-NEXT: vmullb.u32 q4, q2, q1 ; CHECK-NEXT: bfi r6, r5, #0, #8 -; CHECK-NEXT: vmov r8, r5, d11 -; CHECK-NEXT: lsrl r8, r5, #31 -; CHECK-NEXT: subs.w r11, r8, #-1 -; CHECK-NEXT: vmov q3[2], q3[0], r4, r8 -; CHECK-NEXT: sbcs r7, r5, #0 -; CHECK-NEXT: vmov q3[3], q3[1], r9, r5 -; CHECK-NEXT: csetm r7, lo -; CHECK-NEXT: bfi r6, r7, #8, #8 -; CHECK-NEXT: vmov r4, r7, d8 -; CHECK-NEXT: lsrl r4, r7, #31 +; CHECK-NEXT: vmov r4, r5, d11 +; CHECK-NEXT: lsrl r4, r5, #31 +; CHECK-NEXT: subs.w r7, r4, #-1 +; CHECK-NEXT: vmov q3[2], q3[0], r10, r4 +; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: csetm r5, lo +; CHECK-NEXT: bfi r6, r5, #8, #8 +; CHECK-NEXT: vmov r10, r5, d8 +; CHECK-NEXT: lsrl r10, r5, #31 ; CHECK-NEXT: vmsr p0, r6 -; CHECK-NEXT: subs.w r5, r4, #-1 -; CHECK-NEXT: mov.w r6, #0 -; CHECK-NEXT: sbcs r5, r7, #0 +; CHECK-NEXT: subs.w r6, r10, #-1 ; CHECK-NEXT: vpsel q3, q3, q0 +; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: mov.w r6, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #0, #8 -; CHECK-NEXT: vmov r2, r5, d9 -; CHECK-NEXT: lsrl r2, r5, #31 -; CHECK-NEXT: subs.w r3, r2, #-1 -; CHECK-NEXT: vmov q1[2], q1[0], r4, r2 -; CHECK-NEXT: sbcs r3, r5, #0 -; CHECK-NEXT: vmov q1[3], q1[1], r7, r5 -; CHECK-NEXT: csetm r3, lo -; CHECK-NEXT: bfi r6, r3, #8, #8 +; CHECK-NEXT: vmov r4, r5, d9 +; CHECK-NEXT: lsrl r4, r5, #31 +; CHECK-NEXT: subs.w r7, r4, #-1 +; CHECK-NEXT: vmov q1[2], q1[0], r10, r4 +; CHECK-NEXT: sbcs r5, r5, #0 +; CHECK-NEXT: csetm r5, lo +; CHECK-NEXT: bfi r6, r5, #8, #8 ; CHECK-NEXT: vmsr p0, r6 -; CHECK-NEXT: ldr r2, [sp, #12] @ 4-byte Reload ; CHECK-NEXT: vpsel q1, q1, q0 ; CHECK-NEXT: vmov.f32 s5, s6 ; CHECK-NEXT: vmov.f32 s6, s12 @@ -840,26 +820,23 @@ define arm_aapcs_vfpcc void @usatmul_4_q31(ptr nocapture readonly %pSrcA, ptr no ; CHECK-NEXT: vstrb.8 q1, [r2], #16 ; CHECK-NEXT: le lr, .LBB4_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block -; CHECK-NEXT: ldrd r7, r3, [sp, #4] @ 8-byte Folded Reload -; CHECK-NEXT: ldr r1, [sp] @ 4-byte Reload -; CHECK-NEXT: cmp r7, r3 +; CHECK-NEXT: cmp r8, r3 ; CHECK-NEXT: beq .LBB4_8 ; CHECK-NEXT: .LBB4_6: @ %for.body.preheader21 -; CHECK-NEXT: sub.w lr, r3, r7 +; CHECK-NEXT: sub.w lr, r3, r8 ; CHECK-NEXT: .LBB4_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r0, [r12], #4 -; CHECK-NEXT: ldr r2, [r10], #4 -; CHECK-NEXT: umull r0, r3, r2, r0 -; CHECK-NEXT: lsrl r0, r3, #31 +; CHECK-NEXT: ldr r1, [r9], #4 +; CHECK-NEXT: umull r0, r1, r1, r0 +; CHECK-NEXT: lsrl r0, r1, #31 ; CHECK-NEXT: subs.w r2, r0, #-1 -; CHECK-NEXT: sbcs r2, r3, #0 +; CHECK-NEXT: sbcs r1, r1, #0 ; CHECK-NEXT: it hs ; CHECK-NEXT: movhs.w r0, #-1 -; CHECK-NEXT: str r0, [r1], #4 +; CHECK-NEXT: str r0, [r11], #4 ; CHECK-NEXT: le lr, .LBB4_7 ; CHECK-NEXT: .LBB4_8: @ %for.cond.cleanup -; CHECK-NEXT: add sp, #16 ; CHECK-NEXT: vpop {d8, d9, d10, d11} ; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} diff --git a/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll b/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll index d49973a674a21..87df13787c6c8 100644 --- a/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll +++ b/llvm/test/CodeGen/Thumb2/mve-scatter-ind8-unscaled.ll @@ -35,10 +35,15 @@ entry: define arm_aapcs_vfpcc void @unscaled_v2i8_i8(ptr %base, ptr %offptr, <2 x i8> %input) { ; CHECK-LABEL: unscaled_v2i8_i8: ; CHECK: @ %bb.0: @ %entry -; CHECK-NEXT: vmov r3, s0 ; CHECK-NEXT: ldrb r2, [r1] +; CHECK-NEXT: vmov.i32 q1, #0xff ; CHECK-NEXT: ldrb r1, [r1, #1] -; CHECK-NEXT: strb r3, [r0, r2] +; CHECK-NEXT: vmov q2[2], q2[0], r2, r1 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vand q1, q2, q1 +; CHECK-NEXT: vmov r1, s4 +; CHECK-NEXT: strb r2, [r0, r1] +; CHECK-NEXT: vmov r1, s6 ; CHECK-NEXT: vmov r2, s2 ; CHECK-NEXT: strb r2, [r0, r1] ; CHECK-NEXT: bx lr diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll index d2d3912fec65c..f9948db66b3b3 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll @@ -443,7 +443,7 @@ entry: define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %b) { ; CHECK-LABEL: add_v2i16_v2i64_sext: ; CHECK: @ %bb.0: @ %entry -; CHECK-NEXT: vmov.i64 q2, #0xffff +; CHECK-NEXT: vmov.i32 q2, #0xffff ; CHECK-NEXT: movs r1, #0 ; CHECK-NEXT: vand q1, q1, q2 ; CHECK-NEXT: vmov r0, s4 @@ -1363,7 +1363,7 @@ entry: define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %b) { ; CHECK-LABEL: add_v2i8_v2i64_sext: ; CHECK: @ %bb.0: @ %entry -; CHECK-NEXT: vmov.i64 q2, #0xff +; CHECK-NEXT: vmov.i32 q2, #0xff ; CHECK-NEXT: movs r1, #0 ; CHECK-NEXT: vand q1, q1, q2 ; CHECK-NEXT: vmov r0, s4 @@ -1870,7 +1870,7 @@ define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, <2 x i16> %b, ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} -; CHECK-NEXT: vmov.i64 q2, #0xffff +; CHECK-NEXT: vmov.i32 q2, #0xffff ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vand q1, q1, q2 ; CHECK-NEXT: vmov r2, s4 @@ -2544,7 +2544,7 @@ define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, <2 x i8> %b, i6 ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} -; CHECK-NEXT: vmov.i64 q2, #0xff +; CHECK-NEXT: vmov.i32 q2, #0xff ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vand q1, q1, q2 ; CHECK-NEXT: vmov r2, s4 diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll index 76a15f4459afe..63b1431ac0fa4 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll @@ -393,7 +393,7 @@ entry: define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b) { ; CHECK-LABEL: add_v2i16_v2i64_sext: ; CHECK: @ %bb.0: @ %entry -; CHECK-NEXT: vmov.i64 q3, #0xffff +; CHECK-NEXT: vmov.i32 q3, #0xffff ; CHECK-NEXT: movs r1, #0 ; CHECK-NEXT: vand q2, q2, q3 ; CHECK-NEXT: vmov r2, s4 @@ -1587,7 +1587,7 @@ entry: define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b) { ; CHECK-LABEL: add_v2i8_v2i64_sext: ; CHECK: @ %bb.0: @ %entry -; CHECK-NEXT: vmov.i64 q3, #0xff +; CHECK-NEXT: vmov.i32 q3, #0xff ; CHECK-NEXT: movs r1, #0 ; CHECK-NEXT: vand q2, q2, q3 ; CHECK-NEXT: vmov r2, s4 @@ -2020,7 +2020,7 @@ define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, <2 x i16> %y, ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} -; CHECK-NEXT: vmov.i64 q3, #0xffff +; CHECK-NEXT: vmov.i32 q3, #0xffff ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vand q2, q2, q3 ; CHECK-NEXT: vmov r2, s8 @@ -2915,7 +2915,7 @@ define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, <2 x i8> %y, <2 ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} -; CHECK-NEXT: vmov.i64 q3, #0xff +; CHECK-NEXT: vmov.i32 q3, #0xff ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vand q2, q2, q3 ; CHECK-NEXT: vmov r2, s8 diff --git a/llvm/test/CodeGen/Thumb2/mve-vst3.ll b/llvm/test/CodeGen/Thumb2/mve-vst3.ll index ff416dbe3f1a0..1bb799ea1891c 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vst3.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vst3.ll @@ -264,21 +264,21 @@ define void @vst3_v2i16(ptr %src, ptr %dst) { ; CHECK-NEXT: push {r4, lr} ; CHECK-NEXT: ldrh r2, [r0, #10] ; CHECK-NEXT: ldrh r4, [r0, #8] +; CHECK-NEXT: ldrh r3, [r0, #6] ; CHECK-NEXT: ldrh.w r12, [r0, #2] -; CHECK-NEXT: ldrh.w lr, [r0] ; CHECK-NEXT: vmov q0[2], q0[0], r4, r2 -; CHECK-NEXT: ldrh r3, [r0, #6] +; CHECK-NEXT: ldrh.w lr, [r0] ; CHECK-NEXT: ldrh r0, [r0, #4] ; CHECK-NEXT: vmov q1[2], q1[0], r0, r3 -; CHECK-NEXT: vmov q2, q1 -; CHECK-NEXT: vmovnt.i32 q2, q0 +; CHECK-NEXT: vmovnt.i32 q1, q0 ; CHECK-NEXT: vmov q0[2], q0[0], lr, r12 -; CHECK-NEXT: vmov r0, s10 +; CHECK-NEXT: vmov r2, s6 +; CHECK-NEXT: vmov.32 q1[0], r0 ; CHECK-NEXT: vmov.f32 s1, s4 ; CHECK-NEXT: vmov.f32 s3, s2 ; CHECK-NEXT: vmov.32 q0[2], r4 ; CHECK-NEXT: vstrh.32 q0, [r1] -; CHECK-NEXT: str r0, [r1, #8] +; CHECK-NEXT: str r2, [r1, #8] ; CHECK-NEXT: pop {r4, pc} entry: %l1 = load <2 x i16>, ptr %src, align 4 @@ -1178,20 +1178,20 @@ define void @vst3_v2f16(ptr %src, ptr %dst) { ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: ldrd r2, r3, [r0] ; CHECK-NEXT: ldr r0, [r0, #8] -; CHECK-NEXT: vmov.32 q1[0], r2 -; CHECK-NEXT: vmov q0, q1 +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: vmov q1, q0 ; CHECK-NEXT: vmov.32 q2[0], r0 -; CHECK-NEXT: vmov.32 q0[1], r3 -; CHECK-NEXT: vmovx.f16 s2, s4 -; CHECK-NEXT: vmov.f32 s0, s4 -; CHECK-NEXT: vmovx.f16 s4, s8 -; CHECK-NEXT: vins.f16 s8, s2 -; CHECK-NEXT: vmovx.f16 s2, s1 -; CHECK-NEXT: vins.f16 s0, s1 -; CHECK-NEXT: vmov.f32 s1, s8 -; CHECK-NEXT: vins.f16 s2, s4 -; CHECK-NEXT: vmov r3, s2 -; CHECK-NEXT: vmov r0, r2, d0 +; CHECK-NEXT: vmov.32 q1[1], r3 +; CHECK-NEXT: vmovx.f16 s2, s8 +; CHECK-NEXT: vmov.f32 s4, s0 +; CHECK-NEXT: vmovx.f16 s0, s0 +; CHECK-NEXT: vins.f16 s8, s0 +; CHECK-NEXT: vmovx.f16 s6, s5 +; CHECK-NEXT: vins.f16 s4, s5 +; CHECK-NEXT: vmov.f32 s5, s8 +; CHECK-NEXT: vins.f16 s6, s2 +; CHECK-NEXT: vmov r3, s6 +; CHECK-NEXT: vmov r0, r2, d2 ; CHECK-NEXT: stm r1!, {r0, r2, r3} ; CHECK-NEXT: bx lr entry: diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll index eea4261ea30fe..30bf1a261f4b7 100644 --- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll +++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll @@ -6625,7 +6625,7 @@ define i64 @test_mm512_reduce_mul_epi64(<8 x i64> %__W) { ; X64-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 ; X64-NEXT: vpaddq %xmm2, %xmm0, %xmm0 ; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] -; X64-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; X64-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,3,3,3] ; X64-NEXT: vpmuludq %xmm0, %xmm2, %xmm2 ; X64-NEXT: vpsrlq $32, %xmm0, %xmm3 ; X64-NEXT: vpmuludq %xmm3, %xmm1, %xmm3 @@ -6833,7 +6833,7 @@ define i64 @test_mm512_mask_reduce_mul_epi64(i8 zeroext %__M, <8 x i64> %__W) { ; X64-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 ; X64-NEXT: vpaddq %xmm2, %xmm0, %xmm0 ; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] -; X64-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; X64-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,3,3,3] ; X64-NEXT: vpmuludq %xmm0, %xmm2, %xmm2 ; X64-NEXT: vpsrlq $32, %xmm0, %xmm3 ; X64-NEXT: vpmuludq %xmm3, %xmm1, %xmm3 diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll index c48a2e7e81afc..aae48aba93be6 100644 --- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll +++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll @@ -5638,10 +5638,7 @@ define <8 x i64> @test_mask_mul_epu32_rmb(<16 x i32> %a, ptr %ptr_b) { ; X86-LABEL: test_mask_mul_epu32_rmb: ; X86: ## %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08] -; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A] -; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xf4,0xc1] +; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xf4,0x00] ; X86-NEXT: retl ## encoding: [0xc3] ; ; X64-LABEL: test_mask_mul_epu32_rmb: @@ -5660,12 +5657,9 @@ define <8 x i64> @test_mask_mul_epu32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> ; X86-LABEL: test_mask_mul_epu32_rmbk: ; X86: ## %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x10] -; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x58,0xdb,0x15,A,A,A,A] -; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xf4,0xca] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xf4,0x08] ; X86-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1] ; X86-NEXT: retl ## encoding: [0xc3] ; @@ -5687,12 +5681,9 @@ define <8 x i64> @test_mask_mul_epu32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask) ; X86-LABEL: test_mask_mul_epu32_rmbkz: ; X86: ## %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08] -; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A] -; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xf4,0xc1] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xf4,0x00] ; X86-NEXT: retl ## encoding: [0xc3] ; ; X64-LABEL: test_mask_mul_epu32_rmbkz: @@ -7386,10 +7377,7 @@ define <8 x i64> @test_mul_epu32_rmb(<16 x i32> %a, ptr %ptr_b) { ; X86-LABEL: test_mul_epu32_rmb: ; X86: ## %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08] -; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A] -; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xf4,0xc1] +; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xf4,0x00] ; X86-NEXT: retl ## encoding: [0xc3] ; ; X64-LABEL: test_mul_epu32_rmb: @@ -7408,12 +7396,9 @@ define <8 x i64> @test_mul_epu32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %pass ; X86-LABEL: test_mul_epu32_rmbk: ; X86: ## %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x10] -; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x58,0xdb,0x15,A,A,A,A] -; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xf4,0xca] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xf4,0x08] ; X86-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1] ; X86-NEXT: retl ## encoding: [0xc3] ; @@ -7437,12 +7422,9 @@ define <8 x i64> @test_mul_epu32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask) { ; X86-LABEL: test_mul_epu32_rmbkz: ; X86: ## %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08] -; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A] -; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xf4,0xc1] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xf4,0x00] ; X86-NEXT: retl ## encoding: [0xc3] ; ; X64-LABEL: test_mul_epu32_rmbkz: diff --git a/llvm/test/CodeGen/X86/avx512fp16-mov.ll b/llvm/test/CodeGen/X86/avx512fp16-mov.ll index 95e6c67004e7d..d6ac705d543d6 100644 --- a/llvm/test/CodeGen/X86/avx512fp16-mov.ll +++ b/llvm/test/CodeGen/X86/avx512fp16-mov.ll @@ -1487,27 +1487,27 @@ define <8 x half> @movsh(<8 x half> %a, <8 x half> %b) { ; X64-NOVL-NEXT: vaddsh %xmm4, %xmm5, %xmm4 ; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] ; X64-NOVL-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; X64-NOVL-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7] -; X64-NOVL-NEXT: vaddsh %xmm4, %xmm0, %xmm0 -; X64-NOVL-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0] -; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1] +; X64-NOVL-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[3,3,3,3,4,5,6,7] ; X64-NOVL-NEXT: vaddsh %xmm4, %xmm5, %xmm4 -; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3] -; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; X64-NOVL-NEXT: vpsrlq $48, %xmm1, %xmm3 -; X64-NOVL-NEXT: vpsrlq $48, %xmm2, %xmm4 -; X64-NOVL-NEXT: vaddsh %xmm3, %xmm4, %xmm3 -; X64-NOVL-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] -; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,3,3] +; X64-NOVL-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0] +; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1] +; X64-NOVL-NEXT: vaddsh %xmm5, %xmm6, %xmm5 +; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3] +; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; X64-NOVL-NEXT: vpsrlq $48, %xmm1, %xmm4 +; X64-NOVL-NEXT: vpsrlq $48, %xmm2, %xmm5 ; X64-NOVL-NEXT: vaddsh %xmm4, %xmm5, %xmm4 -; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] -; X64-NOVL-NEXT: vaddsh %xmm1, %xmm2, %xmm4 +; X64-NOVL-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3] +; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3] +; X64-NOVL-NEXT: vaddsh %xmm5, %xmm6, %xmm5 +; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3] +; X64-NOVL-NEXT: vaddsh %xmm1, %xmm0, %xmm0 ; X64-NOVL-NEXT: vpsrld $16, %xmm1, %xmm1 ; X64-NOVL-NEXT: vpsrld $16, %xmm2, %xmm2 ; X64-NOVL-NEXT: vaddsh %xmm1, %xmm2, %xmm1 -; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] -; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] -; X64-NOVL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1] +; X64-NOVL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] ; X64-NOVL-NEXT: retq %res1 = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> %res2 = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll index 4ef485b916fe4..e8e22bae23c92 100644 --- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll +++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll @@ -11548,11 +11548,7 @@ define < 2 x i64> @test_mask_mul_epu32_rmb_128(< 4 x i32> %a, ptr %ptr_b) { ; X86-LABEL: test_mask_mul_epu32_rmb_128: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x58,0x08] -; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2] -; X86-NEXT: vpblendd $10, %xmm2, %xmm1, %xmm1 # encoding: [0xc4,0xe3,0x71,0x02,0xca,0x0a] -; X86-NEXT: # xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3] -; X86-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf4,0xc1] +; X86-NEXT: vpmuludq (%eax){1to2}, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x18,0xf4,0x00] ; X86-NEXT: retl # encoding: [0xc3] ; ; X64-LABEL: test_mask_mul_epu32_rmb_128: @@ -11571,13 +11567,9 @@ define < 2 x i64> @test_mask_mul_epu32_rmbk_128(< 4 x i32> %a, ptr %ptr_b, < 2 x ; X86-LABEL: test_mask_mul_epu32_rmbk_128: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x58,0x10] -; X86-NEXT: vpxor %xmm3, %xmm3, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xef,0xdb] -; X86-NEXT: vpblendd $10, %xmm3, %xmm2, %xmm2 # encoding: [0xc4,0xe3,0x69,0x02,0xd3,0x0a] -; X86-NEXT: # xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3] -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xf4,0xca] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to2}, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x19,0xf4,0x08] ; X86-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1] ; X86-NEXT: retl # encoding: [0xc3] ; @@ -11599,13 +11591,9 @@ define < 2 x i64> @test_mask_mul_epu32_rmbkz_128(< 4 x i32> %a, ptr %ptr_b, i8 % ; X86-LABEL: test_mask_mul_epu32_rmbkz_128: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x58,0x08] -; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2] -; X86-NEXT: vpblendd $10, %xmm2, %xmm1, %xmm1 # encoding: [0xc4,0xe3,0x71,0x02,0xca,0x0a] -; X86-NEXT: # xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3] -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xf4,0xc1] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to2}, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x99,0xf4,0x00] ; X86-NEXT: retl # encoding: [0xc3] ; ; X64-LABEL: test_mask_mul_epu32_rmbkz_128: @@ -11728,11 +11716,7 @@ define < 4 x i64> @test_mask_mul_epu32_rmb_256(< 8 x i32> %a, ptr %ptr_b) { ; X86-LABEL: test_mask_mul_epu32_rmb_256: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x58,0x08] -; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2] -; X86-NEXT: vpblendd $170, %ymm2, %ymm1, %ymm1 # encoding: [0xc4,0xe3,0x75,0x02,0xca,0xaa] -; X86-NEXT: # ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7] -; X86-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf4,0xc1] +; X86-NEXT: vpmuludq (%eax){1to4}, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x38,0xf4,0x00] ; X86-NEXT: retl # encoding: [0xc3] ; ; X64-LABEL: test_mask_mul_epu32_rmb_256: @@ -11751,13 +11735,9 @@ define < 4 x i64> @test_mask_mul_epu32_rmbk_256(< 8 x i32> %a, ptr %ptr_b, < 4 x ; X86-LABEL: test_mask_mul_epu32_rmbk_256: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x58,0x10] -; X86-NEXT: vpxor %xmm3, %xmm3, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xef,0xdb] -; X86-NEXT: vpblendd $170, %ymm3, %ymm2, %ymm2 # encoding: [0xc4,0xe3,0x6d,0x02,0xd3,0xaa] -; X86-NEXT: # ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4],ymm3[5],ymm2[6],ymm3[7] -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xf4,0xca] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to4}, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x39,0xf4,0x08] ; X86-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1] ; X86-NEXT: retl # encoding: [0xc3] ; @@ -11779,13 +11759,9 @@ define < 4 x i64> @test_mask_mul_epu32_rmbkz_256(< 8 x i32> %a, ptr %ptr_b, i8 % ; X86-LABEL: test_mask_mul_epu32_rmbkz_256: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X86-NEXT: vpbroadcastd (%eax), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x58,0x08] -; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2] -; X86-NEXT: vpblendd $170, %ymm2, %ymm1, %ymm1 # encoding: [0xc4,0xe3,0x75,0x02,0xca,0xaa] -; X86-NEXT: # ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7] -; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08] -; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8] -; X86-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xf4,0xc1] +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08] +; X86-NEXT: kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9] +; X86-NEXT: vpmuludq (%eax){1to4}, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xb9,0xf4,0x00] ; X86-NEXT: retl # encoding: [0xc3] ; ; X64-LABEL: test_mask_mul_epu32_rmbkz_256: diff --git a/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll b/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll index 384a4c8f889ad..6a2550ede2fb7 100644 --- a/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll +++ b/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll @@ -2698,9 +2698,8 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b) ; NoVLX-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -2724,9 +2723,8 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask_mem(<2 x i64> %__a, ptr %__b) lo ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vmovdqa (%rdi), %xmm1 ; NoVLX-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -2753,9 +2751,8 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i64 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -2784,9 +2781,8 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2 x ; NoVLX-NEXT: vmovdqa (%rsi), %xmm1 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -2814,9 +2810,8 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, ptr %__b) ; NoVLX: # %bb.0: # %entry ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpeqq (%rdi){1to8}, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -2843,9 +2838,8 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <2 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -7502,9 +7496,8 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b) ; NoVLX-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpgtq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -7528,9 +7521,8 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask_mem(<2 x i64> %__a, ptr %__b) l ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vmovdqa (%rdi), %xmm1 ; NoVLX-NEXT: vpcmpgtq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -7557,9 +7549,8 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i6 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpgtq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -7588,9 +7579,8 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2 ; NoVLX-NEXT: vmovdqa (%rsi), %xmm1 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpgtq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -7618,9 +7608,8 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, ptr %__b) ; NoVLX: # %bb.0: # %entry ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpgtq (%rdi){1to8}, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -7647,9 +7636,8 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, < ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -12366,9 +12354,8 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b) ; NoVLX-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpnltq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -12392,9 +12379,8 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask_mem(<2 x i64> %__a, ptr %__b) l ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vmovdqa (%rdi), %xmm1 ; NoVLX-NEXT: vpcmpnltq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -12421,9 +12407,8 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i6 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpnltq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -12452,9 +12437,8 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2 ; NoVLX-NEXT: vmovdqa (%rsi), %xmm1 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpnltq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -12482,9 +12466,8 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, ptr %__b) ; NoVLX: # %bb.0: # %entry ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpnltq (%rdi){1to8}, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -12511,9 +12494,8 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, < ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -17250,9 +17232,8 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b) ; NoVLX-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpltuq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -17276,9 +17257,8 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask_mem(<2 x i64> %__a, ptr %__b) l ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vmovdqa (%rdi), %xmm1 ; NoVLX-NEXT: vpcmpltuq %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -17305,9 +17285,8 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask(i8 zeroext %__u, <2 x i6 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpltuq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -17336,9 +17315,8 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask_mem(i8 zeroext %__u, <2 ; NoVLX-NEXT: vmovdqa (%rsi), %xmm1 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpltuq %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -17366,9 +17344,8 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, ptr %__b) ; NoVLX: # %bb.0: # %entry ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vpcmpltuq (%rdi){1to8}, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -17395,9 +17372,8 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, < ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -21081,9 +21057,8 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask(<2 x i64> %__a, <2 x i64> %__b) ; NoVLX-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vcmpeqpd %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -21107,9 +21082,8 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask_mem(<2 x i64> %__a, ptr %__b) l ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vmovapd (%rdi), %xmm1 ; NoVLX-NEXT: vcmpeqpd %zmm1, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -21133,9 +21107,8 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, ptr %__b) ; NoVLX: # %bb.0: # %entry ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: vcmpeqpd (%rdi){1to8}, %zmm0, %k0 -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -21163,9 +21136,8 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask(i2 zeroext %__u, <2 x i6 ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vcmpeqpd %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -21193,9 +21165,8 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask_mem(i2 zeroext %__u, <2 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vmovapd (%rsi), %xmm1 ; NoVLX-NEXT: vcmpeqpd %zmm1, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: @@ -21223,9 +21194,8 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask_mem_b(i2 zeroext %__u, < ; NoVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; NoVLX-NEXT: kmovw %edi, %k1 ; NoVLX-NEXT: vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1} -; NoVLX-NEXT: kshiftlw $14, %k0, %k0 -; NoVLX-NEXT: kshiftrw $14, %k0, %k0 ; NoVLX-NEXT: kmovw %k0, %eax +; NoVLX-NEXT: andl $3, %eax ; NoVLX-NEXT: vzeroupper ; NoVLX-NEXT: retq entry: diff --git a/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll b/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll index de030f1b78d3d..921cf88518562 100644 --- a/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll +++ b/llvm/test/CodeGen/X86/bitcast-and-setcc-128.ll @@ -195,7 +195,7 @@ define i2 @v2i8(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) { ; SSSE3-NEXT: pcmpgtb %xmm1, %xmm0 ; SSSE3-NEXT: pcmpgtb %xmm3, %xmm2 ; SSSE3-NEXT: pand %xmm0, %xmm2 -; SSSE3-NEXT: pshufb {{.*#+}} xmm2 = xmm2[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1] +; SSSE3-NEXT: pshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,0,u,u,u,u,u,u,u,1] ; SSSE3-NEXT: movmskpd %xmm2, %eax ; SSSE3-NEXT: # kill: def $al killed $al killed $eax ; SSSE3-NEXT: retq @@ -342,25 +342,21 @@ define i2 @v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, <2 x i64> %d) { ; SSE2-SSSE3-NEXT: pxor %xmm4, %xmm1 ; SSE2-SSSE3-NEXT: pxor %xmm4, %xmm0 ; SSE2-SSSE3-NEXT: movdqa %xmm0, %xmm5 -; SSE2-SSSE3-NEXT: pcmpgtd %xmm1, %xmm5 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE2-SSSE3-NEXT: pcmpeqd %xmm1, %xmm0 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-SSSE3-NEXT: pand %xmm6, %xmm0 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,3,3] +; SSE2-SSSE3-NEXT: pcmpeqd %xmm1, %xmm5 +; SSE2-SSSE3-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] +; SSE2-SSSE3-NEXT: pand %xmm5, %xmm1 ; SSE2-SSSE3-NEXT: por %xmm0, %xmm1 ; SSE2-SSSE3-NEXT: pxor %xmm4, %xmm3 ; SSE2-SSSE3-NEXT: pxor %xmm4, %xmm2 ; SSE2-SSSE3-NEXT: movdqa %xmm2, %xmm0 -; SSE2-SSSE3-NEXT: pcmpgtd %xmm3, %xmm0 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] -; SSE2-SSSE3-NEXT: pcmpeqd %xmm3, %xmm2 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE2-SSSE3-NEXT: pand %xmm4, %xmm2 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-SSSE3-NEXT: por %xmm2, %xmm0 -; SSE2-SSSE3-NEXT: pand %xmm1, %xmm0 -; SSE2-SSSE3-NEXT: movmskpd %xmm0, %eax +; SSE2-SSSE3-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE2-SSSE3-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-SSSE3-NEXT: pand %xmm0, %xmm3 +; SSE2-SSSE3-NEXT: por %xmm2, %xmm3 +; SSE2-SSSE3-NEXT: pand %xmm1, %xmm3 +; SSE2-SSSE3-NEXT: movmskpd %xmm3, %eax ; SSE2-SSSE3-NEXT: # kill: def $al killed $al killed $eax ; SSE2-SSSE3-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/bitcast-setcc-128.ll b/llvm/test/CodeGen/X86/bitcast-setcc-128.ll index d050a7f66104c..f21c3f7043e69 100644 --- a/llvm/test/CodeGen/X86/bitcast-setcc-128.ll +++ b/llvm/test/CodeGen/X86/bitcast-setcc-128.ll @@ -157,7 +157,7 @@ define i2 @v2i8(<2 x i8> %a, <2 x i8> %b) { ; SSSE3-LABEL: v2i8: ; SSSE3: # %bb.0: ; SSSE3-NEXT: pcmpgtb %xmm1, %xmm0 -; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1] +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,0,u,u,u,u,u,u,u,1] ; SSSE3-NEXT: movmskpd %xmm0, %eax ; SSSE3-NEXT: # kill: def $al killed $al killed $eax ; SSSE3-NEXT: retq @@ -272,12 +272,10 @@ define i2 @v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE2-SSSE3-NEXT: pxor %xmm2, %xmm1 ; SSE2-SSSE3-NEXT: pxor %xmm2, %xmm0 ; SSE2-SSSE3-NEXT: movdqa %xmm0, %xmm2 -; SSE2-SSSE3-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE2-SSSE3-NEXT: pcmpeqd %xmm1, %xmm0 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-SSSE3-NEXT: pand %xmm3, %xmm0 -; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-SSSE3-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE2-SSSE3-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] +; SSE2-SSSE3-NEXT: pand %xmm2, %xmm1 ; SSE2-SSSE3-NEXT: por %xmm0, %xmm1 ; SSE2-SSSE3-NEXT: movmskpd %xmm1, %eax ; SSE2-SSSE3-NEXT: # kill: def $al killed $al killed $eax diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll index 74ff87911d81d..9042f9e70ac1c 100644 --- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll +++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll @@ -634,11 +634,7 @@ define i1 @trunc_v32i8_cmp(<32 x i8> %a0) nounwind { define i4 @bitcast_v8i64_to_v2i4(<8 x i64> %a0) nounwind { ; SSE-LABEL: bitcast_v8i64_to_v2i4: ; SSE: # %bb.0: -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] ; SSE-NEXT: packssdw %xmm3, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; SSE-NEXT: packssdw %xmm1, %xmm0 ; SSE-NEXT: packssdw %xmm2, %xmm0 ; SSE-NEXT: packsswb %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll b/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll index 07c81c6f8b7a4..deb30a597ca39 100644 --- a/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll +++ b/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll @@ -264,18 +264,16 @@ define i64 @dot_ext_v2i8_v2i64(ptr %a, i64 %a_stride, ptr %b) nounwind { ; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3] ; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: pxor %xmm3, %xmm3 -; SSE2-NEXT: pcmpgtd %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 ; SSE2-NEXT: psrad $24, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] ; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE2-NEXT: pmuludq %xmm0, %xmm3 -; SSE2-NEXT: psllq $32, %xmm3 -; SSE2-NEXT: paddq %xmm1, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3] -; SSE2-NEXT: paddq %xmm3, %xmm0 +; SSE2-NEXT: pmuludq %xmm0, %xmm2 +; SSE2-NEXT: psllq $32, %xmm2 +; SSE2-NEXT: paddq %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] +; SSE2-NEXT: paddq %xmm2, %xmm0 ; SSE2-NEXT: movq %xmm0, %rax ; SSE2-NEXT: retq ; @@ -460,17 +458,15 @@ define i64 @dot_ext_v2i32_v2i64(ptr %a, i64 %a_stride, ptr %b) nounwind { ; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3] ; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: pxor %xmm3, %xmm3 -; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; SSE2-NEXT: pmuludq %xmm1, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE2-NEXT: pmuludq %xmm1, %xmm3 -; SSE2-NEXT: psllq $32, %xmm3 -; SSE2-NEXT: paddq %xmm0, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3] -; SSE2-NEXT: paddq %xmm3, %xmm0 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pmuludq %xmm1, %xmm2 +; SSE2-NEXT: psllq $32, %xmm2 +; SSE2-NEXT: pmuludq %xmm0, %xmm1 +; SSE2-NEXT: paddq %xmm2, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3] +; SSE2-NEXT: paddq %xmm1, %xmm0 ; SSE2-NEXT: movq %xmm0, %rax ; SSE2-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/combine-pmuldq.ll b/llvm/test/CodeGen/X86/combine-pmuldq.ll index cfac763acd04b..70335f834291d 100644 --- a/llvm/test/CodeGen/X86/combine-pmuldq.ll +++ b/llvm/test/CodeGen/X86/combine-pmuldq.ll @@ -44,43 +44,13 @@ define <2 x i64> @combine_shuffle_zext_pmuludq(<4 x i32> %a0, <4 x i32> %a1) { define <2 x i64> @combine_shuffle_zero_pmuludq(<4 x i32> %a0, <4 x i32> %a1) { ; SSE-LABEL: combine_shuffle_zero_pmuludq: ; SSE: # %bb.0: -; SSE-NEXT: pxor %xmm2, %xmm2 -; SSE-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7] -; SSE-NEXT: pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] -; SSE-NEXT: pmuludq %xmm2, %xmm0 +; SSE-NEXT: pmuludq %xmm1, %xmm0 ; SSE-NEXT: retq ; -; AVX1-LABEL: combine_shuffle_zero_pmuludq: -; AVX1: # %bb.0: -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7] -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] -; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: combine_shuffle_zero_pmuludq: -; AVX2: # %bb.0: -; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3] -; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3] -; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: retq -; -; AVX512VL-LABEL: combine_shuffle_zero_pmuludq: -; AVX512VL: # %bb.0: -; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3] -; AVX512VL-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3] -; AVX512VL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 -; AVX512VL-NEXT: retq -; -; AVX512DQVL-LABEL: combine_shuffle_zero_pmuludq: -; AVX512DQVL: # %bb.0: -; AVX512DQVL-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX512DQVL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3] -; AVX512DQVL-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3] -; AVX512DQVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 -; AVX512DQVL-NEXT: retq +; AVX-LABEL: combine_shuffle_zero_pmuludq: +; AVX: # %bb.0: +; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> %2 = shufflevector <4 x i32> %a1, <4 x i32> zeroinitializer, <4 x i32> %3 = bitcast <4 x i32> %1 to <2 x i64> @@ -92,20 +62,12 @@ define <2 x i64> @combine_shuffle_zero_pmuludq(<4 x i32> %a0, <4 x i32> %a1) { define <4 x i64> @combine_shuffle_zero_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) { ; SSE-LABEL: combine_shuffle_zero_pmuludq_256: ; SSE: # %bb.0: -; SSE-NEXT: pxor %xmm4, %xmm4 -; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7] -; SSE-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5],xmm4[6,7] -; SSE-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7] +; SSE-NEXT: pmuludq %xmm2, %xmm0 ; SSE-NEXT: pmuludq %xmm3, %xmm1 -; SSE-NEXT: pblendw {{.*#+}} xmm4 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7] -; SSE-NEXT: pmuludq %xmm4, %xmm0 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_shuffle_zero_pmuludq_256: ; AVX1: # %bb.0: -; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7] -; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7] ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 ; AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm2 @@ -115,25 +77,16 @@ define <4 x i64> @combine_shuffle_zero_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) ; ; AVX2-LABEL: combine_shuffle_zero_pmuludq_256: ; AVX2: # %bb.0: -; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7] ; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; ; AVX512VL-LABEL: combine_shuffle_zero_pmuludq_256: ; AVX512VL: # %bb.0: -; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7] -; AVX512VL-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7] ; AVX512VL-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512DQVL-LABEL: combine_shuffle_zero_pmuludq_256: ; AVX512DQVL: # %bb.0: -; AVX512DQVL-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX512DQVL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7] -; AVX512DQVL-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7] ; AVX512DQVL-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 ; AVX512DQVL-NEXT: retq %1 = shufflevector <8 x i32> %a0, <8 x i32> zeroinitializer, <8 x i32> @@ -147,33 +100,29 @@ define <4 x i64> @combine_shuffle_zero_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) define <8 x i64> @combine_zext_pmuludq_256(<8 x i32> %a) { ; SSE-LABEL: combine_zext_pmuludq_256: ; SSE: # %bb.0: -; SSE-NEXT: movdqa %xmm0, %xmm4 -; SSE-NEXT: pxor %xmm3, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,1,3,3] ; SSE-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero -; SSE-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3] ; SSE-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero -; SSE-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3] -; SSE-NEXT: pmovsxdq {{.*#+}} xmm3 = [715827883,715827883] -; SSE-NEXT: pmuludq %xmm3, %xmm0 -; SSE-NEXT: pmuludq %xmm3, %xmm4 -; SSE-NEXT: pmuludq %xmm3, %xmm2 -; SSE-NEXT: pmuludq %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm4, %xmm1 +; SSE-NEXT: pmovsxdq {{.*#+}} xmm4 = [715827883,715827883] +; SSE-NEXT: pmuludq %xmm4, %xmm0 +; SSE-NEXT: pmuludq %xmm4, %xmm1 +; SSE-NEXT: pmuludq %xmm4, %xmm2 +; SSE-NEXT: pmuludq %xmm4, %xmm3 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_zext_pmuludq_256: ; AVX1: # %bb.0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] -; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3] +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3] ; AVX1-NEXT: vmovddup {{.*#+}} xmm4 = [715827883,715827883] ; AVX1-NEXT: # xmm4 = mem[0,0] -; AVX1-NEXT: vpmuludq %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpmuludq %xmm4, %xmm3, %xmm3 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; AVX1-NEXT: vpmuludq %xmm4, %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 -; AVX1-NEXT: vpmuludq %xmm4, %xmm3, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0 +; AVX1-NEXT: vpmuludq %xmm4, %xmm2, %xmm2 ; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero ; AVX1-NEXT: vpmuludq %xmm4, %xmm1, %xmm1 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll index 045979afc1f53..16bba195b2f5a 100644 --- a/llvm/test/CodeGen/X86/combine-sdiv.ll +++ b/llvm/test/CodeGen/X86/combine-sdiv.ll @@ -1538,7 +1538,7 @@ define <16 x i32> @combine_vec_sdiv_by_pow2b_v16i32(<16 x i32> %x) { define <2 x i64> @combine_vec_sdiv_by_pow2b_v2i64(<2 x i64> %x) { ; SSE2-LABEL: combine_vec_sdiv_by_pow2b_v2i64: ; SSE2: # %bb.0: -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: psrad $31, %xmm1 ; SSE2-NEXT: psrlq $62, %xmm1 ; SSE2-NEXT: paddq %xmm0, %xmm1 @@ -1552,7 +1552,7 @@ define <2 x i64> @combine_vec_sdiv_by_pow2b_v2i64(<2 x i64> %x) { ; ; SSE41-LABEL: combine_vec_sdiv_by_pow2b_v2i64: ; SSE41: # %bb.0: -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: psrad $31, %xmm1 ; SSE41-NEXT: psrlq $62, %xmm1 ; SSE41-NEXT: paddq %xmm0, %xmm1 @@ -1622,7 +1622,7 @@ define <2 x i64> @combine_vec_sdiv_by_pow2b_v2i64(<2 x i64> %x) { define <4 x i64> @combine_vec_sdiv_by_pow2b_v4i64(<4 x i64> %x) { ; SSE2-LABEL: combine_vec_sdiv_by_pow2b_v4i64: ; SSE2: # %bb.0: -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: psrad $31, %xmm2 ; SSE2-NEXT: psrlq $62, %xmm2 ; SSE2-NEXT: paddq %xmm0, %xmm2 @@ -1650,7 +1650,7 @@ define <4 x i64> @combine_vec_sdiv_by_pow2b_v4i64(<4 x i64> %x) { ; ; SSE41-LABEL: combine_vec_sdiv_by_pow2b_v4i64: ; SSE41: # %bb.0: -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: psrad $31, %xmm2 ; SSE41-NEXT: psrlq $62, %xmm2 ; SSE41-NEXT: paddq %xmm0, %xmm2 @@ -1755,7 +1755,7 @@ define <4 x i64> @combine_vec_sdiv_by_pow2b_v4i64(<4 x i64> %x) { define <8 x i64> @combine_vec_sdiv_by_pow2b_v8i64(<8 x i64> %x) { ; SSE2-LABEL: combine_vec_sdiv_by_pow2b_v8i64: ; SSE2: # %bb.0: -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE2-NEXT: movdqa %xmm0, %xmm4 ; SSE2-NEXT: psrad $31, %xmm4 ; SSE2-NEXT: psrlq $62, %xmm4 ; SSE2-NEXT: paddq %xmm0, %xmm4 @@ -1765,7 +1765,7 @@ define <8 x i64> @combine_vec_sdiv_by_pow2b_v8i64(<8 x i64> %x) { ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] ; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3] +; SSE2-NEXT: movdqa %xmm2, %xmm4 ; SSE2-NEXT: psrad $31, %xmm4 ; SSE2-NEXT: psrlq $62, %xmm4 ; SSE2-NEXT: paddq %xmm2, %xmm4 @@ -1806,7 +1806,7 @@ define <8 x i64> @combine_vec_sdiv_by_pow2b_v8i64(<8 x i64> %x) { ; ; SSE41-LABEL: combine_vec_sdiv_by_pow2b_v8i64: ; SSE41: # %bb.0: -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm4 ; SSE41-NEXT: psrad $31, %xmm4 ; SSE41-NEXT: psrlq $62, %xmm4 ; SSE41-NEXT: paddq %xmm0, %xmm4 @@ -1815,7 +1815,7 @@ define <8 x i64> @combine_vec_sdiv_by_pow2b_v8i64(<8 x i64> %x) { ; SSE41-NEXT: psrlq $2, %xmm4 ; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7] ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7] -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3] +; SSE41-NEXT: movdqa %xmm2, %xmm4 ; SSE41-NEXT: psrad $31, %xmm4 ; SSE41-NEXT: psrlq $62, %xmm4 ; SSE41-NEXT: paddq %xmm2, %xmm4 @@ -2187,15 +2187,14 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) { ; SSE41-NEXT: pxor %xmm0, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm3 ; SSE41-NEXT: pcmpgtb %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm4, %xmm4 -; SSE41-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] -; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero -; SSE41-NEXT: paddw %xmm2, %xmm2 -; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2],xmm4[3,4,5],xmm2[6],xmm4[7] -; SSE41-NEXT: psrlw $8, %xmm2 +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero ; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] ; SSE41-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [256,2,2,2,2,128,2,128] ; SSE41-NEXT: psrlw $8, %xmm3 +; SSE41-NEXT: paddw %xmm4, %xmm4 +; SSE41-NEXT: pmovsxbw %xmm1, %xmm2 +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2],xmm2[3,4,5],xmm4[6],xmm2[7] +; SSE41-NEXT: psrlw $8, %xmm2 ; SSE41-NEXT: packuswb %xmm3, %xmm2 ; SSE41-NEXT: paddb %xmm1, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm0 @@ -2223,15 +2222,15 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) { ; AVX1: # %bb.0: ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm2 -; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4 -; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4,5],xmm4[6],xmm3[7] -; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15] ; AVX1-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [256,2,2,2,2,128,2,128] ; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1 -; AVX1-NEXT: vpackuswb %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero +; AVX1-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpmovsxbw %xmm0, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4,5],xmm2[6],xmm3[7] +; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX1-NEXT: vpackuswb %xmm1, %xmm2, %xmm1 ; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm1 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] ; AVX1-NEXT: vpsraw $8, %xmm2, %xmm2 diff --git a/llvm/test/CodeGen/X86/combine-sra.ll b/llvm/test/CodeGen/X86/combine-sra.ll index fa5c5ecded124..c982884314f62 100644 --- a/llvm/test/CodeGen/X86/combine-sra.ll +++ b/llvm/test/CodeGen/X86/combine-sra.ll @@ -724,47 +724,46 @@ define <4 x i64> @combine_vec4i64_ashr_clamped(<4 x i64> %x, <4 x i64> %y) { ; SSE41-LABEL: combine_vec4i64_ashr_clamped: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [2147483711,2147483711,2147483711,2147483711] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: movapd {{.*#+}} xmm8 = [63,63] -; SSE41-NEXT: movapd %xmm8, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm3, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pand %xmm7, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm3, %xmm6 +; SSE41-NEXT: pxor %xmm7, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm8 = [9223372039002259519,9223372039002259519] +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483711,2147483711,2147483711,2147483711] +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm9 = [63,63] +; SSE41-NEXT: movapd %xmm9, %xmm6 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm7, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pand %xmm8, %xmm5 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm9 ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: psrlq %xmm8, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm8[2,3,2,3] -; SSE41-NEXT: movdqa %xmm0, %xmm6 -; SSE41-NEXT: psrlq %xmm3, %xmm6 -; SSE41-NEXT: pblendw {{.*#+}} xmm6 = xmm2[0,1,2,3],xmm6[4,5,6,7] +; SSE41-NEXT: psrlq %xmm9, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm9[2,3,2,3] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: psrlq %xmm3, %xmm5 +; SSE41-NEXT: pblendw {{.*#+}} xmm5 = xmm2[0,1,2,3],xmm5[4,5,6,7] ; SSE41-NEXT: movdqa %xmm4, %xmm2 -; SSE41-NEXT: psrlq %xmm8, %xmm2 +; SSE41-NEXT: psrlq %xmm9, %xmm2 ; SSE41-NEXT: psrlq %xmm3, %xmm4 ; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm2[0,1,2,3],xmm4[4,5,6,7] -; SSE41-NEXT: pxor %xmm6, %xmm4 -; SSE41-NEXT: psubq %xmm6, %xmm4 +; SSE41-NEXT: pxor %xmm5, %xmm4 +; SSE41-NEXT: psubq %xmm5, %xmm4 ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: psrlq %xmm5, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm5[2,3,2,3] +; SSE41-NEXT: psrlq %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm6[2,3,2,3] ; SSE41-NEXT: psrlq %xmm3, %xmm0 ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7] ; SSE41-NEXT: movdqa %xmm1, %xmm2 -; SSE41-NEXT: psrlq %xmm5, %xmm2 +; SSE41-NEXT: psrlq %xmm6, %xmm2 ; SSE41-NEXT: psrlq %xmm3, %xmm1 ; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7] ; SSE41-NEXT: pxor %xmm0, %xmm1 diff --git a/llvm/test/CodeGen/X86/combine-udiv.ll b/llvm/test/CodeGen/X86/combine-udiv.ll index 6230c883cd7c5..55715197830b1 100644 --- a/llvm/test/CodeGen/X86/combine-udiv.ll +++ b/llvm/test/CodeGen/X86/combine-udiv.ll @@ -631,10 +631,7 @@ define <16 x i8> @combine_vec_udiv_nonuniform4(<16 x i8> %x) { ; SSE2-NEXT: pxor %xmm3, %xmm3 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] ; SSE2-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: psrlw $8, %xmm0 -; SSE2-NEXT: packuswb %xmm0, %xmm0 -; SSE2-NEXT: psrlw $7, %xmm0 -; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: psrlw $15, %xmm0 ; SSE2-NEXT: pandn %xmm0, %xmm1 ; SSE2-NEXT: por %xmm2, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm0 diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll index ce68eebd5b752..7a1535bd6b3f1 100644 --- a/llvm/test/CodeGen/X86/extractelement-load.ll +++ b/llvm/test/CodeGen/X86/extractelement-load.ll @@ -558,13 +558,14 @@ define dso_local <2 x float> @multiuse_of_single_value_from_vbroadcast_load(ptr ; X64-SSSE3-NEXT: pushq %rbx ; X64-SSSE3-NEXT: subq $16, %rsp ; X64-SSSE3-NEXT: movq %rsi, %rbx -; X64-SSSE3-NEXT: movddup {{.*#+}} xmm0 = mem[0,0] -; X64-SSSE3-NEXT: movapd %xmm0, (%rsp) # 16-byte Spill -; X64-SSSE3-NEXT: movlpd %xmm0, (%rdi) +; X64-SSSE3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero +; X64-SSSE3-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-SSSE3-NEXT: movsd %xmm0, (%rdi) ; X64-SSSE3-NEXT: movaps 32(%rsi), %xmm0 ; X64-SSSE3-NEXT: callq ccosf@PLT ; X64-SSSE3-NEXT: movlps %xmm0, 32(%rbx) -; X64-SSSE3-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload +; X64-SSSE3-NEXT: movddup (%rsp), %xmm0 # 16-byte Folded Reload +; X64-SSSE3-NEXT: # xmm0 = mem[0,0] ; X64-SSSE3-NEXT: callq ccosf@PLT ; X64-SSSE3-NEXT: addq $16, %rsp ; X64-SSSE3-NEXT: popq %rbx diff --git a/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll index c482ba0268f1c..1886e2911ede8 100644 --- a/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll +++ b/llvm/test/CodeGen/X86/f16c-intrinsics-fast-isel.ll @@ -40,8 +40,6 @@ define i16 @test_cvtss_sh(float %a0) nounwind { ; X86-LABEL: test_cvtss_sh: ; X86: # %bb.0: ; X86-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; X86-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3] ; X86-NEXT: vcvtps2ph $0, %xmm0, %xmm0 ; X86-NEXT: vmovd %xmm0, %eax ; X86-NEXT: # kill: def $ax killed $ax killed $eax @@ -49,8 +47,6 @@ define i16 @test_cvtss_sh(float %a0) nounwind { ; ; X64-LABEL: test_cvtss_sh: ; X64: # %bb.0: -; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; X64-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3] ; X64-NEXT: vcvtps2ph $0, %xmm0, %xmm0 ; X64-NEXT: vmovd %xmm0, %eax ; X64-NEXT: # kill: def $ax killed $ax killed $eax diff --git a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll index 3211002aa8b9f..d59b12c6d1231 100644 --- a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll +++ b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll @@ -141,59 +141,62 @@ declare <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half>, <8 x i16>) define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) { ; CHECK-SSE-LABEL: fmul_pow2_8xhalf: ; CHECK-SSE: # %bb.0: -; CHECK-SSE-NEXT: subq $120, %rsp -; CHECK-SSE-NEXT: .cfi_def_cfa_offset 128 -; CHECK-SSE-NEXT: pxor %xmm1, %xmm1 -; CHECK-SSE-NEXT: movdqa %xmm0, %xmm2 -; CHECK-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; CHECK-SSE-NEXT: pslld $23, %xmm2 -; CHECK-SSE-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] -; CHECK-SSE-NEXT: paddd %xmm3, %xmm2 -; CHECK-SSE-NEXT: cvttps2dq %xmm2, %xmm2 -; CHECK-SSE-NEXT: pslld $16, %xmm2 -; CHECK-SSE-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill -; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; CHECK-SSE-NEXT: subq $104, %rsp +; CHECK-SSE-NEXT: .cfi_def_cfa_offset 112 +; CHECK-SSE-NEXT: movdqa %xmm0, %xmm1 +; CHECK-SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] +; CHECK-SSE-NEXT: pslld $23, %xmm1 +; CHECK-SSE-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] +; CHECK-SSE-NEXT: paddd %xmm2, %xmm1 +; CHECK-SSE-NEXT: cvttps2dq %xmm1, %xmm1 +; CHECK-SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-SSE-NEXT: pslld $16, %xmm1 +; CHECK-SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill +; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; CHECK-SSE-NEXT: pslld $23, %xmm0 -; CHECK-SSE-NEXT: paddd %xmm3, %xmm0 +; CHECK-SSE-NEXT: paddd %xmm2, %xmm0 ; CHECK-SSE-NEXT: cvttps2dq %xmm0, %xmm0 +; CHECK-SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; CHECK-SSE-NEXT: pslld $16, %xmm0 ; CHECK-SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; CHECK-SSE-NEXT: psrld $16, %xmm0 -; CHECK-SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; CHECK-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] ; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; CHECK-SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; CHECK-SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-SSE-NEXT: psrlq $48, %xmm0 ; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; CHECK-SSE-NEXT: cvtdq2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; CHECK-SSE-NEXT: callq __truncsfhf2@PLT -; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; CHECK-SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; CHECK-SSE-NEXT: psrlq $48, %xmm0 +; CHECK-SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; CHECK-SSE-NEXT: xorps %xmm0, %xmm0 +; CHECK-SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; CHECK-SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; CHECK-SSE-NEXT: cvtdq2ps %xmm1, %xmm0 +; CHECK-SSE-NEXT: callq __truncsfhf2@PLT +; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; CHECK-SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; CHECK-SSE-NEXT: psrld $16, %xmm0 -; CHECK-SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; CHECK-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] ; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; CHECK-SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload -; CHECK-SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-SSE-NEXT: psrlq $48, %xmm0 ; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; CHECK-SSE-NEXT: cvtdq2ps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; CHECK-SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; CHECK-SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload -; CHECK-SSE-NEXT: psrlq $48, %xmm0 +; CHECK-SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-SSE-NEXT: pxor %xmm1, %xmm1 +; CHECK-SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; CHECK-SSE-NEXT: cvtdq2ps %xmm0, %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: callq __extendhfsf2@PLT @@ -205,9 +208,9 @@ define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) { ; CHECK-SSE-NEXT: callq __extendhfsf2@PLT ; CHECK-SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT -; CHECK-SSE-NEXT: punpcklwd (%rsp), %xmm0 # 16-byte Folded Reload -; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] -; CHECK-SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill +; CHECK-SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload +; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; CHECK-SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; CHECK-SSE-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload ; CHECK-SSE-NEXT: # xmm0 = mem[0],zero,zero,zero ; CHECK-SSE-NEXT: callq __extendhfsf2@PLT @@ -221,23 +224,23 @@ define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) { ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload ; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] -; CHECK-SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload -; CHECK-SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; CHECK-SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill +; CHECK-SSE-NEXT: punpckldq (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] +; CHECK-SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; CHECK-SSE-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload ; CHECK-SSE-NEXT: # xmm0 = mem[0],zero,zero,zero ; CHECK-SSE-NEXT: callq __extendhfsf2@PLT ; CHECK-SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT -; CHECK-SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill ; CHECK-SSE-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload ; CHECK-SSE-NEXT: # xmm0 = mem[0],zero,zero,zero ; CHECK-SSE-NEXT: callq __extendhfsf2@PLT ; CHECK-SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT -; CHECK-SSE-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] -; CHECK-SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload +; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; CHECK-SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; CHECK-SSE-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload ; CHECK-SSE-NEXT: # xmm0 = mem[0],zero,zero,zero ; CHECK-SSE-NEXT: callq __extendhfsf2@PLT @@ -251,12 +254,11 @@ define <8 x half> @fmul_pow2_8xhalf(<8 x i16> %i) { ; CHECK-SSE-NEXT: callq __truncsfhf2@PLT ; CHECK-SSE-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload ; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] -; CHECK-SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; CHECK-SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; CHECK-SSE-NEXT: punpcklqdq (%rsp), %xmm1 # 16-byte Folded Reload -; CHECK-SSE-NEXT: # xmm1 = xmm1[0],mem[0] -; CHECK-SSE-NEXT: movdqa %xmm1, %xmm0 -; CHECK-SSE-NEXT: addq $120, %rsp +; CHECK-SSE-NEXT: punpckldq (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] +; CHECK-SSE-NEXT: punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; CHECK-SSE-NEXT: # xmm0 = xmm0[0],mem[0] +; CHECK-SSE-NEXT: addq $104, %rsp ; CHECK-SSE-NEXT: .cfi_def_cfa_offset 8 ; CHECK-SSE-NEXT: retq ; @@ -1067,8 +1069,7 @@ define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind { ; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_fail_to_large: ; CHECK-SSE: # %bb.0: ; CHECK-SSE-NEXT: subq $40, %rsp -; CHECK-SSE-NEXT: pxor %xmm1, %xmm1 -; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; CHECK-SSE-NEXT: pslld $23, %xmm0 ; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE-NEXT: cvttps2dq %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll index 3a16c2dd7e801..7001bf7f28071 100644 --- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll +++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll @@ -17,32 +17,32 @@ define <16 x i8> @var_fshl_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou ; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero ; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero ; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] -; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm2 -; GFNISSE-NEXT: movdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216] -; GFNISSE-NEXT: paddd %xmm7, %xmm2 +; GFNISSE-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] +; GFNISSE-NEXT: paddd %xmm6, %xmm2 ; GFNISSE-NEXT: cvttps2dq %xmm2, %xmm2 -; GFNISSE-NEXT: pslld $23, %xmm6 -; GFNISSE-NEXT: paddd %xmm7, %xmm6 -; GFNISSE-NEXT: cvttps2dq %xmm6, %xmm6 -; GFNISSE-NEXT: packusdw %xmm2, %xmm6 -; GFNISSE-NEXT: movdqa %xmm1, %xmm8 -; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15] -; GFNISSE-NEXT: pmullw %xmm6, %xmm8 -; GFNISSE-NEXT: psrlw $8, %xmm8 +; GFNISSE-NEXT: pslld $23, %xmm3 +; GFNISSE-NEXT: paddd %xmm6, %xmm3 +; GFNISSE-NEXT: cvttps2dq %xmm3, %xmm3 +; GFNISSE-NEXT: packusdw %xmm2, %xmm3 +; GFNISSE-NEXT: movdqa %xmm1, %xmm7 +; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15] +; GFNISSE-NEXT: pmullw %xmm3, %xmm7 +; GFNISSE-NEXT: psrlw $8, %xmm7 ; GFNISSE-NEXT: pslld $23, %xmm4 -; GFNISSE-NEXT: paddd %xmm7, %xmm4 +; GFNISSE-NEXT: paddd %xmm6, %xmm4 ; GFNISSE-NEXT: cvttps2dq %xmm4, %xmm2 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm5 -; GFNISSE-NEXT: paddd %xmm7, %xmm5 +; GFNISSE-NEXT: paddd %xmm6, %xmm5 ; GFNISSE-NEXT: cvttps2dq %xmm5, %xmm3 ; GFNISSE-NEXT: packusdw %xmm3, %xmm2 ; GFNISSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; GFNISSE-NEXT: pmullw %xmm1, %xmm2 ; GFNISSE-NEXT: psrlw $8, %xmm2 -; GFNISSE-NEXT: packuswb %xmm8, %xmm2 +; GFNISSE-NEXT: packuswb %xmm7, %xmm2 ; GFNISSE-NEXT: movdqa %xmm2, %xmm0 ; GFNISSE-NEXT: retq ; @@ -50,34 +50,34 @@ define <16 x i8> @var_fshl_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou ; GFNIAVX1: # %bb.0: ; GFNIAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 ; GFNIAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] -; GFNIAVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vcvttps2dq %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm4, %xmm4 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; GFNIAVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vcvttps2dq %xmm4, %xmm4 +; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero +; GFNIAVX1-NEXT: vpslld $23, %xmm3, %xmm3 +; GFNIAVX1-NEXT: vpaddd %xmm5, %xmm3, %xmm3 +; GFNIAVX1-NEXT: vcvttps2dq %xmm3, %xmm3 +; GFNIAVX1-NEXT: vpackusdw %xmm4, %xmm3, %xmm3 +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; GFNIAVX1-NEXT: vpmullw %xmm3, %xmm4, %xmm3 +; GFNIAVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; GFNIAVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 ; GFNIAVX1-NEXT: vcvttps2dq %xmm4, %xmm4 -; GFNIAVX1-NEXT: vpackusdw %xmm5, %xmm4, %xmm4 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] -; GFNIAVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4 -; GFNIAVX1-NEXT: vpsrlw $8, %xmm4, %xmm4 -; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; GFNIAVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vcvttps2dq %xmm5, %xmm5 ; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2 +; GFNIAVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 ; GFNIAVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; GFNIAVX1-NEXT: vpackusdw %xmm2, %xmm5, %xmm2 +; GFNIAVX1-NEXT: vpackusdw %xmm2, %xmm4, %xmm2 ; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; GFNIAVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm0, %xmm0 -; GFNIAVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0 +; GFNIAVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0 ; GFNIAVX1-NEXT: retq ; ; GFNIAVX2-LABEL: var_fshl_v16i8: @@ -541,7 +541,7 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou ; GFNISSE-NEXT: movdqa %xmm4, %xmm10 ; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15] ; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm10 ; GFNISSE-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] ; GFNISSE-NEXT: paddd %xmm4, %xmm10 @@ -557,7 +557,7 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou ; GFNISSE-NEXT: pslld $23, %xmm0 ; GFNISSE-NEXT: paddd %xmm4, %xmm0 ; GFNISSE-NEXT: cvttps2dq %xmm0, %xmm0 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm9 ; GFNISSE-NEXT: paddd %xmm4, %xmm9 ; GFNISSE-NEXT: cvttps2dq %xmm9, %xmm9 @@ -570,23 +570,23 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou ; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero ; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero ; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15] -; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm8 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7] +; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm5 ; GFNISSE-NEXT: paddd %xmm4, %xmm5 ; GFNISSE-NEXT: cvttps2dq %xmm5, %xmm5 -; GFNISSE-NEXT: pslld $23, %xmm8 -; GFNISSE-NEXT: paddd %xmm4, %xmm8 -; GFNISSE-NEXT: cvttps2dq %xmm8, %xmm8 -; GFNISSE-NEXT: packusdw %xmm5, %xmm8 +; GFNISSE-NEXT: pslld $23, %xmm7 +; GFNISSE-NEXT: paddd %xmm4, %xmm7 +; GFNISSE-NEXT: cvttps2dq %xmm7, %xmm7 +; GFNISSE-NEXT: packusdw %xmm5, %xmm7 ; GFNISSE-NEXT: movdqa %xmm3, %xmm5 ; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15] -; GFNISSE-NEXT: pmullw %xmm8, %xmm5 +; GFNISSE-NEXT: pmullw %xmm7, %xmm5 ; GFNISSE-NEXT: psrlw $8, %xmm5 ; GFNISSE-NEXT: pslld $23, %xmm2 ; GFNISSE-NEXT: paddd %xmm4, %xmm2 ; GFNISSE-NEXT: cvttps2dq %xmm2, %xmm2 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm6 ; GFNISSE-NEXT: paddd %xmm4, %xmm6 ; GFNISSE-NEXT: cvttps2dq %xmm6, %xmm4 @@ -601,17 +601,17 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou ; GFNIAVX1-LABEL: var_fshl_v32i8: ; GFNIAVX1: # %bb.0: ; GFNIAVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 -; GFNIAVX1-NEXT: vextractf128 $1, %ymm2, %xmm5 -; GFNIAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] -; GFNIAVX1-NEXT: vpslld $23, %xmm4, %xmm7 -; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 +; GFNIAVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; GFNIAVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5 +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm6[4,4,5,5,6,6,7,7] +; GFNIAVX1-NEXT: vpslld $23, %xmm3, %xmm7 +; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm7, %xmm7 ; GFNIAVX1-NEXT: vcvttps2dq %xmm7, %xmm7 ; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero ; GFNIAVX1-NEXT: vpslld $23, %xmm6, %xmm6 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm6, %xmm6 ; GFNIAVX1-NEXT: vcvttps2dq %xmm6, %xmm6 ; GFNIAVX1-NEXT: vpackusdw %xmm7, %xmm6, %xmm6 ; GFNIAVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 @@ -619,48 +619,48 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou ; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] ; GFNIAVX1-NEXT: vpmullw %xmm6, %xmm9, %xmm6 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 -; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm9 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero +; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm9 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero ; GFNIAVX1-NEXT: vpslld $23, %xmm9, %xmm9 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm9, %xmm9 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm9, %xmm9 ; GFNIAVX1-NEXT: vcvttps2dq %xmm9, %xmm9 -; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7] +; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7] +; GFNIAVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vcvttps2dq %xmm4, %xmm4 +; GFNIAVX1-NEXT: vpackusdw %xmm4, %xmm9, %xmm4 +; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] +; GFNIAVX1-NEXT: vpmullw %xmm4, %xmm7, %xmm4 +; GFNIAVX1-NEXT: vpsrlw $8, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vpackuswb %xmm6, %xmm4, %xmm4 +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm5[4,4,5,5,6,6,7,7] +; GFNIAVX1-NEXT: vpslld $23, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vcvttps2dq %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero ; GFNIAVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm5 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm5, %xmm5 ; GFNIAVX1-NEXT: vcvttps2dq %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpackusdw %xmm5, %xmm9, %xmm5 -; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] -; GFNIAVX1-NEXT: vpmullw %xmm5, %xmm7, %xmm5 +; GFNIAVX1-NEXT: vpackusdw %xmm6, %xmm5, %xmm5 +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; GFNIAVX1-NEXT: vpmullw %xmm5, %xmm6, %xmm5 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] -; GFNIAVX1-NEXT: vpslld $23, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vcvttps2dq %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero +; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero ; GFNIAVX1-NEXT: vpslld $23, %xmm6, %xmm6 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm6, %xmm6 ; GFNIAVX1-NEXT: vcvttps2dq %xmm6, %xmm6 -; GFNIAVX1-NEXT: vpackusdw %xmm7, %xmm6, %xmm6 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] -; GFNIAVX1-NEXT: vpmullw %xmm6, %xmm7, %xmm6 -; GFNIAVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 -; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; GFNIAVX1-NEXT: vpslld $23, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vcvttps2dq %xmm7, %xmm7 ; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2 +; GFNIAVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 ; GFNIAVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; GFNIAVX1-NEXT: vpackusdw %xmm2, %xmm7, %xmm2 +; GFNIAVX1-NEXT: vpackusdw %xmm2, %xmm6, %xmm2 ; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; GFNIAVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm0, %xmm0 -; GFNIAVX1-NEXT: vpackuswb %xmm6, %xmm0, %xmm0 -; GFNIAVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0 +; GFNIAVX1-NEXT: vpackuswb %xmm5, %xmm0, %xmm0 +; GFNIAVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 ; GFNIAVX1-NEXT: retq ; ; GFNIAVX2-LABEL: var_fshl_v32i8: @@ -1365,15 +1365,15 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE: # %bb.0: ; GFNISSE-NEXT: movdqa %xmm1, %xmm8 ; GFNISSE-NEXT: movdqa %xmm0, %xmm1 -; GFNISSE-NEXT: movdqa {{.*#+}} xmm10 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] +; GFNISSE-NEXT: movdqa {{.*#+}} xmm9 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] ; GFNISSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0 -; GFNISSE-NEXT: pand %xmm10, %xmm0 -; GFNISSE-NEXT: pxor %xmm9, %xmm9 +; GFNISSE-NEXT: pand %xmm9, %xmm0 +; GFNISSE-NEXT: pxor %xmm10, %xmm10 ; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm12 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm13 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] +; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15] ; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm14 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm0 ; GFNISSE-NEXT: movdqa {{.*#+}} xmm11 = [1065353216,1065353216,1065353216,1065353216] ; GFNISSE-NEXT: paddd %xmm11, %xmm0 @@ -1389,7 +1389,7 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: pslld $23, %xmm12 ; GFNISSE-NEXT: paddd %xmm11, %xmm12 ; GFNISSE-NEXT: cvttps2dq %xmm12, %xmm0 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm9[4],xmm13[5],xmm9[5],xmm13[6],xmm9[6],xmm13[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm13 ; GFNISSE-NEXT: paddd %xmm11, %xmm13 ; GFNISSE-NEXT: cvttps2dq %xmm13, %xmm12 @@ -1399,12 +1399,12 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: psrlw $8, %xmm0 ; GFNISSE-NEXT: packuswb %xmm15, %xmm0 ; GFNISSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm1 -; GFNISSE-NEXT: pand %xmm10, %xmm1 +; GFNISSE-NEXT: pand %xmm9, %xmm1 ; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero ; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm12 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero -; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm9[8],xmm1[9],xmm9[9],xmm1[10],xmm9[10],xmm1[11],xmm9[11],xmm1[12],xmm9[12],xmm1[13],xmm9[13],xmm1[14],xmm9[14],xmm1[15],xmm9[15] +; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm10[8],xmm1[9],xmm10[9],xmm1[10],xmm10[10],xmm1[11],xmm10[11],xmm1[12],xmm10[12],xmm1[13],xmm10[13],xmm1[14],xmm10[14],xmm1[15],xmm10[15] ; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm13 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm1 ; GFNISSE-NEXT: paddd %xmm11, %xmm1 ; GFNISSE-NEXT: cvttps2dq %xmm1, %xmm1 @@ -1419,7 +1419,7 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: pslld $23, %xmm4 ; GFNISSE-NEXT: paddd %xmm11, %xmm4 ; GFNISSE-NEXT: cvttps2dq %xmm4, %xmm1 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm12 ; GFNISSE-NEXT: paddd %xmm11, %xmm12 ; GFNISSE-NEXT: cvttps2dq %xmm12, %xmm4 @@ -1429,12 +1429,12 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: psrlw $8, %xmm1 ; GFNISSE-NEXT: packuswb %xmm14, %xmm1 ; GFNISSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm4 -; GFNISSE-NEXT: pand %xmm10, %xmm4 +; GFNISSE-NEXT: pand %xmm9, %xmm4 ; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero ; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm8 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero -; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm9[8],xmm4[9],xmm9[9],xmm4[10],xmm9[10],xmm4[11],xmm9[11],xmm4[12],xmm9[12],xmm4[13],xmm9[13],xmm4[14],xmm9[14],xmm4[15],xmm9[15] +; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm10[8],xmm4[9],xmm10[9],xmm4[10],xmm10[10],xmm4[11],xmm10[11],xmm4[12],xmm10[12],xmm4[13],xmm10[13],xmm4[14],xmm10[14],xmm4[15],xmm10[15] ; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm12 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm4 ; GFNISSE-NEXT: paddd %xmm11, %xmm4 ; GFNISSE-NEXT: cvttps2dq %xmm4, %xmm4 @@ -1449,7 +1449,7 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: pslld $23, %xmm5 ; GFNISSE-NEXT: paddd %xmm11, %xmm5 ; GFNISSE-NEXT: cvttps2dq %xmm5, %xmm4 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm8 ; GFNISSE-NEXT: paddd %xmm11, %xmm8 ; GFNISSE-NEXT: cvttps2dq %xmm8, %xmm5 @@ -1458,15 +1458,15 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: pmullw %xmm6, %xmm4 ; GFNISSE-NEXT: psrlw $8, %xmm4 ; GFNISSE-NEXT: packuswb %xmm13, %xmm4 -; GFNISSE-NEXT: pand {{[0-9]+}}(%rsp), %xmm10 -; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero -; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero -; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] -; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm5 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7] -; GFNISSE-NEXT: pslld $23, %xmm10 -; GFNISSE-NEXT: paddd %xmm11, %xmm10 -; GFNISSE-NEXT: cvttps2dq %xmm10, %xmm8 +; GFNISSE-NEXT: pand {{[0-9]+}}(%rsp), %xmm9 +; GFNISSE-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero +; GFNISSE-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero +; GFNISSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm10[8],xmm9[9],xmm10[9],xmm9[10],xmm10[10],xmm9[11],xmm10[11],xmm9[12],xmm10[12],xmm9[13],xmm10[13],xmm9[14],xmm10[14],xmm9[15],xmm10[15] +; GFNISSE-NEXT: pmovzxwd {{.*#+}} xmm5 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4,4,5,5,6,6,7,7] +; GFNISSE-NEXT: pslld $23, %xmm9 +; GFNISSE-NEXT: paddd %xmm11, %xmm9 +; GFNISSE-NEXT: cvttps2dq %xmm9, %xmm8 ; GFNISSE-NEXT: pslld $23, %xmm5 ; GFNISSE-NEXT: paddd %xmm11, %xmm5 ; GFNISSE-NEXT: cvttps2dq %xmm5, %xmm5 @@ -1478,7 +1478,7 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNISSE-NEXT: pslld $23, %xmm2 ; GFNISSE-NEXT: paddd %xmm11, %xmm2 ; GFNISSE-NEXT: cvttps2dq %xmm2, %xmm5 -; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7] +; GFNISSE-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7] ; GFNISSE-NEXT: pslld $23, %xmm6 ; GFNISSE-NEXT: paddd %xmm11, %xmm6 ; GFNISSE-NEXT: cvttps2dq %xmm6, %xmm2 @@ -1496,16 +1496,16 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7] ; GFNIAVX1-NEXT: vandps %ymm7, %ymm4, %ymm8 ; GFNIAVX1-NEXT: vextractf128 $1, %ymm8, %xmm9 -; GFNIAVX1-NEXT: vxorps %xmm4, %xmm4, %xmm4 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7] -; GFNIAVX1-NEXT: vpslld $23, %xmm6, %xmm11 -; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm11, %xmm11 +; GFNIAVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm9[8],xmm6[8],xmm9[9],xmm6[9],xmm9[10],xmm6[10],xmm9[11],xmm6[11],xmm9[12],xmm6[12],xmm9[13],xmm6[13],xmm9[14],xmm6[14],xmm9[15],xmm6[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm10[4,4,5,5,6,6,7,7] +; GFNIAVX1-NEXT: vpslld $23, %xmm4, %xmm11 +; GFNIAVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm11, %xmm11 ; GFNIAVX1-NEXT: vcvttps2dq %xmm11, %xmm11 ; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero ; GFNIAVX1-NEXT: vpslld $23, %xmm10, %xmm10 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm10, %xmm10 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vcvttps2dq %xmm10, %xmm10 ; GFNIAVX1-NEXT: vpackusdw %xmm11, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vextractf128 $1, %ymm0, %xmm11 @@ -1515,26 +1515,26 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNIAVX1-NEXT: vpsrlw $8, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm13 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero ; GFNIAVX1-NEXT: vpslld $23, %xmm13, %xmm13 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm13, %xmm13 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm13, %xmm13 ; GFNIAVX1-NEXT: vcvttps2dq %xmm13, %xmm13 ; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm9[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm9, %xmm9 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm9, %xmm9 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm9, %xmm9 ; GFNIAVX1-NEXT: vcvttps2dq %xmm9, %xmm9 ; GFNIAVX1-NEXT: vpackusdw %xmm9, %xmm13, %xmm9 ; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7] ; GFNIAVX1-NEXT: vpmullw %xmm9, %xmm11, %xmm9 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm9, %xmm9 ; GFNIAVX1-NEXT: vpackuswb %xmm10, %xmm9, %xmm9 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm8[8],xmm4[8],xmm8[9],xmm4[9],xmm8[10],xmm4[10],xmm8[11],xmm4[11],xmm8[12],xmm4[12],xmm8[13],xmm4[13],xmm8[14],xmm4[14],xmm8[15],xmm4[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm11 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7] +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm8[8],xmm6[8],xmm8[9],xmm6[9],xmm8[10],xmm6[10],xmm8[11],xmm6[11],xmm8[12],xmm6[12],xmm8[13],xmm6[13],xmm8[14],xmm6[14],xmm8[15],xmm6[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm11 = xmm10[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm11, %xmm11 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm11, %xmm11 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm11, %xmm11 ; GFNIAVX1-NEXT: vcvttps2dq %xmm11, %xmm11 ; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero ; GFNIAVX1-NEXT: vpslld $23, %xmm10, %xmm10 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm10, %xmm10 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vcvttps2dq %xmm10, %xmm10 ; GFNIAVX1-NEXT: vpackusdw %xmm11, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] @@ -1542,12 +1542,12 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNIAVX1-NEXT: vpsrlw $8, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm11 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero ; GFNIAVX1-NEXT: vpslld $23, %xmm11, %xmm11 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm11, %xmm11 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm11, %xmm11 ; GFNIAVX1-NEXT: vcvttps2dq %xmm11, %xmm11 ; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm8, %xmm8 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm8, %xmm8 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm8, %xmm8 ; GFNIAVX1-NEXT: vcvttps2dq %xmm8, %xmm8 ; GFNIAVX1-NEXT: vpackusdw %xmm8, %xmm11, %xmm8 ; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] @@ -1557,14 +1557,14 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNIAVX1-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm0 ; GFNIAVX1-NEXT: vandps %ymm7, %ymm5, %ymm2 ; GFNIAVX1-NEXT: vextractf128 $1, %ymm2, %xmm5 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm7[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm8, %xmm8 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm8, %xmm8 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm8, %xmm8 ; GFNIAVX1-NEXT: vcvttps2dq %xmm8, %xmm8 ; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero ; GFNIAVX1-NEXT: vpslld $23, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm7, %xmm7 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 ; GFNIAVX1-NEXT: vcvttps2dq %xmm7, %xmm7 ; GFNIAVX1-NEXT: vpackusdw %xmm8, %xmm7, %xmm7 ; GFNIAVX1-NEXT: vextractf128 $1, %ymm1, %xmm8 @@ -1574,45 +1574,45 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou ; GFNIAVX1-NEXT: vpsrlw $8, %xmm7, %xmm7 ; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm10 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero ; GFNIAVX1-NEXT: vpslld $23, %xmm10, %xmm10 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm10, %xmm10 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm10, %xmm10 ; GFNIAVX1-NEXT: vcvttps2dq %xmm10, %xmm10 ; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm5 ; GFNIAVX1-NEXT: vcvttps2dq %xmm5, %xmm5 ; GFNIAVX1-NEXT: vpackusdw %xmm5, %xmm10, %xmm5 ; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7] ; GFNIAVX1-NEXT: vpmullw %xmm5, %xmm8, %xmm5 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 ; GFNIAVX1-NEXT: vpackuswb %xmm7, %xmm5, %xmm5 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15] -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] -; GFNIAVX1-NEXT: vpslld $23, %xmm8, %xmm8 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm8, %xmm8 -; GFNIAVX1-NEXT: vcvttps2dq %xmm8, %xmm8 -; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm6[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm7, %xmm7 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 +; GFNIAVX1-NEXT: vcvttps2dq %xmm7, %xmm7 +; GFNIAVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero +; GFNIAVX1-NEXT: vpslld $23, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vcvttps2dq %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpackusdw %xmm7, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15] +; GFNIAVX1-NEXT: vpmullw %xmm6, %xmm7, %xmm6 +; GFNIAVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 +; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; GFNIAVX1-NEXT: vpslld $23, %xmm7, %xmm7 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 ; GFNIAVX1-NEXT: vcvttps2dq %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpackusdw %xmm8, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15] -; GFNIAVX1-NEXT: vpmullw %xmm7, %xmm8, %xmm7 -; GFNIAVX1-NEXT: vpsrlw $8, %xmm7, %xmm7 -; GFNIAVX1-NEXT: vpmovzxbd {{.*#+}} xmm8 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; GFNIAVX1-NEXT: vpslld $23, %xmm8, %xmm8 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm8, %xmm8 -; GFNIAVX1-NEXT: vcvttps2dq %xmm8, %xmm8 ; GFNIAVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; GFNIAVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; GFNIAVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; GFNIAVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2 +; GFNIAVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2 ; GFNIAVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; GFNIAVX1-NEXT: vpackusdw %xmm2, %xmm8, %xmm2 +; GFNIAVX1-NEXT: vpackusdw %xmm2, %xmm7, %xmm2 ; GFNIAVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7] ; GFNIAVX1-NEXT: vpmullw %xmm2, %xmm1, %xmm1 ; GFNIAVX1-NEXT: vpsrlw $8, %xmm1, %xmm1 -; GFNIAVX1-NEXT: vpackuswb %xmm7, %xmm1, %xmm1 +; GFNIAVX1-NEXT: vpackuswb %xmm6, %xmm1, %xmm1 ; GFNIAVX1-NEXT: vinsertf128 $1, %xmm5, %ymm1, %ymm1 ; GFNIAVX1-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll index b6a4a12eb0fac..a66e584380d26 100644 --- a/llvm/test/CodeGen/X86/half.ll +++ b/llvm/test/CodeGen/X86/half.ll @@ -1114,8 +1114,8 @@ define void @main.45() #0 { ; CHECK-LIBCALL-NEXT: pushq %rbx ; CHECK-LIBCALL-NEXT: subq $16, %rsp ; CHECK-LIBCALL-NEXT: pinsrw $0, (%rax), %xmm0 +; CHECK-LIBCALL-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill ; CHECK-LIBCALL-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,0,0,0,4,5,6,7] -; CHECK-LIBCALL-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; CHECK-LIBCALL-NEXT: movq %xmm1, %rbx ; CHECK-LIBCALL-NEXT: movq %rbx, %r14 ; CHECK-LIBCALL-NEXT: shrq $48, %r14 @@ -1127,14 +1127,14 @@ define void @main.45() #0 { ; CHECK-LIBCALL-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; CHECK-LIBCALL-NEXT: pextrw $0, %xmm0, %eax ; CHECK-LIBCALL-NEXT: movl $32256, %ecx # imm = 0x7E00 -; CHECK-LIBCALL-NEXT: cmovpl %ecx, %eax ; CHECK-LIBCALL-NEXT: cmovpl %ecx, %ebx ; CHECK-LIBCALL-NEXT: cmovpl %ecx, %r15d ; CHECK-LIBCALL-NEXT: cmovpl %ecx, %r14d +; CHECK-LIBCALL-NEXT: cmovpl %ecx, %eax +; CHECK-LIBCALL-NEXT: movw %ax, (%rax) ; CHECK-LIBCALL-NEXT: movw %r14w, (%rax) ; CHECK-LIBCALL-NEXT: movw %r15w, (%rax) ; CHECK-LIBCALL-NEXT: movw %bx, (%rax) -; CHECK-LIBCALL-NEXT: movw %ax, (%rax) ; CHECK-LIBCALL-NEXT: addq $16, %rsp ; CHECK-LIBCALL-NEXT: popq %rbx ; CHECK-LIBCALL-NEXT: popq %r14 diff --git a/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll index eff6bcfe570a1..1a2aac657d30f 100644 --- a/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll +++ b/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll @@ -542,14 +542,16 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; X86-SSE2-LABEL: vec_4xi32_nonsplat_undef0_eq: ; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: movl $1, %eax +; X86-SSE2-NEXT: movd %eax, %xmm2 ; X86-SSE2-NEXT: pslld $23, %xmm1 ; X86-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] -; X86-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; X86-SSE2-NEXT: pand %xmm1, %xmm0 +; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2 +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; X86-SSE2-NEXT: pand %xmm2, %xmm0 ; X86-SSE2-NEXT: pxor %xmm1, %xmm1 ; X86-SSE2-NEXT: pcmpeqd %xmm1, %xmm0 ; X86-SSE2-NEXT: retl @@ -565,14 +567,16 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; X64-SSE2-LABEL: vec_4xi32_nonsplat_undef0_eq: ; X64-SSE2: # %bb.0: +; X64-SSE2-NEXT: movl $1, %eax +; X64-SSE2-NEXT: movd %eax, %xmm2 ; X64-SSE2-NEXT: pslld $23, %xmm1 ; X64-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; X64-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 -; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] -; X64-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; X64-SSE2-NEXT: pand %xmm1, %xmm0 +; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2 +; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; X64-SSE2-NEXT: pand %xmm2, %xmm0 ; X64-SSE2-NEXT: pxor %xmm1, %xmm1 ; X64-SSE2-NEXT: pcmpeqd %xmm1, %xmm0 ; X64-SSE2-NEXT: retq @@ -618,14 +622,16 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; X86-SSE2-LABEL: vec_4xi32_nonsplat_undef2_eq: ; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: movl $1, %eax +; X86-SSE2-NEXT: movd %eax, %xmm2 ; X86-SSE2-NEXT: pslld $23, %xmm1 ; X86-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] -; X86-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; X86-SSE2-NEXT: pand %xmm1, %xmm0 +; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2 +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; X86-SSE2-NEXT: pand %xmm2, %xmm0 ; X86-SSE2-NEXT: pxor %xmm1, %xmm1 ; X86-SSE2-NEXT: pcmpeqd %xmm1, %xmm0 ; X86-SSE2-NEXT: retl @@ -641,14 +647,16 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; X64-SSE2-LABEL: vec_4xi32_nonsplat_undef2_eq: ; X64-SSE2: # %bb.0: +; X64-SSE2-NEXT: movl $1, %eax +; X64-SSE2-NEXT: movd %eax, %xmm2 ; X64-SSE2-NEXT: pslld $23, %xmm1 ; X64-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; X64-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 -; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] -; X64-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; X64-SSE2-NEXT: pand %xmm1, %xmm0 +; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2 +; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; X64-SSE2-NEXT: pand %xmm2, %xmm0 ; X64-SSE2-NEXT: pxor %xmm1, %xmm1 ; X64-SSE2-NEXT: pcmpeqd %xmm1, %xmm0 ; X64-SSE2-NEXT: retq diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll index f6b8839afb2d7..63336ffa7c6c8 100644 --- a/llvm/test/CodeGen/X86/known-never-zero.ll +++ b/llvm/test/CodeGen/X86/known-never-zero.ll @@ -1243,8 +1243,7 @@ define i32 @mul_maybe_zero(i32 %x, i32 %y) { define i32 @bitcast_known_nonzero(<2 x i16> %xx) { ; X86-LABEL: bitcast_known_nonzero: ; X86: # %bb.0: -; X86-NEXT: pxor %xmm1, %xmm1 -; X86-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; X86-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; X86-NEXT: pslld $23, %xmm0 ; X86-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: cvttps2dq %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/known-pow2.ll b/llvm/test/CodeGen/X86/known-pow2.ll index be4605c007092..e183bbc15617d 100644 --- a/llvm/test/CodeGen/X86/known-pow2.ll +++ b/llvm/test/CodeGen/X86/known-pow2.ll @@ -30,12 +30,12 @@ define <4 x i32> @pow2_non_splat_vec_fail0(<4 x i32> %x) { ; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] ; CHECK-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 ; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3] -; CHECK-NEXT: movdqa %xmm1, %xmm4 -; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] -; CHECK-NEXT: psrld $1, %xmm1 -; CHECK-NEXT: movss {{.*#+}} xmm4 = xmm1[0],xmm4[1,2,3] -; CHECK-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 -; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3] +; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; CHECK-NEXT: movdqa %xmm1, %xmm3 +; CHECK-NEXT: psrld $1, %xmm3 +; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3] +; CHECK-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 +; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] ; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] ; CHECK-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 ; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] diff --git a/llvm/test/CodeGen/X86/known-signbits-shl.ll b/llvm/test/CodeGen/X86/known-signbits-shl.ll index 295a2eab029ec..473fecc307ed4 100644 --- a/llvm/test/CodeGen/X86/known-signbits-shl.ll +++ b/llvm/test/CodeGen/X86/known-signbits-shl.ll @@ -70,8 +70,7 @@ define void @computeNumSignBits_shl_zext_vec_1(<2 x i8> %x, ptr %p) nounwind { ; X64-NEXT: movdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4] ; X64-NEXT: pxor %xmm1, %xmm0 ; X64-NEXT: psubb %xmm1, %xmm0 -; X64-NEXT: pxor %xmm1, %xmm1 -; X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; X64-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [2048,8192,u,u,u,u,u,u] ; X64-NEXT: movd %xmm0, (%rdi) ; X64-NEXT: retq diff --git a/llvm/test/CodeGen/X86/known-signbits-vector.ll b/llvm/test/CodeGen/X86/known-signbits-vector.ll index 5368934fa5bf1..45b61155fe626 100644 --- a/llvm/test/CodeGen/X86/known-signbits-vector.ll +++ b/llvm/test/CodeGen/X86/known-signbits-vector.ll @@ -192,10 +192,8 @@ define float @signbits_ashr_shl_extract_sitofp(<2 x i64> %a0) nounwind { ; X86-LABEL: signbits_ashr_shl_extract_sitofp: ; X86: # %bb.0: ; X86-NEXT: pushl %eax -; X86-NEXT: vpsrad $31, %xmm0, %xmm1 ; X86-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; X86-NEXT: vpsrad $29, %xmm0, %xmm0 -; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] ; X86-NEXT: vpsllq $20, %xmm0, %xmm0 ; X86-NEXT: vcvtdq2ps %xmm0, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) @@ -203,25 +201,13 @@ define float @signbits_ashr_shl_extract_sitofp(<2 x i64> %a0) nounwind { ; X86-NEXT: popl %eax ; X86-NEXT: retl ; -; X64-AVX1-LABEL: signbits_ashr_shl_extract_sitofp: -; X64-AVX1: # %bb.0: -; X64-AVX1-NEXT: vpsrad $31, %xmm0, %xmm1 -; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; X64-AVX1-NEXT: vpsrad $29, %xmm0, %xmm0 -; X64-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] -; X64-AVX1-NEXT: vpsllq $20, %xmm0, %xmm0 -; X64-AVX1-NEXT: vcvtdq2ps %xmm0, %xmm0 -; X64-AVX1-NEXT: retq -; -; X64-AVX2-LABEL: signbits_ashr_shl_extract_sitofp: -; X64-AVX2: # %bb.0: -; X64-AVX2-NEXT: vpsrad $31, %xmm0, %xmm1 -; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; X64-AVX2-NEXT: vpsrad $29, %xmm0, %xmm0 -; X64-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] -; X64-AVX2-NEXT: vpsllq $20, %xmm0, %xmm0 -; X64-AVX2-NEXT: vcvtdq2ps %xmm0, %xmm0 -; X64-AVX2-NEXT: retq +; X64-LABEL: signbits_ashr_shl_extract_sitofp: +; X64: # %bb.0: +; X64-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; X64-NEXT: vpsrad $29, %xmm0, %xmm0 +; X64-NEXT: vpsllq $20, %xmm0, %xmm0 +; X64-NEXT: vcvtdq2ps %xmm0, %xmm0 +; X64-NEXT: retq %1 = ashr <2 x i64> %a0, %2 = shl <2 x i64> %1, %3 = extractelement <2 x i64> %2, i32 0 @@ -473,10 +459,8 @@ define <4 x float> @signbits_ashr_sext_select_shuffle_sitofp(<4 x i64> %a0, <4 x ; ; X64-AVX2-LABEL: signbits_ashr_sext_select_shuffle_sitofp: ; X64-AVX2: # %bb.0: -; X64-AVX2-NEXT: vpsrad $31, %ymm2, %ymm4 ; X64-AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,1,3,3,5,5,7,7] ; X64-AVX2-NEXT: vpsrad $1, %ymm2, %ymm2 -; X64-AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3],ymm2[4],ymm4[5],ymm2[6],ymm4[7] ; X64-AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero ; X64-AVX2-NEXT: vpcmpeqq %ymm1, %ymm0, %ymm0 ; X64-AVX2-NEXT: vblendvpd %ymm0, %ymm2, %ymm3, %ymm0 diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll index f4f7f44038153..6722d03d9b856 100644 --- a/llvm/test/CodeGen/X86/masked_store.ll +++ b/llvm/test/CodeGen/X86/masked_store.ll @@ -6178,7 +6178,7 @@ define void @undefshuffle(<8 x i1> %i0, ptr %src, ptr %dst) nounwind { ; AVX2-LABEL: undefshuffle: ; AVX2: ## %bb.0: ; AVX2-NEXT: ## kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,u,u,2,3,u,u,4,5,u,u,6,7,u,u],zero,zero,ymm0[u,u],zero,zero,ymm0[u,u],zero,zero,ymm0[u,u],zero,zero,ymm0[u,u] +; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,u,u,u,2,u,u,u,4,u,u,u,6,u,u,u],zero,ymm0[u,u,u],zero,ymm0[u,u,u],zero,ymm0[u,u,u],zero,ymm0[u,u,u] ; AVX2-NEXT: vpslld $31, %ymm0, %ymm0 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX2-NEXT: vpmaskmovd %ymm1, %ymm0, (%rsi) diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll index 7f50cac5e4290..ddf1a932f772f 100644 --- a/llvm/test/CodeGen/X86/movmsk-cmp.ll +++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll @@ -824,11 +824,7 @@ define i1 @allzeros_v4i64_sign(<4 x i64> %arg) { define i1 @allones_v8i64_sign(<8 x i64> %arg) { ; SSE-LABEL: allones_v8i64_sign: ; SSE: # %bb.0: -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] ; SSE-NEXT: packssdw %xmm3, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; SSE-NEXT: packssdw %xmm1, %xmm0 ; SSE-NEXT: packssdw %xmm2, %xmm0 ; SSE-NEXT: packsswb %xmm0, %xmm0 @@ -893,22 +889,10 @@ define i1 @allones_v8i64_sign(<8 x i64> %arg) { define i1 @allzeros_v8i64_sign(<8 x i64> %arg) { ; SSE-LABEL: allzeros_v8i64_sign: ; SSE: # %bb.0: -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE-NEXT: pxor %xmm4, %xmm4 -; SSE-NEXT: pxor %xmm5, %xmm5 -; SSE-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE-NEXT: pxor %xmm3, %xmm3 -; SSE-NEXT: pcmpgtd %xmm2, %xmm3 -; SSE-NEXT: packssdw %xmm5, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE-NEXT: pxor %xmm2, %xmm2 -; SSE-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE-NEXT: packssdw %xmm2, %xmm4 -; SSE-NEXT: packssdw %xmm3, %xmm4 -; SSE-NEXT: pmovmskb %xmm4, %eax +; SSE-NEXT: packssdw %xmm3, %xmm2 +; SSE-NEXT: packssdw %xmm1, %xmm0 +; SSE-NEXT: packssdw %xmm2, %xmm0 +; SSE-NEXT: pmovmskb %xmm0, %eax ; SSE-NEXT: testl $43690, %eax # imm = 0xAAAA ; SSE-NEXT: sete %al ; SSE-NEXT: retq @@ -2000,7 +1984,6 @@ define i1 @allones_v2i64_and1(<2 x i64> %arg) { ; SSE-LABEL: allones_v2i64_and1: ; SSE: # %bb.0: ; SSE-NEXT: psllq $63, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; SSE-NEXT: movmskpd %xmm0, %eax ; SSE-NEXT: cmpl $3, %eax ; SSE-NEXT: sete %al @@ -2184,15 +2167,11 @@ define i1 @allzeros_v4i64_and1(<4 x i64> %arg) { define i1 @allones_v8i64_and1(<8 x i64> %arg) { ; SSE-LABEL: allones_v8i64_and1: ; SSE: # %bb.0: -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,2,2] -; SSE-NEXT: pslld $31, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2] -; SSE-NEXT: pslld $31, %xmm2 +; SSE-NEXT: psllq $63, %xmm3 +; SSE-NEXT: psllq $63, %xmm2 ; SSE-NEXT: packssdw %xmm3, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] -; SSE-NEXT: pslld $31, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] -; SSE-NEXT: pslld $31, %xmm0 +; SSE-NEXT: psllq $63, %xmm1 +; SSE-NEXT: psllq $63, %xmm0 ; SSE-NEXT: packssdw %xmm1, %xmm0 ; SSE-NEXT: packssdw %xmm2, %xmm0 ; SSE-NEXT: packsswb %xmm0, %xmm0 @@ -3214,7 +3193,6 @@ define i1 @allones_v2i64_and4(<2 x i64> %arg) { ; SSE-LABEL: allones_v2i64_and4: ; SSE: # %bb.0: ; SSE-NEXT: psllq $61, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; SSE-NEXT: movmskpd %xmm0, %eax ; SSE-NEXT: cmpl $3, %eax ; SSE-NEXT: sete %al @@ -3399,14 +3377,10 @@ define i1 @allones_v8i64_and4(<8 x i64> %arg) { ; SSE-LABEL: allones_v8i64_and4: ; SSE: # %bb.0: ; SSE-NEXT: psllq $61, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] ; SSE-NEXT: psllq $61, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] ; SSE-NEXT: packssdw %xmm3, %xmm2 ; SSE-NEXT: psllq $61, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] ; SSE-NEXT: psllq $61, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; SSE-NEXT: packssdw %xmm1, %xmm0 ; SSE-NEXT: packssdw %xmm2, %xmm0 ; SSE-NEXT: packsswb %xmm0, %xmm0 @@ -4543,7 +4517,6 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; SSE2-NEXT: pxor %xmm1, %xmm1 ; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2] -; SSE2-NEXT: pand %xmm1, %xmm0 ; SSE2-NEXT: movmskpd %xmm0, %eax ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: setne %al diff --git a/llvm/test/CodeGen/X86/mulvi32.ll b/llvm/test/CodeGen/X86/mulvi32.ll index e5b0b11204e85..bbda4d68bb685 100644 --- a/llvm/test/CodeGen/X86/mulvi32.ll +++ b/llvm/test/CodeGen/X86/mulvi32.ll @@ -145,14 +145,13 @@ define <4 x i64> @_mul4xi32toi64a(<4 x i32>, <4 x i32>) { ; ; SSE42-LABEL: _mul4xi32toi64a: ; SSE42: # %bb.0: -; SSE42-NEXT: pxor %xmm3, %xmm3 -; SSE42-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero -; SSE42-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; SSE42-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero -; SSE42-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; SSE42-NEXT: pmuludq %xmm0, %xmm1 -; SSE42-NEXT: pmuludq %xmm4, %xmm2 -; SSE42-NEXT: movdqa %xmm2, %xmm0 +; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3] +; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,1,3,3] +; SSE42-NEXT: pmuludq %xmm3, %xmm2 +; SSE42-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero +; SSE42-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; SSE42-NEXT: pmuludq %xmm1, %xmm0 +; SSE42-NEXT: movdqa %xmm2, %xmm1 ; SSE42-NEXT: retq ; ; AVX1-LABEL: _mul4xi32toi64a: diff --git a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll index d5ceff5709974..9e398096bfcc5 100644 --- a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll +++ b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll @@ -130,21 +130,18 @@ define <4 x i1> @p5_vector_urem_by_const__nonsplat(<4 x i32> %x, <4 x i32> %y) { ; SSE2-LABEL: p5_vector_urem_by_const__nonsplat: ; SSE2: # %bb.0: ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177] -; SSE2-NEXT: pmuludq %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] ; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE2-NEXT: psrlq $32, %xmm1 -; SSE2-NEXT: por %xmm2, %xmm1 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE2-NEXT: psrlq $32, %xmm0 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 ; SSE2-NEXT: pxor %xmm1, %xmm0 ; SSE2-NEXT: retq ; @@ -152,7 +149,7 @@ define <4 x i1> @p5_vector_urem_by_const__nonsplat(<4 x i32> %x, <4 x i32> %y) { ; SSE4: # %bb.0: ; SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE4-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE4-NEXT: movdqa {{.*#+}} xmm1 = [1,1,2147483648,1] +; SSE4-NEXT: pmovzxdq {{.*#+}} xmm1 = [1,2147483648] ; SSE4-NEXT: pmuludq %xmm0, %xmm1 ; SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; SSE4-NEXT: psrlq $32, %xmm1 diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll index 2560a93033325..9aee2f11e9ea4 100644 --- a/llvm/test/CodeGen/X86/pmul.ll +++ b/llvm/test/CodeGen/X86/pmul.ll @@ -1007,15 +1007,13 @@ define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) { ; ; SSE41-LABEL: mul_v4i64_zero_upper: ; SSE41: # %bb.0: # %entry -; SSE41-NEXT: pxor %xmm3, %xmm3 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; SSE41-NEXT: pmuludq %xmm0, %xmm1 -; SSE41-NEXT: pmuludq %xmm4, %xmm2 -; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3] -; SSE41-NEXT: movaps %xmm2, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero +; SSE41-NEXT: pmuludq %xmm2, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3] +; SSE41-NEXT: pmuludq %xmm3, %xmm1 +; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] ; SSE41-NEXT: retq ; ; AVX-LABEL: mul_v4i64_zero_upper: @@ -1164,23 +1162,20 @@ define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) { ; ; SSE41-LABEL: mul_v8i64_zero_upper: ; SSE41: # %bb.0: # %entry -; SSE41-NEXT: pxor %xmm6, %xmm6 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm6[2],xmm0[3],xmm6[3] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm8 = xmm1[0],zero,xmm1[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm6[2],xmm1[3],xmm6[3] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3] -; SSE41-NEXT: pmuludq %xmm0, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm6[2],xmm3[3],xmm6[3] -; SSE41-NEXT: pmuludq %xmm1, %xmm3 -; SSE41-NEXT: pmuludq %xmm7, %xmm4 -; SSE41-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,3],xmm2[1,3] -; SSE41-NEXT: pmuludq %xmm8, %xmm5 -; SSE41-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,3],xmm3[1,3] -; SSE41-NEXT: movaps %xmm4, %xmm0 -; SSE41-NEXT: movaps %xmm5, %xmm1 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero +; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm1[2,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero +; SSE41-NEXT: pmuludq %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3] +; SSE41-NEXT: pmuludq %xmm5, %xmm1 +; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero +; SSE41-NEXT: pmuludq %xmm6, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,1,3,3] +; SSE41-NEXT: pmuludq %xmm7, %xmm2 +; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3] ; SSE41-NEXT: retq ; ; AVX2-LABEL: mul_v8i64_zero_upper: @@ -1219,25 +1214,25 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) { ; SSE2-LABEL: mul_v8i64_sext: ; SSE2: # %bb.0: ; SSE2-NEXT: movdqa %xmm1, %xmm4 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7] -; SSE2-NEXT: psrad $16, %xmm8 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] +; SSE2-NEXT: psrad $16, %xmm6 +; SSE2-NEXT: pxor %xmm12, %xmm12 ; SSE2-NEXT: pxor %xmm7, %xmm7 -; SSE2-NEXT: pxor %xmm6, %xmm6 -; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 -; SSE2-NEXT: movdqa %xmm8, %xmm5 -; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm6[2],xmm5[3],xmm6[3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1] +; SSE2-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm7[2],xmm5[3],xmm7[3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1] ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; SSE2-NEXT: psrad $16, %xmm0 -; SSE2-NEXT: pxor %xmm12, %xmm12 -; SSE2-NEXT: pcmpgtd %xmm0, %xmm12 -; SSE2-NEXT: movdqa %xmm0, %xmm11 -; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm12[2],xmm11[3],xmm12[3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1] +; SSE2-NEXT: pxor %xmm11, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm11[2],xmm9[3],xmm11[3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1] ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3] -; SSE2-NEXT: pxor %xmm9, %xmm9 -; SSE2-NEXT: pcmpgtd %xmm3, %xmm9 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1] +; SSE2-NEXT: pxor %xmm8, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm8 +; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1] ; SSE2-NEXT: pxor %xmm10, %xmm10 ; SSE2-NEXT: pcmpgtd %xmm2, %xmm10 ; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1] @@ -1245,43 +1240,40 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) { ; SSE2-NEXT: pxor %xmm13, %xmm13 ; SSE2-NEXT: pcmpgtd %xmm1, %xmm13 ; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm13[0],xmm1[1],xmm13[1] -; SSE2-NEXT: pxor %xmm14, %xmm14 -; SSE2-NEXT: pcmpgtd %xmm4, %xmm14 -; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1] -; SSE2-NEXT: movdqa %xmm12, %xmm15 -; SSE2-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm7[0],xmm15[1],xmm7[1] -; SSE2-NEXT: pmuludq %xmm4, %xmm15 -; SSE2-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm7[0],xmm14[1],xmm7[1] -; SSE2-NEXT: pmuludq %xmm0, %xmm14 -; SSE2-NEXT: paddq %xmm15, %xmm14 -; SSE2-NEXT: psllq $32, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm12 +; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm11[0,1,1,3] +; SSE2-NEXT: pmuludq %xmm4, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,1,3] +; SSE2-NEXT: pmuludq %xmm0, %xmm12 +; SSE2-NEXT: paddq %xmm14, %xmm12 +; SSE2-NEXT: psllq $32, %xmm12 ; SSE2-NEXT: pmuludq %xmm4, %xmm0 -; SSE2-NEXT: paddq %xmm14, %xmm0 -; SSE2-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm7[2],xmm12[3],xmm7[3] -; SSE2-NEXT: pmuludq %xmm1, %xmm12 -; SSE2-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm7[0],xmm13[1],xmm7[1] -; SSE2-NEXT: pmuludq %xmm11, %xmm13 -; SSE2-NEXT: paddq %xmm12, %xmm13 -; SSE2-NEXT: psllq $32, %xmm13 -; SSE2-NEXT: pmuludq %xmm11, %xmm1 -; SSE2-NEXT: paddq %xmm13, %xmm1 -; SSE2-NEXT: movdqa %xmm6, %xmm4 -; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1] +; SSE2-NEXT: paddq %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm11[2,1,3,3] +; SSE2-NEXT: pmuludq %xmm1, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[0,1,1,3] +; SSE2-NEXT: pmuludq %xmm9, %xmm11 +; SSE2-NEXT: paddq %xmm4, %xmm11 +; SSE2-NEXT: psllq $32, %xmm11 +; SSE2-NEXT: pmuludq %xmm9, %xmm1 +; SSE2-NEXT: paddq %xmm11, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,1,1,3] ; SSE2-NEXT: pmuludq %xmm2, %xmm4 -; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1] -; SSE2-NEXT: pmuludq %xmm8, %xmm10 -; SSE2-NEXT: paddq %xmm4, %xmm10 -; SSE2-NEXT: psllq $32, %xmm10 -; SSE2-NEXT: pmuludq %xmm8, %xmm2 -; SSE2-NEXT: paddq %xmm10, %xmm2 -; SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3] -; SSE2-NEXT: pmuludq %xmm3, %xmm6 -; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1] -; SSE2-NEXT: pmuludq %xmm5, %xmm9 -; SSE2-NEXT: paddq %xmm6, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm10[0,1,1,3] +; SSE2-NEXT: pmuludq %xmm6, %xmm9 +; SSE2-NEXT: paddq %xmm4, %xmm9 ; SSE2-NEXT: psllq $32, %xmm9 +; SSE2-NEXT: pmuludq %xmm6, %xmm2 +; SSE2-NEXT: paddq %xmm9, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[2,1,3,3] +; SSE2-NEXT: pmuludq %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm8[0,1,1,3] +; SSE2-NEXT: pmuludq %xmm5, %xmm6 +; SSE2-NEXT: paddq %xmm4, %xmm6 +; SSE2-NEXT: psllq $32, %xmm6 ; SSE2-NEXT: pmuludq %xmm5, %xmm3 -; SSE2-NEXT: paddq %xmm9, %xmm3 +; SSE2-NEXT: paddq %xmm6, %xmm3 ; SSE2-NEXT: retq ; ; SSE41-LABEL: mul_v8i64_sext: diff --git a/llvm/test/CodeGen/X86/pmulh.ll b/llvm/test/CodeGen/X86/pmulh.ll index b54029fa76553..300da68d9a3b3 100644 --- a/llvm/test/CodeGen/X86/pmulh.ll +++ b/llvm/test/CodeGen/X86/pmulh.ll @@ -54,9 +54,9 @@ define <4 x i16> @and_mulhuw_v4i16(<4 x i64> %a, <4 x i64> %b) { ; ; AVX2-LABEL: and_mulhuw_v4i16: ; AVX2: # %bb.0: -; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15] ; AVX2-NEXT: vpmulhuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vpackusdw %xmm0, %xmm0, %xmm0 @@ -65,8 +65,6 @@ define <4 x i16> @and_mulhuw_v4i16(<4 x i64> %a, <4 x i64> %b) { ; ; AVX512-LABEL: and_mulhuw_v4i16: ; AVX512: # %bb.0: -; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15] ; AVX512-NEXT: vpmulhuw %ymm1, %ymm0, %ymm0 ; AVX512-NEXT: vpmovqw %zmm0, %xmm0 ; AVX512-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/pr107423.ll b/llvm/test/CodeGen/X86/pr107423.ll index 5c43276f0dc38..d5119d45f97c0 100644 --- a/llvm/test/CodeGen/X86/pr107423.ll +++ b/llvm/test/CodeGen/X86/pr107423.ll @@ -7,21 +7,21 @@ define void @PR107423(<64 x i8> %arg, ptr %p0) { ; CHECK-NEXT: vextractf128 $1, %ymm1, %xmm2 ; CHECK-NEXT: vpsllw $8, %xmm2, %xmm2 ; CHECK-NEXT: vpsllw $8, %xmm1, %xmm3 -; CHECK-NEXT: vpaddb %xmm0, %xmm2, %xmm4 -; CHECK-NEXT: vpaddb %xmm2, %xmm3, %xmm2 -; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm3 -; CHECK-NEXT: vpaddb %xmm3, %xmm0, %xmm0 -; CHECK-NEXT: vpaddb %xmm1, %xmm3, %xmm1 +; CHECK-NEXT: vpaddb %xmm2, %xmm3, %xmm3 +; CHECK-NEXT: vpaddb %xmm0, %xmm2, %xmm2 +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm4 +; CHECK-NEXT: vpaddb %xmm1, %xmm4, %xmm1 +; CHECK-NEXT: vpaddb %xmm4, %xmm0, %xmm4 +; CHECK-NEXT: vpsllw $8, %xmm4, %xmm4 +; CHECK-NEXT: vpaddb %xmm3, %xmm4, %xmm3 ; CHECK-NEXT: vpsllw $8, %xmm1, %xmm1 -; CHECK-NEXT: vpaddb %xmm4, %xmm1, %xmm1 -; CHECK-NEXT: vpsllw $8, %xmm0, %xmm0 -; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm1, %xmm1 ; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; CHECK-NEXT: vpaddb %xmm0, %xmm2, %xmm0 -; CHECK-NEXT: vpsllw $8, %xmm1, %xmm2 -; CHECK-NEXT: vpaddb %xmm1, %xmm2, %xmm1 -; CHECK-NEXT: vmovdqu %xmm0, (%rdi) -; CHECK-NEXT: vmovdqu %xmm1, 16(%rdi) +; CHECK-NEXT: vpor %xmm3, %xmm2, %xmm2 +; CHECK-NEXT: vpsllw $8, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vmovdqu %xmm0, 16(%rdi) +; CHECK-NEXT: vmovdqu %xmm2, (%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %i3 = bitcast <64 x i8> %arg to <32 x i16> diff --git a/llvm/test/CodeGen/X86/pr35918.ll b/llvm/test/CodeGen/X86/pr35918.ll index f57fab3084a9e..7e63b0abfae69 100644 --- a/llvm/test/CodeGen/X86/pr35918.ll +++ b/llvm/test/CodeGen/X86/pr35918.ll @@ -13,7 +13,7 @@ define void @fetch_r16g16_snorm_unorm8(ptr, ptr, i32, i32, ptr) nounwind { ; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; X86-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 ; X86-NEXT: vpsrlw $7, %xmm0, %xmm0 -; X86-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u] +; X86-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u] ; X86-NEXT: vmovd %xmm0, %ecx ; X86-NEXT: orl $-16777216, %ecx # imm = 0xFF000000 ; X86-NEXT: movl %ecx, (%eax) @@ -25,7 +25,7 @@ define void @fetch_r16g16_snorm_unorm8(ptr, ptr, i32, i32, ptr) nounwind { ; X64-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; X64-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 ; X64-NEXT: vpsrlw $7, %xmm0, %xmm0 -; X64-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u] +; X64-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u] ; X64-NEXT: vmovd %xmm0, %eax ; X64-NEXT: orl $-16777216, %eax # imm = 0xFF000000 ; X64-NEXT: movl %eax, (%rdi) diff --git a/llvm/test/CodeGen/X86/pr41619.ll b/llvm/test/CodeGen/X86/pr41619.ll index 5d11f1c960a8c..ad778b4970cbf 100644 --- a/llvm/test/CodeGen/X86/pr41619.ll +++ b/llvm/test/CodeGen/X86/pr41619.ll @@ -5,8 +5,6 @@ define void @foo(double %arg) { ; CHECK-LABEL: foo: ; CHECK: ## %bb.0: ## %bb -; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; CHECK-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3] ; CHECK-NEXT: vmovq %xmm0, %rax ; CHECK-NEXT: movl %eax, (%rax) ; CHECK-NEXT: movq $0, (%rax) diff --git a/llvm/test/CodeGen/X86/pr42727.ll b/llvm/test/CodeGen/X86/pr42727.ll index 286015840d4c7..cf1fa5a8fc493 100644 --- a/llvm/test/CodeGen/X86/pr42727.ll +++ b/llvm/test/CodeGen/X86/pr42727.ll @@ -7,7 +7,7 @@ define void @_ZN14simd_test_avx216c_imm_v256_alignILi1EEE6c_v256S1_S1_(ptr byval ; CHECK-LABEL: _ZN14simd_test_avx216c_imm_v256_alignILi1EEE6c_v256S1_S1_: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmovdqu {{[0-9]+}}(%esp), %xmm0 -; CHECK-NEXT: vpbroadcastd (%eax), %xmm1 +; CHECK-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero ; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; CHECK-NEXT: vpsllq $56, %ymm0, %ymm0 ; CHECK-NEXT: vmovdqu %ymm0, (%eax) diff --git a/llvm/test/CodeGen/X86/pr45563-2.ll b/llvm/test/CodeGen/X86/pr45563-2.ll index 00430c835837f..72877e1b1d67d 100644 --- a/llvm/test/CodeGen/X86/pr45563-2.ll +++ b/llvm/test/CodeGen/X86/pr45563-2.ll @@ -39,7 +39,7 @@ define <9 x float> @mload_split9(<9 x i1> %mask, ptr %addr, <9 x float> %dst) { ; CHECK-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 ; CHECK-NEXT: vmaskmovps (%rcx), %ymm1, %ymm4 ; CHECK-NEXT: vblendvps %ymm1, %ymm4, %ymm0, %ymm0 -; CHECK-NEXT: vpshufb {{.*#+}} xmm1 = xmm2[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-NEXT: vpshufb {{.*#+}} xmm1 = xmm2[8,u,u,u],zero,xmm2[u,u,u],zero,xmm2[u,u,u],zero,xmm2[u,u,u] ; CHECK-NEXT: vpslld $31, %xmm1, %xmm1 ; CHECK-NEXT: vmaskmovps 32(%rcx), %ymm1, %ymm2 ; CHECK-NEXT: vmovaps %ymm0, (%rdi) diff --git a/llvm/test/CodeGen/X86/pr45833.ll b/llvm/test/CodeGen/X86/pr45833.ll index aa6a38a841f43..04c342b6673ed 100644 --- a/llvm/test/CodeGen/X86/pr45833.ll +++ b/llvm/test/CodeGen/X86/pr45833.ll @@ -29,7 +29,7 @@ define void @mstore_split9(<9 x float> %value, ptr %addr, <9 x i1> %mask) { ; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3 ; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3 ; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3 -; CHECK-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[8,u,u,u],zero,xmm3[u,u,u],zero,xmm3[u,u,u],zero,xmm3[u,u,u] ; CHECK-NEXT: vpslld $31, %xmm4, %xmm4 ; CHECK-NEXT: vmaskmovps %ymm1, %ymm4, 32(%rdi) ; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero diff --git a/llvm/test/CodeGen/X86/pr77459.ll b/llvm/test/CodeGen/X86/pr77459.ll index 602f82b254a1e..9c072e6f5e3fc 100644 --- a/llvm/test/CodeGen/X86/pr77459.ll +++ b/llvm/test/CodeGen/X86/pr77459.ll @@ -80,7 +80,7 @@ define i8 @reverse_cmp_v8i1(<8 x i16> %a0, <8 x i16> %a1) { ; SSE42-LABEL: reverse_cmp_v8i1: ; SSE42: # %bb.0: ; SSE42-NEXT: pcmpeqw %xmm1, %xmm0 -; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1] +; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[u,15,u,13,u,11,u,9,u,7,u,5,u,3,u,1] ; SSE42-NEXT: packsswb %xmm0, %xmm0 ; SSE42-NEXT: pmovmskb %xmm0, %eax ; SSE42-NEXT: # kill: def $al killed $al killed $eax diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll index b93862be0a1b4..e10b360b35b56 100644 --- a/llvm/test/CodeGen/X86/psubus.ll +++ b/llvm/test/CodeGen/X86/psubus.ll @@ -1670,51 +1670,48 @@ define <8 x i16> @psubus_8i64_max(<8 x i16> %x, <8 x i64> %y) nounwind { ; SSE41-LABEL: psubus_8i64_max: ; SSE41: # %bb.0: # %vector.ph ; SSE41-NEXT: movdqa %xmm0, %xmm5 -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm8 = [2147549183,2147549183,2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm8, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd {{.*#+}} xmm7 = [65535,65535] -; SSE41-NEXT: movapd %xmm7, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm9 -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pand %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm7, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm4, %xmm8 +; SSE41-NEXT: pxor %xmm9, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002324991,9223372039002324991] +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm8 = [65535,65535] +; SSE41-NEXT: movapd %xmm8, %xmm10 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm10 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pxor %xmm9, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm4 +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm8, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm4 -; SSE41-NEXT: packusdw %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm8, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm7, %xmm3 +; SSE41-NEXT: packusdw %xmm10, %xmm4 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm9, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm3 +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: movapd %xmm8, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm7 -; SSE41-NEXT: packusdw %xmm3, %xmm7 -; SSE41-NEXT: packusdw %xmm4, %xmm7 -; SSE41-NEXT: psubusw %xmm7, %xmm5 +; SSE41-NEXT: pxor %xmm1, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pand %xmm7, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8 +; SSE41-NEXT: packusdw %xmm3, %xmm8 +; SSE41-NEXT: packusdw %xmm4, %xmm8 +; SSE41-NEXT: psubusw %xmm8, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm0 ; SSE41-NEXT: retq ; @@ -2769,55 +2766,52 @@ define <8 x i32> @test33(<8 x i32> %a0, <8 x i64> %a1) { ; ; SSE41-LABEL: test33: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa %xmm0, %xmm6 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647] -; SSE41-NEXT: movdqa %xmm9, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm10, %xmm0 -; SSE41-NEXT: movapd {{.*#+}} xmm8 = [4294967295,4294967295] -; SSE41-NEXT: movapd %xmm8, %xmm10 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm10 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm9, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm8, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm10[0,2] -; SSE41-NEXT: pmaxud %xmm3, %xmm6 -; SSE41-NEXT: psubd %xmm3, %xmm6 -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm9, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm3, %xmm9 +; SSE41-NEXT: pxor %xmm10, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm8 = [9223372039002259455,9223372039002259455] +; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483647,2147483647,2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm0 +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm9 = [4294967295,4294967295] +; SSE41-NEXT: movapd %xmm9, %xmm11 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm11 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm10, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm3[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm3 +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm12, %xmm0 ; SSE41-NEXT: pand %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm8, %xmm2 +; SSE41-NEXT: movapd %xmm9, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 +; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm11[0,2] +; SSE41-NEXT: pmaxud %xmm3, %xmm7 +; SSE41-NEXT: psubd %xmm3, %xmm7 +; SSE41-NEXT: movdqa %xmm5, %xmm2 +; SSE41-NEXT: pxor %xmm10, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm9, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm2 -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm8 -; SSE41-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,2],xmm2[0,2] -; SSE41-NEXT: pmaxud %xmm8, %xmm1 -; SSE41-NEXT: psubd %xmm8, %xmm1 +; SSE41-NEXT: pxor %xmm4, %xmm10 +; SSE41-NEXT: pcmpeqd %xmm10, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pand %xmm8, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm9 +; SSE41-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,2],xmm2[0,2] +; SSE41-NEXT: pmaxud %xmm9, %xmm1 +; SSE41-NEXT: psubd %xmm9, %xmm1 +; SSE41-NEXT: movdqa %xmm7, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: test33: @@ -2958,53 +2952,50 @@ define <8 x i32> @test34(<8 x i32> %a0, <8 x i64> %a1) { ; SSE41-NEXT: pmovsxbd {{.*#+}} xmm0 = [1,1,1,1] ; SSE41-NEXT: pand %xmm0, %xmm1 ; SSE41-NEXT: pand %xmm0, %xmm6 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647] -; SSE41-NEXT: movdqa %xmm9, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm10, %xmm0 -; SSE41-NEXT: movapd {{.*#+}} xmm8 = [4294967295,4294967295] -; SSE41-NEXT: movapd %xmm8, %xmm10 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm10 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm9, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm8, %xmm3 +; SSE41-NEXT: movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm3, %xmm9 +; SSE41-NEXT: pxor %xmm10, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm8 = [9223372039002259455,9223372039002259455] +; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm7, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm0 +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm9 = [4294967295,4294967295] +; SSE41-NEXT: movapd %xmm9, %xmm11 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm11 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm10, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm3[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm3 +; SSE41-NEXT: movdqa %xmm7, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm12, %xmm0 +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: movapd %xmm9, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm10[0,2] +; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm11[0,2] ; SSE41-NEXT: pmaxud %xmm3, %xmm6 ; SSE41-NEXT: psubd %xmm3, %xmm6 -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm9, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm2 +; SSE41-NEXT: pxor %xmm10, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm7, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm9, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm2 -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm7, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm8 -; SSE41-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,2],xmm2[0,2] -; SSE41-NEXT: pmaxud %xmm8, %xmm1 -; SSE41-NEXT: psubd %xmm8, %xmm1 +; SSE41-NEXT: pxor %xmm4, %xmm10 +; SSE41-NEXT: pcmpeqd %xmm10, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 +; SSE41-NEXT: pand %xmm8, %xmm7 +; SSE41-NEXT: movdqa %xmm7, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm9 +; SSE41-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,2],xmm2[0,2] +; SSE41-NEXT: pmaxud %xmm9, %xmm1 +; SSE41-NEXT: psubd %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm6, %xmm0 ; SSE41-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/rotate-extract-vector.ll b/llvm/test/CodeGen/X86/rotate-extract-vector.ll index fdf43f5ab11f9..7d0ec6435d863 100644 --- a/llvm/test/CodeGen/X86/rotate-extract-vector.ll +++ b/llvm/test/CodeGen/X86/rotate-extract-vector.ll @@ -220,12 +220,10 @@ define <2 x i64> @no_extract_udiv(<2 x i64> %i) nounwind { ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $3, {{[0-9]+}}(%esp) ; X86-NEXT: vmovd %eax, %xmm0 -; X86-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0 ; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: calll __udivdi3 ; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload ; X86-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0 -; X86-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0 ; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload ; X86-NEXT: vextractps $1, %xmm0, {{[0-9]+}}(%esp) diff --git a/llvm/test/CodeGen/X86/sadd_sat_vec.ll b/llvm/test/CodeGen/X86/sadd_sat_vec.ll index a4c94ec2fa883..80b55a364dba9 100644 --- a/llvm/test/CodeGen/X86/sadd_sat_vec.ll +++ b/llvm/test/CodeGen/X86/sadd_sat_vec.ll @@ -1175,25 +1175,22 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pxor %xmm0, %xmm4 ; SSE41-NEXT: paddq %xmm1, %xmm2 ; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pxor %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] -; SSE41-NEXT: blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: por %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movapd {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd %xmm2, %xmm0 ; SSE41-NEXT: retq ; @@ -1355,46 +1352,40 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind { ; SSE41-LABEL: v4i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm5 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm5, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm6, %xmm0 ; SSE41-NEXT: paddq %xmm2, %xmm4 -; SSE41-NEXT: movdqa %xmm4, %xmm6 -; SSE41-NEXT: pxor %xmm5, %xmm6 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: pxor %xmm6, %xmm5 ; SSE41-NEXT: movdqa %xmm0, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pxor %xmm6, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm5 +; SSE41-NEXT: por %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm2, %xmm5 ; SSE41-NEXT: movapd {{.*#+}} xmm7 = [9223372036854775807,9223372036854775807] -; SSE41-NEXT: movapd {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808] -; SSE41-NEXT: movapd %xmm6, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 +; SSE41-NEXT: movapd {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movapd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4 ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 +; SSE41-NEXT: pxor %xmm6, %xmm0 ; SSE41-NEXT: paddq %xmm3, %xmm1 -; SSE41-NEXT: pxor %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm6 +; SSE41-NEXT: pxor %xmm1, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm2 +; SSE41-NEXT: por %xmm0, %xmm2 +; SSE41-NEXT: pxor %xmm3, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 ; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 ; SSE41-NEXT: movapd %xmm4, %xmm0 ; SSE41-NEXT: retq ; @@ -1659,86 +1650,74 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind { ; SSE41-LABEL: v8i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm8 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm9 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm10 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: paddq %xmm4, %xmm8 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pxor %xmm9, %xmm10 +; SSE41-NEXT: movdqa %xmm8, %xmm9 +; SSE41-NEXT: pxor %xmm10, %xmm9 ; SSE41-NEXT: movdqa %xmm0, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm10, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pxor %xmm10, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: movapd {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807] -; SSE41-NEXT: movapd {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808] -; SSE41-NEXT: movapd %xmm10, %xmm12 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm12 -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm8 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm11 +; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm9 +; SSE41-NEXT: por %xmm0, %xmm9 +; SSE41-NEXT: pxor %xmm4, %xmm9 +; SSE41-NEXT: movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807] +; SSE41-NEXT: movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movapd %xmm11, %xmm4 +; SSE41-NEXT: movdqa %xmm8, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm4 +; SSE41-NEXT: movdqa %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm8 ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: paddq %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm4 -; SSE41-NEXT: pxor %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm0, %xmm12 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm13, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] -; SSE41-NEXT: pxor %xmm12, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: movapd %xmm10, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm5 +; SSE41-NEXT: pxor %xmm10, %xmm4 +; SSE41-NEXT: movdqa %xmm0, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm4 +; SSE41-NEXT: por %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm5, %xmm4 +; SSE41-NEXT: movapd %xmm11, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm5 ; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: paddq %xmm6, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm4 -; SSE41-NEXT: pxor %xmm9, %xmm4 +; SSE41-NEXT: pxor %xmm10, %xmm4 ; SSE41-NEXT: movdqa %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: movapd %xmm10, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm4 +; SSE41-NEXT: por %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm6, %xmm4 +; SSE41-NEXT: movapd %xmm11, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm5 ; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: paddq %xmm7, %xmm3 -; SSE41-NEXT: pxor %xmm3, %xmm9 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm10 +; SSE41-NEXT: pxor %xmm3, %xmm10 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm10, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm4 +; SSE41-NEXT: por %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm7, %xmm4 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm11 ; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm3 ; SSE41-NEXT: movapd %xmm8, %xmm0 ; SSE41-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll index 69e6ff7770ebe..b12be7cb129d3 100644 --- a/llvm/test/CodeGen/X86/sat-add.ll +++ b/llvm/test/CodeGen/X86/sat-add.ll @@ -631,13 +631,12 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) { ; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] ; SSE41-NEXT: pxor %xmm1, %xmm0 ; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372034707292117,9223372034707292117] +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm4 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm0 @@ -1164,12 +1163,10 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_min(<2 x i64> %x, <2 x i64> ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372034707292159,9223372034707292159] ; SSE41-NEXT: pxor %xmm1, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 ; SSE41-NEXT: paddq %xmm1, %xmm3 diff --git a/llvm/test/CodeGen/X86/sdiv-exact.ll b/llvm/test/CodeGen/X86/sdiv-exact.ll index 41ae0dd9e8eb1..456819179fcdc 100644 --- a/llvm/test/CodeGen/X86/sdiv-exact.ll +++ b/llvm/test/CodeGen/X86/sdiv-exact.ll @@ -83,12 +83,11 @@ define <4 x i32> @test5(<4 x i32> %x) { ; X86-NEXT: movdqa %xmm0, %xmm1 ; X86-NEXT: psrad $3, %xmm1 ; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3] -; X86-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,2863311531,3264175145,3264175145] -; X86-NEXT: movaps %xmm1, %xmm0 -; X86-NEXT: pmuludq %xmm2, %xmm0 +; X86-NEXT: movdqa {{.*#+}} xmm0 = [2863311531,2863311531,3264175145,3264175145] +; X86-NEXT: pmuludq %xmm1, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,3,3] -; X86-NEXT: pmuludq %xmm2, %xmm1 +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X86-NEXT: retl @@ -109,12 +108,11 @@ define <4 x i32> @test6(<4 x i32> %x) { ; X86-NEXT: psrad $3, %xmm1 ; X86-NEXT: psrad $1, %xmm0 ; X86-NEXT: movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1] -; X86-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,2863311531,3303820997,3303820997] -; X86-NEXT: movapd %xmm0, %xmm1 -; X86-NEXT: pmuludq %xmm2, %xmm1 +; X86-NEXT: movdqa {{.*#+}} xmm1 = [2863311531,2863311531,3303820997,3303820997] +; X86-NEXT: pmuludq %xmm0, %xmm1 ; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,3,3] -; X86-NEXT: pmuludq %xmm2, %xmm0 +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; X86-NEXT: movdqa %xmm1, %xmm0 @@ -132,12 +130,11 @@ define <4 x i32> @test6(<4 x i32> %x) { define <4 x i32> @test7(<4 x i32> %x) { ; X86-LABEL: test7: ; X86: # %bb.0: -; X86-NEXT: movdqa {{.*#+}} xmm1 = [3264175145,3264175145,1749801491,1749801491] -; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; X86-NEXT: pmuludq %xmm1, %xmm0 +; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; X86-NEXT: pmuludq %xmm1, %xmm2 -; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 +; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X86-NEXT: retl ; @@ -155,12 +152,11 @@ define <4 x i32> @test8(<4 x i32> %x) { ; X86-NEXT: movdqa %xmm0, %xmm1 ; X86-NEXT: psrad $3, %xmm1 ; X86-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] -; X86-NEXT: movdqa {{.*#+}} xmm2 = [1,1,2863311531,2863311531] -; X86-NEXT: movapd %xmm1, %xmm0 -; X86-NEXT: pmuludq %xmm2, %xmm0 +; X86-NEXT: movdqa {{.*#+}} xmm0 = [1,1,2863311531,2863311531] +; X86-NEXT: pmuludq %xmm1, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,3,3] -; X86-NEXT: pmuludq %xmm2, %xmm1 +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X86-NEXT: retl diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll index 5f147784a74a3..dbbcc5e9c1b7f 100644 --- a/llvm/test/CodeGen/X86/shrink_vmul.ll +++ b/llvm/test/CodeGen/X86/shrink_vmul.ll @@ -1864,9 +1864,11 @@ define void @mul_2xi16_varconst3(ptr nocapture readonly %a, i64 %index) { ; X86-SSE-NEXT: movl c, %edx ; X86-SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero ; X86-SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] -; X86-SSE-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE-NEXT: psllq $32, %xmm0 -; X86-SSE-NEXT: movq %xmm0, (%edx,%eax,4) +; X86-SSE-NEXT: movl $65536, %ecx # imm = 0x10000 +; X86-SSE-NEXT: movd %ecx, %xmm1 +; X86-SSE-NEXT: pmuludq %xmm0, %xmm1 +; X86-SSE-NEXT: psllq $32, %xmm1 +; X86-SSE-NEXT: movq %xmm1, (%edx,%eax,4) ; X86-SSE-NEXT: retl ; ; X86-AVX-LABEL: mul_2xi16_varconst3: @@ -1885,9 +1887,11 @@ define void @mul_2xi16_varconst3(ptr nocapture readonly %a, i64 %index) { ; X64-SSE-NEXT: movq c(%rip), %rax ; X64-SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero ; X64-SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] -; X64-SSE-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; X64-SSE-NEXT: psllq $32, %xmm0 -; X64-SSE-NEXT: movq %xmm0, (%rax,%rsi,4) +; X64-SSE-NEXT: movl $65536, %ecx # imm = 0x10000 +; X64-SSE-NEXT: movd %ecx, %xmm1 +; X64-SSE-NEXT: pmuludq %xmm0, %xmm1 +; X64-SSE-NEXT: psllq $32, %xmm1 +; X64-SSE-NEXT: movq %xmm1, (%rax,%rsi,4) ; X64-SSE-NEXT: retq ; ; X64-AVX-LABEL: mul_2xi16_varconst3: @@ -1921,11 +1925,12 @@ define void @mul_2xi16_varconst4(ptr nocapture readonly %a, i64 %index) { ; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-SSE-NEXT: movl c, %edx ; X86-SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] ; X86-SSE-NEXT: psrad $16, %xmm0 -; X86-SSE-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE-NEXT: psllq $32, %xmm0 -; X86-SSE-NEXT: movq %xmm0, (%edx,%eax,4) +; X86-SSE-NEXT: movl $32768, %ecx # imm = 0x8000 +; X86-SSE-NEXT: movd %ecx, %xmm1 +; X86-SSE-NEXT: pmuludq %xmm0, %xmm1 +; X86-SSE-NEXT: psllq $32, %xmm1 +; X86-SSE-NEXT: movq %xmm1, (%edx,%eax,4) ; X86-SSE-NEXT: retl ; ; X86-AVX-LABEL: mul_2xi16_varconst4: @@ -1943,11 +1948,12 @@ define void @mul_2xi16_varconst4(ptr nocapture readonly %a, i64 %index) { ; X64-SSE: # %bb.0: # %entry ; X64-SSE-NEXT: movq c(%rip), %rax ; X64-SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] ; X64-SSE-NEXT: psrad $16, %xmm0 -; X64-SSE-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; X64-SSE-NEXT: psllq $32, %xmm0 -; X64-SSE-NEXT: movq %xmm0, (%rax,%rsi,4) +; X64-SSE-NEXT: movl $32768, %ecx # imm = 0x8000 +; X64-SSE-NEXT: movd %ecx, %xmm1 +; X64-SSE-NEXT: pmuludq %xmm0, %xmm1 +; X64-SSE-NEXT: psllq $32, %xmm1 +; X64-SSE-NEXT: movq %xmm1, (%rax,%rsi,4) ; X64-SSE-NEXT: retq ; ; X64-AVX-LABEL: mul_2xi16_varconst4: diff --git a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll index 0b42811f154f8..2d0778853fecd 100644 --- a/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll +++ b/llvm/test/CodeGen/X86/srem-seteq-vec-nonsplat.ll @@ -555,7 +555,7 @@ define <4 x i32> @test_srem_odd_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,268435456,1] +; CHECK-SSE41-NEXT: pmovsxdq {{.*#+}} xmm1 = [1,268435456] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 @@ -1098,7 +1098,7 @@ define <4 x i32> @test_srem_even_INT_MIN(<4 x i32> %X) nounwind { ; CHECK-SSE2-LABEL: test_srem_even_INT_MIN: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pxor %xmm2, %xmm2 -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [3067833783,3067833783,1,3067833783] +; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [3067833783,u,1,u] ; CHECK-SSE2-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] @@ -1354,7 +1354,7 @@ define <4 x i32> @test_srem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,268435456,1] +; CHECK-SSE41-NEXT: pmovsxdq {{.*#+}} xmm1 = [1,268435456] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 @@ -2068,12 +2068,11 @@ define <4 x i32> @test_srem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou ; CHECK-SSE2-LABEL: test_srem_odd_allones_and_poweroftwo_and_one: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [4294967295,0,4294967295,0] -; CHECK-SSE2-NEXT: pand %xmm1, %xmm0 ; CHECK-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pand %xmm0, %xmm1 -; CHECK-SSE2-NEXT: psrlq $32, %xmm0 +; CHECK-SSE2-NEXT: movdqa %xmm0, %xmm1 +; CHECK-SSE2-NEXT: psrlq $32, %xmm1 +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: por %xmm1, %xmm0 ; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 @@ -2140,12 +2139,11 @@ define <4 x i32> @test_srem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no ; CHECK-SSE2-LABEL: test_srem_even_allones_and_poweroftwo_and_one: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [4294967295,0,4294967295,0] -; CHECK-SSE2-NEXT: pand %xmm1, %xmm0 ; CHECK-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pand %xmm0, %xmm1 -; CHECK-SSE2-NEXT: psrlq $32, %xmm0 +; CHECK-SSE2-NEXT: movdqa %xmm0, %xmm1 +; CHECK-SSE2-NEXT: psrlq $32, %xmm1 +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: por %xmm1, %xmm0 ; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 diff --git a/llvm/test/CodeGen/X86/sshl_sat_vec.ll b/llvm/test/CodeGen/X86/sshl_sat_vec.ll index 1ce34bd98f850..a7accb0c9012f 100644 --- a/llvm/test/CodeGen/X86/sshl_sat_vec.ll +++ b/llvm/test/CodeGen/X86/sshl_sat_vec.ll @@ -276,63 +276,63 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind { define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind { ; X64-LABEL: vec_v8i16: ; X64: # %bb.0: -; X64-NEXT: pxor %xmm3, %xmm3 ; X64-NEXT: movdqa %xmm1, %xmm2 -; X64-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; X64-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; X64-NEXT: pslld $23, %xmm2 -; X64-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X64-NEXT: paddd %xmm4, %xmm2 +; X64-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X64-NEXT: paddd %xmm3, %xmm2 ; X64-NEXT: cvttps2dq %xmm2, %xmm2 ; X64-NEXT: pslld $16, %xmm2 ; X64-NEXT: psrad $16, %xmm2 -; X64-NEXT: movdqa %xmm1, %xmm5 -; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3] -; X64-NEXT: pslld $23, %xmm5 -; X64-NEXT: paddd %xmm4, %xmm5 -; X64-NEXT: cvttps2dq %xmm5, %xmm4 -; X64-NEXT: pslld $16, %xmm4 -; X64-NEXT: psrad $16, %xmm4 -; X64-NEXT: packssdw %xmm2, %xmm4 -; X64-NEXT: pmullw %xmm0, %xmm4 +; X64-NEXT: movdqa %xmm1, %xmm4 +; X64-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3] +; X64-NEXT: pslld $23, %xmm4 +; X64-NEXT: paddd %xmm3, %xmm4 +; X64-NEXT: cvttps2dq %xmm4, %xmm3 +; X64-NEXT: pslld $16, %xmm3 +; X64-NEXT: psrad $16, %xmm3 +; X64-NEXT: packssdw %xmm2, %xmm3 +; X64-NEXT: pmullw %xmm0, %xmm3 ; X64-NEXT: psllw $12, %xmm1 ; X64-NEXT: movdqa %xmm1, %xmm2 ; X64-NEXT: psraw $15, %xmm2 -; X64-NEXT: movdqa %xmm4, %xmm5 -; X64-NEXT: psraw $8, %xmm5 -; X64-NEXT: pand %xmm2, %xmm5 -; X64-NEXT: pandn %xmm4, %xmm2 -; X64-NEXT: por %xmm5, %xmm2 +; X64-NEXT: movdqa %xmm3, %xmm4 +; X64-NEXT: psraw $8, %xmm4 +; X64-NEXT: pand %xmm2, %xmm4 +; X64-NEXT: pandn %xmm3, %xmm2 +; X64-NEXT: por %xmm4, %xmm2 ; X64-NEXT: paddw %xmm1, %xmm1 -; X64-NEXT: movdqa %xmm1, %xmm5 -; X64-NEXT: psraw $15, %xmm5 -; X64-NEXT: movdqa %xmm5, %xmm6 -; X64-NEXT: pandn %xmm2, %xmm6 +; X64-NEXT: movdqa %xmm1, %xmm4 +; X64-NEXT: psraw $15, %xmm4 +; X64-NEXT: movdqa %xmm4, %xmm5 +; X64-NEXT: pandn %xmm2, %xmm5 ; X64-NEXT: psraw $4, %xmm2 -; X64-NEXT: pand %xmm5, %xmm2 -; X64-NEXT: por %xmm6, %xmm2 +; X64-NEXT: pand %xmm4, %xmm2 +; X64-NEXT: por %xmm5, %xmm2 ; X64-NEXT: paddw %xmm1, %xmm1 -; X64-NEXT: movdqa %xmm1, %xmm5 -; X64-NEXT: psraw $15, %xmm5 -; X64-NEXT: movdqa %xmm5, %xmm6 -; X64-NEXT: pandn %xmm2, %xmm6 +; X64-NEXT: movdqa %xmm1, %xmm4 +; X64-NEXT: psraw $15, %xmm4 +; X64-NEXT: movdqa %xmm4, %xmm5 +; X64-NEXT: pandn %xmm2, %xmm5 ; X64-NEXT: psraw $2, %xmm2 -; X64-NEXT: pand %xmm5, %xmm2 -; X64-NEXT: por %xmm6, %xmm2 +; X64-NEXT: pand %xmm4, %xmm2 +; X64-NEXT: por %xmm5, %xmm2 ; X64-NEXT: paddw %xmm1, %xmm1 ; X64-NEXT: psraw $15, %xmm1 -; X64-NEXT: movdqa %xmm1, %xmm5 -; X64-NEXT: pandn %xmm2, %xmm5 +; X64-NEXT: movdqa %xmm1, %xmm4 +; X64-NEXT: pandn %xmm2, %xmm4 ; X64-NEXT: psraw $1, %xmm2 ; X64-NEXT: pand %xmm1, %xmm2 -; X64-NEXT: por %xmm5, %xmm2 +; X64-NEXT: por %xmm4, %xmm2 ; X64-NEXT: pcmpeqw %xmm0, %xmm2 -; X64-NEXT: pand %xmm2, %xmm4 -; X64-NEXT: pcmpgtw %xmm0, %xmm3 -; X64-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 +; X64-NEXT: pand %xmm2, %xmm3 +; X64-NEXT: pxor %xmm1, %xmm1 +; X64-NEXT: pcmpgtw %xmm0, %xmm1 +; X64-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; X64-NEXT: por %xmm3, %xmm0 +; X64-NEXT: por %xmm1, %xmm0 ; X64-NEXT: pandn %xmm0, %xmm2 -; X64-NEXT: por %xmm4, %xmm2 +; X64-NEXT: por %xmm3, %xmm2 ; X64-NEXT: movdqa %xmm2, %xmm0 ; X64-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/ssub_sat_vec.ll b/llvm/test/CodeGen/X86/ssub_sat_vec.ll index eb2a1f196774c..eb2ad4fdff92f 100644 --- a/llvm/test/CodeGen/X86/ssub_sat_vec.ll +++ b/llvm/test/CodeGen/X86/ssub_sat_vec.ll @@ -1264,28 +1264,24 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind { ; SSE41-NEXT: movdqa %xmm2, %xmm4 ; SSE41-NEXT: pxor %xmm0, %xmm4 ; SSE41-NEXT: movdqa %xmm3, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm4 ; SSE41-NEXT: por %xmm3, %xmm4 ; SSE41-NEXT: pxor %xmm0, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm1 -; SSE41-NEXT: pxor %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] -; SSE41-NEXT: blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: por %xmm1, %xmm3 +; SSE41-NEXT: pxor %xmm4, %xmm3 +; SSE41-NEXT: movapd {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd %xmm2, %xmm0 ; SSE41-NEXT: retq ; @@ -1479,63 +1475,55 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind { ; SSE41-LABEL: v4i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm5 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm5, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm6, %xmm0 ; SSE41-NEXT: psubq %xmm2, %xmm4 -; SSE41-NEXT: movdqa %xmm4, %xmm6 -; SSE41-NEXT: pxor %xmm5, %xmm6 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: pxor %xmm6, %xmm5 ; SSE41-NEXT: movdqa %xmm0, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm6 -; SSE41-NEXT: pxor %xmm5, %xmm2 -; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm7 ; SSE41-NEXT: pcmpgtd %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3] +; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] ; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm2 +; SSE41-NEXT: por %xmm0, %xmm8 ; SSE41-NEXT: pxor %xmm6, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: movapd {{.*#+}} xmm7 = [9223372036854775807,9223372036854775807] -; SSE41-NEXT: movapd {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808] -; SSE41-NEXT: movapd %xmm6, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 ; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm0, %xmm5 +; SSE41-NEXT: por %xmm2, %xmm5 +; SSE41-NEXT: pxor %xmm8, %xmm5 +; SSE41-NEXT: movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807] +; SSE41-NEXT: movapd {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movapd %xmm7, %xmm2 +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4 ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 +; SSE41-NEXT: pxor %xmm6, %xmm0 ; SSE41-NEXT: psubq %xmm3, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm2 -; SSE41-NEXT: pxor %xmm5, %xmm2 -; SSE41-NEXT: movdqa %xmm0, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm8 -; SSE41-NEXT: pxor %xmm5, %xmm3 +; SSE41-NEXT: pxor %xmm6, %xmm2 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm9 +; SSE41-NEXT: por %xmm0, %xmm9 +; SSE41-NEXT: pxor %xmm6, %xmm3 ; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE41-NEXT: pand %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm0, %xmm2 ; SSE41-NEXT: por %xmm3, %xmm2 -; SSE41-NEXT: pxor %xmm8, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm6 +; SSE41-NEXT: pxor %xmm9, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm7 ; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 ; SSE41-NEXT: movapd %xmm4, %xmm0 ; SSE41-NEXT: retq ; @@ -1856,119 +1844,103 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind { ; SSE41-LABEL: v8i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm8 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm9 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm10 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: psubq %xmm4, %xmm8 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pxor %xmm9, %xmm10 +; SSE41-NEXT: movdqa %xmm8, %xmm9 +; SSE41-NEXT: pxor %xmm10, %xmm9 ; SSE41-NEXT: movdqa %xmm0, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm10, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm10 -; SSE41-NEXT: pxor %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm11 ; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3] +; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2] ; SSE41-NEXT: pand %xmm11, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm12, %xmm4 +; SSE41-NEXT: por %xmm0, %xmm12 ; SSE41-NEXT: pxor %xmm10, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: movapd {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807] -; SSE41-NEXT: movapd {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808] -; SSE41-NEXT: movapd %xmm10, %xmm12 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm12 ; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm8 +; SSE41-NEXT: pcmpeqd %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm0, %xmm9 +; SSE41-NEXT: por %xmm4, %xmm9 +; SSE41-NEXT: pxor %xmm12, %xmm9 +; SSE41-NEXT: movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807] +; SSE41-NEXT: movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movapd %xmm11, %xmm4 +; SSE41-NEXT: movdqa %xmm8, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm4 +; SSE41-NEXT: movdqa %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm8 ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: psubq %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm4 -; SSE41-NEXT: pxor %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm0, %xmm12 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm13, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm12 -; SSE41-NEXT: pxor %xmm9, %xmm5 +; SSE41-NEXT: pxor %xmm10, %xmm4 +; SSE41-NEXT: movdqa %xmm0, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm13 +; SSE41-NEXT: por %xmm0, %xmm13 +; SSE41-NEXT: pxor %xmm10, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm0, %xmm4 ; SSE41-NEXT: por %xmm5, %xmm4 -; SSE41-NEXT: pxor %xmm12, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: movapd %xmm10, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm5 +; SSE41-NEXT: pxor %xmm13, %xmm4 +; SSE41-NEXT: movapd %xmm11, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm5 ; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: psubq %xmm6, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm4 -; SSE41-NEXT: pxor %xmm9, %xmm4 +; SSE41-NEXT: pxor %xmm10, %xmm4 ; SSE41-NEXT: movdqa %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm5 -; SSE41-NEXT: pxor %xmm9, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm9 +; SSE41-NEXT: por %xmm0, %xmm9 +; SSE41-NEXT: pxor %xmm10, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm0, %xmm4 ; SSE41-NEXT: por %xmm6, %xmm4 -; SSE41-NEXT: pxor %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: movapd %xmm10, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm5 +; SSE41-NEXT: pxor %xmm9, %xmm4 +; SSE41-NEXT: movapd %xmm11, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm5 ; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: pxor %xmm10, %xmm0 ; SSE41-NEXT: psubq %xmm7, %xmm3 ; SSE41-NEXT: movdqa %xmm3, %xmm4 -; SSE41-NEXT: pxor %xmm9, %xmm4 +; SSE41-NEXT: pxor %xmm10, %xmm4 ; SSE41-NEXT: movdqa %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm5 -; SSE41-NEXT: pxor %xmm9, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm6 +; SSE41-NEXT: por %xmm0, %xmm6 +; SSE41-NEXT: pxor %xmm10, %xmm7 ; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm4 -; SSE41-NEXT: pxor %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm10 +; SSE41-NEXT: pcmpeqd %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm0, %xmm4 +; SSE41-NEXT: por %xmm7, %xmm4 +; SSE41-NEXT: pxor %xmm6, %xmm4 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm11 ; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm3 ; SSE41-NEXT: movapd %xmm8, %xmm0 ; SSE41-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/test-shrink-bug.ll b/llvm/test/CodeGen/X86/test-shrink-bug.ll index eb44e10b188ac..953a0d65c5386 100644 --- a/llvm/test/CodeGen/X86/test-shrink-bug.ll +++ b/llvm/test/CodeGen/X86/test-shrink-bug.ll @@ -65,7 +65,7 @@ define dso_local void @fail(i16 %a, <2 x i8> %b) { ; ; CHECK-X64-LABEL: fail: ; CHECK-X64: # %bb.0: -; CHECK-X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; CHECK-X64-NEXT: pslld $8, %xmm0 ; CHECK-X64-NEXT: pcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-X64-NEXT: pextrw $1, %xmm0, %eax ; CHECK-X64-NEXT: xorb $1, %al diff --git a/llvm/test/CodeGen/X86/udiv-exact.ll b/llvm/test/CodeGen/X86/udiv-exact.ll index 6d255f6bcf303..271d11edff9a7 100644 --- a/llvm/test/CodeGen/X86/udiv-exact.ll +++ b/llvm/test/CodeGen/X86/udiv-exact.ll @@ -83,12 +83,11 @@ define <4 x i32> @test5(<4 x i32> %x) { ; X86-NEXT: movdqa %xmm0, %xmm1 ; X86-NEXT: psrld $3, %xmm1 ; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3] -; X86-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,2863311531,3264175145,3264175145] -; X86-NEXT: movaps %xmm1, %xmm0 -; X86-NEXT: pmuludq %xmm2, %xmm0 +; X86-NEXT: movdqa {{.*#+}} xmm0 = [2863311531,2863311531,3264175145,3264175145] +; X86-NEXT: pmuludq %xmm1, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,3,3] -; X86-NEXT: pmuludq %xmm2, %xmm1 +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X86-NEXT: retl @@ -109,12 +108,11 @@ define <4 x i32> @test6(<4 x i32> %x) { ; X86-NEXT: psrld $3, %xmm1 ; X86-NEXT: psrld $1, %xmm0 ; X86-NEXT: movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1] -; X86-NEXT: movdqa {{.*#+}} xmm2 = [2863311531,2863311531,3303820997,3303820997] -; X86-NEXT: movapd %xmm0, %xmm1 -; X86-NEXT: pmuludq %xmm2, %xmm1 +; X86-NEXT: movdqa {{.*#+}} xmm1 = [2863311531,2863311531,3303820997,3303820997] +; X86-NEXT: pmuludq %xmm0, %xmm1 ; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,3,3] -; X86-NEXT: pmuludq %xmm2, %xmm0 +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; X86-NEXT: movdqa %xmm1, %xmm0 @@ -132,12 +130,11 @@ define <4 x i32> @test6(<4 x i32> %x) { define <4 x i32> @test7(<4 x i32> %x) { ; X86-LABEL: test7: ; X86: # %bb.0: -; X86-NEXT: movdqa {{.*#+}} xmm1 = [3264175145,3264175145,1749801491,1749801491] -; X86-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; X86-NEXT: pmuludq %xmm1, %xmm0 +; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; X86-NEXT: pmuludq %xmm1, %xmm2 -; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 +; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X86-NEXT: retl ; @@ -155,12 +152,11 @@ define <4 x i32> @test8(<4 x i32> %x) { ; X86-NEXT: movdqa %xmm0, %xmm1 ; X86-NEXT: psrld $3, %xmm1 ; X86-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] -; X86-NEXT: movdqa {{.*#+}} xmm2 = [1,1,2863311531,2863311531] -; X86-NEXT: movapd %xmm1, %xmm0 -; X86-NEXT: pmuludq %xmm2, %xmm0 +; X86-NEXT: movdqa {{.*#+}} xmm0 = [1,1,2863311531,2863311531] +; X86-NEXT: pmuludq %xmm1, %xmm0 ; X86-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; X86-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,3,3] -; X86-NEXT: pmuludq %xmm2, %xmm1 +; X86-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; X86-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X86-NEXT: retl diff --git a/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll b/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll index 2e4fe6a4c120c..7c1a1e285ca05 100644 --- a/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll +++ b/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll @@ -137,15 +137,17 @@ define <3 x i1> @test_urem_vec(<3 x i11> %X) nounwind { ; SSE2-NEXT: movd %edx, %xmm0 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; SSE2-NEXT: psubd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [683,1463,819,u] +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [683,u,819,u] ; SSE2-NEXT: pmuludq %xmm1, %xmm0 ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1] -; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE2-NEXT: movl $1463, %eax # imm = 0x5B7 +; SSE2-NEXT: movd %eax, %xmm3 +; SSE2-NEXT: pmuludq %xmm1, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] ; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2047,2047,2047,2047] -; SSE2-NEXT: xorps %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm3, %xmm3 ; SSE2-NEXT: movss {{.*#+}} xmm3 = xmm0[0],xmm3[1,2,3] ; SSE2-NEXT: pand %xmm1, %xmm0 ; SSE2-NEXT: psrld $1, %xmm0 diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll index 11808ea1def8e..838086e366fbf 100644 --- a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll +++ b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll @@ -11,17 +11,17 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_even: @@ -160,22 +160,18 @@ define <4 x i32> @test_urem_even_allones_eq(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] -; CHECK-SSE2-NEXT: pmuludq %xmm3, %xmm2 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3] +; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; CHECK-SSE2-NEXT: por %xmm1, %xmm0 -; CHECK-SSE2-NEXT: pxor %xmm3, %xmm0 +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: por %xmm4, %xmm0 +; CHECK-SSE2-NEXT: pxor %xmm2, %xmm0 ; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: retq @@ -242,22 +238,18 @@ define <4 x i32> @test_urem_even_allones_ne(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] -; CHECK-SSE2-NEXT: pmuludq %xmm3, %xmm2 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3] +; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; CHECK-SSE2-NEXT: por %xmm1, %xmm0 -; CHECK-SSE2-NEXT: pxor %xmm3, %xmm0 +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: por %xmm4, %xmm0 +; CHECK-SSE2-NEXT: pxor %xmm2, %xmm0 ; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: psrld $31, %xmm0 ; CHECK-SSE2-NEXT: retq @@ -326,17 +318,17 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_even_allones_eq: @@ -399,17 +391,17 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: psrld $31, %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: psrld $31, %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_even_allones_ne: @@ -475,11 +467,9 @@ define <4 x i32> @test_urem_odd_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE2-LABEL: test_urem_odd_poweroftwo: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] @@ -493,7 +483,7 @@ define <4 x i32> @test_urem_odd_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE41-LABEL: test_urem_odd_poweroftwo: ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,268435456,1] +; CHECK-SSE41-NEXT: pmovsxdq {{.*#+}} xmm1 = [1,268435456] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 @@ -547,22 +537,18 @@ define <4 x i32> @test_urem_even_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] -; CHECK-SSE2-NEXT: pmuludq %xmm3, %xmm2 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3] +; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; CHECK-SSE2-NEXT: por %xmm1, %xmm0 -; CHECK-SSE2-NEXT: pxor %xmm3, %xmm0 +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: por %xmm4, %xmm0 +; CHECK-SSE2-NEXT: pxor %xmm2, %xmm0 ; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: retq @@ -631,13 +617,9 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] @@ -841,17 +823,17 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_even_one: @@ -917,11 +899,9 @@ define <4 x i32> @test_urem_odd_INT_MIN(<4 x i32> %X) nounwind { ; CHECK-SSE2-LABEL: test_urem_odd_INT_MIN: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] @@ -935,7 +915,7 @@ define <4 x i32> @test_urem_odd_INT_MIN(<4 x i32> %X) nounwind { ; CHECK-SSE41-LABEL: test_urem_odd_INT_MIN: ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [1,1,2,1] +; CHECK-SSE41-NEXT: pmovsxbq {{.*#+}} xmm1 = [1,2] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 @@ -989,22 +969,18 @@ define <4 x i32> @test_urem_even_INT_MIN(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] -; CHECK-SSE2-NEXT: pmuludq %xmm3, %xmm2 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3] +; CHECK-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; CHECK-SSE2-NEXT: pmuludq %xmm2, %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; CHECK-SSE2-NEXT: por %xmm1, %xmm0 -; CHECK-SSE2-NEXT: pxor %xmm3, %xmm0 +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: por %xmm4, %xmm0 +; CHECK-SSE2-NEXT: pxor %xmm2, %xmm0 ; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: retq @@ -1073,13 +1049,9 @@ define <4 x i32> @test_urem_odd_even_INT_MIN(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] @@ -1157,11 +1129,9 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE2-LABEL: test_urem_odd_allones_and_poweroftwo: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] @@ -1175,7 +1145,7 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE41-LABEL: test_urem_odd_allones_and_poweroftwo: ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,268435456,1] +; CHECK-SSE41-NEXT: pmovsxdq {{.*#+}} xmm1 = [1,268435456] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 @@ -1229,13 +1199,9 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] @@ -1312,13 +1278,9 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] @@ -1434,13 +1396,9 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] @@ -1517,17 +1475,17 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_even_allones_and_one: @@ -1594,17 +1552,17 @@ define <4 x i32> @test_urem_odd_poweroftwo_and_one(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_poweroftwo_and_one: @@ -1669,13 +1627,9 @@ define <4 x i32> @test_urem_even_poweroftwo_and_one(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3] -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] @@ -1752,17 +1706,17 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind { ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: por %xmm2, %xmm0 -; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; CHECK-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: por %xmm0, %xmm1 +; CHECK-SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; CHECK-SSE2-NEXT: movdqa %xmm1, %xmm0 ; CHECK-SSE2-NEXT: retq ; ; CHECK-SSE41-LABEL: test_urem_odd_even_poweroftwo_and_one: @@ -1827,11 +1781,9 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou ; CHECK-SSE2-LABEL: test_urem_odd_allones_and_poweroftwo_and_one: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] @@ -1845,7 +1797,7 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou ; CHECK-SSE41-LABEL: test_urem_odd_allones_and_poweroftwo_and_one: ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,268435456,1] +; CHECK-SSE41-NEXT: pmovsxdq {{.*#+}} xmm1 = [1,268435456] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 @@ -1897,11 +1849,9 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no ; CHECK-SSE2-LABEL: test_urem_even_allones_and_poweroftwo_and_one: ; CHECK-SSE2: # %bb.0: ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] ; CHECK-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] @@ -1915,7 +1865,7 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no ; CHECK-SSE41-LABEL: test_urem_even_allones_and_poweroftwo_and_one: ; CHECK-SSE41: # %bb.0: ; CHECK-SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483648,1,268435456,1] +; CHECK-SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = [2147483648,268435456] ; CHECK-SSE41-NEXT: pmuludq %xmm0, %xmm1 ; CHECK-SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] ; CHECK-SSE41-NEXT: psrlq $32, %xmm1 diff --git a/llvm/test/CodeGen/X86/ushl_sat_vec.ll b/llvm/test/CodeGen/X86/ushl_sat_vec.ll index ed523e31b0569..ebb5e135eacd0 100644 --- a/llvm/test/CodeGen/X86/ushl_sat_vec.ll +++ b/llvm/test/CodeGen/X86/ushl_sat_vec.ll @@ -222,23 +222,22 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind { define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind { ; X64-LABEL: vec_v8i16: ; X64: # %bb.0: -; X64-NEXT: pxor %xmm2, %xmm2 -; X64-NEXT: movdqa %xmm1, %xmm3 -; X64-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; X64-NEXT: pslld $23, %xmm3 -; X64-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X64-NEXT: paddd %xmm4, %xmm3 -; X64-NEXT: cvttps2dq %xmm3, %xmm3 -; X64-NEXT: pslld $16, %xmm3 -; X64-NEXT: psrad $16, %xmm3 -; X64-NEXT: movdqa %xmm1, %xmm5 -; X64-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3] -; X64-NEXT: pslld $23, %xmm5 -; X64-NEXT: paddd %xmm4, %xmm5 -; X64-NEXT: cvttps2dq %xmm5, %xmm2 +; X64-NEXT: movdqa %xmm1, %xmm2 +; X64-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; X64-NEXT: pslld $23, %xmm2 +; X64-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X64-NEXT: paddd %xmm3, %xmm2 +; X64-NEXT: cvttps2dq %xmm2, %xmm4 +; X64-NEXT: pslld $16, %xmm4 +; X64-NEXT: psrad $16, %xmm4 +; X64-NEXT: movdqa %xmm1, %xmm2 +; X64-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] +; X64-NEXT: pslld $23, %xmm2 +; X64-NEXT: paddd %xmm3, %xmm2 +; X64-NEXT: cvttps2dq %xmm2, %xmm2 ; X64-NEXT: pslld $16, %xmm2 ; X64-NEXT: psrad $16, %xmm2 -; X64-NEXT: packssdw %xmm3, %xmm2 +; X64-NEXT: packssdw %xmm4, %xmm2 ; X64-NEXT: pmullw %xmm0, %xmm2 ; X64-NEXT: psllw $12, %xmm1 ; X64-NEXT: movdqa %xmm1, %xmm3 diff --git a/llvm/test/CodeGen/X86/vec_minmax_sint.ll b/llvm/test/CodeGen/X86/vec_minmax_sint.ll index c8d988cb011ae..853e29b8acfcd 100644 --- a/llvm/test/CodeGen/X86/vec_minmax_sint.ll +++ b/llvm/test/CodeGen/X86/vec_minmax_sint.ll @@ -34,17 +34,15 @@ define <2 x i64> @max_gt_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: max_gt_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -127,24 +125,20 @@ define <4 x i64> @max_gt_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm4, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -421,17 +415,15 @@ define <2 x i64> @max_ge_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: max_ge_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -514,24 +506,20 @@ define <4 x i64> @max_ge_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm4, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -808,17 +796,14 @@ define <2 x i64> @min_lt_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: min_lt_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -901,24 +886,20 @@ define <4 x i64> @min_lt_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm2, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -1196,17 +1177,14 @@ define <2 x i64> @min_le_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: min_le_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -1289,24 +1267,20 @@ define <4 x i64> @min_le_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm2, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 diff --git a/llvm/test/CodeGen/X86/vec_minmax_uint.ll b/llvm/test/CodeGen/X86/vec_minmax_uint.ll index c9bfff4b7dfd7..9b4da3f9b817f 100644 --- a/llvm/test/CodeGen/X86/vec_minmax_uint.ll +++ b/llvm/test/CodeGen/X86/vec_minmax_uint.ll @@ -34,17 +34,15 @@ define <2 x i64> @max_gt_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: max_gt_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -138,24 +136,20 @@ define <4 x i64> @max_gt_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm4, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -448,17 +442,15 @@ define <2 x i64> @max_ge_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: max_ge_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -552,24 +544,20 @@ define <4 x i64> @max_ge_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm4, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -862,17 +850,14 @@ define <2 x i64> @min_lt_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: min_lt_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -965,24 +950,20 @@ define <4 x i64> @min_lt_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm2, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -1279,17 +1260,14 @@ define <2 x i64> @min_le_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE41-LABEL: min_le_v2i64: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm0 @@ -1382,24 +1360,20 @@ define <4 x i64> @min_le_v4i64(<4 x i64> %a, <4 x i64> %b) { ; SSE41-NEXT: movdqa %xmm2, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm2, %xmm0 diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll index 0567cb2ac74b6..1a1a50689c87f 100644 --- a/llvm/test/CodeGen/X86/vec_smulo.ll +++ b/llvm/test/CodeGen/X86/vec_smulo.ll @@ -435,12 +435,12 @@ define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero ; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm1[0] -; SSE2-NEXT: movd %r9d, %xmm3 -; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE2-NEXT: movd %r9d, %xmm2 ; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero ; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero +; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] ; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx ; SSE2-NEXT: pxor %xmm4, %xmm4 ; SSE2-NEXT: pxor %xmm5, %xmm5 @@ -467,26 +467,23 @@ define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSE2-NEXT: pcmpeqd %xmm5, %xmm5 ; SSE2-NEXT: pxor %xmm5, %xmm6 ; SSE2-NEXT: pxor %xmm7, %xmm7 -; SSE2-NEXT: pcmpgtd %xmm2, %xmm7 -; SSE2-NEXT: pand %xmm3, %xmm7 -; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE2-NEXT: pand %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm7 +; SSE2-NEXT: pand %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm4 +; SSE2-NEXT: pand %xmm1, %xmm4 ; SSE2-NEXT: paddd %xmm7, %xmm4 -; SSE2-NEXT: pmuludq %xmm3, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,0,1,1] -; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] -; SSE2-NEXT: psubd %xmm4, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1] -; SSE2-NEXT: pmuludq %xmm7, %xmm0 +; SSE2-NEXT: pmuludq %xmm2, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] +; SSE2-NEXT: pmuludq %xmm3, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSE2-NEXT: psubd %xmm4, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; SSE2-NEXT: movq %xmm1, 16(%rcx) ; SSE2-NEXT: psrad $31, %xmm1 -; SSE2-NEXT: pcmpeqd %xmm3, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm1 ; SSE2-NEXT: pxor %xmm5, %xmm1 ; SSE2-NEXT: movq %xmm1, 16(%rdi) ; SSE2-NEXT: movdqa %xmm6, (%rdi) @@ -509,12 +506,12 @@ define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSSE3-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero ; SSSE3-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] ; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm1[0] -; SSSE3-NEXT: movd %r9d, %xmm3 -; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero -; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSSE3-NEXT: movd %r9d, %xmm2 ; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSSE3-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero ; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSSE3-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero +; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] ; SSSE3-NEXT: movq {{[0-9]+}}(%rsp), %rcx ; SSSE3-NEXT: pxor %xmm4, %xmm4 ; SSSE3-NEXT: pxor %xmm5, %xmm5 @@ -541,26 +538,23 @@ define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSSE3-NEXT: pcmpeqd %xmm5, %xmm5 ; SSSE3-NEXT: pxor %xmm5, %xmm6 ; SSSE3-NEXT: pxor %xmm7, %xmm7 -; SSSE3-NEXT: pcmpgtd %xmm2, %xmm7 -; SSSE3-NEXT: pand %xmm3, %xmm7 -; SSSE3-NEXT: pcmpgtd %xmm3, %xmm4 -; SSSE3-NEXT: pand %xmm2, %xmm4 +; SSSE3-NEXT: pcmpgtd %xmm1, %xmm7 +; SSSE3-NEXT: pand %xmm2, %xmm7 +; SSSE3-NEXT: pcmpgtd %xmm2, %xmm4 +; SSSE3-NEXT: pand %xmm1, %xmm4 ; SSSE3-NEXT: paddd %xmm7, %xmm4 -; SSSE3-NEXT: pmuludq %xmm3, %xmm2 -; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3] -; SSSE3-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,0,1,1] -; SSSE3-NEXT: pmuludq %xmm0, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] -; SSSE3-NEXT: psubd %xmm4, %xmm3 -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1] -; SSSE3-NEXT: pmuludq %xmm7, %xmm0 +; SSSE3-NEXT: pmuludq %xmm2, %xmm1 +; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3] +; SSSE3-NEXT: pmuludq %xmm3, %xmm0 +; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSSE3-NEXT: psubd %xmm4, %xmm2 +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; SSSE3-NEXT: movq %xmm1, 16(%rcx) ; SSSE3-NEXT: psrad $31, %xmm1 -; SSSE3-NEXT: pcmpeqd %xmm3, %xmm1 +; SSSE3-NEXT: pcmpeqd %xmm2, %xmm1 ; SSSE3-NEXT: pxor %xmm5, %xmm1 ; SSSE3-NEXT: movq %xmm1, 16(%rdi) ; SSSE3-NEXT: movdqa %xmm6, (%rdi) diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll index 2b76f02b57e3c..1df40e773246a 100644 --- a/llvm/test/CodeGen/X86/vec_umulo.ll +++ b/llvm/test/CodeGen/X86/vec_umulo.ll @@ -366,9 +366,9 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero ; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero -; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0] +; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] ; SSE2-NEXT: movd %r8d, %xmm0 ; SSE2-NEXT: movd %ecx, %xmm1 ; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] @@ -376,42 +376,37 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSE2-NEXT: movd %esi, %xmm3 ; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0] -; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero -; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: movd %r9d, %xmm0 -; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1] -; SSE2-NEXT: pmuludq %xmm2, %xmm0 +; SSE2-NEXT: movd %r9d, %xmm1 ; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx -; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm4, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm6, %xmm4 -; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,3,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1] -; SSE2-NEXT: pxor %xmm6, %xmm6 -; SSE2-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 -; SSE2-NEXT: pxor %xmm7, %xmm2 +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pmuludq %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm2, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm4, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] +; SSE2-NEXT: pxor %xmm4, %xmm4 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm5, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm1 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,0,1,1] -; SSE2-NEXT: pmuludq %xmm1, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] -; SSE2-NEXT: pcmpeqd %xmm6, %xmm4 -; SSE2-NEXT: pxor %xmm7, %xmm4 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1] -; SSE2-NEXT: pmuludq %xmm8, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero +; SSE2-NEXT: pmuludq %xmm2, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm7 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1] ; SSE2-NEXT: movq %xmm0, 16(%rcx) ; SSE2-NEXT: movdqa %xmm3, (%rcx) -; SSE2-NEXT: movq %xmm4, 16(%rdi) -; SSE2-NEXT: movdqa %xmm2, (%rdi) +; SSE2-NEXT: movq %xmm7, 16(%rdi) +; SSE2-NEXT: movdqa %xmm1, (%rdi) ; SSE2-NEXT: retq ; ; SSSE3-LABEL: umulo_v6i32: @@ -421,9 +416,9 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero ; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSSE3-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero -; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0] +; SSSE3-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] ; SSSE3-NEXT: movd %r8d, %xmm0 ; SSSE3-NEXT: movd %ecx, %xmm1 ; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] @@ -431,42 +426,37 @@ define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; SSSE3-NEXT: movd %esi, %xmm3 ; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] ; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0] -; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero -; SSSE3-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero -; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSSE3-NEXT: movd %r9d, %xmm0 -; SSSE3-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero -; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1] -; SSSE3-NEXT: pmuludq %xmm2, %xmm0 +; SSSE3-NEXT: movd %r9d, %xmm1 ; SSSE3-NEXT: movq {{[0-9]+}}(%rsp), %rcx -; SSSE3-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3] -; SSSE3-NEXT: pmuludq %xmm4, %xmm3 -; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3] -; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSSE3-NEXT: pmuludq %xmm6, %xmm4 -; SSSE3-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,3,2,3] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1] -; SSSE3-NEXT: pxor %xmm6, %xmm6 -; SSSE3-NEXT: pcmpeqd %xmm6, %xmm2 -; SSSE3-NEXT: pcmpeqd %xmm7, %xmm7 -; SSSE3-NEXT: pxor %xmm7, %xmm2 +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pmuludq %xmm1, %xmm0 +; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3] +; SSSE3-NEXT: pmuludq %xmm2, %xmm3 +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSSE3-NEXT: pmuludq %xmm4, %xmm2 +; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] +; SSSE3-NEXT: pxor %xmm4, %xmm4 +; SSSE3-NEXT: pcmpeqd %xmm4, %xmm1 +; SSSE3-NEXT: pcmpeqd %xmm5, %xmm5 +; SSSE3-NEXT: pxor %xmm5, %xmm1 ; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] -; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3] -; SSSE3-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,0,1,1] -; SSSE3-NEXT: pmuludq %xmm1, %xmm5 -; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,3,2,3] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] -; SSSE3-NEXT: pcmpeqd %xmm6, %xmm4 -; SSSE3-NEXT: pxor %xmm7, %xmm4 -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1] -; SSSE3-NEXT: pmuludq %xmm8, %xmm1 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; SSSE3-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSSE3-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero +; SSSE3-NEXT: pmuludq %xmm2, %xmm6 +; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1] +; SSSE3-NEXT: pcmpeqd %xmm4, %xmm7 +; SSSE3-NEXT: pxor %xmm5, %xmm7 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1] ; SSSE3-NEXT: movq %xmm0, 16(%rcx) ; SSSE3-NEXT: movdqa %xmm3, (%rcx) -; SSSE3-NEXT: movq %xmm4, 16(%rdi) -; SSSE3-NEXT: movdqa %xmm2, (%rdi) +; SSSE3-NEXT: movq %xmm7, 16(%rdi) +; SSSE3-NEXT: movdqa %xmm1, (%rdi) ; SSSE3-NEXT: retq ; ; SSE41-LABEL: umulo_v6i32: diff --git a/llvm/test/CodeGen/X86/vector-compare-all_of.ll b/llvm/test/CodeGen/X86/vector-compare-all_of.ll index 8995989ef4474..bf027a7346deb 100644 --- a/llvm/test/CodeGen/X86/vector-compare-all_of.ll +++ b/llvm/test/CodeGen/X86/vector-compare-all_of.ll @@ -272,12 +272,10 @@ define i64 @test_v2i64_sext(<2 x i64> %a0, <2 x i64> %a1) { ; SSE2-NEXT: pxor %xmm2, %xmm1 ; SSE2-NEXT: pxor %xmm2, %xmm0 ; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-NEXT: pand %xmm3, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] +; SSE2-NEXT: pand %xmm2, %xmm1 ; SSE2-NEXT: por %xmm0, %xmm1 ; SSE2-NEXT: movmskpd %xmm1, %ecx ; SSE2-NEXT: xorl %eax, %eax @@ -319,25 +317,21 @@ define i64 @test_v4i64_sext(<4 x i64> %a0, <4 x i64> %a1) { ; SSE2-NEXT: pxor %xmm4, %xmm3 ; SSE2-NEXT: pxor %xmm4, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm5 -; SSE2-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE2-NEXT: pcmpeqd %xmm3, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE2-NEXT: pand %xmm6, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] +; SSE2-NEXT: pand %xmm5, %xmm3 ; SSE2-NEXT: por %xmm1, %xmm3 ; SSE2-NEXT: pxor %xmm4, %xmm2 ; SSE2-NEXT: pxor %xmm4, %xmm0 ; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: pcmpgtd %xmm2, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2] -; SSE2-NEXT: pcmpeqd %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-NEXT: pand %xmm4, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: pand %xmm3, %xmm1 -; SSE2-NEXT: movmskpd %xmm1, %ecx +; SSE2-NEXT: pcmpeqd %xmm2, %xmm1 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] +; SSE2-NEXT: pand %xmm1, %xmm2 +; SSE2-NEXT: por %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm3, %xmm2 +; SSE2-NEXT: movmskpd %xmm2, %ecx ; SSE2-NEXT: xorl %eax, %eax ; SSE2-NEXT: cmpl $3, %ecx ; SSE2-NEXT: sete %al diff --git a/llvm/test/CodeGen/X86/vector-compare-any_of.ll b/llvm/test/CodeGen/X86/vector-compare-any_of.ll index 24fee35c73404..2df39d69dbb75 100644 --- a/llvm/test/CodeGen/X86/vector-compare-any_of.ll +++ b/llvm/test/CodeGen/X86/vector-compare-any_of.ll @@ -223,12 +223,10 @@ define i64 @test_v2i64_sext(<2 x i64> %a0, <2 x i64> %a1) { ; SSE2-NEXT: pxor %xmm2, %xmm1 ; SSE2-NEXT: pxor %xmm2, %xmm0 ; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-NEXT: pand %xmm3, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] +; SSE2-NEXT: pand %xmm2, %xmm1 ; SSE2-NEXT: por %xmm0, %xmm1 ; SSE2-NEXT: movmskpd %xmm1, %ecx ; SSE2-NEXT: xorl %eax, %eax @@ -268,25 +266,21 @@ define i64 @test_v4i64_sext(<4 x i64> %a0, <4 x i64> %a1) { ; SSE2-NEXT: pxor %xmm4, %xmm3 ; SSE2-NEXT: pxor %xmm4, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm5 -; SSE2-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE2-NEXT: pcmpeqd %xmm3, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE2-NEXT: pand %xmm6, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] +; SSE2-NEXT: pand %xmm5, %xmm3 ; SSE2-NEXT: por %xmm1, %xmm3 ; SSE2-NEXT: pxor %xmm4, %xmm2 ; SSE2-NEXT: pxor %xmm4, %xmm0 ; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: pcmpgtd %xmm2, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2] -; SSE2-NEXT: pcmpeqd %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-NEXT: pand %xmm4, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: por %xmm3, %xmm1 -; SSE2-NEXT: movmskpd %xmm1, %ecx +; SSE2-NEXT: pcmpeqd %xmm2, %xmm1 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2] +; SSE2-NEXT: pand %xmm1, %xmm2 +; SSE2-NEXT: por %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movmskpd %xmm2, %ecx ; SSE2-NEXT: xorl %eax, %eax ; SSE2-NEXT: negl %ecx ; SSE2-NEXT: sbbq %rax, %rax diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll index f33bab2b2b0d3..6b8a03ba5eb76 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll @@ -429,25 +429,24 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; SSE2-NEXT: movdqa %xmm1, %xmm3 ; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE2-NEXT: pxor %xmm4, %xmm4 -; SSE2-NEXT: movdqa %xmm2, %xmm5 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] -; SSE2-NEXT: pslld $23, %xmm5 -; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm6, %xmm5 -; SSE2-NEXT: cvttps2dq %xmm5, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm5, %xmm3 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm4 +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm5, %xmm4 +; SSE2-NEXT: cvttps2dq %xmm4, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm4, %xmm3 ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm7, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm6, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] ; SSE2-NEXT: psrad $16, %xmm3 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm2 -; SSE2-NEXT: paddd %xmm6, %xmm2 +; SSE2-NEXT: paddd %xmm5, %xmm2 ; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3] ; SSE2-NEXT: pmuludq %xmm2, %xmm1 @@ -465,19 +464,18 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; SSE41-NEXT: movdqa %xmm1, %xmm3 ; SSE41-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] ; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE41-NEXT: pxor %xmm4, %xmm4 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm4, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm5, %xmm2 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: pmulld %xmm3, %xmm2 ; SSE41-NEXT: psrld $16, %xmm2 ; SSE41-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE41-NEXT: pslld $23, %xmm5 -; SSE41-NEXT: paddd %xmm4, %xmm5 -; SSE41-NEXT: cvttps2dq %xmm5, %xmm0 +; SSE41-NEXT: pslld $23, %xmm4 +; SSE41-NEXT: paddd %xmm5, %xmm4 +; SSE41-NEXT: cvttps2dq %xmm4, %xmm0 ; SSE41-NEXT: pmulld %xmm1, %xmm0 ; SSE41-NEXT: psrld $16, %xmm0 ; SSE41-NEXT: packusdw %xmm2, %xmm0 @@ -487,8 +485,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; AVX1: # %bb.0: ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 @@ -619,25 +616,24 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; X86-SSE2-NEXT: movdqa %xmm1, %xmm3 ; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 -; X86-SSE2-NEXT: pxor %xmm4, %xmm4 -; X86-SSE2-NEXT: movdqa %xmm2, %xmm6 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] -; X86-SSE2-NEXT: pslld $23, %xmm6 -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE2-NEXT: paddd %xmm5, %xmm6 -; X86-SSE2-NEXT: cvttps2dq %xmm6, %xmm6 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3] -; X86-SSE2-NEXT: pmuludq %xmm6, %xmm3 +; X86-SSE2-NEXT: movdqa %xmm2, %xmm5 +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] +; X86-SSE2-NEXT: pslld $23, %xmm5 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE2-NEXT: paddd %xmm4, %xmm5 +; X86-SSE2-NEXT: cvttps2dq %xmm5, %xmm5 +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3] +; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; X86-SSE2-NEXT: pmuludq %xmm7, %xmm6 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1] +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; X86-SSE2-NEXT: pmuludq %xmm6, %xmm5 +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3] +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1] ; X86-SSE2-NEXT: psrad $16, %xmm3 ; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: pslld $23, %xmm2 -; X86-SSE2-NEXT: paddd %xmm5, %xmm2 +; X86-SSE2-NEXT: paddd %xmm4, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3] ; X86-SSE2-NEXT: pmuludq %xmm2, %xmm1 @@ -657,47 +653,47 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) ; SSE2-LABEL: var_funnnel_v16i8: ; SSE2: # %bb.0: ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE2-NEXT: pxor %xmm3, %xmm3 -; SSE2-NEXT: movdqa %xmm2, %xmm5 -; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15] -; SSE2-NEXT: movdqa %xmm5, %xmm6 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] +; SSE2-NEXT: movdqa %xmm4, %xmm6 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7] ; SSE2-NEXT: pslld $23, %xmm6 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm6 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm6 ; SSE2-NEXT: cvttps2dq %xmm6, %xmm6 ; SSE2-NEXT: pslld $16, %xmm6 ; SSE2-NEXT: psrad $16, %xmm6 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3] -; SSE2-NEXT: pslld $23, %xmm5 -; SSE2-NEXT: paddd %xmm4, %xmm5 -; SSE2-NEXT: cvttps2dq %xmm5, %xmm7 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3] +; SSE2-NEXT: pslld $23, %xmm4 +; SSE2-NEXT: paddd %xmm3, %xmm4 +; SSE2-NEXT: cvttps2dq %xmm4, %xmm7 ; SSE2-NEXT: pslld $16, %xmm7 ; SSE2-NEXT: psrad $16, %xmm7 ; SSE2-NEXT: packssdw %xmm6, %xmm7 -; SSE2-NEXT: movdqa %xmm1, %xmm5 -; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] -; SSE2-NEXT: pmullw %xmm7, %xmm5 -; SSE2-NEXT: psrlw $8, %xmm5 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] -; SSE2-NEXT: movdqa %xmm2, %xmm6 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] -; SSE2-NEXT: pslld $23, %xmm6 -; SSE2-NEXT: paddd %xmm4, %xmm6 -; SSE2-NEXT: cvttps2dq %xmm6, %xmm6 -; SSE2-NEXT: pslld $16, %xmm6 -; SSE2-NEXT: psrad $16, %xmm6 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] +; SSE2-NEXT: pmullw %xmm7, %xmm4 +; SSE2-NEXT: psrlw $8, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm5 +; SSE2-NEXT: paddd %xmm3, %xmm5 +; SSE2-NEXT: cvttps2dq %xmm5, %xmm5 +; SSE2-NEXT: pslld $16, %xmm5 +; SSE2-NEXT: psrad $16, %xmm5 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm2 -; SSE2-NEXT: paddd %xmm4, %xmm2 +; SSE2-NEXT: paddd %xmm3, %xmm2 ; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE2-NEXT: pslld $16, %xmm2 ; SSE2-NEXT: psrad $16, %xmm2 -; SSE2-NEXT: packssdw %xmm6, %xmm2 +; SSE2-NEXT: packssdw %xmm5, %xmm2 ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; SSE2-NEXT: pmullw %xmm1, %xmm2 ; SSE2-NEXT: psrlw $8, %xmm2 -; SSE2-NEXT: packuswb %xmm5, %xmm2 +; SSE2-NEXT: packuswb %xmm4, %xmm2 ; SSE2-NEXT: movdqa %xmm2, %xmm0 ; SSE2-NEXT: retq ; @@ -708,32 +704,32 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) ; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero ; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm7, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm6, %xmm2 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 -; SSE41-NEXT: pslld $23, %xmm6 -; SSE41-NEXT: paddd %xmm7, %xmm6 -; SSE41-NEXT: cvttps2dq %xmm6, %xmm6 -; SSE41-NEXT: packusdw %xmm2, %xmm6 -; SSE41-NEXT: movdqa %xmm1, %xmm8 -; SSE41-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15] -; SSE41-NEXT: pmullw %xmm6, %xmm8 -; SSE41-NEXT: psrlw $8, %xmm8 +; SSE41-NEXT: pslld $23, %xmm3 +; SSE41-NEXT: paddd %xmm6, %xmm3 +; SSE41-NEXT: cvttps2dq %xmm3, %xmm3 +; SSE41-NEXT: packusdw %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm1, %xmm7 +; SSE41-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15] +; SSE41-NEXT: pmullw %xmm3, %xmm7 +; SSE41-NEXT: psrlw $8, %xmm7 ; SSE41-NEXT: pslld $23, %xmm4 -; SSE41-NEXT: paddd %xmm7, %xmm4 +; SSE41-NEXT: paddd %xmm6, %xmm4 ; SSE41-NEXT: cvttps2dq %xmm4, %xmm2 -; SSE41-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7] +; SSE41-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm5 -; SSE41-NEXT: paddd %xmm7, %xmm5 +; SSE41-NEXT: paddd %xmm6, %xmm5 ; SSE41-NEXT: cvttps2dq %xmm5, %xmm3 ; SSE41-NEXT: packusdw %xmm3, %xmm2 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; SSE41-NEXT: pmullw %xmm1, %xmm2 ; SSE41-NEXT: psrlw $8, %xmm2 -; SSE41-NEXT: packuswb %xmm8, %xmm2 +; SSE41-NEXT: packuswb %xmm7, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; @@ -741,34 +737,34 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) ; AVX1: # %bb.0: ; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] -; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 -; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 -; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 -; AVX1-NEXT: vpaddd %xmm6, %xmm4, %xmm4 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero +; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 +; AVX1-NEXT: vpackusdw %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; AVX1-NEXT: vpmullw %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 ; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 -; AVX1-NEXT: vpackusdw %xmm5, %xmm4, %xmm4 -; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] -; AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4 -; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4 -; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 -; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; AVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 ; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; AVX1-NEXT: vpackusdw %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpackusdw %xmm2, %xmm4, %xmm2 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0 -; AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_funnnel_v16i8: @@ -920,47 +916,47 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) ; X86-SSE2-LABEL: var_funnnel_v16i8: ; X86-SSE2: # %bb.0: ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 -; X86-SSE2-NEXT: pxor %xmm3, %xmm3 -; X86-SSE2-NEXT: movdqa %xmm2, %xmm5 -; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15] -; X86-SSE2-NEXT: movdqa %xmm5, %xmm6 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] +; X86-SSE2-NEXT: pxor %xmm5, %xmm5 +; X86-SSE2-NEXT: movdqa %xmm2, %xmm4 +; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] +; X86-SSE2-NEXT: movdqa %xmm4, %xmm6 +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: pslld $23, %xmm6 -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE2-NEXT: paddd %xmm4, %xmm6 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE2-NEXT: paddd %xmm3, %xmm6 ; X86-SSE2-NEXT: cvttps2dq %xmm6, %xmm6 ; X86-SSE2-NEXT: pslld $16, %xmm6 ; X86-SSE2-NEXT: psrad $16, %xmm6 -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3] -; X86-SSE2-NEXT: pslld $23, %xmm5 -; X86-SSE2-NEXT: paddd %xmm4, %xmm5 -; X86-SSE2-NEXT: cvttps2dq %xmm5, %xmm7 +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3] +; X86-SSE2-NEXT: pslld $23, %xmm4 +; X86-SSE2-NEXT: paddd %xmm3, %xmm4 +; X86-SSE2-NEXT: cvttps2dq %xmm4, %xmm7 ; X86-SSE2-NEXT: pslld $16, %xmm7 ; X86-SSE2-NEXT: psrad $16, %xmm7 ; X86-SSE2-NEXT: packssdw %xmm6, %xmm7 -; X86-SSE2-NEXT: movdqa %xmm1, %xmm5 -; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15] -; X86-SSE2-NEXT: pmullw %xmm7, %xmm5 -; X86-SSE2-NEXT: psrlw $8, %xmm5 -; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] -; X86-SSE2-NEXT: movdqa %xmm2, %xmm6 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] -; X86-SSE2-NEXT: pslld $23, %xmm6 -; X86-SSE2-NEXT: paddd %xmm4, %xmm6 -; X86-SSE2-NEXT: cvttps2dq %xmm6, %xmm6 -; X86-SSE2-NEXT: pslld $16, %xmm6 -; X86-SSE2-NEXT: psrad $16, %xmm6 -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; X86-SSE2-NEXT: movdqa %xmm1, %xmm4 +; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] +; X86-SSE2-NEXT: pmullw %xmm7, %xmm4 +; X86-SSE2-NEXT: psrlw $8, %xmm4 +; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] +; X86-SSE2-NEXT: movdqa %xmm2, %xmm5 +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7] +; X86-SSE2-NEXT: pslld $23, %xmm5 +; X86-SSE2-NEXT: paddd %xmm3, %xmm5 +; X86-SSE2-NEXT: cvttps2dq %xmm5, %xmm5 +; X86-SSE2-NEXT: pslld $16, %xmm5 +; X86-SSE2-NEXT: psrad $16, %xmm5 +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: pslld $23, %xmm2 -; X86-SSE2-NEXT: paddd %xmm4, %xmm2 +; X86-SSE2-NEXT: paddd %xmm3, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: pslld $16, %xmm2 ; X86-SSE2-NEXT: psrad $16, %xmm2 -; X86-SSE2-NEXT: packssdw %xmm6, %xmm2 +; X86-SSE2-NEXT: packssdw %xmm5, %xmm2 ; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; X86-SSE2-NEXT: pmullw %xmm1, %xmm2 ; X86-SSE2-NEXT: psrlw $8, %xmm2 -; X86-SSE2-NEXT: packuswb %xmm5, %xmm2 +; X86-SSE2-NEXT: packuswb %xmm4, %xmm2 ; X86-SSE2-NEXT: movdqa %xmm2, %xmm0 ; X86-SSE2-NEXT: retl %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) @@ -1323,34 +1319,19 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> % } define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind { -; SSE2-LABEL: splatvar_funnnel_v16i8: -; SSE2: # %bb.0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: movdqa %xmm1, %xmm3 -; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] -; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE2-NEXT: psllw %xmm2, %xmm3 -; SSE2-NEXT: psrlw $8, %xmm3 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE2-NEXT: psllw %xmm2, %xmm1 -; SSE2-NEXT: psrlw $8, %xmm1 -; SSE2-NEXT: packuswb %xmm3, %xmm1 -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; SSE41-LABEL: splatvar_funnnel_v16i8: -; SSE41: # %bb.0: -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] -; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE41-NEXT: psllw %xmm2, %xmm3 -; SSE41-NEXT: psrlw $8, %xmm3 -; SSE41-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE41-NEXT: psllw %xmm2, %xmm1 -; SSE41-NEXT: psrlw $8, %xmm1 -; SSE41-NEXT: packuswb %xmm3, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: retq +; SSE-LABEL: splatvar_funnnel_v16i8: +; SSE: # %bb.0: +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 +; SSE-NEXT: psllw %xmm2, %xmm3 +; SSE-NEXT: psrlw $8, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE-NEXT: psllw %xmm2, %xmm1 +; SSE-NEXT: psrlw $8, %xmm1 +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq ; ; AVX-LABEL: splatvar_funnnel_v16i8: ; AVX: # %bb.0: @@ -1448,7 +1429,6 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> % ; ; X86-SSE2-LABEL: splatvar_funnnel_v16i8: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: movdqa %xmm1, %xmm3 ; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll index c4ce5d2260d25..6fbc10307e0bb 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll @@ -320,33 +320,32 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> % ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] ; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 ; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 -; AVX1-NEXT: vpxor %xmm7, %xmm7, %xmm7 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] -; AVX1-NEXT: vpslld $23, %xmm8, %xmm8 -; AVX1-NEXT: vbroadcastss {{.*#+}} xmm9 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm9, %xmm8, %xmm8 -; AVX1-NEXT: vcvttps2dq %xmm8, %xmm8 -; AVX1-NEXT: vpmulld %xmm8, %xmm5, %xmm5 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm6[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm7, %xmm7 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm8 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm7, %xmm8, %xmm7 +; AVX1-NEXT: vcvttps2dq %xmm7, %xmm7 +; AVX1-NEXT: vpmulld %xmm7, %xmm5, %xmm5 ; AVX1-NEXT: vpsrld $16, %xmm5, %xmm5 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3] ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero ; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 -; AVX1-NEXT: vpaddd %xmm4, %xmm9, %xmm4 +; AVX1-NEXT: vpaddd %xmm4, %xmm8, %xmm4 ; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 ; AVX1-NEXT: vpmulld %xmm4, %xmm3, %xmm3 ; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3 ; AVX1-NEXT: vpackusdw %xmm5, %xmm3, %xmm3 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; AVX1-NEXT: vpaddd %xmm5, %xmm9, %xmm5 +; AVX1-NEXT: vpaddd %xmm5, %xmm8, %xmm5 ; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 ; AVX1-NEXT: vpmulld %xmm5, %xmm4, %xmm4 ; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4 ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero ; AVX1-NEXT: vpslld $23, %xmm1, %xmm1 -; AVX1-NEXT: vpaddd %xmm1, %xmm9, %xmm1 +; AVX1-NEXT: vpaddd %xmm1, %xmm8, %xmm1 ; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1 ; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 @@ -481,17 +480,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt) ; AVX1-LABEL: var_funnnel_v32i8: ; AVX1: # %bb.0: ; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 -; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5 -; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15] -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] -; AVX1-NEXT: vpslld $23, %xmm4, %xmm7 -; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm6[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm3, %xmm7 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm3, %xmm7, %xmm7 ; AVX1-NEXT: vcvttps2dq %xmm7, %xmm7 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero ; AVX1-NEXT: vpslld $23, %xmm6, %xmm6 -; AVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm6 +; AVX1-NEXT: vpaddd %xmm3, %xmm6, %xmm6 ; AVX1-NEXT: vcvttps2dq %xmm6, %xmm6 ; AVX1-NEXT: vpackusdw %xmm7, %xmm6, %xmm6 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 @@ -499,48 +498,48 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt) ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] ; AVX1-NEXT: vpmullw %xmm6, %xmm9, %xmm6 ; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 -; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm9 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm9 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero ; AVX1-NEXT: vpslld $23, %xmm9, %xmm9 -; AVX1-NEXT: vpaddd %xmm4, %xmm9, %xmm9 +; AVX1-NEXT: vpaddd %xmm3, %xmm9, %xmm9 ; AVX1-NEXT: vcvttps2dq %xmm9, %xmm9 -; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7] +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; AVX1-NEXT: vpaddd %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 +; AVX1-NEXT: vpackusdw %xmm4, %xmm9, %xmm4 +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] +; AVX1-NEXT: vpmullw %xmm4, %xmm7, %xmm4 +; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4 +; AVX1-NEXT: vpackuswb %xmm6, %xmm4, %xmm4 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm5[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm6, %xmm6 +; AVX1-NEXT: vpaddd %xmm3, %xmm6, %xmm6 +; AVX1-NEXT: vcvttps2dq %xmm6, %xmm6 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero ; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm5 +; AVX1-NEXT: vpaddd %xmm3, %xmm5, %xmm5 ; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 -; AVX1-NEXT: vpackusdw %xmm5, %xmm9, %xmm5 -; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] -; AVX1-NEXT: vpmullw %xmm5, %xmm7, %xmm5 +; AVX1-NEXT: vpackusdw %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; AVX1-NEXT: vpmullw %xmm5, %xmm6, %xmm5 ; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 -; AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5 -; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15] -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7] -; AVX1-NEXT: vpslld $23, %xmm7, %xmm7 -; AVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 -; AVX1-NEXT: vcvttps2dq %xmm7, %xmm7 -; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero ; AVX1-NEXT: vpslld $23, %xmm6, %xmm6 -; AVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm6 +; AVX1-NEXT: vpaddd %xmm3, %xmm6, %xmm6 ; AVX1-NEXT: vcvttps2dq %xmm6, %xmm6 -; AVX1-NEXT: vpackusdw %xmm7, %xmm6, %xmm6 -; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] -; AVX1-NEXT: vpmullw %xmm6, %xmm7, %xmm6 -; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 -; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; AVX1-NEXT: vpslld $23, %xmm7, %xmm7 -; AVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 -; AVX1-NEXT: vcvttps2dq %xmm7, %xmm7 ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 ; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; AVX1-NEXT: vpackusdw %xmm2, %xmm7, %xmm2 +; AVX1-NEXT: vpackusdw %xmm2, %xmm6, %xmm2 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] ; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0 -; AVX1-NEXT: vpackuswb %xmm6, %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0 +; AVX1-NEXT: vpackuswb %xmm5, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll index 4e619c07774ec..e60b56551e58d 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll @@ -306,22 +306,21 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; SSE2-LABEL: var_funnnel_v8i16: ; SSE2: # %bb.0: ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: movdqa %xmm1, %xmm3 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE2-NEXT: pslld $23, %xmm3 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm3 -; SSE2-NEXT: cvttps2dq %xmm3, %xmm3 -; SSE2-NEXT: pslld $16, %xmm3 -; SSE2-NEXT: psrad $16, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 +; SSE2-NEXT: pslld $16, %xmm2 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm1 -; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: paddd %xmm3, %xmm1 ; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: packssdw %xmm3, %xmm1 +; SSE2-NEXT: packssdw %xmm2, %xmm1 ; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: pmulhuw %xmm1, %xmm2 ; SSE2-NEXT: pmullw %xmm1, %xmm0 @@ -331,16 +330,15 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; SSE41-LABEL: var_funnnel_v8i16: ; SSE41: # %bb.0: ; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm1 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm3, %xmm1 ; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE41-NEXT: pslld $23, %xmm3 -; SSE41-NEXT: paddd %xmm2, %xmm3 -; SSE41-NEXT: cvttps2dq %xmm3, %xmm2 +; SSE41-NEXT: pslld $23, %xmm2 +; SSE41-NEXT: paddd %xmm3, %xmm2 +; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: packusdw %xmm1, %xmm2 ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: pmulhuw %xmm2, %xmm1 @@ -351,8 +349,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; AVX1-LABEL: var_funnnel_v8i16: ; AVX1: # %bb.0: ; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 @@ -456,18 +453,17 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; X86-SSE2-LABEL: var_funnnel_v8i16: ; X86-SSE2: # %bb.0: ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 -; X86-SSE2-NEXT: pxor %xmm3, %xmm3 ; X86-SSE2-NEXT: movdqa %xmm1, %xmm2 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: pslld $23, %xmm2 -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE2-NEXT: paddd %xmm4, %xmm2 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE2-NEXT: paddd %xmm3, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: pslld $16, %xmm2 ; X86-SSE2-NEXT: psrad $16, %xmm2 -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: pslld $23, %xmm1 -; X86-SSE2-NEXT: paddd %xmm4, %xmm1 +; X86-SSE2-NEXT: paddd %xmm3, %xmm1 ; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE2-NEXT: pslld $16, %xmm1 ; X86-SSE2-NEXT: psrad $16, %xmm1 @@ -1060,32 +1056,18 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind } define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind { -; SSE2-LABEL: splatvar_funnnel_v16i8: -; SSE2: # %bb.0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: psllw %xmm1, %xmm2 -; SSE2-NEXT: psrlw $8, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: psllw %xmm1, %xmm0 -; SSE2-NEXT: psrlw $8, %xmm0 -; SSE2-NEXT: packuswb %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; SSE41-LABEL: splatvar_funnnel_v16i8: -; SSE41: # %bb.0: -; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; SSE41-NEXT: psllw %xmm1, %xmm2 -; SSE41-NEXT: psrlw $8, %xmm2 -; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE41-NEXT: psllw %xmm1, %xmm0 -; SSE41-NEXT: psrlw $8, %xmm0 -; SSE41-NEXT: packuswb %xmm2, %xmm0 -; SSE41-NEXT: retq +; SSE-LABEL: splatvar_funnnel_v16i8: +; SSE: # %bb.0: +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] +; SSE-NEXT: psllw %xmm1, %xmm2 +; SSE-NEXT: psrlw $8, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: psllw %xmm1, %xmm0 +; SSE-NEXT: psrlw $8, %xmm0 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: retq ; ; AVX-LABEL: splatvar_funnnel_v16i8: ; AVX: # %bb.0: @@ -1186,10 +1168,9 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind ; ; X86-SSE2-LABEL: splatvar_funnnel_v16i8: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: movdqa %xmm0, %xmm2 ; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: psllw %xmm1, %xmm2 ; X86-SSE2-NEXT: psrlw $8, %xmm2 ; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll index 523621f459535..11a02f8cf754c 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll @@ -234,29 +234,28 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind { ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] ; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 -; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] -; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 -; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; AVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 ; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; AVX1-NEXT: vpackusdw %xmm5, %xmm2, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 -; AVX1-NEXT: vpmulhuw %xmm2, %xmm5, %xmm7 -; AVX1-NEXT: vpmullw %xmm2, %xmm5, %xmm2 -; AVX1-NEXT: vpor %xmm7, %xmm2, %xmm2 +; AVX1-NEXT: vpackusdw %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vpmulhuw %xmm2, %xmm4, %xmm6 +; AVX1-NEXT: vpmullw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpor %xmm6, %xmm2, %xmm2 ; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 -; AVX1-NEXT: vpaddd %xmm6, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm5, %xmm3, %xmm3 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero ; AVX1-NEXT: vpslld $23, %xmm1, %xmm1 -; AVX1-NEXT: vpaddd %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm5, %xmm1, %xmm1 ; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1 ; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vpmulhuw %xmm1, %xmm0, %xmm3 diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll index 8ca5e9e7da9f7..bf525442a419b 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll @@ -461,24 +461,23 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; SSE2-NEXT: pand %xmm4, %xmm3 ; SSE2-NEXT: por %xmm1, %xmm3 ; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: movdqa %xmm2, %xmm4 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7] -; SSE2-NEXT: pslld $23, %xmm4 -; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm5, %xmm4 -; SSE2-NEXT: cvttps2dq %xmm4, %xmm4 -; SSE2-NEXT: pslld $16, %xmm4 -; SSE2-NEXT: psrad $16, %xmm4 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: pslld $23, %xmm2 -; SSE2-NEXT: paddd %xmm5, %xmm2 -; SSE2-NEXT: cvttps2dq %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: packssdw %xmm4, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: paddd %xmm4, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 +; SSE2-NEXT: pslld $16, %xmm2 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: packssdw %xmm1, %xmm2 ; SSE2-NEXT: paddw %xmm0, %xmm0 -; SSE2-NEXT: pmullw %xmm1, %xmm0 +; SSE2-NEXT: pmullw %xmm2, %xmm0 ; SSE2-NEXT: por %xmm3, %xmm0 ; SSE2-NEXT: retq ; @@ -512,16 +511,15 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: pblendvb %xmm0, %xmm6, %xmm1 ; SSE41-NEXT: pandn %xmm5, %xmm2 -; SSE41-NEXT: pxor %xmm0, %xmm0 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm4, %xmm2 ; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 -; SSE41-NEXT: pslld $23, %xmm4 -; SSE41-NEXT: paddd %xmm0, %xmm4 -; SSE41-NEXT: cvttps2dq %xmm4, %xmm0 +; SSE41-NEXT: pslld $23, %xmm0 +; SSE41-NEXT: paddd %xmm4, %xmm0 +; SSE41-NEXT: cvttps2dq %xmm0, %xmm0 ; SSE41-NEXT: packusdw %xmm2, %xmm0 ; SSE41-NEXT: paddw %xmm3, %xmm3 ; SSE41-NEXT: pmullw %xmm0, %xmm3 @@ -548,8 +546,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; AVX1-NEXT: vpaddw %xmm5, %xmm5, %xmm5 ; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm1, %xmm1 ; AVX1-NEXT: vpandn %xmm3, %xmm2, %xmm2 -; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 @@ -707,18 +704,17 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt) ; X86-SSE2-NEXT: pand %xmm4, %xmm3 ; X86-SSE2-NEXT: por %xmm1, %xmm3 ; X86-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 -; X86-SSE2-NEXT: pxor %xmm4, %xmm4 ; X86-SSE2-NEXT: movdqa %xmm2, %xmm1 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: pslld $23, %xmm1 -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE2-NEXT: paddd %xmm5, %xmm1 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE2-NEXT: paddd %xmm4, %xmm1 ; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE2-NEXT: pslld $16, %xmm1 ; X86-SSE2-NEXT: psrad $16, %xmm1 -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: pslld $23, %xmm2 -; X86-SSE2-NEXT: paddd %xmm5, %xmm2 +; X86-SSE2-NEXT: paddd %xmm4, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: pslld $16, %xmm2 ; X86-SSE2-NEXT: psrad $16, %xmm2 @@ -1451,7 +1447,6 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> % define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind { ; SSE2-LABEL: splatvar_funnnel_v16i8: ; SSE2: # %bb.0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSE2-NEXT: movdqa %xmm1, %xmm4 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 @@ -1593,7 +1588,6 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> % ; ; X86-SSE2-LABEL: splatvar_funnnel_v16i8: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: movdqa %xmm1, %xmm4 ; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15] ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll index 944dbe3de3eec..b0a1a91bdccc5 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll @@ -339,37 +339,36 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> % ; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm6 ; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm5, %xmm5 ; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm6 -; AVX1-NEXT: vpxor %xmm7, %xmm7, %xmm7 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] -; AVX1-NEXT: vpslld $23, %xmm4, %xmm8 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm4, %xmm7 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm4, %xmm8, %xmm8 -; AVX1-NEXT: vcvttps2dq %xmm8, %xmm8 +; AVX1-NEXT: vpaddd %xmm4, %xmm7, %xmm7 +; AVX1-NEXT: vcvttps2dq %xmm7, %xmm7 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero ; AVX1-NEXT: vpslld $23, %xmm6, %xmm6 ; AVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm6 ; AVX1-NEXT: vcvttps2dq %xmm6, %xmm6 -; AVX1-NEXT: vpackusdw %xmm8, %xmm6, %xmm6 -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm8 -; AVX1-NEXT: vpaddw %xmm8, %xmm8, %xmm8 -; AVX1-NEXT: vpmullw %xmm6, %xmm8, %xmm6 +; AVX1-NEXT: vpackusdw %xmm7, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vpaddw %xmm7, %xmm7, %xmm7 +; AVX1-NEXT: vpmullw %xmm6, %xmm7, %xmm6 ; AVX1-NEXT: vpor %xmm5, %xmm6, %xmm5 ; AVX1-NEXT: vpsllw $12, %xmm2, %xmm6 -; AVX1-NEXT: vpsllw $4, %xmm2, %xmm8 -; AVX1-NEXT: vpor %xmm6, %xmm8, %xmm6 -; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm8 -; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm9 -; AVX1-NEXT: vpblendvb %xmm6, %xmm9, %xmm1, %xmm1 +; AVX1-NEXT: vpsllw $4, %xmm2, %xmm7 +; AVX1-NEXT: vpor %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm7 +; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm8 +; AVX1-NEXT: vpblendvb %xmm6, %xmm8, %xmm1, %xmm1 ; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm6 -; AVX1-NEXT: vpblendvb %xmm8, %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpblendvb %xmm7, %xmm6, %xmm1, %xmm1 ; AVX1-NEXT: vpsrlw $2, %xmm1, %xmm6 -; AVX1-NEXT: vpaddw %xmm8, %xmm8, %xmm8 -; AVX1-NEXT: vpblendvb %xmm8, %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpaddw %xmm7, %xmm7, %xmm7 +; AVX1-NEXT: vpblendvb %xmm7, %xmm6, %xmm1, %xmm1 ; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm6 -; AVX1-NEXT: vpaddw %xmm8, %xmm8, %xmm8 -; AVX1-NEXT: vpblendvb %xmm8, %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpaddw %xmm7, %xmm7, %xmm7 +; AVX1-NEXT: vpblendvb %xmm7, %xmm6, %xmm1, %xmm1 ; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 ; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll index b462700c1c75d..d9799975cd37a 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll @@ -318,21 +318,20 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; SSE2-LABEL: var_funnnel_v8i16: ; SSE2: # %bb.0: ; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: pxor %xmm3, %xmm3 -; SSE2-NEXT: psubw %xmm1, %xmm3 -; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE2-NEXT: movdqa %xmm3, %xmm1 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: psubw %xmm1, %xmm2 +; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; SSE2-NEXT: pslld $23, %xmm1 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm1 ; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSE2-NEXT: pslld $23, %xmm3 -; SSE2-NEXT: paddd %xmm4, %xmm3 -; SSE2-NEXT: cvttps2dq %xmm3, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE2-NEXT: pslld $16, %xmm2 ; SSE2-NEXT: psrad $16, %xmm2 ; SSE2-NEXT: packssdw %xmm1, %xmm2 @@ -345,19 +344,18 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; SSE41-LABEL: var_funnnel_v8i16: ; SSE41: # %bb.0: ; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pxor %xmm3, %xmm3 -; SSE41-NEXT: psubw %xmm1, %xmm3 -; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE41-NEXT: pslld $23, %xmm3 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm2, %xmm3 -; SSE41-NEXT: cvttps2dq %xmm3, %xmm3 +; SSE41-NEXT: psubw %xmm1, %xmm2 +; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; SSE41-NEXT: pslld $23, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm3, %xmm2 +; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: pslld $23, %xmm1 -; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: paddd %xmm3, %xmm1 ; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE41-NEXT: packusdw %xmm3, %xmm1 +; SSE41-NEXT: packusdw %xmm2, %xmm1 ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pmulhuw %xmm1, %xmm2 ; SSE41-NEXT: pmullw %xmm1, %xmm0 @@ -369,7 +367,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; AVX1-NEXT: vpsubw %xmm1, %xmm2, %xmm1 ; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 @@ -474,21 +472,20 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind { ; ; X86-SSE2-LABEL: var_funnnel_v8i16: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: pxor %xmm3, %xmm3 ; X86-SSE2-NEXT: pxor %xmm2, %xmm2 ; X86-SSE2-NEXT: psubw %xmm1, %xmm2 ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2 ; X86-SSE2-NEXT: movdqa %xmm2, %xmm1 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: pslld $23, %xmm1 -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE2-NEXT: paddd %xmm4, %xmm1 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE2-NEXT: paddd %xmm3, %xmm1 ; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE2-NEXT: pslld $16, %xmm1 ; X86-SSE2-NEXT: psrad $16, %xmm1 -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: pslld $23, %xmm2 -; X86-SSE2-NEXT: paddd %xmm4, %xmm2 +; X86-SSE2-NEXT: paddd %xmm3, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: pslld $16, %xmm2 ; X86-SSE2-NEXT: psrad $16, %xmm2 @@ -1104,10 +1101,9 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind { ; SSE2-LABEL: splatvar_funnnel_v16i8: ; SSE2: # %bb.0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ; SSE2-NEXT: psrlw %xmm1, %xmm2 ; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] ; SSE2-NEXT: pand %xmm3, %xmm2 @@ -1249,10 +1245,9 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind ; ; X86-SSE2-LABEL: splatvar_funnnel_v16i8: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: movdqa %xmm0, %xmm2 ; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: psrlw %xmm1, %xmm2 ; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] ; X86-SSE2-NEXT: pand %xmm3, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll index 0cf5e511e5a25..15e09c3b6737e 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll @@ -249,7 +249,7 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind { ; AVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15] ; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 @@ -265,7 +265,7 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind { ; AVX1-NEXT: vpor %xmm7, %xmm2, %xmm2 ; AVX1-NEXT: vpsubw %xmm1, %xmm3, %xmm1 ; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 ; AVX1-NEXT: vpaddd %xmm6, %xmm3, %xmm3 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll index 038c73bd9fed2..c82e2b322e55a 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll @@ -2803,17 +2803,16 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] ; AVX2-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6],ymm11[7] ; AVX2-NEXT: vpblendd {{.*#+}} ymm11 = ymm2[0,1],ymm3[2],ymm2[3,4,5],ymm3[6],ymm2[7] -; AVX2-NEXT: vpshufb {{.*#+}} ymm12 = ymm11[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] -; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1,2,3,4,5,6,7],ymm12[8],ymm10[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm13, %xmm14 -; AVX2-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[3,1,2,3,4,5,6,7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,2,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3] -; AVX2-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6],ymm12[7,8,9,10,11,12,13],ymm11[14],ymm12[15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3] +; AVX2-NEXT: vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] +; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1,2,3,4,5,6,7],ymm11[8],ymm10[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5],mem[6],xmm11[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm12 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm12, %xmm13 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[3,1,2,3,4,5,6,7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3] +; AVX2-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7] ; AVX2-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm11, %xmm12 @@ -2971,9 +2970,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[3,1,2,3,4,5,6,7] ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm14 = ymm2[0,1],ymm3[2],ymm2[3,4,5],ymm3[6],ymm2[7] -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1] ; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm15 = ymm14[0,1,2,3,4,5],ymm15[6],ymm14[7,8,9,10,11,12,13],ymm15[14],ymm14[15] +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm15 = xmm14[0,1,2,3,4,5],mem[6],xmm14[7] ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,3] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm15 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm15, %xmm4 @@ -5563,112 +5561,108 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-LABEL: load_i16_stride7_vf32: ; AVX2: # %bb.0: ; AVX2-NEXT: subq $520, %rsp # imm = 0x208 -; AVX2-NEXT: vmovdqa 288(%rdi), %ymm14 -; AVX2-NEXT: vmovdqa 320(%rdi), %ymm9 +; AVX2-NEXT: vmovdqa 288(%rdi), %ymm10 +; AVX2-NEXT: vmovdqa 320(%rdi), %ymm14 ; AVX2-NEXT: vmovdqa 256(%rdi), %ymm5 -; AVX2-NEXT: vmovdqa 224(%rdi), %ymm13 -; AVX2-NEXT: vmovdqa (%rdi), %ymm7 +; AVX2-NEXT: vmovdqa 224(%rdi), %ymm11 +; AVX2-NEXT: vmovdqa (%rdi), %ymm15 ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm6 -; AVX2-NEXT: vmovdqa 64(%rdi), %ymm11 +; AVX2-NEXT: vmovdqa 64(%rdi), %ymm9 ; AVX2-NEXT: vmovdqa 96(%rdi), %ymm8 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0],ymm11[1],ymm8[2,3,4],ymm11[5],ymm8[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0],ymm9[1],ymm8[2,3,4],ymm9[5],ymm8[6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,2,2,3] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,14,15,12,13,14,15,2,3,6,7,12,13,2,3,16,17,30,31,28,29,30,31,18,19,22,23,28,29,18,19] ; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2 -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1],ymm6[2],ymm7[3,4,5],ymm6[6],ymm7[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1],ymm6[2],ymm15[3,4,5],ymm6[6],ymm15[7] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4],xmm1[5],xmm3[6],xmm1[7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,14,15,12,13,10,11,8,9,8,9,8,9,8,9] ; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm3 -; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm12 = [65535,65535,65535,65535,65535,0,0,0] -; AVX2-NEXT: vpblendvb %ymm12, %ymm3, %ymm2, %ymm1 -; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,0,0] +; AVX2-NEXT: vpblendvb %ymm1, %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vmovdqa %ymm5, %ymm3 -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1],ymm5[2],ymm13[3,4,5],ymm5[6],ymm13[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm5[2],ymm11[3,4,5],ymm5[6],ymm11[7] ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm5 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4],xmm2[5],xmm5[6],xmm2[7] ; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm2 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0],ymm14[1],ymm9[2,3,4],ymm14[5],ymm9[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0],ymm10[1],ymm14[2,3,4],ymm10[5],ymm14[6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] ; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0 -; AVX2-NEXT: vpblendvb %ymm12, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1],ymm14[2],ymm9[3,4],ymm14[5],ymm9[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1],ymm10[2],ymm14[3,4],ymm10[5],ymm14[6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6,7,8,9,10],ymm2[11],ymm0[12,13,14,15] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,2,3,2,3,2,3,8,9,8,9,6,7,4,5,18,19,18,19,18,19,18,19,24,25,24,25,22,23,20,21] ; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7] -; AVX2-NEXT: vmovdqa %ymm13, %ymm1 +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7] ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3,4,5],xmm4[6],xmm5[7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = [2,3,0,1,14,15,12,13,10,11,10,11,10,11,10,11] ; AVX2-NEXT: vpshufb %xmm5, %xmm4, %xmm4 -; AVX2-NEXT: vpblendvb %ymm12, %ymm4, %ymm0, %ymm0 +; AVX2-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1],ymm11[2],ymm8[3,4],ymm11[5],ymm8[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3],ymm0[4,5,6,7,8,9,10],ymm4[11],ymm0[12,13,14,15] ; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0 -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2],ymm6[3],ymm7[4,5],ymm6[6],ymm7[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1,2],ymm6[3],ymm15[4,5],ymm6[6],ymm15[7] ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm2[1],xmm4[2,3,4,5],xmm2[6],xmm4[7] ; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm2 -; AVX2-NEXT: vpblendvb %ymm12, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm11[2,3,0,1] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1],ymm11[2,3],ymm8[4,5],ymm11[6,7] -; AVX2-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm9[2,3,0,1] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1],ymm9[2,3],ymm8[4,5],ymm9[6,7] +; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0],ymm15[1],ymm6[2,3],ymm15[4],ymm6[5,6,7] ; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3,4,5],xmm4[6],xmm2[7] -; AVX2-NEXT: vmovdqa {{.*#+}} ymm15 = [4,5,4,5,4,5,4,5,8,9,10,11,8,9,6,7,20,21,20,21,20,21,20,21,24,25,26,27,24,25,22,23] -; AVX2-NEXT: vpshufb %ymm15, %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [4,5,4,5,4,5,4,5,8,9,10,11,8,9,6,7,20,21,20,21,20,21,20,21,24,25,26,27,24,25,22,23] +; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,2,3,0,1,14,15,12,13,12,13,12,13,12,13] ; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm2 -; AVX2-NEXT: vpblendvb %ymm12, %ymm2, %ymm0, %ymm13 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6,7] -; AVX2-NEXT: vmovdqa %ymm1, %ymm4 -; AVX2-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill -; AVX2-NEXT: vmovdqa %ymm3, %ymm1 +; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm13 +; AVX2-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0],ymm11[1],ymm3[2,3],ymm11[4],ymm3[5,6,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7] ; AVX2-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX2-NEXT: vmovdqa %ymm9, %ymm3 -; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1],ymm14[2,3],ymm9[4,5],ymm14[6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm14[2,3,0,1] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm14[0,1],ymm10[2,3],ymm14[4,5],ymm10[6,7] ; AVX2-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm10[2,3,0,1] +; AVX2-NEXT: vmovdqu %ymm10, (%rsp) # 32-byte Spill ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7,8,9,10,11],ymm5[12],ymm2[13,14,15] -; AVX2-NEXT: vpshufb %ymm15, %ymm2, %ymm2 -; AVX2-NEXT: vpblendvb %ymm12, %ymm0, %ymm2, %ymm10 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0],ymm7[1],ymm6[2,3,4],ymm7[5],ymm6[6,7] -; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2 +; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm12 +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0],ymm15[1],ymm6[2,3,4],ymm15[5],ymm6[6,7] +; AVX2-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4,5,6,7] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1,2],ymm11[3],ymm8[4,5],ymm11[6],ymm8[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1,2],ymm9[3],ymm8[4,5],ymm9[6],ymm8[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3] -; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [6,7,6,7,6,7,6,7,8,9,4,5,10,11,0,1,22,23,22,23,22,23,22,23,24,25,20,21,26,27,16,17] -; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [6,7,6,7,6,7,6,7,8,9,4,5,10,11,0,1,22,23,22,23,22,23,22,23,24,25,20,21,26,27,16,17] +; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] -; AVX2-NEXT: vpblendvb %ymm12, %ymm0, %ymm2, %ymm9 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm4[1],ymm1[2,3,4],ymm4[5],ymm1[6,7] +; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm9 +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0],ymm11[1],ymm3[2,3,4],ymm11[5],ymm3[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4,5,6,7] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm14[3],ymm3[4,5],ymm14[6],ymm3[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm14[0,1,2],ymm10[3],ymm14[4,5],ymm10[6],ymm14[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3] -; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2 +; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] -; AVX2-NEXT: vpblendvb %ymm12, %ymm0, %ymm2, %ymm8 -; AVX2-NEXT: vmovdqa 128(%rdi), %ymm12 -; AVX2-NEXT: vmovdqa 160(%rdi), %ymm15 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2],ymm12[3],ymm15[4,5],ymm12[6],ymm15[7] +; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm8 +; AVX2-NEXT: vmovdqa 128(%rdi), %ymm7 +; AVX2-NEXT: vmovdqa 160(%rdi), %ymm11 +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm7[3],ymm11[4,5],ymm7[6],ymm11[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,6,4,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 ; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] @@ -5713,7 +5707,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1],ymm15[2],ymm12[3,4,5],ymm15[6],ymm12[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1],ymm11[2],ymm7[3,4,5],ymm11[6],ymm7[7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshufb %xmm14, %xmm0, %xmm0 @@ -5725,7 +5719,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2],ymm15[3],ymm12[4,5],ymm15[6],ymm12[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2],ymm11[3],ymm7[4,5],ymm11[6],ymm7[7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5],xmm0[6],xmm1[7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,0,1,14,15,12,13,10,11,12,13,14,15] @@ -5747,64 +5741,76 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,0,3,4,5,4,7] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0],ymm0[1,2,3,4,5,6,7],ymm10[8],ymm0[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0],ymm0[1,2,3,4,5,6,7],ymm12[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0],ymm12[1],ymm15[2,3],ymm12[4],ymm15[5,6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0],ymm7[1],ymm11[2,3],ymm7[4],ymm11[5,6,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm5[0,1,1,3] -; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm2[0,1,2,0,4,5,6,4] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] +; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm5[0,1,1,3] +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[0,1,2,0,4,5,6,4] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm9[0],ymm0[1,2,3,4,5,6,7],ymm9[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0],ymm6[1],ymm4[2,3],ymm6[4],ymm4[5,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7] +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7] -; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm2 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm3[0,1,1,3] ; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,1,2,0,4,5,6,4] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2,3,4,5,6,7],ymm8[8],ymm1[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0],ymm12[1],ymm15[2,3,4],ymm12[5],ymm15[6,7] -; AVX2-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 -; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4,5,6,7] -; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] -; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm3 -; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 -; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,1,4,5,6,5] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7] -; AVX2-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm3 # 32-byte Folded Reload -; AVX2-NEXT: # ymm3 = mem[0,1],ymm7[2],mem[3,4],ymm7[5],mem[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5 +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm3[7] +; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm2[1,2,3,4,5,6,7],ymm8[8],ymm2[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm2[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0],ymm7[1],ymm11[2,3,4],ymm7[5],ymm11[6,7] +; AVX2-NEXT: vmovdqa %ymm7, %ymm9 +; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 +; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4,5,6,7] +; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] +; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2 +; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7] +; AVX2-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm2 # 32-byte Folded Reload +; AVX2-NEXT: # ymm2 = mem[0,1],ymm15[2],mem[3,4],ymm15[5],mem[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm5 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3] +; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX2-NEXT: vpblendd $187, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm5 # 32-byte Folded Reload -; AVX2-NEXT: # ymm5 = mem[0,1],ymm14[2],mem[3,4,5],ymm14[6],mem[7] +; AVX2-NEXT: vpblendd $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm5 # 32-byte Folded Reload +; AVX2-NEXT: # ymm5 = ymm14[0,1],mem[2],ymm14[3,4,5],mem[6],ymm14[7] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] -; AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm8 -; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm2[1,2,3,4,5,6,7],ymm8[8],ymm2[9,10,11,12,13,14,15] -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6],ymm8[7,8,9,10,11,12,13],ymm5[14],ymm8[15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu (%rsp), %ymm13 # 32-byte Reload +; AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm5 +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm1[1,2,3,4,5,6,7],ymm5[8],ymm1[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],mem[6],xmm5[7] +; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4,5,6,7] +; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 +; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX2-NEXT: vmovdqu (%rsp), %ymm5 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm10[2],ymm5[3,4,5],ymm10[6],ymm5[7] +; AVX2-NEXT: vpshufb %ymm7, %ymm1, %ymm1 +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1],ymm13[2],ymm8[3,4],ymm13[5],ymm8[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 @@ -5812,25 +5818,12 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7] ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1],ymm10[2],ymm9[3,4,5],ymm10[6],ymm9[7] -; AVX2-NEXT: vpshufb %ymm7, %ymm3, %ymm5 -; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6],ymm5[7,8,9,10,11,12,13],ymm3[14],ymm5[15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm7 -; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0],xmm3[1],xmm7[2],xmm3[3],xmm7[4,5,6,7] -; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm1 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,1,2,1,4,5,6,5] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2,3,4,5,6,7],ymm5[8],ymm0[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],mem[6],xmm1[7] +; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm10[3],ymm5[4,5],ymm10[6],ymm5[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7,8],ymm0[9,10,11,12,13,14],ymm1[15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7] @@ -5859,7 +5852,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm7[0,1],xmm1[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1],ymm12[2],ymm15[3,4],ymm12[5],ymm15[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1],ymm9[2],ymm11[3,4],ymm9[5],ymm11[6,7] +; AVX2-NEXT: vmovdqa %ymm11, %ymm15 ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm7 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1],xmm1[2],xmm7[3],xmm1[4],xmm7[5,6,7] ; AVX2-NEXT: vpshufb %xmm5, %xmm1, %xmm1 @@ -5872,14 +5866,14 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12 ; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm12[5,6,7],ymm5[8,9,10,11,12],ymm12[13,14,15] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm12 = ymm7[0,1,2],ymm14[3],ymm7[4,5],ymm14[6],ymm7[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm12 = ymm14[0,1,2],ymm7[3],ymm14[4,5],ymm7[6],ymm14[7] ; AVX2-NEXT: vmovdqa %ymm14, %ymm13 ; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm12[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm14[0],ymm12[1,2,3,4,5,6],ymm14[7,8],ymm12[9,10,11,12,13,14],ymm14[15] ; AVX2-NEXT: vpshufb %ymm8, %ymm12, %ymm8 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm12 = ymm9[0,1],ymm2[2,3],ymm9[4,5],ymm2[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0,1],ymm9[2,3],ymm2[4,5],ymm9[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm12, %xmm14 ; AVX2-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[2,1,2,3] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[0,2,2,3,4,5,6,7] @@ -5889,7 +5883,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm8[0],ymm5[1,2,3,4,5,6,7],ymm8[8],ymm5[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7] -; AVX2-NEXT: vpblendd $18, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm8 # 32-byte Folded Reload +; AVX2-NEXT: vpblendd $18, (%rsp), %ymm10, %ymm8 # 32-byte Folded Reload ; AVX2-NEXT: # ymm8 = ymm10[0],mem[1],ymm10[2,3],mem[4],ymm10[5,6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm8[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm10[1],ymm8[2,3,4,5,6,7,8],ymm10[9],ymm8[10,11,12,13,14,15] @@ -5904,7 +5898,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7],ymm4[8,9,10,11,12],ymm0[13,14,15] -; AVX2-NEXT: vmovdqu (%rsp), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload ; AVX2-NEXT: # ymm4 = mem[0,1,2],ymm4[3],mem[4,5],ymm4[6],mem[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm4[0,1,2,3,6,4,6,7] @@ -5928,11 +5922,11 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2,3,4],ymm1[5,6,7],ymm3[8,9,10,11,12],ymm1[13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0],ymm7[1],ymm13[2,3],ymm7[4],ymm13[5,6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0],ymm13[1],ymm7[2,3],ymm13[4],ymm7[5,6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4,5,6,7,8],ymm4[9],ymm3[10,11,12,13,14,15] ; AVX2-NEXT: vpshufb %ymm6, %ymm3, %ymm3 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2],ymm2[3],ymm9[4,5],ymm2[6],ymm9[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1,2],ymm9[3],ymm2[4,5],ymm9[6],ymm2[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm4[0,1,2,3,6,4,6,7] ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm4 ; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,1] @@ -5975,10 +5969,10 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-LABEL: load_i16_stride7_vf32: ; AVX2-FP: # %bb.0: ; AVX2-FP-NEXT: subq $552, %rsp # imm = 0x228 -; AVX2-FP-NEXT: vmovdqa 288(%rdi), %ymm11 -; AVX2-FP-NEXT: vmovdqa 320(%rdi), %ymm13 +; AVX2-FP-NEXT: vmovdqa 288(%rdi), %ymm10 +; AVX2-FP-NEXT: vmovdqa 320(%rdi), %ymm11 ; AVX2-FP-NEXT: vmovdqa 256(%rdi), %ymm5 -; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm12 +; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm15 ; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm8 ; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm7 ; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm6 @@ -5992,30 +5986,30 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4],xmm1[5],xmm3[6],xmm1[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,14,15,12,13,10,11,8,9,8,9,8,9,8,9] ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm1, %xmm3 -; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} xmm14 = [65535,65535,65535,65535,65535,0,0,0] -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm3, %ymm2, %ymm1 -; AVX2-FP-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill -; AVX2-FP-NEXT: vmovdqa %ymm5, %ymm1 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm12[0,1],ymm5[2],ymm12[3,4,5],ymm5[6],ymm12[7] +; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,0,0] +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm3, %ymm2, %ymm14 +; AVX2-FP-NEXT: vmovdqa %ymm5, %ymm3 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1],ymm5[2],ymm15[3,4,5],ymm5[6],ymm15[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm5 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4],xmm2[5],xmm5[6],xmm2[7] ; AVX2-FP-NEXT: vpshufb %xmm4, %xmm2, %xmm2 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0],ymm11[1],ymm13[2,3,4],ymm11[5],ymm13[6,7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0],ymm10[1],ymm11[2,3,4],ymm10[5],ymm11[6,7] ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] ; AVX2-FP-NEXT: vpshufb %ymm0, %ymm4, %ymm0 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm2, %ymm0, %ymm0 +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm11[2],ymm13[3,4],ymm11[5],ymm13[6,7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1],ymm10[2],ymm11[3,4],ymm10[5],ymm11[6,7] ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1] ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6,7,8,9,10],ymm2[11],ymm0[12,13,14,15] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,2,3,2,3,2,3,8,9,8,9,6,7,4,5,18,19,18,19,18,19,18,19,24,25,24,25,22,23,20,21] ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm0, %ymm0 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2],ymm1[3],ymm12[4,5],ymm1[6],ymm12[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm15[0,1,2],ymm3[3],ymm15[4,5],ymm3[6],ymm15[7] +; AVX2-FP-NEXT: vmovdqa %ymm3, %ymm13 ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm5 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3,4,5],xmm4[6],xmm5[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [2,3,0,1,14,15,12,13,10,11,10,11,10,11,10,11] ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm4 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm4, %ymm0, %ymm0 +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1],ymm6[2],ymm9[3,4],ymm6[5],ymm9[6,7] ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1] @@ -6025,7 +6019,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm4 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm2[1],xmm4[2,3,4,5],xmm2[6],xmm4[7] ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm2, %xmm2 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm2, %ymm0, %ymm0 +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm6[2,3,0,1] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1],ymm6[2,3],ymm9[4,5],ymm6[6,7] @@ -6035,30 +6029,29 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vmovdqa %ymm7, %ymm3 +; AVX2-FP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm4 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3,4,5],xmm4[6],xmm2[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm4 = [4,5,4,5,4,5,4,5,8,9,10,11,8,9,6,7,20,21,20,21,20,21,20,21,24,25,26,27,24,25,22,23] ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm0, %ymm0 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,2,3,0,1,14,15,12,13,12,13,12,13,12,13] ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm2, %xmm2 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm2, %ymm0, %ymm0 +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm12[1],ymm1[2,3],ymm12[4],ymm1[5,6,7] -; AVX2-FP-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0],ymm15[1],ymm13[2,3],ymm15[4],ymm13[5,6,7] +; AVX2-FP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-FP-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7] ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX2-FP-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1],ymm11[2,3],ymm13[4,5],ymm11[6,7] -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm11[2,3,0,1] -; AVX2-FP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-FP-NEXT: vmovdqu %ymm11, (%rsp) # 32-byte Spill +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm10[2,3],ymm11[4,5],ymm10[6,7] +; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm10[2,3,0,1] +; AVX2-FP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4],ymm2[5,6,7,8,9,10,11],ymm5[12],ymm2[13,14,15] ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm2, %ymm2 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm0, %ymm2, %ymm7 +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm7 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0],ymm8[1],ymm3[2,3,4],ymm8[5],ymm3[6,7] -; AVX2-FP-NEXT: vmovdqa %ymm3, %ymm15 -; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4,5,6,7] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm6[3],ymm9[4,5],ymm6[6],ymm9[7] @@ -6067,18 +6060,18 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm2, %ymm2 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [6,7,4,5,2,3,0,1,14,15,14,15,14,15,14,15] ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm0, %ymm2, %ymm10 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm12[1],ymm1[2,3,4],ymm12[5],ymm1[6,7] +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm12 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0],ymm15[1],ymm13[2,3,4],ymm15[5],ymm13[6,7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4,5,6,7] ; AVX2-FP-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1,2],ymm11[3],ymm13[4,5],ymm11[6],ymm13[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0,1,2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7] ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3] ; AVX2-FP-NEXT: vpshufb %ymm4, %ymm2, %ymm2 -; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm0, %ymm2, %ymm8 -; AVX2-FP-NEXT: vmovdqa 128(%rdi), %ymm6 -; AVX2-FP-NEXT: vmovdqa 160(%rdi), %ymm11 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm6[3],ymm11[4,5],ymm6[6],ymm11[7] +; AVX2-FP-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm8 +; AVX2-FP-NEXT: vmovdqa 128(%rdi), %ymm11 +; AVX2-FP-NEXT: vmovdqa 160(%rdi), %ymm13 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm11[3],ymm13[4,5],ymm11[6],ymm13[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,9,10,11,6,7,6,7] ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1 @@ -6090,12 +6083,11 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27] ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm0, %ymm4 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm4[7] -; AVX2-FP-NEXT: vpblendd $31, (%rsp), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX2-FP-NEXT: # ymm1 = mem[0,1,2,3,4],ymm1[5,6,7] -; AVX2-FP-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill -; AVX2-FP-NEXT: vmovdqa 352(%rdi), %ymm9 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2,3,4],ymm1[5,6,7] +; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-FP-NEXT: vmovdqa 352(%rdi), %ymm6 ; AVX2-FP-NEXT: vmovdqa 384(%rdi), %ymm4 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm9[3],ymm4[4,5],ymm9[6],ymm4[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm6[3],ymm4[4,5],ymm6[6],ymm4[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm14 ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm14, %xmm3 ; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,4,7] @@ -6103,28 +6095,28 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3 ; AVX2-FP-NEXT: vmovdqa 416(%rdi), %ymm1 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm14 = ymm1[0,1,0,2] -; AVX2-FP-NEXT: vpshufb %ymm2, %ymm14, %ymm13 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm13[7] +; AVX2-FP-NEXT: vpshufb %ymm2, %ymm14, %ymm15 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm15[7] ; AVX2-FP-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm2 # 32-byte Folded Reload ; AVX2-FP-NEXT: # ymm2 = mem[0,1,2,3,4],ymm3[5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1],ymm4[2],ymm9[3,4,5],ymm4[6],ymm9[7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm13 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm13[4],xmm3[5],xmm13[6],xmm3[7] -; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm13 = [0,1,0,1,14,15,12,13,10,11,8,9,12,13,14,15] -; AVX2-FP-NEXT: vpshufb %xmm13, %xmm3, %xmm3 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm4[2],ymm6[3,4,5],ymm4[6],ymm6[7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm15 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm15[4],xmm3[5],xmm15[6],xmm3[7] +; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm15 = [0,1,0,1,14,15,12,13,10,11,8,9,12,13,14,15] +; AVX2-FP-NEXT: vpshufb %xmm15, %xmm3, %xmm3 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 ; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm2 = [22,23,28,29,22,23,28,29,22,23,28,29,22,23,28,29,22,23,28,29,22,23,28,29,22,23,28,29,22,23,28,29] ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm14, %ymm14 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm14[7] -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm12[0],ymm3[1,2,3,4,5,6,7],ymm12[8],ymm3[9,10,11,12,13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2,3],ymm3[4,5,6,7] +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm10[0],ymm3[1,2,3,4,5,6,7],ymm10[8],ymm3[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm10[0,1,2,3],ymm3[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm11[2],ymm6[3,4,5],ymm11[6],ymm6[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm13[2],ymm11[3,4,5],ymm13[6],ymm11[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm14 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm14[4],xmm3[5],xmm14[6],xmm3[7] -; AVX2-FP-NEXT: vpshufb %xmm13, %xmm3, %xmm3 +; AVX2-FP-NEXT: vpshufb %xmm15, %xmm3, %xmm3 ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm0, %ymm0 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm2 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7] @@ -6132,32 +6124,33 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm11[3],ymm6[4,5],ymm11[6],ymm6[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm13[3],ymm11[4,5],ymm13[6],ymm11[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5],xmm0[6],xmm2[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,0,1,14,15,12,13,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm5[0,1,1,2] -; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm13 = [16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31] -; AVX2-FP-NEXT: vpshufb %ymm13, %ymm3, %ymm3 +; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm14 = [16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31,16,17,30,31] +; AVX2-FP-NEXT: vpshufb %ymm14, %ymm3, %ymm3 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7] ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4,5,6,7],ymm3[8],ymm0[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm4[3],ymm9[4,5],ymm4[6],ymm9[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm3 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1],xmm3[2,3,4,5],xmm0[6],xmm3[7] ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0 ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[0,1,1,2] -; AVX2-FP-NEXT: vpshufb %ymm13, %ymm2, %ymm2 +; AVX2-FP-NEXT: vpshufb %ymm14, %ymm2, %ymm2 ; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7] ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0],ymm0[1,2,3,4,5,6,7],ymm7[8],ymm0[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0],ymm6[1],ymm11[2,3],ymm6[4],ymm11[5,6,7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0],ymm11[1],ymm13[2,3],ymm11[4],ymm13[5,6,7] +; AVX2-FP-NEXT: vmovdqa %ymm11, %ymm14 ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,2,3,0,1,14,15,12,13,12,13,14,15] @@ -6167,10 +6160,10 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm3 = [18,19,24,25,18,19,24,25,18,19,24,25,18,19,24,25,18,19,24,25,18,19,24,25,18,19,24,25,18,19,24,25] ; AVX2-FP-NEXT: vpshufb %ymm3, %ymm7, %ymm5 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm5[7] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0],ymm0[1,2,3,4,5,6,7],ymm10[8],ymm0[9,10,11,12,13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0],ymm0[1,2,3,4,5,6,7],ymm12[8],ymm0[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0],ymm9[1],ymm4[2,3],ymm9[4],ymm4[5,6,7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0],ymm6[1],ymm4[2,3],ymm6[4],ymm4[5,6,7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm5 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3,4,5],xmm5[6],xmm0[7] ; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm0 @@ -6181,120 +6174,118 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2,3,4,5,6,7],ymm8[8],ymm1[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm2 # 32-byte Folded Reload -; AVX2-FP-NEXT: # ymm2 = ymm15[0,1],mem[2],ymm15[3,4],mem[5],ymm15[6,7] -; AVX2-FP-NEXT: vmovd {{.*#+}} xmm14 = [8,9,4,5,0,0,0,0,0,0,0,0,0,0,0,0] -; AVX2-FP-NEXT: vpshufb %xmm14, %xmm2, %xmm3 +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX2-FP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX2-FP-NEXT: # ymm2 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7] +; AVX2-FP-NEXT: vmovd {{.*#+}} xmm5 = [8,9,4,5,0,0,0,0,0,0,0,0,0,0,0,0] +; AVX2-FP-NEXT: vpshufb %xmm5, %xmm2, %xmm3 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm2 ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd $187, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload -; AVX2-FP-NEXT: # ymm3 = mem[0,1],ymm1[2],mem[3,4,5],ymm1[6],mem[7] -; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm5 = [u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] -; AVX2-FP-NEXT: vpshufb %ymm5, %ymm3, %ymm8 -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm8[0,1,2,3,4,5],ymm3[6],ymm8[7,8,9,10,11,12,13],ymm3[14],ymm8[15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm11[0],ymm6[1],ymm11[2,3,4],ymm6[5],ymm11[6,7] -; AVX2-FP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm10 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm10[0],xmm3[1],xmm10[2],xmm3[3],xmm10[4,5,6,7] -; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] -; AVX2-FP-NEXT: vpshufb %xmm10, %xmm3, %xmm3 -; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 -; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm13 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27] -; AVX2-FP-NEXT: vpshufb %ymm13, %ymm7, %ymm7 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm7[7] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm8[0],ymm3[1,2,3,4,5,6,7],ymm8[8],ymm3[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm9[2],ymm11[3,4,5],ymm9[6],ymm11[7] +; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm3[0,1,2,3,4,5],mem[6],xmm3[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm8[2,3] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0],ymm14[1],ymm13[2,3,4],ymm14[5],ymm13[6,7] +; AVX2-FP-NEXT: vmovdqa %ymm14, %ymm15 +; AVX2-FP-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-FP-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm10 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0],xmm8[1],xmm10[2],xmm8[3],xmm10[4,5,6,7] +; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] +; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8 +; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm10 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27] +; AVX2-FP-NEXT: vpshufb %ymm10, %ymm7, %ymm7 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5,6],ymm7[7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm7[1,2,3,4,5,6,7],ymm3[8],ymm7[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm3[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1],ymm12[2],ymm15[3,4],ymm12[5],ymm15[6,7] -; AVX2-FP-NEXT: vpshufb %xmm14, %xmm2, %xmm3 +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1],ymm12[2],ymm1[3,4],ymm12[5],ymm1[6,7] +; AVX2-FP-NEXT: vpshufb %xmm5, %xmm2, %xmm3 ; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm2 ; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] ; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1],ymm1[2],ymm8[3,4,5],ymm1[6],ymm8[7] -; AVX2-FP-NEXT: vpshufb %ymm5, %ymm3, %ymm5 -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6],ymm5[7,8,9,10,11,12,13],ymm3[14],ymm5[15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm9[1],ymm4[2,3,4],ymm9[5],ymm4[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm7 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0],xmm3[1],xmm7[2],xmm3[3],xmm7[4,5,6,7] -; AVX2-FP-NEXT: vpshufb %xmm10, %xmm3, %xmm3 -; AVX2-FP-NEXT: vpshufb %ymm13, %ymm0, %ymm0 -; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5,6],ymm0[7] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2,3,4,5,6,7],ymm5[8],ymm0[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vmovdqu (%rsp), %ymm8 # 32-byte Reload +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm14[0,1],ymm8[2],ymm14[3,4,5],ymm8[6],ymm14[7] +; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,5],mem[6],xmm3[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm5, %xmm7 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0],xmm5[1],xmm7[2],xmm5[3],xmm7[4,5,6,7] +; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] +; AVX2-FP-NEXT: vpshufb %ymm10, %ymm0, %ymm0 +; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4,5,6,7],ymm3[8],ymm0[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm1[3],ymm8[4,5],ymm1[6],ymm8[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2],ymm8[3],ymm14[4,5],ymm8[6],ymm14[7] ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1] ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6],ymm2[7,8],ymm0[9,10,11,12,13,14],ymm2[15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm9[2],ymm4[3,4],ymm9[5],ymm4[6,7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm3 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3],xmm0[4],xmm3[5,6,7] -; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,8,9,6,7,4,5,2,3,2,3,2,3,2,3] -; AVX2-FP-NEXT: vpshufb %xmm7, %xmm0, %xmm0 -; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm8 +; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [8,9,8,9,6,7,4,5,2,3,2,3,2,3,2,3] +; AVX2-FP-NEXT: vpshufb %xmm5, %xmm0, %xmm0 +; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm7 ; AVX2-FP-NEXT: vmovdqa 432(%rdi), %xmm3 ; AVX2-FP-NEXT: vmovdqa 416(%rdi), %xmm0 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm10 = xmm0[0,1,2,3,4,5],xmm3[6],xmm0[7] -; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm13 = [0,1,2,3,4,5,6,7,0,1,0,1,14,15,12,13] -; AVX2-FP-NEXT: vpshufb %xmm13, %xmm10, %xmm10 -; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10 -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7],ymm8[8,9,10,11,12],ymm10[13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm10 = ymm15[0,1],ymm12[2,3],ymm15[4,5],ymm12[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm10, %xmm14 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0,1,2,3,4,5],xmm3[6],xmm0[7] +; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,2,3,4,5,6,7,0,1,0,1,14,15,12,13] +; AVX2-FP-NEXT: vpshufb %xmm10, %xmm8, %xmm8 +; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8 +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm8[5,6,7],ymm7[8,9,10,11,12],ymm8[13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1],ymm12[2,3],ymm1[4,5],ymm12[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm14 ; AVX2-FP-NEXT: vmovd {{.*#+}} xmm1 = [8,9,4,5,0,0,0,0,0,0,0,0,0,0,0,0] ; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14 -; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm10 = xmm10[10,11,6,7,u,u,u,u,u,u,u,u,u,u,u,u] -; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3] +; AVX2-FP-NEXT: vmovd {{.*#+}} xmm12 = [10,11,6,7,0,0,0,0,0,0,0,0,0,0,0,0] +; AVX2-FP-NEXT: vpshufb %xmm12, %xmm8, %xmm8 +; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm14 = [u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29] ; AVX2-FP-NEXT: vpshufb %ymm14, %ymm2, %ymm2 -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm2[0],ymm8[1,2,3,4,5,6,7],ymm2[8],ymm8[9,10,11,12,13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm10[0,1],xmm2[2,3] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm8[4,5,6,7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm2[0],ymm7[1,2,3,4,5,6,7],ymm2[8],ymm7[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm8[0,1],xmm2[2,3] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm8 = ymm11[0,1],ymm6[2],ymm11[3,4],ymm6[5],ymm11[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm8, %xmm10 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0,1],xmm8[2],xmm10[3],xmm8[4],xmm10[5,6,7] -; AVX2-FP-NEXT: vpshufb %xmm7, %xmm8, %xmm10 -; AVX2-FP-NEXT: vmovdqa 208(%rdi), %xmm15 -; AVX2-FP-NEXT: vmovdqa 192(%rdi), %xmm12 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm12[0,1,2,3,4,5],xmm15[6],xmm12[7] -; AVX2-FP-NEXT: vpshufb %xmm13, %xmm5, %xmm5 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm13[0,1],ymm15[2],ymm13[3,4],ymm15[5],ymm13[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm7, %xmm8 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1],xmm7[2],xmm8[3],xmm7[4],xmm8[5,6,7] +; AVX2-FP-NEXT: vpshufb %xmm5, %xmm7, %xmm8 +; AVX2-FP-NEXT: vmovdqa 208(%rdi), %xmm5 +; AVX2-FP-NEXT: vmovdqa 192(%rdi), %xmm7 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm15 = xmm7[0,1,2,3,4,5],xmm5[6],xmm7[7] +; AVX2-FP-NEXT: vpshufb %xmm10, %xmm15, %xmm10 +; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8 ; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10 -; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm10[0,1,2,3,4],ymm5[5,6,7],ymm10[8,9,10,11,12],ymm5[13,14,15] -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2],ymm7[3],ymm11[4,5],ymm7[6],ymm11[7] -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm10[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm13[0],ymm10[1,2,3,4,5,6],ymm13[7,8],ymm10[9,10,11,12,13,14],ymm13[15] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7],ymm8[8,9,10,11,12],ymm10[13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2],ymm9[3],ymm11[4,5],ymm9[6],ymm11[7] +; AVX2-FP-NEXT: vmovdqa %ymm11, %ymm9 +; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm15[0],ymm10[1,2,3,4,5,6],ymm15[7,8],ymm10[9,10,11,12,13,14],ymm15[15] ; AVX2-FP-NEXT: vpshufb %ymm14, %ymm10, %ymm10 -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm13 = ymm2[0,1],ymm8[2,3],ymm2[4,5],ymm8[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm13, %xmm14 -; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm14 -; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm13[10,11,6,7,u,u,u,u,u,u,u,u,u,u,u,u] -; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm14[0],xmm1[1],xmm14[1],xmm1[2],xmm14[2],xmm1[3],xmm14[3] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm10[0],ymm5[1,2,3,4,5,6,7],ymm10[8],ymm5[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm14 = ymm11[0,1],ymm2[2,3],ymm11[4,5],ymm2[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm14, %xmm15 +; AVX2-FP-NEXT: vpshufb %xmm1, %xmm15, %xmm13 +; AVX2-FP-NEXT: vpshufb %xmm12, %xmm14, %xmm1 +; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm13[0],xmm1[1],xmm13[1],xmm1[2],xmm13[2],xmm1[3],xmm13[3] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm10[0],ymm8[1,2,3,4,5,6,7],ymm10[8],ymm8[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm10[2,3] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7] -; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd $18, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload -; AVX2-FP-NEXT: # ymm5 = ymm5[0],mem[1],ymm5[2,3],mem[4],ymm5[5,6,7] -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm10 = ymm5[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm10[1],ymm5[2,3,4,5,6,7,8],ymm10[9],ymm5[10,11,12,13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm9[2,3],ymm4[4,5],ymm9[6,7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7] +; AVX2-FP-NEXT: vmovdqu (%rsp), %ymm8 # 32-byte Reload +; AVX2-FP-NEXT: vpblendd $18, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload +; AVX2-FP-NEXT: # ymm8 = ymm8[0],mem[1],ymm8[2,3],mem[4],ymm8[5,6,7] +; AVX2-FP-NEXT: vpermq {{.*#+}} ymm10 = ymm8[2,3,0,1] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm10[1],ymm8[2,3,4,5,6,7,8],ymm10[9],ymm8[10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5],ymm6[6,7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm6 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3],xmm6[4],xmm4[5],xmm6[6,7] ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1],xmm3[2,3,4,5,6,7] @@ -6309,33 +6300,34 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload ; AVX2-FP-NEXT: # ymm4 = mem[0,1,2],ymm4[3],mem[4,5],ymm4[6],mem[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm10 -; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} xmm9 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7] -; AVX2-FP-NEXT: vpshufb %xmm9, %xmm10, %xmm10 +; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} xmm12 = [10,11,6,7,10,11,6,7,10,11,6,7,10,11,6,7] +; AVX2-FP-NEXT: vpshufb %xmm12, %xmm10, %xmm10 ; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,4,6,7] ; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31] -; AVX2-FP-NEXT: vpshufb %ymm10, %ymm5, %ymm5 -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2,3,4,5,6,7],ymm5[8],ymm0[9,10,11,12,13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3] +; AVX2-FP-NEXT: vpshufb %ymm10, %ymm8, %ymm8 +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm8[0],ymm0[1,2,3,4,5,6,7],ymm8[8],ymm0[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm8[2,3] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX2-FP-NEXT: vpblendd $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload ; AVX2-FP-NEXT: # ymm4 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3],xmm5[4],xmm4[5],xmm5[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm8 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm8[0,1,2],xmm4[3],xmm8[4],xmm4[5],xmm8[6,7] ; AVX2-FP-NEXT: vpshufb %xmm3, %xmm4, %xmm3 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm15[0],xmm12[1],xmm15[2,3,4,5,6,7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0],xmm7[1],xmm5[2,3,4,5,6,7] ; AVX2-FP-NEXT: vpshufb %xmm6, %xmm4, %xmm4 ; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 ; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4 ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5,6,7],ymm3[8,9,10,11,12],ymm4[13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0],ymm11[1],ymm7[2,3],ymm11[4],ymm7[5,6,7] +; AVX2-FP-NEXT: vpblendd $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm4 # 32-byte Folded Reload +; AVX2-FP-NEXT: # ymm4 = mem[0],ymm9[1],mem[2,3],ymm9[4],mem[5,6,7] ; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1] ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3,4,5,6,7,8],ymm5[9],ymm4[10,11,12,13,14,15] ; AVX2-FP-NEXT: vpshufb %ymm10, %ymm4, %ymm4 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1,2],ymm8[3],ymm2[4,5],ymm8[6],ymm2[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm11[0,1,2],ymm2[3],ymm11[4,5],ymm2[6],ymm11[7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm5, %xmm6 -; AVX2-FP-NEXT: vpshufb %xmm9, %xmm6, %xmm6 +; AVX2-FP-NEXT: vpshufb %xmm12, %xmm6, %xmm6 ; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,4,6,7] ; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7] ; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6,7],ymm4[8],ymm3[9,10,11,12,13,14,15] @@ -6343,7 +6335,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rsi) -; AVX2-FP-NEXT: vmovups (%rsp), %ymm2 # 32-byte Reload +; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX2-FP-NEXT: vmovaps %ymm2, (%rsi) ; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rdx) @@ -11461,13 +11453,14 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; ; AVX2-LABEL: load_i16_stride7_vf64: ; AVX2: # %bb.0: -; AVX2-NEXT: subq $1448, %rsp # imm = 0x5A8 +; AVX2-NEXT: subq $1464, %rsp # imm = 0x5B8 ; AVX2-NEXT: vmovdqa 256(%rdi), %ymm13 -; AVX2-NEXT: vmovdqa 224(%rdi), %ymm14 -; AVX2-NEXT: vmovdqa 512(%rdi), %ymm15 -; AVX2-NEXT: vmovdqa 544(%rdi), %ymm12 -; AVX2-NEXT: vmovdqa 480(%rdi), %ymm9 -; AVX2-NEXT: vmovdqa 448(%rdi), %ymm10 +; AVX2-NEXT: vmovdqa 224(%rdi), %ymm9 +; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqa 512(%rdi), %ymm8 +; AVX2-NEXT: vmovdqa 544(%rdi), %ymm7 +; AVX2-NEXT: vmovdqa 480(%rdi), %ymm11 +; AVX2-NEXT: vmovdqa 448(%rdi), %ymm12 ; AVX2-NEXT: vmovdqa (%rdi), %ymm3 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vmovdqa 32(%rdi), %ymm4 @@ -11487,207 +11480,207 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm4 ; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,0,0] ; AVX2-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa %ymm1, %ymm10 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm10[0,1],ymm9[2],ymm10[3,4,5],ymm9[6],ymm10[7] -; AVX2-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1],ymm11[2],ymm12[3,4,5],ymm11[6],ymm12[7] +; AVX2-NEXT: vmovdqu %ymm11, (%rsp) # 32-byte Spill ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5],xmm5[6],xmm4[7] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm12[0],ymm15[1],ymm12[2,3,4],ymm15[5],ymm12[6,7] -; AVX2-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0],ymm8[1],ymm7[2,3,4],ymm8[5],ymm7[6,7] +; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqa %ymm7, %ymm0 +; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3] ; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm5 -; AVX2-NEXT: vpblendvb %ymm1, %ymm4, %ymm5, %ymm0 -; AVX2-NEXT: vmovdqa %ymm1, %ymm8 -; AVX2-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1],ymm13[2],ymm14[3,4,5],ymm13[6],ymm14[7] -; AVX2-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendvb %ymm1, %ymm4, %ymm5, %ymm1 +; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1],ymm13[2],ymm9[3,4,5],ymm13[6],ymm9[7] ; AVX2-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5],xmm5[6],xmm4[7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4],xmm4[5],xmm6[6],xmm4[7] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 -; AVX2-NEXT: vmovdqa 288(%rdi), %ymm7 -; AVX2-NEXT: vmovdqa 320(%rdi), %ymm1 -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7] -; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqa 288(%rdi), %ymm9 +; AVX2-NEXT: vmovdqa 320(%rdi), %ymm5 +; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm5[0],ymm9[1],ymm5[2,3,4],ymm9[5],ymm5[6,7] +; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3] +; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 +; AVX2-NEXT: vpblendvb %ymm10, %ymm4, %ymm6, %ymm1 ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3] -; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm5 -; AVX2-NEXT: vpblendvb %ymm8, %ymm4, %ymm5, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa 704(%rdi), %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa 672(%rdi), %ymm8 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm0[2],ymm8[3,4,5],ymm0[6],ymm8[7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm5[4],xmm4[5],xmm5[6],xmm4[7] +; AVX2-NEXT: vmovdqa 704(%rdi), %ymm1 +; AVX2-NEXT: vmovdqa 672(%rdi), %ymm10 +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm10[0,1],ymm1[2],ymm10[3,4,5],ymm1[6],ymm10[7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4],xmm4[5],xmm6[6],xmm4[7] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 -; AVX2-NEXT: vmovdqa 736(%rdi), %ymm11 -; AVX2-NEXT: vmovdqa 768(%rdi), %ymm0 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm0[0],ymm11[1],ymm0[2,3,4],ymm11[5],ymm0[6,7] +; AVX2-NEXT: vmovdqa 736(%rdi), %ymm15 +; AVX2-NEXT: vmovdqa 768(%rdi), %ymm14 +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0],ymm15[1],ymm14[2,3,4],ymm15[5],ymm14[6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] ; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2 -; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0] -; AVX2-NEXT: vpblendvb %ymm6, %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm7 = [65535,65535,65535,65535,65535,0,0,0] +; AVX2-NEXT: vpblendvb %ymm7, %ymm3, %ymm2, %ymm2 ; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm12[0,1],ymm15[2],ymm12[3,4],ymm15[5],ymm12[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1],ymm8[2],ymm0[3,4],ymm8[5],ymm0[6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5,6,7,8,9,10],ymm3[11],ymm2[12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm10[0,1,2],ymm9[3],ymm10[4,5],ymm9[6],ymm10[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0,1,2],ymm11[3],ymm12[4,5],ymm11[6],ymm12[7] ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm3[1],xmm4[2,3,4,5],xmm3[6],xmm4[7] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,2,3,2,3,2,3,8,9,8,9,6,7,4,5,18,19,18,19,18,19,18,19,24,25,24,25,22,23,20,21] -; AVX2-NEXT: vpshufb %ymm3, %ymm2, %ymm5 +; AVX2-NEXT: vpshufb %ymm3, %ymm2, %ymm6 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [2,3,0,1,14,15,12,13,10,11,10,11,10,11,10,11] ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm4 -; AVX2-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm4 -; AVX2-NEXT: vmovdqa %ymm6, %ymm9 -; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm7[2],ymm1[3,4],ymm7[5],ymm1[6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5,6,7,8,9,10],ymm5[11],ymm4[12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2],ymm13[3],ymm14[4,5],ymm13[6],ymm14[7] -; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 -; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3,4,5],xmm5[6],xmm6[7] +; AVX2-NEXT: vpblendvb %ymm7, %ymm4, %ymm6, %ymm0 +; AVX2-NEXT: vmovdqa %ymm7, %ymm8 +; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm9[2],ymm5[3,4],ymm9[5],ymm5[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3],ymm4[4,5,6,7,8,9,10],ymm6[11],ymm4[12,13,14,15] +; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm6 # 32-byte Folded Reload +; AVX2-NEXT: # ymm6 = mem[0,1,2],ymm13[3],mem[4,5],ymm13[6],mem[7] +; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7 +; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2,3,4,5],xmm6[6],xmm7[7] ; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm4 -; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5 -; AVX2-NEXT: vpblendvb %ymm9, %ymm5, %ymm4, %ymm4 -; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa %ymm11, %ymm15 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm0[0,1],ymm11[2],ymm0[3,4],ymm11[5],ymm0[6,7] -; AVX2-NEXT: vmovdqa %ymm0, %ymm14 +; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm6 +; AVX2-NEXT: vpblendvb %ymm8, %ymm6, %ymm4, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5,6,7,8,9,10],ymm5[11],ymm4[12,13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2],ymm12[3],ymm8[4,5],ymm12[6],ymm8[7] -; AVX2-NEXT: vmovdqa %ymm8, %ymm13 -; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 -; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3,4,5],xmm5[6],xmm6[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3],ymm4[4,5,6,7,8,9,10],ymm6[11],ymm4[12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm10[0,1,2],ymm1[3],ymm10[4,5],ymm1[6],ymm10[7] +; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqa %ymm10, %ymm5 +; AVX2-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7 +; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2,3,4,5],xmm6[6],xmm7[7] ; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm4 -; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm5 -; AVX2-NEXT: vmovdqa %ymm9, %ymm7 -; AVX2-NEXT: vpblendvb %ymm9, %ymm5, %ymm4, %ymm0 +; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm6 +; AVX2-NEXT: vmovdqa %ymm8, %ymm13 +; AVX2-NEXT: vpblendvb %ymm8, %ymm6, %ymm4, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1],ymm1[2],ymm9[3,4],ymm1[5],ymm9[6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5,6,7,8,9,10],ymm5[11],ymm4[12,13,14,15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1],ymm8[2],ymm11[3,4],ymm8[5],ymm11[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3],ymm4[4,5,6,7,8,9,10],ymm6[11],ymm4[12,13,14,15] ; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2,3,4,5],xmm4[6],xmm5[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm10[0,1,2],ymm9[3],ymm10[4,5],ymm9[6],ymm10[7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0],xmm4[1],xmm6[2,3,4,5],xmm4[6],xmm6[7] ; AVX2-NEXT: vpshufb %xmm2, %xmm4, %xmm2 -; AVX2-NEXT: vpblendvb %ymm7, %ymm2, %ymm3, %ymm0 +; AVX2-NEXT: vpblendvb %ymm13, %ymm2, %ymm3, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1] -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm9[0,1],ymm1[2,3],ymm9[4,5],ymm1[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm8[2,3,0,1] +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm8[2,3],ymm11[4,5],ymm8[6,7] ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm2[4],ymm3[5,6,7,8,9,10,11],ymm2[12],ymm3[13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5,6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0],ymm10[1],ymm9[2,3],ymm10[4],ymm9[5,6,7] +; AVX2-NEXT: vmovdqa %ymm9, %ymm11 ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm4[1],xmm2[2,3,4,5],xmm4[6],xmm2[7] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [4,5,4,5,4,5,4,5,8,9,10,11,8,9,6,7,20,21,20,21,20,21,20,21,24,25,26,27,24,25,22,23] -; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm5 +; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm6 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,2,3,0,1,14,15,12,13,12,13,12,13,12,13] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 -; AVX2-NEXT: vpblendvb %ymm7, %ymm4, %ymm5, %ymm0 +; AVX2-NEXT: vpblendvb %ymm13, %ymm4, %ymm6, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0],ymm1[1],ymm7[2,3],ymm1[4],ymm7[5,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5],xmm5[6],xmm4[7] +; AVX2-NEXT: vmovdqa %ymm12, %ymm0 +; AVX2-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqu (%rsp), %ymm8 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0],ymm12[1],ymm8[2,3],ymm12[4],ymm8[5,6,7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1],xmm4[2,3,4,5],xmm6[6],xmm4[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1],ymm9[2,3],ymm8[4,5],ymm9[6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm9[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4],ymm5[5,6,7,8,9,10,11],ymm6[12],ymm5[13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1],ymm12[2,3],ymm9[4,5],ymm12[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm12[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4],ymm6[5,6,7,8,9,10,11],ymm7[12],ymm6[13,14,15] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 -; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm5 -; AVX2-NEXT: vpmovsxbw {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0] -; AVX2-NEXT: vpblendvb %ymm0, %ymm4, %ymm5, %ymm4 +; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 +; AVX2-NEXT: vpblendvb %ymm13, %ymm4, %ymm6, %ymm4 ; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0],ymm13[1],ymm12[2,3],ymm13[4],ymm12[5,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5],xmm5[6],xmm4[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0],ymm5[1],ymm1[2,3],ymm5[4],ymm1[5,6,7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1],xmm4[2,3,4,5],xmm6[6],xmm4[7] +; AVX2-NEXT: vmovdqa %ymm14, %ymm12 +; AVX2-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1],ymm15[2,3],ymm14[4,5],ymm15[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm15[2,3,0,1] +; AVX2-NEXT: vmovdqa %ymm15, %ymm5 ; AVX2-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1],ymm15[2,3],ymm14[4,5],ymm15[6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm15[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4],ymm5[5,6,7,8,9,10,11],ymm6[12],ymm5[13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4],ymm6[5,6,7,8,9,10,11],ymm7[12],ymm6[13,14,15] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 -; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm5 -; AVX2-NEXT: vpblendvb %ymm0, %ymm4, %ymm5, %ymm4 -; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 +; AVX2-NEXT: vmovdqa %ymm13, %ymm7 +; AVX2-NEXT: vpblendvb %ymm13, %ymm4, %ymm6, %ymm1 +; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload ; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0],ymm14[1],ymm13[2,3],ymm14[4],ymm13[5,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5],xmm5[6],xmm4[7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm6[1],xmm4[2,3,4,5],xmm6[6],xmm4[7] ; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm15[2,3],ymm6[4,5],ymm15[6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm15[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6,7,8,9,10,11],ymm5[12],ymm4[13,14,15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm15[2,3],ymm1[4,5],ymm15[6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm15[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4],ymm4[5,6,7,8,9,10,11],ymm6[12],ymm4[13,14,15] ; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2 -; AVX2-NEXT: vmovdqa %ymm0, %ymm5 -; AVX2-NEXT: vpblendvb %ymm0, %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpblendvb %ymm7, %ymm3, %ymm2, %ymm2 ; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0],ymm11[1],ymm10[2,3,4],ymm11[5],ymm10[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0],ymm10[1],ymm11[2,3,4],ymm10[5],ymm11[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4,5,6,7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX2-NEXT: vpblendd $72, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload -; AVX2-NEXT: # ymm2 = ymm0[0,1,2],mem[3],ymm0[4,5],mem[6],ymm0[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX2-NEXT: # ymm2 = mem[0,1,2],ymm2[3],mem[4,5],ymm2[6],mem[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm2[1,3,2,3] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [6,7,6,7,6,7,6,7,8,9,4,5,10,11,0,1,22,23,22,23,22,23,22,23,24,25,20,21,26,27,16,17] ; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm4 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,7,7,7] -; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vpblendvb %ymm7, %ymm3, %ymm4, %ymm3 ; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0],ymm1[1],ymm7[2,3,4],ymm1[5],ymm7[6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0],ymm0[1],ymm8[2,3,4],ymm0[5],ymm8[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3],xmm4[4,5,6,7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,7,7,7] -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2],ymm9[3],ymm8[4,5],ymm9[6],ymm8[7] +; AVX2-NEXT: vpblendd $72, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm4 # 32-byte Folded Reload +; AVX2-NEXT: # ymm4 = ymm9[0,1,2],mem[3],ymm9[4,5],mem[6],ymm9[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[1,3,2,3] ; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm4 -; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm0 +; AVX2-NEXT: vpblendvb %ymm7, %ymm3, %ymm4, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0],ymm14[1],ymm13[2,3,4],ymm14[5],ymm13[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3],xmm4[4,5,6,7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,7,7,7] -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2],ymm15[3],ymm6[4,5],ymm15[6],ymm6[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1,2],ymm15[3],ymm1[4,5],ymm15[6],ymm1[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[1,3,2,3] ; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm4 -; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm0 +; AVX2-NEXT: vpblendvb %ymm7, %ymm3, %ymm4, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm3 # 32-byte Folded Reload -; AVX2-NEXT: # ymm3 = ymm12[0],mem[1],ymm12[2,3,4],mem[5],ymm12[6,7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX2-NEXT: vpblendd $221, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload +; AVX2-NEXT: # ymm3 = mem[0],ymm0[1],mem[2,3,4],ymm0[5],mem[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3],xmm4[4,5,6,7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload -; AVX2-NEXT: # ymm4 = mem[0,1,2],ymm0[3],mem[4,5],ymm0[6],mem[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[1,3,2,3] ; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2 ; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,7,7,7] -; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm2, %ymm0 +; AVX2-NEXT: vpblendvb %ymm7, %ymm3, %ymm2, %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa 128(%rdi), %ymm12 -; AVX2-NEXT: vmovdqa 160(%rdi), %ymm10 -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm12[3],ymm10[4,5],ymm12[6],ymm10[7] +; AVX2-NEXT: vmovdqa 128(%rdi), %ymm11 +; AVX2-NEXT: vmovdqa 160(%rdi), %ymm9 +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2],ymm11[3],ymm9[4,5],ymm11[6],ymm9[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,6,4,7] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1] @@ -11696,16 +11689,16 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX2-NEXT: vmovdqa 192(%rdi), %ymm0 ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm0[0,1,0,2] -; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm8[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,1,0,2] +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm6[0,1,2,1,4,5,6,5] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] ; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7] ; AVX2-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload ; AVX2-NEXT: # ymm0 = mem[0,1,2,3,4],ymm1[5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa 576(%rdi), %ymm13 -; AVX2-NEXT: vmovdqa 608(%rdi), %ymm11 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm13[3],ymm11[4,5],ymm13[6],ymm11[7] +; AVX2-NEXT: vmovdqa 576(%rdi), %ymm14 +; AVX2-NEXT: vmovdqa 608(%rdi), %ymm13 +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm14[3],ymm13[4,5],ymm14[6],ymm13[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,6,4,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 ; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] @@ -11714,67 +11707,53 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-NEXT: vmovdqa 640(%rdi), %ymm1 ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,1,0,2] -; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm4[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,1,0,2] +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,1,2,1,4,5,6,5] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7] -; AVX2-NEXT: vpblendd $31, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload +; AVX2-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX2-NEXT: # ymm0 = mem[0,1,2,3,4],ymm0[5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa 352(%rdi), %ymm5 -; AVX2-NEXT: vmovdqa 384(%rdi), %ymm6 -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7] -; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqa 352(%rdi), %ymm8 +; AVX2-NEXT: vmovdqa 384(%rdi), %ymm12 +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2],ymm8[3],ymm12[4,5],ymm8[6],ymm12[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,4,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 ; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm2 -; AVX2-NEXT: vmovdqa 416(%rdi), %ymm14 -; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm14[0,1,0,2] -; AVX2-NEXT: vpshufd {{.*#+}} ymm9 = ymm3[0,1,2,1,4,5,6,5] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm9[7] -; AVX2-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload -; AVX2-NEXT: # ymm0 = mem[0,1,2,3,4],ymm2[5,6,7] +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm5 +; AVX2-NEXT: vmovdqa 416(%rdi), %ymm7 +; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm7[0,1,0,2] +; AVX2-NEXT: vpshufd {{.*#+}} ymm10 = ymm2[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm10[7] +; AVX2-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm0 # 32-byte Folded Reload +; AVX2-NEXT: # ymm0 = mem[0,1,2,3,4],ymm5[5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa 800(%rdi), %ymm7 -; AVX2-NEXT: vmovdqa 832(%rdi), %ymm9 -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm2[0,1,2,3,4,6,4,7] -; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2 -; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; AVX2-NEXT: vmovdqa 800(%rdi), %ymm4 +; AVX2-NEXT: vmovdqa 832(%rdi), %ymm10 +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2],ymm4[3],ymm10[4,5],ymm4[6],ymm10[7] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm5[0,1,2,3,4,6,4,7] +; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm5 +; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,1] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,7] +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vmovdqa 864(%rdi), %ymm2 -; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm2[0,1,0,2] +; AVX2-NEXT: vmovdqa 864(%rdi), %ymm5 +; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm5[0,1,0,2] ; AVX2-NEXT: vpshufd {{.*#+}} ymm15 = ymm1[0,1,2,1,4,5,6,5] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm15 = ymm15[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm15[7] ; AVX2-NEXT: vpblendd $31, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX2-NEXT: # ymm0 = mem[0,1,2,3,4],ymm0[5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm11[2],ymm13[3,4,5],ymm11[6],ymm13[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1],ymm13[2],ymm14[3,4,5],ymm13[6],ymm14[7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm15 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm15[4],xmm0[5],xmm15[6],xmm0[7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm15 = [0,1,0,1,14,15,12,13,10,11,8,9,12,13,14,15] ; AVX2-NEXT: vpshufb %xmm15, %xmm0, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[0,1,1,3,4,5,5,7] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm4[7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2,3,4,5,6,7],ymm4[8],ymm0[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm6[2],ymm5[3,4,5],ymm6[6],ymm5[7] -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 -; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4],xmm0[5],xmm4[6],xmm0[7] -; AVX2-NEXT: vpshufb %xmm15, %xmm0, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,1,3,4,5,5,7] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7] @@ -11782,11 +11761,23 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4,5,6,7],ymm3[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1],ymm9[2],ymm7[3,4,5],ymm9[6],ymm7[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1],ymm12[2],ymm8[3,4,5],ymm12[6],ymm8[7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4],xmm0[5],xmm3[6],xmm0[7] ; AVX2-NEXT: vpshufb %xmm15, %xmm0, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,1,3,4,5,5,7] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm10[2],ymm4[3,4,5],ymm10[6],ymm4[7] +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4],xmm0[5],xmm2[6],xmm0[7] +; AVX2-NEXT: vpshufb %xmm15, %xmm0, %xmm0 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] @@ -11794,73 +11785,68 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1],ymm10[2],ymm12[3,4,5],ymm10[6],ymm12[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1],ymm9[2],ymm11[3,4,5],ymm9[6],ymm11[7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshufb %xmm15, %xmm0, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm8[0,1,1,3,4,5,5,7] +; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm6[0,1,1,3,4,5,5,7] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm9[3],ymm11[4,5],ymm9[6],ymm11[7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2,3,4,5],xmm0[6],xmm1[7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,0,1,14,15,12,13,10,11,12,13,14,15] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm8[0,1,1,2] -; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,0,3,4,5,4,7] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1,2,3,4,5,6,7],ymm3[8],ymm1[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm3[0,1,1,2] +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,0,3,4,5,4,7] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqa %ymm11, %ymm6 -; AVX2-NEXT: vmovdqa %ymm13, %ymm4 -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2],ymm11[3],ymm13[4,5],ymm11[6],ymm13[7] -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 -; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3,4,5],xmm1[6],xmm3[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm14[0,1,2],ymm13[3],ymm14[4,5],ymm13[6],ymm14[7] +; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5],xmm1[6],xmm2[7] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm13[0,1,1,2] -; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,0,3,4,5,4,7] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1,2,3,4,5,6,7],ymm3[8],ymm1[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm15[0,1,1,2] +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,0,3,4,5,4,7] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2],ymm9[3],ymm7[4,5],ymm9[6],ymm7[7] -; AVX2-NEXT: vmovdqa %ymm7, %ymm5 -; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu %ymm9, (%rsp) # 32-byte Spill -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 -; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3,4,5],xmm1[6],xmm3[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm10[3],ymm4[4,5],ymm10[6],ymm4[7] +; AVX2-NEXT: vmovdqa %ymm4, %ymm6 +; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5],xmm1[6],xmm2[7] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,1,2] -; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,0,3,4,5,4,7] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1,2,3,4,5,6,7],ymm3[8],ymm1[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm5[0,1,1,2] +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,0,3,4,5,4,7] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2],ymm11[3],ymm7[4,5],ymm11[6],ymm7[7] -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 -; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3,4,5],xmm1[6],xmm3[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2],ymm12[3],ymm8[4,5],ymm12[6],ymm8[7] +; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5],xmm1[6],xmm2[7] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm14[0,1,1,2] +; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm7[0,1,1,2] ; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,0,3,4,5,4,7] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] @@ -11868,105 +11854,104 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0],ymm12[1],ymm10[2,3],ymm12[4],ymm10[5,6,7] -; AVX2-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0],ymm11[1],ymm9[2,3],ymm11[4],ymm9[5,6,7] +; AVX2-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm8[0,1,1,3] -; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] +; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,1,3] +; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm3[0,1,2,0,4,5,6,4] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0],ymm4[1],ymm6[2,3],ymm4[4],ymm6[5,6,7] -; AVX2-NEXT: vmovdqa %ymm4, %ymm8 -; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0],ymm14[1],ymm13[2,3],ymm14[4],ymm13[5,6,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm13[0,1,1,3] -; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm15[0,1,2,0,4,5,6,4] +; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm15[0,1,1,3] +; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm4[0,1,2,0,4,5,6,4] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0],ymm7[1],ymm11[2,3],ymm7[4],ymm11[5,6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0],ymm8[1],ymm12[2,3],ymm8[4],ymm12[5,6,7] +; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm14[0,1,1,3] -; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[0,1,2,0,4,5,6,4] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3,4,5,6,7],ymm3[8],ymm0[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm7[0,1,1,3] +; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm2[0,1,2,0,4,5,6,4] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0],ymm5[1],ymm9[2,3],ymm5[4],ymm9[5,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 -; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3,4,5],xmm3[6],xmm0[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0],ymm6[1],ymm10[2,3],ymm6[4],ymm10[5,6,7] +; AVX2-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7] -; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm3 -; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm2[0,1,1,3] -; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm0[0,1,2,0,4,5,6,4] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3,4,5,6,7],ymm3[8],ymm2[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX2-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload -; AVX2-NEXT: # ymm2 = ymm2[0,1],mem[2],ymm2[3,4],mem[5],ymm2[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 -; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm1 +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm5[0,1,1,3] +; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = ymm0[0,1,2,0,4,5,6,4] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm5[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm1[1,2,3,4,5,6,7],ymm5[8],ymm1[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0],ymm14[1],ymm13[2,3,4],ymm14[5],ymm13[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5 +; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4,5,6,7] +; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] +; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm5 +; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 +; AVX2-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm4[7] +; AVX2-NEXT: vmovdqu (%rsp), %ymm4 # 32-byte Reload +; AVX2-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload +; AVX2-NEXT: # ymm4 = ymm4[0,1],mem[2],ymm4[3,4],mem[5],ymm4[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm6 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1],ymm13[2],ymm14[3,4,5],ymm13[6],ymm14[7] -; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] -; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm5 -; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6],ymm5[7,8,9,10,11,12,13],ymm4[14],ymm5[15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm3[0,1],xmm4[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0],ymm8[1],ymm6[2,3,4],ymm8[5],ymm6[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm6 -; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0],xmm3[1],xmm6[2],xmm3[3],xmm6[4,5,6,7] -; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,6,7,4,5,2,3,0,1,14,15,12,13,14,15] -; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6 -; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6 -; AVX2-NEXT: vpshufd {{.*#+}} ymm7 = ymm15[0,1,2,1,4,5,6,5] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5,6],ymm7[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm6[1,2,3,4,5,6,7],ymm5[8],ymm6[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm10[0],ymm12[1],ymm10[2,3,4],ymm12[5],ymm10[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2],xmm4[3],xmm5[4,5,6,7] -; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 -; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4 -; AVX2-NEXT: vpshufd $100, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload -; AVX2-NEXT: # ymm5 = mem[0,1,2,1,4,5,6,5] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm5[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm7 = ymm14[0,1],ymm15[2],ymm14[3,4,5],ymm15[6],ymm14[7] +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] +; AVX2-NEXT: vpshufb %ymm4, %ymm7, %ymm7 +; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1,2,3,4,5,6,7],ymm7[8],ymm5[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5],mem[6],xmm7[7] +; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0],ymm11[1],ymm9[2,3,4],ymm11[5],ymm9[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2],xmm5[3],xmm6[4,5,6,7] +; AVX2-NEXT: vpshufb %xmm1, %xmm5, %xmm5 +; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 +; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5,6],ymm3[7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload ; AVX2-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload ; AVX2-NEXT: # ymm5 = mem[0,1],ymm5[2],mem[3,4],ymm5[5],mem[6,7] @@ -11978,88 +11963,83 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload ; AVX2-NEXT: vpblendd $187, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload ; AVX2-NEXT: # ymm6 = mem[0,1],ymm6[2],mem[3,4,5],ymm6[6],mem[7] -; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm7 -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm7[0],ymm4[1,2,3,4,5,6,7],ymm7[8],ymm4[9,10,11,12,13,14,15] -; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6],ymm7[7,8,9,10,11,12,13],ymm6[14],ymm7[15] +; AVX2-NEXT: vpshufb %ymm4, %ymm6, %ymm6 +; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1,2,3,4,5,6,7],ymm6[8],ymm3[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5],mem[6],xmm6[7] ; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload -; AVX2-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm4 # 32-byte Folded Reload -; AVX2-NEXT: # ymm4 = ymm15[0,1],mem[2],ymm15[3,4],mem[5],ymm15[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm12[0,1],ymm10[2],ymm12[3,4,5],ymm10[6],ymm12[7] -; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm6 -; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6],ymm6[7,8,9,10,11,12,13],ymm5[14],ymm6[15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX2-NEXT: vmovdqa %ymm11, %ymm8 -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm11[0],ymm9[1],ymm11[2,3,4],ymm9[5],ymm11[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm7 -; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0],xmm5[1],xmm7[2],xmm5[3],xmm7[4,5,6,7] -; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5 -; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 -; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,1,4,5,6,5] -; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm6[0],ymm1[1,2,3,4,5,6,7],ymm6[8],ymm1[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX2-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX2-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm4 -; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX2-NEXT: vpblendd $187, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm4 # 32-byte Folded Reload -; AVX2-NEXT: # ymm4 = mem[0,1],ymm11[2],mem[3,4,5],ymm11[6],mem[7] -; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2 -; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm2[0,1,2,3,4,5],ymm4[6],ymm2[7,8,9,10,11,12,13],ymm4[14],ymm2[15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX2-NEXT: vpblendd $221, (%rsp), %ymm7, %ymm4 # 32-byte Folded Reload -; AVX2-NEXT: # ymm4 = mem[0],ymm7[1],mem[2,3,4],ymm7[5],mem[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0],xmm4[1],xmm5[2],xmm4[3],xmm5[4,5,6,7] -; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm12[0],ymm8[1],ymm12[2,3,4],ymm8[5],ymm12[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5 +; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0],xmm3[1],xmm5[2],xmm3[3],xmm5[4,5,6,7] +; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm3 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 +; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,1,4,5,6,5] +; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm13[0,1],ymm6[2],ymm13[3,4,5],ymm6[6],ymm13[7] +; AVX2-NEXT: vpshufb %ymm4, %ymm5, %ymm5 +; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0],ymm2[1,2,3,4,5,6,7],ymm5[8],ymm2[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],mem[6],xmm5[7] +; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX2-NEXT: vpblendd $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm2 # 32-byte Folded Reload +; AVX2-NEXT: # ymm2 = ymm10[0],mem[1],ymm10[2,3,4],mem[5],ymm10[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 +; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4,5,6,7] +; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm1 +; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,1,2,1,4,5,6,5] ; AVX2-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5,6],ymm0[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1],ymm7[2],ymm11[3,4,5],ymm7[6],ymm11[7] +; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1 +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3 +; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],mem[6],xmm1[7] +; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2],ymm13[3],ymm14[4,5],ymm13[6],ymm14[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2],ymm15[3],ymm14[4,5],ymm15[6],ymm14[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7,8],ymm0[9,10,11,12,13,14],ymm1[15] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX2-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX2-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7] +; AVX2-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX2-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3],xmm1[4],xmm2[5,6,7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [8,9,8,9,6,7,4,5,2,3,2,3,2,3,2,3] ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX2-NEXT: vmovdqa 656(%rdi), %xmm13 -; AVX2-NEXT: vmovdqa 640(%rdi), %xmm14 -; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm14[0,1,2,3,4,5],xmm13[6],xmm14[7] +; AVX2-NEXT: vmovdqa 656(%rdi), %xmm2 +; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovdqa 640(%rdi), %xmm15 +; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm15[0,1,2,3,4,5],xmm2[6],xmm15[7] ; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,7,6] ; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7],ymm1[8,9,10,11,12],ymm2[13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX2-NEXT: vmovdqu (%rsp), %ymm2 # 32-byte Reload ; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload ; AVX2-NEXT: # ymm2 = ymm2[0,1],mem[2,3],ymm2[4,5],mem[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4 @@ -12074,96 +12054,95 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm6[3],ymm13[4,5],ymm6[6],ymm13[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7,8],ymm0[9,10,11,12,13,14],ymm1[15] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX2-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX2-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm4 ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1],xmm1[2],xmm4[3],xmm1[4],xmm4[5,6,7] ; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX2-NEXT: vmovdqa 432(%rdi), %xmm4 -; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX2-NEXT: vmovdqa 416(%rdi), %xmm5 +; AVX2-NEXT: vmovdqa 432(%rdi), %xmm5 ; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5],xmm4[6],xmm5[7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,0,3] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,7,6] -; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5,6,7],ymm1[8,9,10,11,12],ymm5[13,14,15] -; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm5 # 32-byte Folded Reload -; AVX2-NEXT: # ymm5 = ymm15[0,1],mem[2,3],ymm15[4,5],mem[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 -; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vmovdqa 416(%rdi), %xmm4 +; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5],xmm5[6],xmm4[7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,3] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,7,6] +; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4 +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5,6,7],ymm1[8,9,10,11,12],ymm4[13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm9[2,3],ymm8[4,5],ymm9[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 ; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3] ; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm5[0,1],xmm0[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm11[3],ymm9[4,5],ymm11[6],ymm9[7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm7[3],ymm11[4,5],ymm7[6],ymm11[7] ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7,8],ymm0[9,10,11,12,13,14],ymm1[15] -; AVX2-NEXT: vmovdqu (%rsp), %ymm8 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1],ymm7[2],ymm8[3,4],ymm7[5],ymm8[6,7] +; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm1[0],ymm0[1,2,3,4,5,6],ymm1[7,8],ymm0[9,10,11,12,13,14],ymm1[15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1],ymm11[2],ymm7[3,4],ymm11[5],ymm7[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7] ; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm6 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm5 ; AVX2-NEXT: vmovdqa 880(%rdi), %xmm1 ; AVX2-NEXT: vmovdqa 864(%rdi), %xmm0 -; AVX2-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,5],xmm1[6],xmm0[7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,0,3] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,7,6] -; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7 -; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm7[5,6,7],ymm6[8,9,10,11,12],ymm7[13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0,1,2,3,4,5],xmm1[6],xmm0[7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,7,6] +; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6 +; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5,6,7],ymm5[8,9,10,11,12],ymm6[13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm10[0,1],ymm12[2,3],ymm10[4,5],ymm12[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm8 +; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3] +; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm4 +; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm4[0],ymm5[1,2,3,4,5,6,7],ymm4[8],ymm5[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7] +; AVX2-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm7 # 32-byte Folded Reload -; AVX2-NEXT: # ymm7 = ymm4[0,1],mem[2,3],ymm4[4,5],mem[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm15 -; AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[0,2,2,3,4,5,6,7] -; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm15[0],xmm7[1],xmm15[1],xmm7[2],xmm15[2],xmm7[3],xmm15[3] -; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm5 -; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0],ymm6[1,2,3,4,5,6,7],ymm5[8],ymm6[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7] -; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX2-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload -; AVX2-NEXT: # ymm5 = ymm5[0,1],mem[2],ymm5[3,4],mem[5],ymm5[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 -; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2],xmm6[3],xmm5[4],xmm6[5,6,7] -; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm3 +; AVX2-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload +; AVX2-NEXT: # ymm4 = ymm4[0,1],mem[2],ymm4[3,4],mem[5],ymm4[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 +; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3],xmm4[4],xmm5[5,6,7] +; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm5 -; AVX2-NEXT: vmovdqa 208(%rdi), %xmm15 +; AVX2-NEXT: vmovdqa 208(%rdi), %xmm4 ; AVX2-NEXT: vmovdqa 192(%rdi), %xmm3 -; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1,2,3,4,5],xmm15[6],xmm3[7] +; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1,2,3,4,5],xmm4[6],xmm3[7] ; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,7,6] ; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6 ; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5,6,7],ymm5[8,9,10,11,12],ymm6[13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX2-NEXT: vpblendd $72, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload -; AVX2-NEXT: # ymm6 = ymm6[0,1,2],mem[3],ymm6[4,5],mem[6],ymm6[7] -; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0],ymm6[1,2,3,4,5,6],ymm7[7,8],ymm6[9,10,11,12,13,14],ymm7[15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm6 # 32-byte Folded Reload +; AVX2-NEXT: # ymm6 = mem[0,1,2],ymm13[3],mem[4,5],ymm13[6],mem[7] +; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm8[0],ymm6[1,2,3,4,5,6],ymm8[7,8],ymm6[9,10,11,12,13,14],ymm8[15] ; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2 +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1],ymm11[2,3],ymm12[4,5],ymm11[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7 -; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,1,2,3] -; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,2,2,3,4,5,6,7] +; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1],ymm12[2,3],ymm14[4,5],ymm12[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm8 +; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[2,1,2,3] +; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,3,4,5,6,7] ; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[2,1,2,3] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3] +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3] ; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0],ymm5[1,2,3,4,5,6,7],ymm2[8],ymm5[9,10,11,12,13,14,15] ; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm6[0,1],xmm2[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7] @@ -12172,39 +12151,40 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpblendd $18, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload ; AVX2-NEXT: # ymm2 = ymm2[0],mem[1],ymm2[2,3],mem[4],ymm2[5,6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm2[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm2[0],ymm5[1],ymm2[2,3,4,5,6,7,8],ymm5[9],ymm2[10,11,12,13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX2-NEXT: vpblendd $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload -; AVX2-NEXT: # ymm2 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm5 -; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3],xmm5[4],xmm2[5],xmm5[6,7] -; AVX2-NEXT: vpblendw {{.*#+}} xmm7 = xmm13[0],xmm14[1],xmm13[2,3,4,5,6,7] +; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm5[1],ymm2[2,3,4,5,6,7,8],ymm5[9],ymm2[10,11,12,13,14,15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload +; AVX2-NEXT: # ymm5 = ymm5[0,1],mem[2,3],ymm5[4,5],mem[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 +; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm5[3],xmm6[4],xmm5[5],xmm6[6,7] +; AVX2-NEXT: vpblendw $253, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm8 # 16-byte Folded Reload +; AVX2-NEXT: # xmm8 = mem[0],xmm15[1],mem[2,3,4,5,6,7] ; AVX2-NEXT: vmovdqa {{.*#+}} xmm5 = [8,9,10,11,8,9,6,7,4,5,4,5,4,5,4,5] -; AVX2-NEXT: vpshufb %xmm5, %xmm2, %xmm2 -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2 -; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,0,3] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,4,7] -; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7 -; AVX2-NEXT: vpblendw {{.*#+}} ymm7 = ymm2[0,1,2,3,4],ymm7[5,6,7],ymm2[8,9,10,11,12],ymm7[13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX2-NEXT: vpblendd $72, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload -; AVX2-NEXT: # ymm2 = ymm2[0,1,2],mem[3],ymm2[4,5],mem[6],ymm2[7] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm10 = xmm2[0,1,2,3,6,4,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2 -; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,7,6,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31] -; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 -; AVX2-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0],ymm7[1,2,3,4,5,6,7],ymm6[8],ymm7[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm10[0,1],xmm6[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7] -; AVX2-NEXT: vpblendd $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm7 # 32-byte Folded Reload -; AVX2-NEXT: # ymm7 = mem[0],ymm9[1],mem[2,3],ymm9[4],mem[5,6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm7[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0],ymm10[1],ymm7[2,3,4,5,6,7,8],ymm10[9],ymm7[10,11,12,13,14,15] -; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm9 # 32-byte Folded Reload -; AVX2-NEXT: # ymm9 = ymm8[0,1],mem[2,3],ymm8[4,5],mem[6,7] +; AVX2-NEXT: vpshufb %xmm5, %xmm6, %xmm6 +; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6 +; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,0,3] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,4,7] +; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8 +; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm6[0,1,2,3,4],ymm8[5,6,7],ymm6[8,9,10,11,12],ymm8[13,14,15] +; AVX2-NEXT: vmovdqu (%rsp), %ymm6 # 32-byte Reload +; AVX2-NEXT: vpblendd $72, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload +; AVX2-NEXT: # ymm6 = ymm6[0,1,2],mem[3],ymm6[4,5],mem[6],ymm6[7] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm6[0,1,2,3,6,4,6,7] +; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm6 +; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,1,2,1] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,7,6,7] +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7] +; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31] +; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm2 +; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm2[0],ymm8[1,2,3,4,5,6,7],ymm2[8],ymm8[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1],xmm2[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm8[4,5,6,7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX2-NEXT: vpblendd $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload +; AVX2-NEXT: # ymm8 = mem[0],ymm8[1],mem[2,3],ymm8[4],mem[5,6,7] +; AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3,4,5,6,7,8],ymm9[9],ymm8[10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm7[0,1],ymm11[2,3],ymm7[4,5],ymm11[6,7] ; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm10 ; AVX2-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3],xmm10[4],xmm9[5],xmm10[6,7] ; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6,7] @@ -12214,130 +12194,130 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,7] ; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15] -; AVX2-NEXT: vpblendd $72, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm1 # 32-byte Folded Reload -; AVX2-NEXT: # ymm1 = ymm4[0,1,2],mem[3],ymm4[4,5],mem[6],ymm4[7] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX2-NEXT: # ymm1 = mem[0,1,2],ymm1[3],mem[4,5],ymm1[6],mem[7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm1[0,1,2,3,6,4,6,7] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,7] ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7] -; AVX2-NEXT: vpshufb %ymm2, %ymm7, %ymm7 -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0],ymm0[1,2,3,4,5,6,7],ymm7[8],ymm0[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm7[2,3] +; AVX2-NEXT: vpshufb %ymm6, %ymm8, %ymm8 +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm8[0],ymm0[1,2,3,4,5,6,7],ymm8[8],ymm0[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX2-NEXT: vpblendd $18, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload ; AVX2-NEXT: # ymm1 = ymm1[0],mem[1],ymm1[2,3],mem[4],ymm1[5,6,7] -; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm1[2,3,0,1] -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4,5,6,7,8],ymm7[9],ymm1[10,11,12,13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX2-NEXT: vpblendd $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm7 # 32-byte Folded Reload -; AVX2-NEXT: # ymm7 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm9 -; AVX2-NEXT: vpblendw {{.*#+}} xmm7 = xmm9[0,1,2],xmm7[3],xmm9[4],xmm7[5],xmm9[6,7] -; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; AVX2-NEXT: vpblendw $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload -; AVX2-NEXT: # xmm4 = xmm4[0],mem[1],xmm4[2,3,4,5,6,7] -; AVX2-NEXT: vpshufb %xmm5, %xmm7, %xmm7 +; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1] +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3,4,5,6,7,8],ymm8[9],ymm1[10,11,12,13,14,15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload +; AVX2-NEXT: vpblendd $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm8 # 32-byte Folded Reload +; AVX2-NEXT: # ymm8 = mem[0,1],ymm7[2,3],mem[4,5],ymm7[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm9 +; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3],xmm9[4],xmm8[5],xmm9[6,7] +; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; AVX2-NEXT: vpblendw $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload +; AVX2-NEXT: # xmm7 = xmm7[0],mem[1],xmm7[2,3,4,5,6,7] +; AVX2-NEXT: vpshufb %xmm5, %xmm8, %xmm8 +; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8 +; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,0,3] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,4,7] ; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7 -; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,3] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7] -; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4 -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm7[0,1,2,3,4],ymm4[5,6,7],ymm7[8,9,10,11,12],ymm4[13,14,15] +; AVX2-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0,1,2,3,4],ymm7[5,6,7],ymm8[8,9,10,11,12],ymm7[13,14,15] +; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload +; AVX2-NEXT: # ymm8 = mem[0,1,2],ymm8[3],mem[4,5],ymm8[6],mem[7] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm8[0,1,2,3,6,4,6,7] +; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm8 +; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,2,1] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,7,6,7] +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7] +; AVX2-NEXT: vpshufb %ymm6, %ymm1, %ymm1 +; AVX2-NEXT: vpblendw {{.*#+}} ymm7 = ymm1[0],ymm7[1,2,3,4,5,6,7],ymm1[8],ymm7[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm8[0,1],xmm1[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm7[4,5,6,7] ; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX2-NEXT: vpblendd $183, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload -; AVX2-NEXT: # ymm7 = mem[0,1,2],ymm7[3],mem[4,5],ymm7[6],mem[7] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,6,4,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm7 -; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,5,7,6,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] -; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1 -; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm1[0],ymm4[1,2,3,4,5,6,7],ymm1[8],ymm4[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm7[0,1],xmm1[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload -; AVX2-NEXT: # ymm4 = ymm4[0,1],mem[2,3],ymm4[4,5],mem[6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm7 -; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1,2],xmm4[3],xmm7[4],xmm4[5],xmm7[6,7] -; AVX2-NEXT: vpshufb %xmm5, %xmm4, %xmm4 -; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm15[0],xmm3[1],xmm15[2,3,4,5,6,7] -; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4 +; AVX2-NEXT: vpblendd $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload +; AVX2-NEXT: # ymm7 = ymm7[0,1],mem[2,3],ymm7[4,5],mem[6,7] +; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm8 +; AVX2-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2],xmm7[3],xmm8[4],xmm7[5],xmm8[6,7] +; AVX2-NEXT: vpshufb %xmm5, %xmm7, %xmm5 +; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3,4,5,6,7] +; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm4 ; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,3] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,4,7] ; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3 ; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5,6,7],ymm4[8,9,10,11,12],ymm3[13,14,15] -; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX2-NEXT: vpblendd $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload -; AVX2-NEXT: # ymm4 = mem[0],ymm4[1],mem[2,3],ymm4[4],mem[5,6,7] +; AVX2-NEXT: vpblendd $18, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm4 # 32-byte Folded Reload +; AVX2-NEXT: # ymm4 = ymm13[0],mem[1],ymm13[2,3],mem[4],ymm13[5,6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1] ; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3,4,5,6,7,8],ymm5[9],ymm4[10,11,12,13,14,15] -; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2 -; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2],ymm11[3],ymm12[4,5],ymm11[6],ymm12[7] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,6,4,6,7] -; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm4 -; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,1] -; AVX2-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] -; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm3[1,2,3,4,5,6,7],ymm2[8],ymm3[9,10,11,12,13,14,15] -; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3] -; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7] -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 96(%rsi) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 32(%rsi) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 64(%rsi) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, (%rsi) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 96(%rdx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 32(%rdx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 64(%rdx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, (%rdx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 32(%rcx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 96(%rcx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 64(%rcx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, (%rcx) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 96(%r8) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 32(%r8) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 64(%r8) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, (%r8) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 96(%r9) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 32(%r9) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, (%r9) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 64(%r9) +; AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2],ymm12[3],ymm14[4,5],ymm12[6],ymm14[7] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,6,4,6,7] +; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm5 +; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,1] +; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,7,6,7] +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] +; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1,2,3,4,5,6,7],ymm4[8],ymm3[9,10,11,12,13,14,15] +; AVX2-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3] +; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 96(%rsi) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 32(%rsi) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 64(%rsi) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, (%rsi) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 96(%rdx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 32(%rdx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 64(%rdx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, (%rdx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 32(%rcx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 96(%rcx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 64(%rcx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, (%rcx) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 96(%r8) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 32(%r8) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 64(%r8) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, (%r8) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 96(%r9) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 32(%r9) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, (%r9) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 64(%r9) ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 96(%rax) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 32(%rax) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, 64(%rax) -; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX2-NEXT: vmovaps %ymm3, (%rax) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 96(%rax) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 32(%rax) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, 64(%rax) +; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX2-NEXT: vmovaps %ymm4, (%rax) ; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX2-NEXT: vmovdqa %ymm1, 32(%rax) -; AVX2-NEXT: vmovdqa %ymm2, (%rax) +; AVX2-NEXT: vmovdqa %ymm3, (%rax) ; AVX2-NEXT: vmovdqa %ymm0, 96(%rax) -; AVX2-NEXT: vmovdqa %ymm6, 64(%rax) -; AVX2-NEXT: addq $1448, %rsp # imm = 0x5A8 +; AVX2-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX2-NEXT: addq $1464, %rsp # imm = 0x5B8 ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; @@ -12806,10 +12786,9 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendd $187, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm6 # 32-byte Folded Reload ; AVX2-FP-NEXT: # ymm6 = mem[0,1],ymm13[2],mem[3,4,5],ymm13[6],mem[7] ; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27] -; AVX2-FP-NEXT: vpshufb %ymm2, %ymm6, %ymm7 -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6],ymm7[7,8,9,10,11,12,13],ymm6[14],ymm7[15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm4[0,1],xmm6[2,3] +; AVX2-FP-NEXT: vpshufb %ymm2, %ymm6, %ymm6 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5],mem[6],xmm6[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm7 = xmm4[0,1],xmm7[2,3] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0],ymm8[1],ymm11[2,3,4],ymm8[5],ymm11[6,7] ; AVX2-FP-NEXT: vextracti128 $1, %ymm4, %xmm8 ; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm4[1],xmm8[2],xmm4[3],xmm8[4,5,6,7] @@ -12819,8 +12798,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} ymm9 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27] ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm5, %ymm5 ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5,6],ymm5[7] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1,2,3,4,5,6,7],ymm7[8],ymm5[9,10,11,12,13,14,15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1,2,3,4,5,6,7],ymm6[8],ymm5[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm5[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload ; AVX2-FP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload @@ -12832,20 +12811,19 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vmovdqu (%rsp), %ymm6 # 32-byte Reload ; AVX2-FP-NEXT: vpblendd $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload ; AVX2-FP-NEXT: # ymm6 = ymm6[0,1],mem[2],ymm6[3,4,5],mem[6],ymm6[7] -; AVX2-FP-NEXT: vpshufb %ymm2, %ymm6, %ymm7 -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6],ymm7[7,8,9,10,11,12,13],ymm6[14],ymm7[15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3] -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm10[0],ymm14[1],ymm10[2,3,4],ymm14[5],ymm10[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm6, %xmm8 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm6 = xmm8[0],xmm6[1],xmm8[2],xmm6[3],xmm8[4,5,6,7] -; AVX2-FP-NEXT: vpshufb %xmm4, %xmm6, %xmm6 -; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6 +; AVX2-FP-NEXT: vpshufb %ymm2, %ymm6, %ymm6 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5],mem[6],xmm6[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],xmm7[2,3] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0],ymm14[1],ymm10[2,3,4],ymm14[5],ymm10[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm7, %xmm8 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0],xmm7[1],xmm8[2],xmm7[3],xmm8[4,5,6,7] +; AVX2-FP-NEXT: vpshufb %xmm4, %xmm7, %xmm7 +; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7 ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload ; AVX2-FP-NEXT: vpshufb %ymm9, %ymm3, %ymm3 ; AVX2-FP-NEXT: vmovdqa %ymm9, %ymm8 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5,6],ymm3[7] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1,2,3,4,5,6,7],ymm7[8],ymm3[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5,6],ymm3[7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1,2,3,4,5,6,7],ymm6[8],ymm3[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload @@ -12859,20 +12837,19 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm12[0,1],ymm11[2],ymm12[3,4,5],ymm11[6],ymm12[7] -; AVX2-FP-NEXT: vpshufb %ymm2, %ymm5, %ymm6 -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6],ymm6[7,8,9,10,11,12,13],ymm5[14],ymm6[15] -; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3] +; AVX2-FP-NEXT: vpshufb %ymm2, %ymm5, %ymm5 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,5],mem[6],xmm5[7] +; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3] ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0],ymm10[1],ymm9[2,3,4],ymm10[5],ymm9[6,7] -; AVX2-FP-NEXT: vextracti128 $1, %ymm5, %xmm7 -; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0],xmm5[1],xmm7[2],xmm5[3],xmm7[4,5,6,7] -; AVX2-FP-NEXT: vpshufb %xmm4, %xmm5, %xmm5 -; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5 +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0],ymm10[1],ymm9[2,3,4],ymm10[5],ymm9[6,7] +; AVX2-FP-NEXT: vextracti128 $1, %ymm6, %xmm7 +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0],xmm6[1],xmm7[2],xmm6[3],xmm7[4,5,6,7] +; AVX2-FP-NEXT: vpshufb %xmm4, %xmm6, %xmm6 +; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6 ; AVX2-FP-NEXT: vpshufb %ymm8, %ymm1, %ymm1 -; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm6[0],ymm1[1,2,3,4,5,6,7],ymm6[8],ymm1[9,10,11,12,13,14,15] +; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5,6],ymm1[7] +; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm5[0],ymm1[1,2,3,4,5,6,7],ymm5[8],ymm1[9,10,11,12,13,14,15] ; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7] ; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload @@ -12886,8 +12863,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX2-FP-NEXT: vpblendd $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm3 # 32-byte Folded Reload ; AVX2-FP-NEXT: # ymm3 = ymm14[0,1],mem[2],ymm14[3,4,5],mem[6],ymm14[7] ; AVX2-FP-NEXT: vpshufb %ymm2, %ymm3, %ymm2 -; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1] -; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3,4,5],ymm3[6],ymm2[7,8,9,10,11,12,13],ymm3[14],ymm2[15] +; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,5],mem[6],xmm2[7] ; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3] ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload ; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll index 09b97df9afead..3acc94d6e1fc4 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll @@ -3422,9 +3422,9 @@ define void @store_i8_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512-NEXT: vmovdqa (%r10), %xmm1 -; AVX512-NEXT: vmovdqa 16(%r10), %xmm12 +; AVX512-NEXT: vmovdqa 16(%r10), %xmm11 ; AVX512-NEXT: vmovdqa (%rax), %xmm5 -; AVX512-NEXT: vmovdqa 16(%rax), %xmm13 +; AVX512-NEXT: vmovdqa 16(%rax), %xmm12 ; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15] ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm22 ; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] @@ -3436,117 +3436,114 @@ define void @store_i8_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: vmovdqa (%r8), %xmm7 ; AVX512-NEXT: vmovdqa 16(%r8), %xmm15 ; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,5,5,7] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,5,5,7] ; AVX512-NEXT: vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,6,5,7,7] -; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm4, %ymm4 -; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[0,2,2,3,4,6,6,7] -; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7],ymm4[8,9,10],ymm1[11],ymm4[12,13,14],ymm1[15] -; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm0[0,0,2,1,4,5,6,7] +; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm3, %ymm3 +; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7] +; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7],ymm3[8,9,10],ymm1[11],ymm3[12,13,14],ymm1[15] +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[0,0,2,1,4,5,6,7] ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm4, %ymm0 +; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0 ; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5] -; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[0,1,1,3,4,5,6,7] +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,1,3,4,5,6,7] ; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm4, %ymm2 +; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2 ; AVX512-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,0,2,1,4,4,6,5] ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7],ymm2[8,9,10],ymm0[11],ymm2[12,13,14],ymm0[15] -; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm17 +; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16 ; AVX512-NEXT: vmovdqa (%rsi), %xmm1 ; AVX512-NEXT: vmovdqa (%rdi), %xmm2 ; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15] ; AVX512-NEXT: vmovdqa64 %xmm2, %xmm20 ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm21 +; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; AVX512-NEXT: vmovdqa (%rcx), %xmm8 ; AVX512-NEXT: vmovdqa (%rdx), %xmm9 -; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm9[8],xmm8[8],xmm9[9],xmm8[9],xmm9[10],xmm8[10],xmm9[11],xmm8[11],xmm9[12],xmm8[12],xmm9[13],xmm8[13],xmm9[14],xmm8[14],xmm9[15],xmm8[15] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,4,6,5] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm11 = xmm1[0,1,2,3,4,6,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm4, %ymm4 -; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm11 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,0,1,4,5,2,3,0,1,4,5,4,5,6,7] -; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm11[0],ymm1[1],ymm11[2,3,4],ymm1[5],ymm11[6,7,8],ymm1[9],ymm11[10,11,12],ymm1[13],ymm11[14,15] -; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[2,1,3,3,6,5,7,7] +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm9[8],xmm8[8],xmm9[9],xmm8[9],xmm9[10],xmm8[10],xmm9[11],xmm8[11],xmm9[12],xmm8[12],xmm9[13],xmm8[13],xmm9[14],xmm8[14],xmm9[15],xmm8[15] +; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm10[0,0,1,1,2,2,3,3] +; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm13 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero +; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1],ymm3[2,3,4],ymm13[5],ymm3[6,7,8],ymm13[9],ymm3[10,11,12],ymm13[13],ymm3[14,15] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm13 = xmm10[0,1,2,3,4,4,6,5] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,6,6,7] +; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm13, %ymm10 +; AVX512-NEXT: vpshufd {{.*#+}} ymm10 = ymm10[2,1,3,3,6,5,7,7] ; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] ; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3,4],ymm4[5],ymm0[6,7,8],ymm4[9],ymm0[10,11,12],ymm4[13],ymm0[14,15] -; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm19 -; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7] +; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm10[1],ymm0[2,3,4],ymm10[5],ymm0[6,7,8],ymm10[9],ymm0[10,11,12],ymm10[13],ymm0[14,15] +; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm18 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm3[0,1,2,3,4,4,6,5] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,4,6,6,7] +; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm13 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm13[0,1,2,3,4,5,5,7] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm13[0,1,2,3,6,5,7,7] +; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 +; AVX512-NEXT: vmovdqa 16(%rcx), %xmm2 +; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm3[0,0,2,1,4,5,6,7] +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7] +; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0 +; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm10[0,2,2,3,4,6,6,7] +; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7] +; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3],ymm1[4,5,6],ymm3[7],ymm1[8,9,10],ymm3[11],ymm1[12,13,14],ymm3[15] +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[0,1,1,3,4,5,6,7] +; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm13[2,1,3,3,4,5,6,7] +; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm3, %ymm3 +; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5] +; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,0,2,1,4,4,6,5] +; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6],ymm0[7],ymm3[8,9,10],ymm0[11],ymm3[12,13,14],ymm0[15] +; AVX512-NEXT: vmovdqa 16(%rdx), %xmm10 +; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm17 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,6,6,7] +; AVX512-NEXT: vinserti32x4 $1, %xmm3, %ymm1, %ymm19 +; AVX512-NEXT: vmovdqa 16(%rsi), %xmm3 +; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512-NEXT: vmovdqa 16(%rdi), %xmm13 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3],xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7] +; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1],ymm4[2,3,4],ymm0[5],ymm4[6,7,8],ymm0[9],ymm4[10,11,12],ymm0[13],ymm4[14,15] +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm19[2,1,3,3,6,5,7,7] +; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3,4],ymm4[5],ymm1[6,7,8],ymm4[9],ymm1[10,11,12],ymm4[13],ymm1[14,15] +; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm19 +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15] ; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] ; AVX512-NEXT: vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,6,6,7] ; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1 -; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7] +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15] ; AVX512-NEXT: vpshufhw {{.*#+}} xmm11 = xmm4[0,1,2,3,4,5,5,7] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm4[0,1,2,3,6,5,7,7] -; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm11, %ymm10 -; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm0[0,0,2,1,4,5,6,7] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm12 = xmm4[0,1,2,3,6,5,7,7] +; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm11, %ymm11 +; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm0[0,0,2,1,4,5,6,7] ; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm11, %ymm0 +; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm12, %ymm0 ; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7] -; AVX512-NEXT: vpshufd {{.*#+}} ymm10 = ymm10[0,2,2,3,4,6,6,7] -; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1,2],ymm1[3],ymm10[4,5,6],ymm1[7],ymm10[8,9,10],ymm1[11],ymm10[12,13,14],ymm1[15] -; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,1,1,3,4,5,6,7] +; AVX512-NEXT: vpshufd {{.*#+}} ymm11 = ymm11[0,2,2,3,4,6,6,7] +; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm11[0,1,2],ymm1[3],ymm11[4,5,6],ymm1[7],ymm11[8,9,10],ymm1[11],ymm11[12,13,14],ymm1[15] +; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm4[0,1,1,3,4,5,6,7] ; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,3,3,4,5,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1 +; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm11, %ymm4 ; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5] -; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,0,2,1,4,4,6,5] -; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7],ymm1[8,9,10],ymm0[11],ymm1[12,13,14],ymm0[15] -; AVX512-NEXT: vmovdqa 16(%rcx), %xmm2 -; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm0, %zmm18 -; AVX512-NEXT: vmovdqa 16(%rdx), %xmm11 -; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3],xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,4,6,5] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm0[0,1,2,3,4,6,6,7] -; AVX512-NEXT: vinserti32x4 $1, %xmm10, %ymm4, %ymm16 -; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0 -; AVX512-NEXT: vmovdqa64 %xmm3, %xmm23 -; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX512-NEXT: vmovdqa 16(%rsi), %xmm4 -; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero -; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3,4],ymm0[5],ymm3[6,7,8],ymm0[9],ymm3[10,11,12],ymm0[13],ymm3[14,15] -; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm10[2,3,2,3] -; AVX512-NEXT: vpshufd {{.*#+}} ymm10 = ymm16[2,1,3,3,6,5,7,7] -; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero -; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm10[1],ymm3[2,3,4],ymm10[5],ymm3[6,7,8],ymm10[9],ymm3[10,11,12],ymm10[13],ymm3[14,15] -; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm16 -; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,4,6,5] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm0[0,1,2,3,4,6,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm3, %ymm3 -; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm12 = xmm10[0,1,2,3,4,5,5,7] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm13 = xmm10[0,1,2,3,6,5,7,7] -; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm12, %ymm12 -; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm0[0,0,2,1,4,5,6,7] -; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm13, %ymm0 -; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,2,2,3,4,6,6,7] -; AVX512-NEXT: vpshufd {{.*#+}} ymm12 = ymm12[0,2,2,3,4,6,6,7] -; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm12[0,1,2],ymm3[3],ymm12[4,5,6],ymm3[7],ymm12[8,9,10],ymm3[11],ymm12[12,13,14],ymm3[15] -; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm10[0,1,1,3,4,5,6,7] -; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[2,1,3,3,4,5,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm12, %ymm10 -; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5] -; AVX512-NEXT: vpshufd {{.*#+}} ymm10 = ymm10[0,0,2,1,4,4,6,5] -; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3],ymm10[4,5,6],ymm0[7],ymm10[8,9,10],ymm0[11],ymm10[12,13,14],ymm0[15] -; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm12 -; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5] -; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,6,6,7] -; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 -; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15] -; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero -; AVX512-NEXT: vmovdqa64 %xmm23, %xmm10 -; AVX512-NEXT: vpshufb %xmm10, %xmm0, %xmm0 +; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[0,0,2,1,4,4,6,5] +; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0,1,2],ymm0[3],ymm4[4,5,6],ymm0[7],ymm4[8,9,10],ymm0[11],ymm4[12,13,14],ymm0[15] +; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm11 +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] +; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7] +; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm13[8],xmm3[8],xmm13[9],xmm3[9],xmm13[10],xmm3[10],xmm13[11],xmm3[11],xmm13[12],xmm3[12],xmm13[13],xmm3[13],xmm13[14],xmm3[14],xmm13[15],xmm3[15] +; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3,4],ymm0[5],ymm3[6,7,8],ymm0[9],ymm3[10,11,12],ymm0[13],ymm3[14,15] -; AVX512-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,1,3,3,6,5,7,7] -; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] -; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero -; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7,8],ymm2[9],ymm1[10,11,12],ymm2[13],ymm1[14,15] +; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[2,1,3,3,6,5,7,7] +; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3] +; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7,8],ymm1[9],ymm2[10,11,12],ymm1[13],ymm2[14,15] ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 ; AVX512-NEXT: vmovdqa64 %xmm22, %xmm1 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7] @@ -3577,8 +3574,8 @@ define void @store_i8_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: vmovdqa64 %xmm20, %xmm4 ; AVX512-NEXT: vmovdqa64 %xmm21, %xmm5 ; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] -; AVX512-NEXT: vpshufb %xmm10, %xmm2, %xmm2 ; AVX512-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero +; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] ; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero ; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0],ymm2[1],ymm5[2,3,4],ymm2[5],ymm5[6,7,8],ymm2[9],ymm5[10,11,12],ymm2[13],ymm5[14,15] ; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,1,3,3,6,5,7,7] @@ -3589,14 +3586,14 @@ define void @store_i8_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512-NEXT: movw $-21846, %cx # imm = 0xAAAA ; AVX512-NEXT: kmovw %ecx, %k1 +; AVX512-NEXT: vmovdqa32 %zmm16, %zmm18 {%k1} ; AVX512-NEXT: vmovdqa32 %zmm17, %zmm19 {%k1} -; AVX512-NEXT: vmovdqa32 %zmm18, %zmm16 {%k1} -; AVX512-NEXT: vmovdqa32 %zmm12, %zmm0 {%k1} +; AVX512-NEXT: vmovdqa32 %zmm11, %zmm0 {%k1} ; AVX512-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1} ; AVX512-NEXT: vmovdqa64 %zmm2, (%rax) ; AVX512-NEXT: vmovdqa64 %zmm0, 192(%rax) -; AVX512-NEXT: vmovdqa64 %zmm16, 128(%rax) -; AVX512-NEXT: vmovdqa64 %zmm19, 64(%rax) +; AVX512-NEXT: vmovdqa64 %zmm19, 128(%rax) +; AVX512-NEXT: vmovdqa64 %zmm18, 64(%rax) ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq ; @@ -3769,165 +3766,160 @@ define void @store_i8_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10 ; AVX512DQ-NEXT: vmovdqa (%r10), %xmm0 -; AVX512DQ-NEXT: vmovdqa 16(%r10), %xmm10 +; AVX512DQ-NEXT: vmovdqa 16(%r10), %xmm9 ; AVX512DQ-NEXT: vmovdqa (%rax), %xmm2 -; AVX512DQ-NEXT: vmovdqa 16(%rax), %xmm11 +; AVX512DQ-NEXT: vmovdqa 16(%rax), %xmm10 ; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm23 -; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm24 +; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm22 +; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm23 ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,4,6,5] ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm4 = xmm1[0,1,2,3,4,6,6,7] ; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0 ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vmovdqa (%r9), %xmm2 -; AVX512DQ-NEXT: vmovdqa 16(%r9), %xmm12 +; AVX512DQ-NEXT: vmovdqa (%r9), %xmm4 +; AVX512DQ-NEXT: vmovdqa 16(%r9), %xmm11 ; AVX512DQ-NEXT: vmovdqa (%r8), %xmm6 -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] -; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm25 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15] ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm5[0,1,2,3,4,5,5,7] ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm8 = xmm5[0,1,2,3,6,5,7,7] ; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm7, %ymm7 ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm0[3],ymm7[4,5,6],ymm0[7],ymm7[8,9,10],ymm0[11],ymm7[12,13,14],ymm0[15] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm7[0,1,2],ymm0[3],ymm7[4,5,6],ymm0[7],ymm7[8,9,10],ymm0[11],ymm7[12,13,14],ymm0[15] ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm1[0,0,2,1,4,5,6,7] ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm7, %ymm13 -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm5[0,1,1,3,4,5,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm7, %ymm1 +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,0,2,1,4,4,6,5] +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm5[0,1,1,3,4,5,6,7] ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[2,1,3,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm1, %ymm5 -; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm0 -; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm1 -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm14 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] -; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm21 -; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm22 +; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm7, %ymm5 +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[0,0,2,1,4,4,6,5] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2],ymm1[3],ymm5[4,5,6],ymm1[7],ymm5[8,9,10],ymm1[11],ymm5[12,13,14],ymm1[15] +; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm1 +; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm2 ; AVX512DQ-NEXT: vmovdqa (%rcx), %xmm7 ; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm8 -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm15 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm15[0,1,2,3,4,4,6,5] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm15[0,1,2,3,4,6,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 -; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero -; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,0,1,4,5,2,3,0,1,4,5,4,5,6,7] -; AVX512DQ-NEXT: vpshufb %xmm4, %xmm15, %xmm15 -; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm15 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7,8],ymm15[9],ymm1[10,11,12],ymm15[13],ymm1[14,15] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,1,3,3,6,5,7,7] +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm12 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm13 = xmm12[0,1,2,3,4,4,6,5] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm14 = xmm12[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm13, %ymm13 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm14 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15] +; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm24 +; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm25 +; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm15 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3] +; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm12 = xmm12[0],zero,xmm12[1],zero,xmm12[2],zero,xmm12[3],zero +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm12 = ymm15[0],ymm12[1],ymm15[2,3,4],ymm12[5],ymm15[6,7,8],ymm12[9],ymm15[10,11,12],ymm12[13],ymm15[14,15] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm13 = ymm13[2,1,3,3,6,5,7,7] ; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[2,3,2,3] ; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm14 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1],ymm14[2,3,4],ymm0[5],ymm14[6,7,8],ymm0[9],ymm14[10,11,12],ymm0[13],ymm14[14,15] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm13 = ymm14[0],ymm13[1],ymm14[2,3,4],ymm13[5],ymm14[6,7,8],ymm13[9],ymm14[10,11,12],ymm13[13],ymm14[14,15] ; AVX512DQ-NEXT: vmovdqa 16(%r8), %xmm14 -; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm16 -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm13[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm5[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7],ymm1[8,9,10],ymm0[11],ymm1[12,13,14],ymm0[15] +; AVX512DQ-NEXT: vinserti64x4 $1, %ymm13, %zmm12, %zmm16 ; AVX512DQ-NEXT: movw $-21846, %ax # imm = 0xAAAA ; AVX512DQ-NEXT: kmovw %eax, %k1 -; AVX512DQ-NEXT: vinserti32x8 $1, %ymm2, %zmm0, %zmm16 {%k1} -; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7] -; AVX512DQ-NEXT: vinserti32x4 $1, %xmm2, %ymm1, %ymm20 -; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3],xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,5,5,7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm13 = xmm2[0,1,2,3,6,5,7,7] -; AVX512DQ-NEXT: vinserti32x4 $1, %xmm13, %ymm5, %ymm18 -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[0,0,2,1,4,5,6,7] -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti32x4 $1, %xmm0, %ymm5, %ymm19 -; AVX512DQ-NEXT: vmovdqa 16(%rcx), %xmm15 -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm2[0,1,1,3,4,5,6,7] -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2 -; AVX512DQ-NEXT: vmovdqa 16(%rdx), %xmm3 -; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3],xmm3[4],xmm15[4],xmm3[5],xmm15[5],xmm3[6],xmm15[6],xmm3[7],xmm15[7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm13 = xmm5[0,1,2,3,4,4,6,5] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm9 = xmm5[0,1,2,3,4,6,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm13, %ymm9 -; AVX512DQ-NEXT: vpshufb %xmm4, %xmm5, %xmm5 -; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm26 -; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero -; AVX512DQ-NEXT: vmovdqa 16(%rsi), %xmm5 -; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm13 -; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3],xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7] -; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3,4],ymm1[5],ymm4[6,7,8],ymm1[9],ymm4[10,11,12],ymm1[13],ymm4[14,15] -; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm4 = ymm9[2,1,3,3,6,5,7,7] -; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3,4],ymm4[5],ymm0[6,7,8],ymm4[9],ymm0[10,11,12],ymm4[13],ymm0[14,15] -; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm17 -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm20[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm18[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7],ymm1[8,9,10],ymm0[11],ymm1[12,13,14],ymm0[15] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm19[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7],ymm2[8,9,10],ymm1[11],ymm2[12,13,14],ymm1[15] -; AVX512DQ-NEXT: vinserti32x8 $1, %ymm0, %zmm1, %zmm17 {%k1} -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm14[8],xmm12[8],xmm14[9],xmm12[9],xmm14[10],xmm12[10],xmm14[11],xmm12[11],xmm14[12],xmm12[12],xmm14[13],xmm12[13],xmm14[14],xmm12[14],xmm14[15],xmm12[15] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,5,5,7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,6,5,7,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm4, %ymm4 -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm0[0,0,2,1,4,5,6,7] +; AVX512DQ-NEXT: vinserti32x8 $1, %ymm0, %zmm5, %zmm16 {%k1} +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm0[0,1,2,3,4,4,6,5] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm12 = xmm0[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti32x4 $1, %xmm12, %ymm5, %ymm17 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3],xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm12 = xmm5[0,1,2,3,4,5,5,7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm13 = xmm5[0,1,2,3,6,5,7,7] +; AVX512DQ-NEXT: vinserti32x4 $1, %xmm13, %ymm12, %ymm18 +; AVX512DQ-NEXT: vmovdqa 16(%rcx), %xmm13 +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm12 = xmm0[0,0,2,1,4,5,6,7] ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm9, %ymm0 -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm2[0,1,1,3,4,5,6,7] -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm9, %ymm2 -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm15[8],xmm3[9],xmm15[9],xmm3[10],xmm15[10],xmm3[11],xmm15[11],xmm3[12],xmm15[12],xmm3[13],xmm15[13],xmm3[14],xmm15[14],xmm3[15],xmm15[15] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm9 = xmm3[0,1,2,3,4,4,6,5] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti32x4 $1, %xmm0, %ymm12, %ymm19 +; AVX512DQ-NEXT: vmovdqa 16(%rdx), %xmm15 +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,1,1,3,4,5,6,7] +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[2,1,3,3,4,5,6,7] +; AVX512DQ-NEXT: vinserti32x4 $1, %xmm5, %ymm0, %ymm20 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3],xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm5[0,1,2,3,4,4,6,5] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm12 = xmm5[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti32x4 $1, %xmm12, %ymm0, %ymm21 +; AVX512DQ-NEXT: vmovdqa 16(%rsi), %xmm1 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3] +; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero +; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm5 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm12 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7] +; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero,xmm12[2],zero,zero,zero,xmm12[3],zero,zero,zero +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3,4],ymm0[5],ymm2[6,7,8],ymm0[9],ymm2[10,11,12],ymm0[13],ymm2[14,15] +; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[2,3,2,3] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm12 = ymm21[2,1,3,3,6,5,7,7] +; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2,3,4],ymm12[5],ymm2[6,7,8],ymm12[9],ymm2[10,11,12],ymm12[13],ymm2[14,15] +; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm12 +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm17[0,2,2,3,4,6,6,7] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[0,2,2,3,4,6,6,7] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7],ymm2[8,9,10],ymm0[11],ymm2[12,13,14],ymm0[15] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm19[0,0,2,1,4,4,6,5] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm20[0,0,2,1,4,4,6,5] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7],ymm3[8,9,10],ymm2[11],ymm3[12,13,14],ymm2[15] +; AVX512DQ-NEXT: vinserti32x8 $1, %ymm0, %zmm2, %zmm12 {%k1} +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm14[8],xmm11[8],xmm14[9],xmm11[9],xmm14[10],xmm11[10],xmm14[11],xmm11[11],xmm14[12],xmm11[12],xmm14[13],xmm11[13],xmm14[14],xmm11[14],xmm14[15],xmm11[15] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm9 = xmm3[0,1,2,3,4,5,5,7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,6,5,7,7] ; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm9, %ymm9 -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7],ymm4[8,9,10],ymm1[11],ymm4[12,13,14],ymm1[15] -; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15] -; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero -; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm10 -; AVX512DQ-NEXT: vpshufb %xmm10, %xmm3, %xmm3 -; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3,4],ymm3[5],ymm5[6,7,8],ymm3[9],ymm5[10,11,12],ymm3[13],ymm5[14,15] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm5 = ymm9[2,1,3,3,6,5,7,7] -; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3] -; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3,4],ymm5[5],ymm4[6,7,8],ymm5[9],ymm4[10,11,12],ymm5[13],ymm4[14,15] -; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm0[0,0,2,1,4,5,6,7] +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm10, %ymm0 +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm3[0,1,1,3,4,5,6,7] +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,3,3,4,5,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm10, %ymm3 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm11 = xmm10[0,1,2,3,4,4,6,5] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm13 = xmm10[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm11, %ymm11 +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm9 = ymm9[0,2,2,3,4,6,6,7] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3],ymm9[4,5,6],ymm2[7],ymm9[8,9,10],ymm2[11],ymm9[12,13,14],ymm2[15] +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15] +; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0,0,1,1,2,2,3,3] +; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm9[1],ymm5[2,3,4],ymm9[5],ymm5[6,7,8],ymm9[9],ymm5[10,11,12],ymm9[13],ymm5[14,15] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm9 = ymm11[2,1,3,3,6,5,7,7] +; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1],ymm1[2,3,4],ymm9[5],ymm1[6,7,8],ymm9[9],ymm1[10,11,12],ymm9[13],ymm1[14,15] +; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm5, %zmm1 ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7],ymm2[8,9,10],ymm0[11],ymm2[12,13,14],ymm0[15] -; AVX512DQ-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1} -; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm0 -; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm1 -; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,4,6,5] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 -; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm2 -; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,5,5,7] -; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,6,5,7,7] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,0,2,1,4,4,6,5] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6],ymm0[7],ymm3[8,9,10],ymm0[11],ymm3[12,13,14],ymm0[15] +; AVX512DQ-NEXT: vinserti32x8 $1, %ymm2, %zmm0, %zmm1 {%k1} +; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm0 +; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm2 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,6,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm4 = xmm3[0,1,2,3,4,5,5,7] +; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,6,5,7,7] ; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4 ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[0,0,2,1,4,5,6,7] ; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] ; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm5, %ymm0 -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,1,1,3,4,5,6,7] -; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7] -; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2 +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm3[0,1,1,3,4,5,6,7] +; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,3,3,4,5,6,7] +; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm5, %ymm3 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,4,6,5] ; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm5[0,1,2,3,4,6,6,7] ; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm6 -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,2,2,3,4,6,6,7] +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7] ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm4 = ymm4[0,2,2,3,4,6,6,7] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7],ymm4[8,9,10],ymm1[11],ymm4[12,13,14],ymm1[15] -; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm4 -; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm7 +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7],ymm4[8,9,10],ymm2[11],ymm4[12,13,14],ymm2[15] +; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm4 +; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm7 ; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3],xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7] -; AVX512DQ-NEXT: vpshufb %xmm10, %xmm5, %xmm5 ; AVX512DQ-NEXT: vpmovzxwq {{.*#+}} ymm7 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3] ; AVX512DQ-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3,4],ymm5[5],ymm7[6,7,8],ymm5[9],ymm7[10,11,12],ymm5[13],ymm7[14,15] ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[2,1,3,3,6,5,7,7] @@ -3936,13 +3928,13 @@ define void @store_i8_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7,8],ymm6[9],ymm4[10,11,12],ymm6[13],ymm4[14,15] ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4 ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,0,2,1,4,4,6,5] -; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7],ymm2[8,9,10],ymm0[11],ymm2[12,13,14],ymm0[15] -; AVX512DQ-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm4 {%k1} +; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,0,2,1,4,4,6,5] +; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6],ymm0[7],ymm3[8,9,10],ymm0[11],ymm3[12,13,14],ymm0[15] +; AVX512DQ-NEXT: vinserti32x8 $1, %ymm2, %zmm0, %zmm4 {%k1} ; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-NEXT: vmovdqa64 %zmm4, (%rax) -; AVX512DQ-NEXT: vmovdqa64 %zmm3, 192(%rax) -; AVX512DQ-NEXT: vmovdqa64 %zmm17, 128(%rax) +; AVX512DQ-NEXT: vmovdqa64 %zmm1, 192(%rax) +; AVX512DQ-NEXT: vmovdqa64 %zmm12, 128(%rax) ; AVX512DQ-NEXT: vmovdqa64 %zmm16, 64(%rax) ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-mul.ll b/llvm/test/CodeGen/X86/vector-mul.ll index b5ac1ee706a54..13b21a747878b 100644 --- a/llvm/test/CodeGen/X86/vector-mul.ll +++ b/llvm/test/CodeGen/X86/vector-mul.ll @@ -1393,29 +1393,29 @@ define <2 x i64> @mul_v2i64_15_63(<2 x i64> %a0) nounwind { define <2 x i64> @mul_v2i64_neg_15_63(<2 x i64> %a0) nounwind { ; X86-SSE2-LABEL: mul_v2i64_neg_15_63: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [4294967281,4294967295,4294967233,4294967295] +; X86-SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1 ; X86-SSE2-NEXT: movdqa %xmm0, %xmm2 -; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2 -; X86-SSE2-NEXT: movdqa %xmm0, %xmm3 -; X86-SSE2-NEXT: psrlq $32, %xmm3 -; X86-SSE2-NEXT: pmuludq %xmm1, %xmm3 -; X86-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE2-NEXT: paddq %xmm3, %xmm0 -; X86-SSE2-NEXT: psllq $32, %xmm0 +; X86-SSE2-NEXT: psrlq $32, %xmm2 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [4294967281,4294967295,4294967233,4294967295] +; X86-SSE2-NEXT: pmuludq %xmm3, %xmm2 +; X86-SSE2-NEXT: paddq %xmm1, %xmm2 +; X86-SSE2-NEXT: psllq $32, %xmm2 +; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0 ; X86-SSE2-NEXT: paddq %xmm2, %xmm0 ; X86-SSE2-NEXT: retl ; ; X86-SSE4-LABEL: mul_v2i64_neg_15_63: ; X86-SSE4: # %bb.0: -; X86-SSE4-NEXT: pmovsxbq {{.*#+}} xmm1 = [18446744073709551601,18446744073709551553] +; X86-SSE4-NEXT: pcmpeqd %xmm1, %xmm1 +; X86-SSE4-NEXT: pmuludq %xmm0, %xmm1 ; X86-SSE4-NEXT: movdqa %xmm0, %xmm2 -; X86-SSE4-NEXT: pmuludq %xmm1, %xmm2 -; X86-SSE4-NEXT: movdqa %xmm0, %xmm3 -; X86-SSE4-NEXT: psrlq $32, %xmm3 -; X86-SSE4-NEXT: pmuludq %xmm1, %xmm3 -; X86-SSE4-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE4-NEXT: paddq %xmm3, %xmm0 -; X86-SSE4-NEXT: psllq $32, %xmm0 +; X86-SSE4-NEXT: psrlq $32, %xmm2 +; X86-SSE4-NEXT: pmovsxbq {{.*#+}} xmm3 = [18446744073709551601,18446744073709551553] +; X86-SSE4-NEXT: pmuludq %xmm3, %xmm2 +; X86-SSE4-NEXT: paddq %xmm1, %xmm2 +; X86-SSE4-NEXT: psllq $32, %xmm2 +; X86-SSE4-NEXT: pmuludq %xmm3, %xmm0 ; X86-SSE4-NEXT: paddq %xmm2, %xmm0 ; X86-SSE4-NEXT: retl ; @@ -1482,29 +1482,29 @@ define <2 x i64> @mul_v2i64_neg_15_63(<2 x i64> %a0) nounwind { define <2 x i64> @mul_v2i64_neg_17_65(<2 x i64> %a0) nounwind { ; X86-SSE2-LABEL: mul_v2i64_neg_17_65: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [4294967279,4294967295,4294967231,4294967295] +; X86-SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1 ; X86-SSE2-NEXT: movdqa %xmm0, %xmm2 -; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2 -; X86-SSE2-NEXT: movdqa %xmm0, %xmm3 -; X86-SSE2-NEXT: psrlq $32, %xmm3 -; X86-SSE2-NEXT: pmuludq %xmm1, %xmm3 -; X86-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE2-NEXT: paddq %xmm3, %xmm0 -; X86-SSE2-NEXT: psllq $32, %xmm0 +; X86-SSE2-NEXT: psrlq $32, %xmm2 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [4294967279,4294967295,4294967231,4294967295] +; X86-SSE2-NEXT: pmuludq %xmm3, %xmm2 +; X86-SSE2-NEXT: paddq %xmm1, %xmm2 +; X86-SSE2-NEXT: psllq $32, %xmm2 +; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0 ; X86-SSE2-NEXT: paddq %xmm2, %xmm0 ; X86-SSE2-NEXT: retl ; ; X86-SSE4-LABEL: mul_v2i64_neg_17_65: ; X86-SSE4: # %bb.0: -; X86-SSE4-NEXT: pmovsxbq {{.*#+}} xmm1 = [18446744073709551599,18446744073709551551] +; X86-SSE4-NEXT: pcmpeqd %xmm1, %xmm1 +; X86-SSE4-NEXT: pmuludq %xmm0, %xmm1 ; X86-SSE4-NEXT: movdqa %xmm0, %xmm2 -; X86-SSE4-NEXT: pmuludq %xmm1, %xmm2 -; X86-SSE4-NEXT: movdqa %xmm0, %xmm3 -; X86-SSE4-NEXT: psrlq $32, %xmm3 -; X86-SSE4-NEXT: pmuludq %xmm1, %xmm3 -; X86-SSE4-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE4-NEXT: paddq %xmm3, %xmm0 -; X86-SSE4-NEXT: psllq $32, %xmm0 +; X86-SSE4-NEXT: psrlq $32, %xmm2 +; X86-SSE4-NEXT: pmovsxbq {{.*#+}} xmm3 = [18446744073709551599,18446744073709551551] +; X86-SSE4-NEXT: pmuludq %xmm3, %xmm2 +; X86-SSE4-NEXT: paddq %xmm1, %xmm2 +; X86-SSE4-NEXT: psllq $32, %xmm2 +; X86-SSE4-NEXT: pmuludq %xmm3, %xmm0 ; X86-SSE4-NEXT: paddq %xmm2, %xmm0 ; X86-SSE4-NEXT: retl ; @@ -2027,8 +2027,8 @@ define <2 x i64> @mul_v2i64_zext_cross_bb(ptr %in, ptr %y) { ; X86-SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; X86-SSE2-NEXT: pxor %xmm1, %xmm1 ; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; X86-SSE2-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero -; X86-SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1,1,3] +; X86-SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0,0,1,1] ; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0 ; X86-SSE2-NEXT: retl ; diff --git a/llvm/test/CodeGen/X86/vector-pcmp.ll b/llvm/test/CodeGen/X86/vector-pcmp.ll index 084a1faa516fe..30eb2279bda85 100644 --- a/llvm/test/CodeGen/X86/vector-pcmp.ll +++ b/llvm/test/CodeGen/X86/vector-pcmp.ll @@ -473,10 +473,9 @@ define <2 x i64> @cmpgt_zext_v2i64(<2 x i64> %a, <2 x i64> %b) { ; SSE2-NEXT: pxor %xmm2, %xmm0 ; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] ; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; SSE2-NEXT: pand %xmm3, %xmm1 +; SSE2-NEXT: pand %xmm2, %xmm1 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] ; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll index b0ad27e3a790a..008e3e4c217cb 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-fmaximum.ll @@ -1087,13 +1087,11 @@ define double @test_v4f64(<4 x double> %a0) { ; ; SSE41-LABEL: test_v4f64: ; SSE41: # %bb.0: -; SSE41-NEXT: movaps %xmm0, %xmm2 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: movaps %xmm2, %xmm4 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm0, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm0, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm3 -; SSE41-NEXT: maxpd %xmm4, %xmm3 +; SSE41-NEXT: maxpd %xmm2, %xmm3 ; SSE41-NEXT: movapd %xmm1, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 @@ -1273,45 +1271,45 @@ define double @test_v8f64(<8 x double> %a0) { ; ; SSE41-LABEL: test_v8f64: ; SSE41: # %bb.0: -; SSE41-NEXT: movaps %xmm0, %xmm4 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: movaps %xmm1, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 +; SSE41-NEXT: movapd %xmm0, %xmm4 +; SSE41-NEXT: movapd %xmm1, %xmm6 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 -; SSE41-NEXT: movapd %xmm3, %xmm1 -; SSE41-NEXT: maxpd %xmm5, %xmm1 +; SSE41-NEXT: movapd %xmm3, %xmm5 +; SSE41-NEXT: maxpd %xmm6, %xmm5 ; SSE41-NEXT: movapd %xmm3, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: movaps %xmm4, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 +; SSE41-NEXT: movapd %xmm4, %xmm3 +; SSE41-NEXT: movapd %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm4 -; SSE41-NEXT: maxpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm2, %xmm1 +; SSE41-NEXT: maxpd %xmm3, %xmm1 ; SSE41-NEXT: movapd %xmm2, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm2, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: movapd %xmm4, %xmm2 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm1 -; SSE41-NEXT: movapd %xmm1, %xmm3 -; SSE41-NEXT: maxpd %xmm2, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: cmpunordpd %xmm1, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 -; SSE41-NEXT: movapd %xmm3, %xmm2 -; SSE41-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1] -; SSE41-NEXT: movq %xmm3, %rax +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 +; SSE41-NEXT: movapd %xmm5, %xmm1 +; SSE41-NEXT: maxpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm5, %xmm0 +; SSE41-NEXT: cmpunordpd %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm2 +; SSE41-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1] +; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: testq %rax, %rax ; SSE41-NEXT: js .LBB7_1 ; SSE41-NEXT: # %bb.2: -; SSE41-NEXT: movapd %xmm3, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm0 ; SSE41-NEXT: jmp .LBB7_3 ; SSE41-NEXT: .LBB7_1: ; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: .LBB7_3: ; SSE41-NEXT: movapd %xmm2, %xmm1 ; SSE41-NEXT: maxsd %xmm0, %xmm1 @@ -1570,81 +1568,82 @@ define double @test_v16f64(<16 x double> %a0) { ; ; SSE41-LABEL: test_v16f64: ; SSE41: # %bb.0: -; SSE41-NEXT: movaps %xmm0, %xmm8 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: movaps %xmm3, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9 +; SSE41-NEXT: movapd %xmm1, %xmm8 +; SSE41-NEXT: movapd %xmm0, %xmm1 +; SSE41-NEXT: movapd %xmm3, %xmm10 +; SSE41-NEXT: movapd %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm10 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 -; SSE41-NEXT: movapd %xmm7, %xmm3 -; SSE41-NEXT: maxpd %xmm9, %xmm3 +; SSE41-NEXT: movapd %xmm7, %xmm9 +; SSE41-NEXT: maxpd %xmm10, %xmm9 ; SSE41-NEXT: movapd %xmm7, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm7, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm3 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: movaps %xmm1, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9 +; SSE41-NEXT: movapd %xmm8, %xmm7 +; SSE41-NEXT: movapd %xmm8, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm1 -; SSE41-NEXT: maxpd %xmm7, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm5 +; SSE41-NEXT: movapd %xmm5, %xmm3 +; SSE41-NEXT: maxpd %xmm7, %xmm3 ; SSE41-NEXT: movapd %xmm5, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm5, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: movapd %xmm1, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 -; SSE41-NEXT: movapd %xmm3, %xmm1 -; SSE41-NEXT: maxpd %xmm5, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm3 +; SSE41-NEXT: movapd %xmm3, %xmm5 ; SSE41-NEXT: movapd %xmm3, %xmm0 -; SSE41-NEXT: cmpunordpd %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: movaps %xmm2, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm9 +; SSE41-NEXT: movapd %xmm9, %xmm3 +; SSE41-NEXT: maxpd %xmm5, %xmm3 +; SSE41-NEXT: movapd %xmm9, %xmm0 +; SSE41-NEXT: cmpunordpd %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm5 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 ; SSE41-NEXT: movapd %xmm6, %xmm2 -; SSE41-NEXT: maxpd %xmm3, %xmm2 +; SSE41-NEXT: maxpd %xmm5, %xmm2 ; SSE41-NEXT: movapd %xmm6, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm2 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: movaps %xmm8, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 -; SSE41-NEXT: movapd %xmm4, %xmm5 -; SSE41-NEXT: maxpd %xmm3, %xmm5 +; SSE41-NEXT: movapd %xmm1, %xmm5 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm1 +; SSE41-NEXT: maxpd %xmm5, %xmm1 ; SSE41-NEXT: movapd %xmm4, %xmm0 ; SSE41-NEXT: cmpunordpd %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm5 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: movapd %xmm5, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm4 -; SSE41-NEXT: maxpd %xmm3, %xmm4 -; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: cmpunordpd %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm4 +; SSE41-NEXT: movapd %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4 -; SSE41-NEXT: movshdup {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: movapd %xmm4, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm1 -; SSE41-NEXT: movapd %xmm1, %xmm3 -; SSE41-NEXT: maxpd %xmm2, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm1 +; SSE41-NEXT: maxpd %xmm4, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: cmpunordpd %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: cmpunordpd %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 -; SSE41-NEXT: movapd %xmm3, %xmm2 -; SSE41-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1] -; SSE41-NEXT: movq %xmm3, %rax +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: maxpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm3, %xmm0 +; SSE41-NEXT: cmpunordpd %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm2 +; SSE41-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1] +; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: testq %rax, %rax ; SSE41-NEXT: js .LBB8_1 ; SSE41-NEXT: # %bb.2: -; SSE41-NEXT: movapd %xmm3, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm0 ; SSE41-NEXT: jmp .LBB8_3 ; SSE41-NEXT: .LBB8_1: ; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: .LBB8_3: ; SSE41-NEXT: movapd %xmm2, %xmm1 ; SSE41-NEXT: maxsd %xmm0, %xmm1 diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll index 5e3ef32ef7e4a..f9b37a068862c 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll @@ -19,8 +19,7 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: psrlq $32, %xmm2 ; SSE-NEXT: pmuludq %xmm1, %xmm2 -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; SSE-NEXT: pmuludq %xmm0, %xmm3 ; SSE-NEXT: paddq %xmm2, %xmm3 ; SSE-NEXT: psllq $32, %xmm3 @@ -34,7 +33,7 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; AVX1OR2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX1OR2-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX1OR2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX1OR2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1OR2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX1OR2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX1OR2-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX1OR2-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -48,7 +47,7 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -62,7 +61,7 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -107,8 +106,7 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: psrlq $32, %xmm2 ; SSE-NEXT: pmuludq %xmm1, %xmm2 -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; SSE-NEXT: pmuludq %xmm0, %xmm3 ; SSE-NEXT: paddq %xmm2, %xmm3 ; SSE-NEXT: psllq $32, %xmm3 @@ -131,7 +129,7 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -155,7 +153,7 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -179,7 +177,7 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -203,7 +201,7 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -274,8 +272,7 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: psrlq $32, %xmm2 ; SSE-NEXT: pmuludq %xmm1, %xmm2 -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; SSE-NEXT: pmuludq %xmm0, %xmm3 ; SSE-NEXT: paddq %xmm2, %xmm3 ; SSE-NEXT: psllq $32, %xmm3 @@ -315,7 +312,7 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -347,7 +344,7 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -380,7 +377,7 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -413,7 +410,7 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -527,8 +524,7 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: psrlq $32, %xmm2 ; SSE-NEXT: pmuludq %xmm1, %xmm2 -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; SSE-NEXT: pmuludq %xmm0, %xmm3 ; SSE-NEXT: paddq %xmm2, %xmm3 ; SSE-NEXT: psllq $32, %xmm3 @@ -602,7 +598,7 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX1-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -650,7 +646,7 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX2-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -691,7 +687,7 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -732,7 +728,7 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2 ; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2 -; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] ; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 ; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2 ; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2 @@ -808,10 +804,8 @@ define i32 @test_v4i32(<4 x i32> %a0) { ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] ; SSE2-NEXT: pmuludq %xmm2, %xmm3 ; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0] -; SSE2-NEXT: pmuludq %xmm1, %xmm0 -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pmuludq %xmm3, %xmm1 +; SSE2-NEXT: movd %xmm1, %eax ; SSE2-NEXT: retq ; ; SSE41-LABEL: test_v4i32: @@ -838,21 +832,15 @@ define i32 @test_v4i32(<4 x i32> %a0) { define i32 @test_v8i32(<8 x i32> %a0) { ; SSE2-LABEL: test_v8i32: ; SSE2: # %bb.0: -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm2, %xmm3 ; SSE2-NEXT: pmuludq %xmm1, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm2, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3] -; SSE2-NEXT: pmuludq %xmm0, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,2,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,0,0] -; SSE2-NEXT: pmuludq %xmm2, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3] +; SSE2-NEXT: pmuludq %xmm3, %xmm0 +; SSE2-NEXT: pmuludq %xmm1, %xmm0 ; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: retq ; @@ -910,32 +898,18 @@ define i32 @test_v16i32(<16 x i32> %a0) { ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3] ; SSE2-NEXT: pmuludq %xmm3, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm4, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] ; SSE2-NEXT: pmuludq %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm4, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1] ; SSE2-NEXT: pmuludq %xmm1, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm4, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm5, %xmm2 ; SSE2-NEXT: pmuludq %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2] -; SSE2-NEXT: pmuludq %xmm0, %xmm2 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] +; SSE2-NEXT: pmuludq %xmm2, %xmm0 ; SSE2-NEXT: pmuludq %xmm1, %xmm0 ; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: retq @@ -1000,64 +974,34 @@ define i32 @test_v16i32(<16 x i32> %a0) { define i32 @test_v32i32(<32 x i32> %a0) { ; SSE2-LABEL: test_v32i32: ; SSE2: # %bb.0: -; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm2[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm8, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm8, %xmm10 +; SSE2-NEXT: pmuludq %xmm9, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm8, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm1[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm8, %xmm11 +; SSE2-NEXT: pmuludq %xmm9, %xmm11 +; SSE2-NEXT: pmuludq %xmm10, %xmm11 ; SSE2-NEXT: pmuludq %xmm6, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm2[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm8, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] ; SSE2-NEXT: pmuludq %xmm4, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm8, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1] -; SSE2-NEXT: pmuludq %xmm9, %xmm6 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm2, %xmm0 ; SSE2-NEXT: pmuludq %xmm7, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm4, %xmm7 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3] ; SSE2-NEXT: pmuludq %xmm5, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm4, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] ; SSE2-NEXT: pmuludq %xmm3, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2] -; SSE2-NEXT: pmuludq %xmm3, %xmm4 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm6[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] -; SSE2-NEXT: pmuludq %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE2-NEXT: pmuludq %xmm1, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] -; SSE2-NEXT: pmuludq %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] -; SSE2-NEXT: pmuludq %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] +; SSE2-NEXT: pmuludq %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3] ; SSE2-NEXT: pmuludq %xmm1, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] -; SSE2-NEXT: pmuludq %xmm2, %xmm0 -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm11[2,3,2,3] +; SSE2-NEXT: pmuludq %xmm11, %xmm1 +; SSE2-NEXT: pmuludq %xmm0, %xmm1 +; SSE2-NEXT: movd %xmm1, %eax ; SSE2-NEXT: retq ; ; SSE41-LABEL: test_v32i32: diff --git a/llvm/test/CodeGen/X86/vector-reduce-smax.ll b/llvm/test/CodeGen/X86/vector-reduce-smax.ll index 95aea6f524023..322fdde106dcf 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-smax.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-smax.ll @@ -42,12 +42,10 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; SSE41-NEXT: pxor %xmm0, %xmm3 ; SSE41-NEXT: pxor %xmm2, %xmm0 ; SSE41-NEXT: movdqa %xmm3, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pmovsxdq %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax @@ -135,26 +133,22 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm2, %xmm4 ; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm1, %xmm4 +; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: pmovsxdq %xmm4, %xmm0 +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax ; SSE41-NEXT: retq @@ -295,12 +289,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm4, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 @@ -308,12 +300,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm1, %xmm4 ; SSE41-NEXT: pxor %xmm5, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm3, %xmm0 @@ -321,26 +311,22 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movapd %xmm2, %xmm1 ; SSE41-NEXT: xorpd %xmm5, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3] -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: pxor %xmm5, %xmm2 ; SSE41-NEXT: pxor %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 ; SSE41-NEXT: pcmpgtd %xmm5, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm4 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pmovsxdq %xmm2, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: retq @@ -558,12 +544,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm1, %xmm10 ; SSE41-NEXT: pxor %xmm9, %xmm10 ; SSE41-NEXT: movdqa %xmm10, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm11 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 ; SSE41-NEXT: por %xmm10, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm7, %xmm0 @@ -571,12 +555,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm3, %xmm1 ; SSE41-NEXT: pxor %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm11, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 ; SSE41-NEXT: movdqa %xmm4, %xmm0 @@ -584,12 +566,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm8, %xmm1 ; SSE41-NEXT: pxor %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm10, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 ; SSE41-NEXT: movdqa %xmm6, %xmm0 @@ -597,12 +577,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm2, %xmm1 ; SSE41-NEXT: pxor %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 ; SSE41-NEXT: movapd %xmm6, %xmm0 @@ -610,12 +588,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm4, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 ; SSE41-NEXT: movapd %xmm7, %xmm0 @@ -623,12 +599,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm5, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 ; SSE41-NEXT: movapd %xmm7, %xmm0 @@ -636,26 +610,22 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm6, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3] -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: movdqa %xmm7, %xmm2 +; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: pxor %xmm1, %xmm9 -; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 ; SSE41-NEXT: pcmpgtd %xmm9, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: pmovsxdq %xmm2, %xmm0 +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll index f86d3c1dd6e3f..bb87740c21538 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll @@ -37,17 +37,14 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pmovsxdq %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax @@ -135,12 +132,10 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm1, %xmm4 ; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] @@ -148,12 +143,10 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE41-NEXT: pxor %xmm3, %xmm0 ; SSE41-NEXT: pxor %xmm2, %xmm3 ; SSE41-NEXT: movdqa %xmm3, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pmovsxdq %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax @@ -296,12 +289,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm3, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movdqa %xmm4, %xmm0 @@ -309,12 +300,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm2, %xmm1 ; SSE41-NEXT: pxor %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -322,12 +311,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movapd %xmm3, %xmm1 ; SSE41-NEXT: xorpd %xmm5, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3] @@ -335,12 +322,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm4 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pmovsxdq %xmm5, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax @@ -559,12 +544,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm6, %xmm10 ; SSE41-NEXT: pxor %xmm9, %xmm10 ; SSE41-NEXT: movdqa %xmm10, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm11 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 ; SSE41-NEXT: por %xmm10, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 ; SSE41-NEXT: movdqa %xmm8, %xmm0 @@ -572,12 +555,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm4, %xmm2 ; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm11, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 ; SSE41-NEXT: movdqa %xmm3, %xmm0 @@ -585,12 +566,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm7, %xmm2 ; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm10, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 ; SSE41-NEXT: movdqa %xmm1, %xmm0 @@ -598,12 +577,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm5, %xmm2 ; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 ; SSE41-NEXT: movapd %xmm5, %xmm0 @@ -611,12 +588,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm7, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 ; SSE41-NEXT: movapd %xmm4, %xmm0 @@ -624,12 +599,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm6, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 ; SSE41-NEXT: movapd %xmm6, %xmm0 @@ -637,12 +610,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm7, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3] @@ -650,13 +621,11 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: pxor %xmm9, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm9 ; SSE41-NEXT: movdqa %xmm9, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm9 +; SSE41-NEXT: pmovsxdq %xmm9, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-reduce-umax.ll b/llvm/test/CodeGen/X86/vector-reduce-umax.ll index 2f3c1e09ea78e..b355c3dee5309 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-umax.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-umax.ll @@ -42,12 +42,10 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; SSE41-NEXT: pxor %xmm0, %xmm3 ; SSE41-NEXT: pxor %xmm2, %xmm0 ; SSE41-NEXT: movdqa %xmm3, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pmovsxdq %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax @@ -153,26 +151,22 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm2, %xmm4 ; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm1, %xmm4 +; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: pmovsxdq %xmm4, %xmm0 +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax ; SSE41-NEXT: retq @@ -330,12 +324,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm4, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movdqa %xmm3, %xmm0 @@ -343,12 +335,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm1, %xmm4 ; SSE41-NEXT: pxor %xmm5, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movapd %xmm3, %xmm0 @@ -356,26 +346,22 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movapd %xmm2, %xmm1 ; SSE41-NEXT: xorpd %xmm5, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3] -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: pxor %xmm5, %xmm2 ; SSE41-NEXT: pxor %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 ; SSE41-NEXT: pcmpgtd %xmm5, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm4 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pmovsxdq %xmm2, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: retq @@ -621,12 +607,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm1, %xmm10 ; SSE41-NEXT: pxor %xmm9, %xmm10 ; SSE41-NEXT: movdqa %xmm10, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm11 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 ; SSE41-NEXT: por %xmm10, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm7, %xmm0 @@ -634,12 +618,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm3, %xmm1 ; SSE41-NEXT: pxor %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm11, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 ; SSE41-NEXT: movdqa %xmm4, %xmm0 @@ -647,12 +629,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm8, %xmm1 ; SSE41-NEXT: pxor %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm10, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 ; SSE41-NEXT: movdqa %xmm6, %xmm0 @@ -660,12 +640,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm2, %xmm1 ; SSE41-NEXT: pxor %xmm9, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 ; SSE41-NEXT: movapd %xmm6, %xmm0 @@ -673,12 +651,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm4, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 ; SSE41-NEXT: movapd %xmm7, %xmm0 @@ -686,12 +662,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm5, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 ; SSE41-NEXT: movapd %xmm7, %xmm0 @@ -699,26 +673,22 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm6, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3] -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: movdqa %xmm7, %xmm2 +; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: pxor %xmm1, %xmm9 -; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 ; SSE41-NEXT: pcmpgtd %xmm9, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: pmovsxdq %xmm2, %xmm0 +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-reduce-umin.ll b/llvm/test/CodeGen/X86/vector-reduce-umin.ll index dc8d0d53a91a1..2d68cf9d6374d 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-umin.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-umin.ll @@ -37,17 +37,14 @@ define i64 @test_v2i64(<2 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm3 -; SSE41-NEXT: pxor %xmm0, %xmm3 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pmovsxdq %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax @@ -153,12 +150,10 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm1, %xmm4 ; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: movdqa %xmm4, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] @@ -166,12 +161,10 @@ define i64 @test_v4i64(<4 x i64> %a0) { ; SSE41-NEXT: pxor %xmm3, %xmm0 ; SSE41-NEXT: pxor %xmm2, %xmm3 ; SSE41-NEXT: movdqa %xmm3, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pmovsxdq %xmm4, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pmovsxdq %xmm3, %xmm0 +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movq %xmm2, %rax @@ -332,12 +325,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm3, %xmm6 ; SSE41-NEXT: pxor %xmm5, %xmm6 ; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3 ; SSE41-NEXT: movdqa %xmm4, %xmm0 @@ -345,12 +336,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm2, %xmm1 ; SSE41-NEXT: pxor %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: movapd %xmm2, %xmm0 @@ -358,12 +347,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: movapd %xmm3, %xmm1 ; SSE41-NEXT: xorpd %xmm5, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm6, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3] @@ -371,12 +358,10 @@ define i64 @test_v8i64(<8 x i64> %a0) { ; SSE41-NEXT: pxor %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm5 ; SSE41-NEXT: movdqa %xmm5, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm4 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE41-NEXT: pmovsxdq %xmm5, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax @@ -625,12 +610,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm6, %xmm10 ; SSE41-NEXT: pxor %xmm9, %xmm10 ; SSE41-NEXT: movdqa %xmm10, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] -; SSE41-NEXT: pand %xmm12, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm11 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 ; SSE41-NEXT: por %xmm10, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 ; SSE41-NEXT: movdqa %xmm8, %xmm0 @@ -638,12 +621,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm4, %xmm2 ; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm11, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm10 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 ; SSE41-NEXT: movdqa %xmm3, %xmm0 @@ -651,12 +632,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm7, %xmm2 ; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm10, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 ; SSE41-NEXT: movdqa %xmm1, %xmm0 @@ -664,12 +643,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm5, %xmm2 ; SSE41-NEXT: pxor %xmm9, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 ; SSE41-NEXT: movapd %xmm5, %xmm0 @@ -677,12 +654,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm7, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 ; SSE41-NEXT: movapd %xmm4, %xmm0 @@ -690,12 +665,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm6, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 ; SSE41-NEXT: movapd %xmm6, %xmm0 @@ -703,12 +676,10 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: movapd %xmm7, %xmm1 ; SSE41-NEXT: xorpd %xmm9, %xmm1 ; SSE41-NEXT: movapd %xmm1, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3] @@ -716,13 +687,11 @@ define i64 @test_v16i64(<16 x i64> %a0) { ; SSE41-NEXT: pxor %xmm9, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm9 ; SSE41-NEXT: movdqa %xmm9, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pmovsxdq %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm9 +; SSE41-NEXT: pmovsxdq %xmm9, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 ; SSE41-NEXT: movq %xmm1, %rax ; SSE41-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll index 700e97821cd88..6c79be75550ed 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-128.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll @@ -236,22 +236,21 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: var_rotate_v8i16: ; SSE2: # %bb.0: ; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: movdqa %xmm1, %xmm3 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE2-NEXT: pslld $23, %xmm3 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm3 -; SSE2-NEXT: cvttps2dq %xmm3, %xmm3 -; SSE2-NEXT: pslld $16, %xmm3 -; SSE2-NEXT: psrad $16, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 +; SSE2-NEXT: pslld $16, %xmm2 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm1 -; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: paddd %xmm3, %xmm1 ; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: packssdw %xmm3, %xmm1 +; SSE2-NEXT: packssdw %xmm2, %xmm1 ; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: pmulhuw %xmm1, %xmm2 ; SSE2-NEXT: pmullw %xmm1, %xmm0 @@ -261,16 +260,15 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE41-LABEL: var_rotate_v8i16: ; SSE41: # %bb.0: ; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm1 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm3, %xmm1 ; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE41-NEXT: pslld $23, %xmm3 -; SSE41-NEXT: paddd %xmm2, %xmm3 -; SSE41-NEXT: cvttps2dq %xmm3, %xmm2 +; SSE41-NEXT: pslld $23, %xmm2 +; SSE41-NEXT: paddd %xmm3, %xmm2 +; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: packusdw %xmm1, %xmm2 ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: pmulhuw %xmm2, %xmm1 @@ -281,8 +279,7 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; AVX1-LABEL: var_rotate_v8i16: ; AVX1: # %bb.0: ; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 @@ -386,18 +383,17 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; X86-SSE2-LABEL: var_rotate_v8i16: ; X86-SSE2: # %bb.0: ; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 -; X86-SSE2-NEXT: pxor %xmm3, %xmm3 ; X86-SSE2-NEXT: movdqa %xmm1, %xmm2 -; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; X86-SSE2-NEXT: pslld $23, %xmm2 -; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE2-NEXT: paddd %xmm4, %xmm2 +; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE2-NEXT: paddd %xmm3, %xmm2 ; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE2-NEXT: pslld $16, %xmm2 ; X86-SSE2-NEXT: psrad $16, %xmm2 -; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; X86-SSE2-NEXT: pslld $23, %xmm1 -; X86-SSE2-NEXT: paddd %xmm4, %xmm1 +; X86-SSE2-NEXT: paddd %xmm3, %xmm1 ; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE2-NEXT: pslld $16, %xmm1 ; X86-SSE2-NEXT: psrad $16, %xmm1 @@ -933,32 +929,18 @@ define <8 x i16> @splatvar_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { } define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { -; SSE2-LABEL: splatvar_rotate_v16i8: -; SSE2: # %bb.0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: psllw %xmm1, %xmm2 -; SSE2-NEXT: psrlw $8, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: psllw %xmm1, %xmm0 -; SSE2-NEXT: psrlw $8, %xmm0 -; SSE2-NEXT: packuswb %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; SSE41-LABEL: splatvar_rotate_v16i8: -; SSE41: # %bb.0: -; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; SSE41-NEXT: psllw %xmm1, %xmm2 -; SSE41-NEXT: psrlw $8, %xmm2 -; SSE41-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE41-NEXT: psllw %xmm1, %xmm0 -; SSE41-NEXT: psrlw $8, %xmm0 -; SSE41-NEXT: packuswb %xmm2, %xmm0 -; SSE41-NEXT: retq +; SSE-LABEL: splatvar_rotate_v16i8: +; SSE: # %bb.0: +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] +; SSE-NEXT: psllw %xmm1, %xmm2 +; SSE-NEXT: psrlw $8, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: psllw %xmm1, %xmm0 +; SSE-NEXT: psrlw $8, %xmm0 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: retq ; ; AVX-LABEL: splatvar_rotate_v16i8: ; AVX: # %bb.0: @@ -999,10 +981,9 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; ; X86-SSE2-LABEL: splatvar_rotate_v16i8: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: movdqa %xmm0, %xmm2 ; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] -; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 ; X86-SSE2-NEXT: psllw %xmm1, %xmm2 ; X86-SSE2-NEXT: psrlw $8, %xmm2 ; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll index 0d3a00d64dc1f..684721f434ebd 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-256.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll @@ -175,29 +175,28 @@ define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15] ; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 -; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] -; AVX1-NEXT: vpslld $23, %xmm5, %xmm5 -; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 -; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; AVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 ; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; AVX1-NEXT: vpackusdw %xmm5, %xmm2, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 -; AVX1-NEXT: vpmulhuw %xmm2, %xmm5, %xmm7 -; AVX1-NEXT: vpmullw %xmm2, %xmm5, %xmm2 -; AVX1-NEXT: vpor %xmm7, %xmm2, %xmm2 +; AVX1-NEXT: vpackusdw %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vpmulhuw %xmm2, %xmm4, %xmm6 +; AVX1-NEXT: vpmullw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpor %xmm6, %xmm2, %xmm2 ; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 -; AVX1-NEXT: vpaddd %xmm6, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm5, %xmm3, %xmm3 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero ; AVX1-NEXT: vpslld $23, %xmm1, %xmm1 -; AVX1-NEXT: vpaddd %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm5, %xmm1, %xmm1 ; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1 ; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vpmulhuw %xmm1, %xmm0, %xmm3 diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll index 85f38deed700d..2b1cf5b671e53 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll @@ -157,45 +157,42 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: var_shift_v8i16: ; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: movdqa %xmm1, %xmm3 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE2-NEXT: pslld $23, %xmm3 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm3 -; SSE2-NEXT: cvttps2dq %xmm3, %xmm3 -; SSE2-NEXT: pslld $16, %xmm3 -; SSE2-NEXT: psrad $16, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 +; SSE2-NEXT: pslld $16, %xmm2 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm1 -; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: paddd %xmm3, %xmm1 ; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: packssdw %xmm3, %xmm1 +; SSE2-NEXT: packssdw %xmm2, %xmm1 ; SSE2-NEXT: pmullw %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; SSE41-LABEL: var_shift_v8i16: ; SSE41: # %bb.0: -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm1 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm3, %xmm1 ; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE41-NEXT: pslld $23, %xmm3 -; SSE41-NEXT: paddd %xmm2, %xmm3 -; SSE41-NEXT: cvttps2dq %xmm3, %xmm2 +; SSE41-NEXT: pslld $23, %xmm2 +; SSE41-NEXT: paddd %xmm3, %xmm2 +; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: packusdw %xmm1, %xmm2 ; SSE41-NEXT: pmullw %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: var_shift_v8i16: ; AVX1: # %bb.0: -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 @@ -259,18 +256,17 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; ; X86-SSE-LABEL: var_shift_v8i16: ; X86-SSE: # %bb.0: -; X86-SSE-NEXT: pxor %xmm3, %xmm3 ; X86-SSE-NEXT: movdqa %xmm1, %xmm2 -; X86-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; X86-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; X86-SSE-NEXT: pslld $23, %xmm2 -; X86-SSE-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE-NEXT: paddd %xmm4, %xmm2 +; X86-SSE-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE-NEXT: paddd %xmm3, %xmm2 ; X86-SSE-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE-NEXT: pslld $16, %xmm2 ; X86-SSE-NEXT: psrad $16, %xmm2 -; X86-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; X86-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; X86-SSE-NEXT: pslld $23, %xmm1 -; X86-SSE-NEXT: paddd %xmm4, %xmm1 +; X86-SSE-NEXT: paddd %xmm3, %xmm1 ; X86-SSE-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE-NEXT: pslld $16, %xmm1 ; X86-SSE-NEXT: psrad $16, %xmm1 diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll index 902672c96180e..b56a8b5b2294c 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll @@ -158,26 +158,25 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: var_shift_v16i16: ; AVX1: # %bb.0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 -; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] -; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 -; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] -; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 -; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 -; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2 ; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2 -; AVX1-NEXT: vpackusdw %xmm4, %xmm2, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 -; AVX1-NEXT: vpmullw %xmm2, %xmm4, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmullw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm3, %xmm3 -; AVX1-NEXT: vpaddd %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 ; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 ; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero ; AVX1-NEXT: vpslld $23, %xmm1, %xmm1 -; AVX1-NEXT: vpaddd %xmm5, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm4, %xmm1, %xmm1 ; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1 ; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0 @@ -248,20 +247,19 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; X86-AVX1-LABEL: var_shift_v16i16: ; X86-AVX1: # %bb.0: ; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 -; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4 -; X86-AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] -; X86-AVX1-NEXT: vpslld $23, %xmm2, %xmm5 +; X86-AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4,4,5,5,6,6,7,7] +; X86-AVX1-NEXT: vpslld $23, %xmm2, %xmm4 ; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; X86-AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm5 -; X86-AVX1-NEXT: vcvttps2dq %xmm5, %xmm5 +; X86-AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm4 +; X86-AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 ; X86-AVX1-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero ; X86-AVX1-NEXT: vpslld $23, %xmm3, %xmm3 ; X86-AVX1-NEXT: vpaddd %xmm2, %xmm3, %xmm3 ; X86-AVX1-NEXT: vcvttps2dq %xmm3, %xmm3 -; X86-AVX1-NEXT: vpackusdw %xmm5, %xmm3, %xmm3 -; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 -; X86-AVX1-NEXT: vpmullw %xmm3, %xmm5, %xmm3 -; X86-AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; X86-AVX1-NEXT: vpackusdw %xmm4, %xmm3, %xmm3 +; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; X86-AVX1-NEXT: vpmullw %xmm3, %xmm4, %xmm3 +; X86-AVX1-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4,4,5,5,6,6,7,7] ; X86-AVX1-NEXT: vpslld $23, %xmm4, %xmm4 ; X86-AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm4 ; X86-AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll index 3996d7f09f01c..d245bdca6ee29 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll @@ -93,45 +93,42 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind { define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind { ; SSE2-LABEL: var_shift_v4i16: ; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: movdqa %xmm1, %xmm3 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE2-NEXT: pslld $23, %xmm3 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm3 -; SSE2-NEXT: cvttps2dq %xmm3, %xmm3 -; SSE2-NEXT: pslld $16, %xmm3 -; SSE2-NEXT: psrad $16, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 +; SSE2-NEXT: pslld $16, %xmm2 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm1 -; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: paddd %xmm3, %xmm1 ; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: packssdw %xmm3, %xmm1 +; SSE2-NEXT: packssdw %xmm2, %xmm1 ; SSE2-NEXT: pmullw %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; SSE41-LABEL: var_shift_v4i16: ; SSE41: # %bb.0: -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm1 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm3, %xmm1 ; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE41-NEXT: pslld $23, %xmm3 -; SSE41-NEXT: paddd %xmm2, %xmm3 -; SSE41-NEXT: cvttps2dq %xmm3, %xmm2 +; SSE41-NEXT: pslld $23, %xmm2 +; SSE41-NEXT: paddd %xmm3, %xmm2 +; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: packusdw %xmm1, %xmm2 ; SSE41-NEXT: pmullw %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: var_shift_v4i16: ; AVX1: # %bb.0: -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 @@ -195,18 +192,17 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind { ; ; X86-SSE-LABEL: var_shift_v4i16: ; X86-SSE: # %bb.0: -; X86-SSE-NEXT: pxor %xmm3, %xmm3 ; X86-SSE-NEXT: movdqa %xmm1, %xmm2 -; X86-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; X86-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; X86-SSE-NEXT: pslld $23, %xmm2 -; X86-SSE-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE-NEXT: paddd %xmm4, %xmm2 +; X86-SSE-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE-NEXT: paddd %xmm3, %xmm2 ; X86-SSE-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE-NEXT: pslld $16, %xmm2 ; X86-SSE-NEXT: psrad $16, %xmm2 -; X86-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; X86-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; X86-SSE-NEXT: pslld $23, %xmm1 -; X86-SSE-NEXT: paddd %xmm4, %xmm1 +; X86-SSE-NEXT: paddd %xmm3, %xmm1 ; X86-SSE-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE-NEXT: pslld $16, %xmm1 ; X86-SSE-NEXT: psrad $16, %xmm1 @@ -220,45 +216,42 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind { define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind { ; SSE2-LABEL: var_shift_v2i16: ; SSE2: # %bb.0: -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: movdqa %xmm1, %xmm3 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE2-NEXT: pslld $23, %xmm3 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; SSE2-NEXT: paddd %xmm4, %xmm3 -; SSE2-NEXT: cvttps2dq %xmm3, %xmm3 -; SSE2-NEXT: pslld $16, %xmm3 -; SSE2-NEXT: psrad $16, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] +; SSE2-NEXT: pslld $23, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: cvttps2dq %xmm2, %xmm2 +; SSE2-NEXT: pslld $16, %xmm2 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; SSE2-NEXT: pslld $23, %xmm1 -; SSE2-NEXT: paddd %xmm4, %xmm1 +; SSE2-NEXT: paddd %xmm3, %xmm1 ; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 ; SSE2-NEXT: pslld $16, %xmm1 ; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: packssdw %xmm3, %xmm1 +; SSE2-NEXT: packssdw %xmm2, %xmm1 ; SSE2-NEXT: pmullw %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; SSE41-LABEL: var_shift_v2i16: ; SSE41: # %bb.0: -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] ; SSE41-NEXT: pslld $23, %xmm1 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216] -; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; SSE41-NEXT: paddd %xmm3, %xmm1 ; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE41-NEXT: pslld $23, %xmm3 -; SSE41-NEXT: paddd %xmm2, %xmm3 -; SSE41-NEXT: cvttps2dq %xmm3, %xmm2 +; SSE41-NEXT: pslld $23, %xmm2 +; SSE41-NEXT: paddd %xmm3, %xmm2 +; SSE41-NEXT: cvttps2dq %xmm2, %xmm2 ; SSE41-NEXT: packusdw %xmm1, %xmm2 ; SSE41-NEXT: pmullw %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: var_shift_v2i16: ; AVX1: # %bb.0: -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpslld $23, %xmm2, %xmm2 ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] ; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2 @@ -322,18 +315,17 @@ define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind { ; ; X86-SSE-LABEL: var_shift_v2i16: ; X86-SSE: # %bb.0: -; X86-SSE-NEXT: pxor %xmm3, %xmm3 ; X86-SSE-NEXT: movdqa %xmm1, %xmm2 -; X86-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] +; X86-SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] ; X86-SSE-NEXT: pslld $23, %xmm2 -; X86-SSE-NEXT: movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216] -; X86-SSE-NEXT: paddd %xmm4, %xmm2 +; X86-SSE-NEXT: movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216] +; X86-SSE-NEXT: paddd %xmm3, %xmm2 ; X86-SSE-NEXT: cvttps2dq %xmm2, %xmm2 ; X86-SSE-NEXT: pslld $16, %xmm2 ; X86-SSE-NEXT: psrad $16, %xmm2 -; X86-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; X86-SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] ; X86-SSE-NEXT: pslld $23, %xmm1 -; X86-SSE-NEXT: paddd %xmm4, %xmm1 +; X86-SSE-NEXT: paddd %xmm3, %xmm1 ; X86-SSE-NEXT: cvttps2dq %xmm1, %xmm1 ; X86-SSE-NEXT: pslld $16, %xmm1 ; X86-SSE-NEXT: psrad $16, %xmm1 diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll index de6e79550e869..2df013d0ff3e3 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll @@ -377,9 +377,9 @@ define void @PR39483() { ; X86-AVX1-NEXT: vmovups 64, %ymm1 ; X86-AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm1[2,3,0,1] ; X86-AVX1-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,1],ymm1[0,3],ymm2[4,5],ymm1[4,7] -; X86-AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7] -; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3 -; X86-AVX1-NEXT: vshufps {{.*#+}} ymm2 = ymm3[1,0],ymm2[2,0],ymm3[5,4],ymm2[6,4] +; X86-AVX1-NEXT: vmovups 16, %xmm2 +; X86-AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7] +; X86-AVX1-NEXT: vshufps {{.*#+}} ymm2 = ymm2[1,0],ymm3[2,0],ymm2[5,4],ymm3[6,4] ; X86-AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[0,3],ymm2[6,4],ymm0[4,7] ; X86-AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; X86-AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1 @@ -417,9 +417,9 @@ define void @PR39483() { ; X64-AVX1-NEXT: vmovups 64, %ymm1 ; X64-AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm1[2,3,0,1] ; X64-AVX1-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,1],ymm1[0,3],ymm2[4,5],ymm1[4,7] -; X64-AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7] -; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3 -; X64-AVX1-NEXT: vshufps {{.*#+}} ymm2 = ymm3[1,0],ymm2[2,0],ymm3[5,4],ymm2[6,4] +; X64-AVX1-NEXT: vmovups 16, %xmm2 +; X64-AVX1-NEXT: vblendps {{.*#+}} ymm3 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7] +; X64-AVX1-NEXT: vshufps {{.*#+}} ymm2 = ymm2[1,0],ymm3[2,0],ymm2[5,4],ymm3[6,4] ; X64-AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[0,3],ymm2[6,4],ymm0[4,7] ; X64-AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; X64-AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1 diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll index 37de6b6f160f5..bd2710139d584 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll @@ -846,20 +846,23 @@ define <16 x i8> @constant_fold_pshufb_2() { define i32 @mask_zzz3_v16i8(<16 x i8> %a0) { ; SSSE3-LABEL: mask_zzz3_v16i8: ; SSSE3: # %bb.0: -; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = zero,zero,zero,xmm0[14,u,u,u,u,u,u,u,u,u,u,u,u] +; SSSE3-NEXT: psrldq {{.*#+}} xmm0 = xmm0[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; SSSE3-NEXT: movd %xmm0, %eax +; SSSE3-NEXT: andl $-16777216, %eax # imm = 0xFF000000 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: mask_zzz3_v16i8: ; SSE41: # %bb.0: -; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[14] +; SSE41-NEXT: psllw $8, %xmm0 ; SSE41-NEXT: pextrd $3, %xmm0, %eax +; SSE41-NEXT: andl $-16777216, %eax # imm = 0xFF000000 ; SSE41-NEXT: retq ; ; AVX-LABEL: mask_zzz3_v16i8: ; AVX: # %bb.0: -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[14] +; AVX-NEXT: vpsllw $8, %xmm0, %xmm0 ; AVX-NEXT: vpextrd $3, %xmm0, %eax +; AVX-NEXT: andl $-16777216, %eax # imm = 0xFF000000 ; AVX-NEXT: retq %1 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> ) %2 = bitcast <16 x i8> %1 to <4 x i32> diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll index b18f2dc1324c7..5b61de5a3b772 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll @@ -2854,16 +2854,13 @@ define <4 x float> @PR30264(<4 x float> %x) { define <8 x i16> @PR39549(<16 x i8> %x) { ; SSE-LABEL: PR39549: ; SSE: # %bb.0: -; SSE-NEXT: pxor %xmm1, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] -; SSE-NEXT: psraw $8, %xmm1 -; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; SSE-NEXT: psraw $8, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: PR39549: ; AVX: # %bb.0: -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; AVX-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] ; AVX-NEXT: vpsraw $8, %xmm0, %xmm0 ; AVX-NEXT: retq %a = shufflevector <16 x i8> %x, <16 x i8> undef, <16 x i32> diff --git a/llvm/test/CodeGen/X86/vector-trunc-packus.ll b/llvm/test/CodeGen/X86/vector-trunc-packus.ll index 9481d9ae70471..da8a3f3fa0d4e 100644 --- a/llvm/test/CodeGen/X86/vector-trunc-packus.ll +++ b/llvm/test/CodeGen/X86/vector-trunc-packus.ll @@ -56,28 +56,25 @@ define <2 x i32> @trunc_packus_v2i64_v2i32(<2 x i64> %a0) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 -; SSE41-NEXT: pmovsxdq {{.*#+}} xmm6 = [2147483647,2147483647] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: pmovsxdq {{.*#+}} xmm4 = [2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3] +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: retq ; ; AVX-LABEL: trunc_packus_v2i64_v2i32: @@ -177,28 +174,25 @@ define void @trunc_packus_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 -; SSE41-NEXT: pmovsxdq {{.*#+}} xmm6 = [2147483647,2147483647] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: pmovsxdq {{.*#+}} xmm4 = [2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3] +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: movq %xmm0, (%rdi) ; SSE41-NEXT: retq ; @@ -320,57 +314,51 @@ define <4 x i32> @trunc_packus_v4i64_v4i32(<4 x i64> %a0) { ; ; SSE41-LABEL: trunc_packus_v4i64_v4i32: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: movapd {{.*#+}} xmm5 = [4294967295,4294967295] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm6 -; SSE41-NEXT: pmovsxdq {{.*#+}} xmm7 = [2147483647,2147483647] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm6 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [4294967295,4294967295] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovsxdq {{.*#+}} xmm6 = [2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 +; SSE41-NEXT: movapd %xmm5, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm2 -; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[0,2] -; SSE41-NEXT: movaps %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2] +; SSE41-NEXT: movaps %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_packus_v4i64_v4i32: @@ -591,106 +579,91 @@ define <8 x i32> @trunc_packus_v8i64_v8i32(ptr %p0) "min-legal-vector-width"="25 ; SSE41-LABEL: trunc_packus_v8i64_v8i32: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa (%rdi), %xmm5 -; SSE41-NEXT: movdqa 16(%rdi), %xmm9 -; SSE41-NEXT: movdqa 32(%rdi), %xmm8 -; SSE41-NEXT: movdqa 48(%rdi), %xmm6 +; SSE41-NEXT: movdqa 16(%rdi), %xmm8 +; SSE41-NEXT: movdqa 32(%rdi), %xmm7 +; SSE41-NEXT: movdqa 48(%rdi), %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [4294967295,4294967295] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm4 -; SSE41-NEXT: pmovsxdq {{.*#+}} xmm7 = [2147483647,2147483647] -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm5, %xmm4 +; SSE41-NEXT: pxor %xmm3, %xmm4 +; SSE41-NEXT: pmovsxdq {{.*#+}} xmm6 = [2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 -; SSE41-NEXT: movdqa %xmm9, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm5 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm8, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm5 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm9 -; SSE41-NEXT: movdqa %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm5 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm3, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm1 -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm2 +; SSE41-NEXT: movapd %xmm1, %xmm6 +; SSE41-NEXT: xorpd %xmm3, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm7 ; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 -; SSE41-NEXT: movapd %xmm9, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 +; SSE41-NEXT: movapd %xmm8, %xmm1 +; SSE41-NEXT: xorpd %xmm3, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm7 ; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 ; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2] -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 +; SSE41-NEXT: movapd %xmm5, %xmm6 +; SSE41-NEXT: xorpd %xmm3, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm7 ; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm6 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm5 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: xorpd %xmm3, %xmm5 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm7 ; SSE41-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm6[0,2] ; SSE41-NEXT: movaps %xmm2, %xmm0 @@ -811,28 +784,25 @@ define <2 x i16> @trunc_packus_v2i64_v2i16(<2 x i64> %a0) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147549183,2147549183] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3] +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] ; SSE41-NEXT: retq ; @@ -957,28 +927,25 @@ define void @trunc_packus_v2i64_v2i16_store(<2 x i64> %a0, ptr%p1) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147549183,2147549183] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3] +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] ; SSE41-NEXT: movd %xmm0, (%rdi) ; SSE41-NEXT: retq @@ -1130,58 +1097,52 @@ define <4 x i16> @trunc_packus_v4i64_v4i16(<4 x i64> %a0) { ; ; SSE41-LABEL: trunc_packus_v4i64_v4i16: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: movapd {{.*#+}} xmm5 = [65535,65535] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm6 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm6 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [65535,65535] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 +; SSE41-NEXT: movapd %xmm5, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm2 -; SSE41-NEXT: packusdw %xmm1, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm2 -; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: packusdw %xmm1, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_packus_v4i64_v4i16: @@ -1327,57 +1288,51 @@ define void @trunc_packus_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) { ; SSE41-LABEL: trunc_packus_v4i64_v4i16_store: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movapd {{.*#+}} xmm5 = [65535,65535] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm3, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm2, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [65535,65535] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 +; SSE41-NEXT: movapd %xmm5, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm3 -; SSE41-NEXT: packusdw %xmm1, %xmm3 -; SSE41-NEXT: packusdw %xmm3, %xmm3 -; SSE41-NEXT: movq %xmm3, (%rdi) +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: packusdw %xmm1, %xmm1 +; SSE41-NEXT: movq %xmm1, (%rdi) ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_packus_v4i64_v4i16_store: @@ -1580,109 +1535,94 @@ define <8 x i16> @trunc_packus_v8i64_v8i16(ptr %p0) "min-legal-vector-width"="25 ; SSE41-LABEL: trunc_packus_v8i64_v8i16: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa (%rdi), %xmm7 -; SSE41-NEXT: movdqa 16(%rdi), %xmm6 -; SSE41-NEXT: movdqa 32(%rdi), %xmm5 -; SSE41-NEXT: movdqa 48(%rdi), %xmm9 +; SSE41-NEXT: movdqa 16(%rdi), %xmm5 +; SSE41-NEXT: movdqa 32(%rdi), %xmm4 +; SSE41-NEXT: movdqa 48(%rdi), %xmm8 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [65535,65535] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm8 = [2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 +; SSE41-NEXT: movdqa %xmm8, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm4 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 -; SSE41-NEXT: movdqa %xmm9, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm5 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm5 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm2, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: xorpd %xmm5, %xmm5 +; SSE41-NEXT: movapd %xmm1, %xmm6 +; SSE41-NEXT: xorpd %xmm2, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm7, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm1 -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 -; SSE41-NEXT: movapd %xmm9, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm8, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 ; SSE41-NEXT: packusdw %xmm6, %xmm1 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: xorpd %xmm2, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm6 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 -; SSE41-NEXT: packusdw %xmm6, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: xorpd %xmm2, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 +; SSE41-NEXT: packusdw %xmm6, %xmm5 +; SSE41-NEXT: packusdw %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; @@ -2248,31 +2188,28 @@ define <2 x i8> @trunc_packus_v2i64_v2i8(<2 x i64> %a0) { ; ; SSE41-LABEL: trunc_packus_v2i64_v2i8: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: pcmpeqd %xmm1, %xmm5 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483903,2147483903] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm7, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: movapd %xmm3, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u] ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq @@ -2407,31 +2344,28 @@ define void @trunc_packus_v2i64_v2i8_store(<2 x i64> %a0, ptr%p1) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm3, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483903,2147483903] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 -; SSE41-NEXT: movapd %xmm2, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: pshufb {{.*#+}} xmm3 = xmm3[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u] -; SSE41-NEXT: pextrw $0, %xmm3, (%rdi) +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 +; SSE41-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u] +; SSE41-NEXT: pextrw $0, %xmm1, (%rdi) ; SSE41-NEXT: retq ; ; AVX-LABEL: trunc_packus_v2i64_v2i8_store: @@ -2556,59 +2490,53 @@ define <4 x i8> @trunc_packus_v4i64_v4i8(<4 x i64> %a0) { ; ; SSE41-LABEL: trunc_packus_v4i64_v4i8: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: movapd {{.*#+}} xmm5 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm6 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm6 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm3, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm3 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 +; SSE41-NEXT: movapd %xmm5, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm2 -; SSE41-NEXT: packusdw %xmm1, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm2 -; SSE41-NEXT: packuswb %xmm2, %xmm2 -; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: packusdw %xmm1, %xmm1 +; SSE41-NEXT: packuswb %xmm1, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_packus_v4i64_v4i8: @@ -2757,58 +2685,52 @@ define void @trunc_packus_v4i64_v4i8_store(<4 x i64> %a0, ptr%p1) { ; SSE41-LABEL: trunc_packus_v4i64_v4i8_store: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movapd {{.*#+}} xmm5 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm3, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm2, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 ; SSE41-NEXT: por %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm4, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm4, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 +; SSE41-NEXT: movapd %xmm5, %xmm4 +; SSE41-NEXT: xorpd %xmm3, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm3 -; SSE41-NEXT: packusdw %xmm1, %xmm3 -; SSE41-NEXT: packusdw %xmm3, %xmm3 -; SSE41-NEXT: packuswb %xmm3, %xmm3 -; SSE41-NEXT: movd %xmm3, (%rdi) +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: packusdw %xmm1, %xmm1 +; SSE41-NEXT: packuswb %xmm1, %xmm1 +; SSE41-NEXT: movd %xmm1, (%rdi) ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_packus_v4i64_v4i8_store: @@ -3014,109 +2936,94 @@ define <8 x i8> @trunc_packus_v8i64_v8i8(ptr %p0) "min-legal-vector-width"="256" ; SSE41-LABEL: trunc_packus_v8i64_v8i8: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa (%rdi), %xmm7 -; SSE41-NEXT: movdqa 16(%rdi), %xmm6 -; SSE41-NEXT: movdqa 32(%rdi), %xmm5 -; SSE41-NEXT: movdqa 48(%rdi), %xmm9 +; SSE41-NEXT: movdqa 16(%rdi), %xmm5 +; SSE41-NEXT: movdqa 32(%rdi), %xmm4 +; SSE41-NEXT: movdqa 48(%rdi), %xmm8 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm8 = [2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 +; SSE41-NEXT: movdqa %xmm8, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm4 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 -; SSE41-NEXT: movdqa %xmm9, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm5 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm5 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm2, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 +; SSE41-NEXT: xorpd %xmm5, %xmm5 +; SSE41-NEXT: movapd %xmm1, %xmm6 +; SSE41-NEXT: xorpd %xmm2, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm7, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm1 -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 -; SSE41-NEXT: movapd %xmm9, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm8, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 ; SSE41-NEXT: packusdw %xmm6, %xmm1 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: xorpd %xmm2, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm6 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 -; SSE41-NEXT: packusdw %xmm6, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: xorpd %xmm2, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 +; SSE41-NEXT: packusdw %xmm6, %xmm5 +; SSE41-NEXT: packusdw %xmm5, %xmm1 ; SSE41-NEXT: packuswb %xmm1, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq @@ -3316,111 +3223,96 @@ define void @trunc_packus_v8i64_v8i8_store(ptr %p0, ptr%p1) "min-legal-vector-wi ; SSE41-LABEL: trunc_packus_v8i64_v8i8_store: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa (%rdi), %xmm7 -; SSE41-NEXT: movdqa 16(%rdi), %xmm6 -; SSE41-NEXT: movdqa 32(%rdi), %xmm4 -; SSE41-NEXT: movdqa 48(%rdi), %xmm9 -; SSE41-NEXT: movapd {{.*#+}} xmm5 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: pcmpeqd %xmm1, %xmm3 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm8 = [2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm3 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 -; SSE41-NEXT: movdqa %xmm9, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm1, %xmm4 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm4 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm1, %xmm9 -; SSE41-NEXT: movdqa %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa 16(%rdi), %xmm5 +; SSE41-NEXT: movdqa 32(%rdi), %xmm3 +; SSE41-NEXT: movdqa 48(%rdi), %xmm8 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255] +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: pxor %xmm1, %xmm2 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 +; SSE41-NEXT: movdqa %xmm8, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm3 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm1, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 +; SSE41-NEXT: xorpd %xmm5, %xmm5 +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: xorpd %xmm1, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm1, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm7, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm5 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm6, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm6 -; SSE41-NEXT: movapd %xmm9, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm5 -; SSE41-NEXT: packusdw %xmm6, %xmm5 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6 -; SSE41-NEXT: movapd %xmm3, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm2, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm7, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm2, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1 -; SSE41-NEXT: packusdw %xmm6, %xmm1 -; SSE41-NEXT: packusdw %xmm1, %xmm5 -; SSE41-NEXT: packuswb %xmm5, %xmm5 -; SSE41-NEXT: movq %xmm5, (%rsi) +; SSE41-NEXT: movapd %xmm8, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: pxor %xmm4, %xmm4 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 +; SSE41-NEXT: packusdw %xmm6, %xmm4 +; SSE41-NEXT: movapd %xmm3, %xmm6 +; SSE41-NEXT: xorpd %xmm1, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pxor %xmm6, %xmm6 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm6 +; SSE41-NEXT: movapd %xmm2, %xmm3 +; SSE41-NEXT: xorpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm3, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 +; SSE41-NEXT: packusdw %xmm6, %xmm5 +; SSE41-NEXT: packusdw %xmm5, %xmm4 +; SSE41-NEXT: packuswb %xmm4, %xmm4 +; SSE41-NEXT: movq %xmm4, (%rsi) ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_packus_v8i64_v8i8_store: @@ -3723,214 +3615,183 @@ define <16 x i8> @trunc_packus_v16i64_v16i8(ptr %p0) "min-legal-vector-width"="2 ; ; SSE41-LABEL: trunc_packus_v16i64_v16i8: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm9 -; SSE41-NEXT: movdqa 16(%rdi), %xmm8 -; SSE41-NEXT: movdqa 32(%rdi), %xmm13 -; SSE41-NEXT: movdqa 48(%rdi), %xmm12 -; SSE41-NEXT: movdqa 80(%rdi), %xmm11 -; SSE41-NEXT: movdqa 64(%rdi), %xmm7 -; SSE41-NEXT: movdqa 112(%rdi), %xmm6 -; SSE41-NEXT: movdqa 96(%rdi), %xmm5 +; SSE41-NEXT: movdqa (%rdi), %xmm8 +; SSE41-NEXT: movdqa 16(%rdi), %xmm7 +; SSE41-NEXT: movdqa 32(%rdi), %xmm12 +; SSE41-NEXT: movdqa 48(%rdi), %xmm11 +; SSE41-NEXT: movdqa 80(%rdi), %xmm10 +; SSE41-NEXT: movdqa 64(%rdi), %xmm6 +; SSE41-NEXT: movdqa 112(%rdi), %xmm5 +; SSE41-NEXT: movdqa 96(%rdi), %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [255,255] -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm4 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm10 = [2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm9 = [2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm3 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 +; SSE41-NEXT: movdqa %xmm5, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm4 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm5 -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm6, %xmm5 +; SSE41-NEXT: pxor %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm5 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm6 -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm10, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm6 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm6 -; SSE41-NEXT: movdqa %xmm11, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm7 -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm7, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm7 -; SSE41-NEXT: movdqa %xmm13, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm11 -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm11, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm11 -; SSE41-NEXT: blendvpd %xmm0, %xmm13, %xmm11 -; SSE41-NEXT: movdqa %xmm12, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm13 -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm13, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm13 -; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm13 -; SSE41-NEXT: movdqa %xmm9, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm2, %xmm12 -; SSE41-NEXT: movdqa %xmm10, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm12, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm6 +; SSE41-NEXT: movdqa %xmm12, %xmm10 +; SSE41-NEXT: pxor %xmm2, %xmm10 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm10 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm10 +; SSE41-NEXT: movdqa %xmm11, %xmm12 +; SSE41-NEXT: pxor %xmm2, %xmm12 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm12, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm12 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm12, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm12 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm12 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm12 +; SSE41-NEXT: movdqa %xmm8, %xmm11 +; SSE41-NEXT: pxor %xmm2, %xmm11 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm11 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm11 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm11 +; SSE41-NEXT: movdqa %xmm7, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm9 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 +; SSE41-NEXT: xorpd %xmm7, %xmm7 +; SSE41-NEXT: movapd %xmm1, %xmm8 +; SSE41-NEXT: xorpd %xmm2, %xmm8 +; SSE41-NEXT: movapd %xmm8, %xmm9 ; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm14, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm8, %xmm0 ; SSE41-NEXT: pxor %xmm8, %xmm8 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8 -; SSE41-NEXT: movapd %xmm12, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: movapd %xmm11, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm1 ; SSE41-NEXT: packusdw %xmm8, %xmm1 -; SSE41-NEXT: movapd %xmm13, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: movapd %xmm12, %xmm8 +; SSE41-NEXT: xorpd %xmm2, %xmm8 +; SSE41-NEXT: movapd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm8, %xmm0 ; SSE41-NEXT: pxor %xmm8, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm13, %xmm8 -; SSE41-NEXT: movapd %xmm11, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm10, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3] -; SSE41-NEXT: por %xmm12, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm8 +; SSE41-NEXT: movapd %xmm10, %xmm9 +; SSE41-NEXT: xorpd %xmm2, %xmm9 +; SSE41-NEXT: movapd %xmm9, %xmm11 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm11 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm9 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: pxor %xmm9, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm9 ; SSE41-NEXT: packusdw %xmm8, %xmm9 ; SSE41-NEXT: packusdw %xmm9, %xmm1 -; SSE41-NEXT: movapd %xmm7, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: movapd %xmm6, %xmm8 +; SSE41-NEXT: xorpd %xmm2, %xmm8 +; SSE41-NEXT: movapd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm8, %xmm0 ; SSE41-NEXT: pxor %xmm8, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: pxor %xmm7, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 -; SSE41-NEXT: packusdw %xmm8, %xmm7 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8 +; SSE41-NEXT: movapd %xmm5, %xmm6 +; SSE41-NEXT: xorpd %xmm2, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: pxor %xmm6, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm6 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: movapd %xmm0, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pand %xmm8, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 -; SSE41-NEXT: packusdw %xmm6, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm7 -; SSE41-NEXT: packuswb %xmm7, %xmm1 +; SSE41-NEXT: packusdw %xmm8, %xmm6 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: xorpd %xmm2, %xmm5 +; SSE41-NEXT: movapd %xmm5, %xmm8 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pxor %xmm5, %xmm5 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm5 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: xorpd %xmm2, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm8 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 +; SSE41-NEXT: packusdw %xmm5, %xmm7 +; SSE41-NEXT: packusdw %xmm7, %xmm6 +; SSE41-NEXT: packuswb %xmm6, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll index 5bfe2c1702880..d0cdbf1e3f08d 100644 --- a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll +++ b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll @@ -58,26 +58,23 @@ define <2 x i32> @trunc_ssat_v2i64_v2i32(<2 x i64> %a0) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [2147483647,2147483647] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pmovsxbd {{.*#+}} xmm4 = [4294967295,0,4294967295,0] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968] ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm0 = [0,4294967295,0,4294967295] +; SSE41-NEXT: movdqa %xmm3, %xmm4 ; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: retq @@ -184,26 +181,23 @@ define void @trunc_ssat_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [2147483647,2147483647] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pmovsxbd {{.*#+}} xmm4 = [4294967295,0,4294967295,0] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968] ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm0 = [0,4294967295,0,4294967295] +; SSE41-NEXT: movdqa %xmm3, %xmm4 ; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: movq %xmm0, (%rdi) @@ -340,51 +334,45 @@ define <4 x i32> @trunc_ssat_v4i64_v4i32(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [2147483647,2147483647] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm6, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm6 = [4294967295,0,4294967295,0] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm7 = [4294967295,0,4294967295,0] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968] -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 -; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm7 = [0,4294967295,0,4294967295] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm6 = [0,4294967295,0,4294967295] +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: xorpd %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2] ; SSE41-NEXT: movaps %xmm1, %xmm0 @@ -616,101 +604,88 @@ define <8 x i32> @trunc_ssat_v8i64_v8i32(ptr %p0) "min-legal-vector-width"="256" ; SSE41-NEXT: movdqa 48(%rdi), %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [2147483647,2147483647] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm9, %xmm9 -; SSE41-NEXT: pcmpeqd %xmm9, %xmm4 +; SSE41-NEXT: movdqa %xmm5, %xmm4 +; SSE41-NEXT: pxor %xmm3, %xmm4 ; SSE41-NEXT: pmovsxbd {{.*#+}} xmm6 = [4294967295,0,4294967295,0] -; SSE41-NEXT: movdqa %xmm6, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm5 -; SSE41-NEXT: movdqa %xmm6, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm8, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm5 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm8 -; SSE41-NEXT: movdqa %xmm6, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm3, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm8 ; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 ; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm7 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm6[0,0,2,2] -; SSE41-NEXT: pand %xmm7, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [18446744071562067968,18446744071562067968] -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE41-NEXT: movapd %xmm1, %xmm7 +; SSE41-NEXT: xorpd %xmm3, %xmm7 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm6 = [0,4294967295,0,4294967295] +; SSE41-NEXT: movapd %xmm7, %xmm9 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm7 = [0,4294967295,0,4294967295] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm9 -; SSE41-NEXT: movapd %xmm8, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm1, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm7 +; SSE41-NEXT: movapd %xmm8, %xmm1 +; SSE41-NEXT: xorpd %xmm3, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: movapd %xmm2, %xmm1 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 -; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm9[0,2] -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] +; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm7[0,2] +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: xorpd %xmm3, %xmm7 +; SSE41-NEXT: movapd %xmm7, %xmm8 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 ; SSE41-NEXT: xorpd %xmm4, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 -; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm8[0,2] +; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm7[0,2] ; SSE41-NEXT: movaps %xmm2, %xmm0 ; SSE41-NEXT: retq ; @@ -830,26 +805,23 @@ define <2 x i16> @trunc_ssat_v2i64_v2i16(<2 x i64> %a0) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [32767,32767] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147516415,2147516415] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848] ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200] +; SSE41-NEXT: movdqa %xmm3, %xmm4 ; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] @@ -967,26 +939,23 @@ define void @trunc_ssat_v2i64_v2i16_store(<2 x i64> %a0, ptr%p1) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [32767,32767] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147516415,2147516415] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848] ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [18446744071562035200,18446744071562035200] +; SSE41-NEXT: movdqa %xmm3, %xmm4 ; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] ; SSE41-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] @@ -1137,51 +1106,45 @@ define <4 x i16> @trunc_ssat_v4i64_v4i16(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [32767,32767] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm6, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147516415,2147516415] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147516415,2147516415] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848] -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 -; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562035200,18446744071562035200] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200] +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: xorpd %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: packssdw %xmm2, %xmm1 ; SSE41-NEXT: packssdw %xmm1, %xmm1 @@ -1327,51 +1290,45 @@ define void @trunc_ssat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) { ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [32767,32767] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm6, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147516415,2147516415] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147516415,2147516415] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848] -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 -; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562035200,18446744071562035200] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200] +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: xorpd %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: packssdw %xmm2, %xmm1 ; SSE41-NEXT: packssdw %xmm1, %xmm1 @@ -1573,107 +1530,94 @@ define <8 x i16> @trunc_ssat_v8i64_v8i16(ptr %p0) "min-legal-vector-width"="256" ; ; SSE41-LABEL: trunc_ssat_v8i64_v8i16: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm6 +; SSE41-NEXT: movdqa (%rdi), %xmm7 ; SSE41-NEXT: movdqa 16(%rdi), %xmm5 ; SSE41-NEXT: movdqa 32(%rdi), %xmm4 ; SSE41-NEXT: movdqa 48(%rdi), %xmm8 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [32767,32767] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm9, %xmm9 -; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147516415,2147516415] -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147516415,2147516415] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm8, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm4 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm2, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 ; SSE41-NEXT: movdqa %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm5 = [18446744073709518848,18446744073709518848] -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE41-NEXT: movapd %xmm1, %xmm7 +; SSE41-NEXT: xorpd %xmm2, %xmm7 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562035200,18446744071562035200] +; SSE41-NEXT: movapd %xmm7, %xmm9 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562035200,18446744071562035200] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm9 -; SSE41-NEXT: movapd %xmm8, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm1, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm7 +; SSE41-NEXT: movapd %xmm8, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: movapd %xmm5, %xmm1 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 -; SSE41-NEXT: packssdw %xmm9, %xmm1 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] +; SSE41-NEXT: packssdw %xmm7, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm7 +; SSE41-NEXT: xorpd %xmm2, %xmm7 +; SSE41-NEXT: movapd %xmm7, %xmm8 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm7 ; SSE41-NEXT: xorpd %xmm3, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 -; SSE41-NEXT: packssdw %xmm8, %xmm5 +; SSE41-NEXT: packssdw %xmm7, %xmm5 ; SSE41-NEXT: packssdw %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq @@ -2011,26 +1955,23 @@ define <2 x i8> @trunc_ssat_v2i64_v2i8(<2 x i64> %a0) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483775,2147483775] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488] ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movdqa %xmm3, %xmm4 ; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u] ; SSE41-NEXT: movdqa %xmm1, %xmm0 @@ -2160,26 +2101,23 @@ define void @trunc_ssat_v2i64_v2i8_store(<2 x i64> %a0, ptr%p1) { ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm5, %xmm5 -; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = [2147483775,2147483775] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm5 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] -; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488] ; SSE41-NEXT: pxor %xmm2, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movdqa %xmm3, %xmm4 ; SSE41-NEXT: pcmpeqd %xmm0, %xmm4 -; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 ; SSE41-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u] ; SSE41-NEXT: pextrw $0, %xmm1, (%rdi) @@ -2304,51 +2242,45 @@ define <4 x i8> @trunc_ssat_v4i64_v4i8(<4 x i64> %a0) { ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm6, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483775,2147483775] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147483775,2147483775] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488] -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 -; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562067840,18446744071562067840] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: xorpd %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: packssdw %xmm2, %xmm1 ; SSE41-NEXT: packssdw %xmm1, %xmm1 @@ -2498,51 +2430,45 @@ define void @trunc_ssat_v4i64_v4i8_store(<4 x i64> %a0, ptr%p1) { ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648] -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm6, %xmm6 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483775,2147483775] +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147483775,2147483775] -; SSE41-NEXT: movdqa %xmm7, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 ; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488] -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 -; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562067840,18446744071562067840] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: xorpd %xmm3, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2 ; SSE41-NEXT: xorpd %xmm5, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: packssdw %xmm2, %xmm1 ; SSE41-NEXT: packssdw %xmm1, %xmm1 @@ -2748,107 +2674,94 @@ define <8 x i8> @trunc_ssat_v8i64_v8i8(ptr %p0) "min-legal-vector-width"="256" { ; ; SSE41-LABEL: trunc_ssat_v8i64_v8i8: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm6 +; SSE41-NEXT: movdqa (%rdi), %xmm7 ; SSE41-NEXT: movdqa 16(%rdi), %xmm5 ; SSE41-NEXT: movdqa 32(%rdi), %xmm4 ; SSE41-NEXT: movdqa 48(%rdi), %xmm8 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm9, %xmm9 -; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147483775,2147483775] -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm3 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483775,2147483775] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm4 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm8, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm4 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm2, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 ; SSE41-NEXT: movdqa %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm5 = [18446744073709551488,18446744073709551488] -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE41-NEXT: movapd %xmm1, %xmm7 +; SSE41-NEXT: xorpd %xmm2, %xmm7 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movapd %xmm7, %xmm9 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562067840,18446744071562067840] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm9 -; SSE41-NEXT: movapd %xmm8, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm1, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm7 +; SSE41-NEXT: movapd %xmm8, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: movapd %xmm5, %xmm1 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 -; SSE41-NEXT: packssdw %xmm9, %xmm1 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] +; SSE41-NEXT: packssdw %xmm7, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm7 +; SSE41-NEXT: xorpd %xmm2, %xmm7 +; SSE41-NEXT: movapd %xmm7, %xmm8 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm7 ; SSE41-NEXT: xorpd %xmm3, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm5 -; SSE41-NEXT: packssdw %xmm8, %xmm5 +; SSE41-NEXT: packssdw %xmm7, %xmm5 ; SSE41-NEXT: packssdw %xmm5, %xmm1 ; SSE41-NEXT: packsswb %xmm1, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 @@ -3054,107 +2967,94 @@ define void @trunc_ssat_v8i64_v8i8_store(ptr %p0, ptr%p1) "min-legal-vector-widt ; ; SSE41-LABEL: trunc_ssat_v8i64_v8i8_store: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm6 +; SSE41-NEXT: movdqa (%rdi), %xmm7 ; SSE41-NEXT: movdqa 16(%rdi), %xmm5 ; SSE41-NEXT: movdqa 32(%rdi), %xmm3 ; SSE41-NEXT: movdqa 48(%rdi), %xmm8 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm9, %xmm9 -; SSE41-NEXT: pcmpeqd %xmm9, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm7 = [2147483775,2147483775] -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm2, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: pxor %xmm1, %xmm2 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = [2147483775,2147483775] +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm2, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm2 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm8, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm3 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm1, %xmm8 +; SSE41-NEXT: movdqa %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: movapd %xmm4, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 ; SSE41-NEXT: movdqa %xmm5, %xmm0 ; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm9, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm5 = [18446744073709551488,18446744073709551488] -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE41-NEXT: movapd %xmm4, %xmm7 +; SSE41-NEXT: xorpd %xmm1, %xmm7 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movapd %xmm7, %xmm9 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 -; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [18446744071562067840,18446744071562067840] -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm9 -; SSE41-NEXT: movapd %xmm8, %xmm0 -; SSE41-NEXT: xorpd %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm4 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm7 +; SSE41-NEXT: movapd %xmm8, %xmm4 +; SSE41-NEXT: xorpd %xmm1, %xmm4 +; SSE41-NEXT: movapd %xmm4, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm4, %xmm0 ; SSE41-NEXT: movapd %xmm5, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4 -; SSE41-NEXT: packssdw %xmm9, %xmm4 -; SSE41-NEXT: movapd %xmm3, %xmm0 -; SSE41-NEXT: xorpd %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] +; SSE41-NEXT: packssdw %xmm7, %xmm4 +; SSE41-NEXT: movapd %xmm3, %xmm7 +; SSE41-NEXT: xorpd %xmm1, %xmm7 +; SSE41-NEXT: movapd %xmm7, %xmm8 ; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm5, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm8 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm5, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 ; SSE41-NEXT: xorpd %xmm2, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm5 -; SSE41-NEXT: packssdw %xmm8, %xmm5 +; SSE41-NEXT: packssdw %xmm7, %xmm5 ; SSE41-NEXT: packssdw %xmm5, %xmm4 ; SSE41-NEXT: packsswb %xmm4, %xmm4 ; SSE41-NEXT: movq %xmm4, (%rsi) @@ -3470,202 +3370,177 @@ define <16 x i8> @trunc_ssat_v16i64_v16i8(ptr %p0) "min-legal-vector-width"="256 ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa (%rdi), %xmm8 ; SSE41-NEXT: movdqa 16(%rdi), %xmm7 -; SSE41-NEXT: movdqa 32(%rdi), %xmm13 -; SSE41-NEXT: movdqa 48(%rdi), %xmm12 +; SSE41-NEXT: movdqa 32(%rdi), %xmm12 +; SSE41-NEXT: movdqa 48(%rdi), %xmm11 ; SSE41-NEXT: movdqa 80(%rdi), %xmm10 ; SSE41-NEXT: movdqa 64(%rdi), %xmm6 ; SSE41-NEXT: movdqa 112(%rdi), %xmm5 ; SSE41-NEXT: movdqa 96(%rdi), %xmm4 ; SSE41-NEXT: movapd {{.*#+}} xmm1 = [127,127] ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = [2147483648,2147483648] -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pxor %xmm11, %xmm11 -; SSE41-NEXT: pcmpeqd %xmm11, %xmm3 +; SSE41-NEXT: movdqa %xmm4, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm3 ; SSE41-NEXT: pmovzxdq {{.*#+}} xmm9 = [2147483775,2147483775] -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm3, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm3 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm4 -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm4, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm5, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm4 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm4 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm4 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm5 -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm6, %xmm5 +; SSE41-NEXT: pxor %xmm2, %xmm5 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm5 -; SSE41-NEXT: movdqa %xmm10, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm6 -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm10, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm6 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm6 -; SSE41-NEXT: movdqa %xmm13, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm10 -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm10, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: movdqa %xmm12, %xmm10 +; SSE41-NEXT: pxor %xmm2, %xmm10 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm10 -; SSE41-NEXT: blendvpd %xmm0, %xmm13, %xmm10 -; SSE41-NEXT: movdqa %xmm12, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm13 -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm13, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm13 -; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm13 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm12 -; SSE41-NEXT: movdqa %xmm9, %xmm14 -; SSE41-NEXT: pcmpgtd %xmm0, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] -; SSE41-NEXT: pand %xmm12, %xmm15 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm14[1,1,3,3] -; SSE41-NEXT: por %xmm15, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm10 +; SSE41-NEXT: movdqa %xmm11, %xmm12 +; SSE41-NEXT: pxor %xmm2, %xmm12 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm12, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm12 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm12, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 ; SSE41-NEXT: movapd %xmm1, %xmm12 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm12 +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm12 +; SSE41-NEXT: movdqa %xmm8, %xmm11 +; SSE41-NEXT: pxor %xmm2, %xmm11 +; SSE41-NEXT: movdqa %xmm9, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm11 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 +; SSE41-NEXT: por %xmm13, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm11 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm11 ; SSE41-NEXT: movdqa %xmm7, %xmm0 ; SSE41-NEXT: pxor %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm11, %xmm8 +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm8 ; SSE41-NEXT: pcmpgtd %xmm0, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2] -; SSE41-NEXT: pand %xmm8, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm7 = [18446744073709551488,18446744073709551488] -; SSE41-NEXT: movapd %xmm1, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm8, %xmm8 -; SSE41-NEXT: pcmpeqd %xmm8, %xmm11 -; SSE41-NEXT: movdqa {{.*#+}} xmm9 = [18446744071562067840,18446744071562067840] -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm11, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm14, %xmm0 -; SSE41-NEXT: movapd %xmm7, %xmm11 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm11 -; SSE41-NEXT: movapd %xmm12, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm8, %xmm1 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm1, %xmm14 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm14, %xmm0 +; SSE41-NEXT: movapd %xmm1, %xmm9 +; SSE41-NEXT: xorpd %xmm2, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm8 = [18446744071562067840,18446744071562067840] +; SSE41-NEXT: movapd %xmm9, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm8, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm13, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm7, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm9 +; SSE41-NEXT: movapd %xmm11, %xmm1 +; SSE41-NEXT: xorpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm13 +; SSE41-NEXT: pcmpeqd %xmm8, %xmm13 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE41-NEXT: pand %xmm13, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: movapd %xmm7, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm1 -; SSE41-NEXT: packssdw %xmm11, %xmm1 -; SSE41-NEXT: movapd %xmm13, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3] +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm1 +; SSE41-NEXT: packssdw %xmm9, %xmm1 +; SSE41-NEXT: movapd %xmm12, %xmm9 +; SSE41-NEXT: xorpd %xmm2, %xmm9 +; SSE41-NEXT: movapd %xmm9, %xmm11 ; SSE41-NEXT: pcmpeqd %xmm8, %xmm11 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm11, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm12, %xmm0 -; SSE41-NEXT: movapd %xmm7, %xmm11 -; SSE41-NEXT: blendvpd %xmm0, %xmm13, %xmm11 -; SSE41-NEXT: movapd %xmm10, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm11, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm7, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm9 +; SSE41-NEXT: movapd %xmm10, %xmm11 +; SSE41-NEXT: xorpd %xmm2, %xmm11 +; SSE41-NEXT: movapd %xmm11, %xmm12 ; SSE41-NEXT: pcmpeqd %xmm8, %xmm12 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm12, %xmm13 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm13, %xmm0 -; SSE41-NEXT: movapd %xmm7, %xmm12 -; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm12 -; SSE41-NEXT: packssdw %xmm11, %xmm12 -; SSE41-NEXT: packssdw %xmm12, %xmm1 -; SSE41-NEXT: movapd %xmm6, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm8, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm10, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm11, %xmm0 -; SSE41-NEXT: movapd %xmm7, %xmm10 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm10 -; SSE41-NEXT: movapd %xmm5, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm8, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm6, %xmm11 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm11[0,0,2,2] +; SSE41-NEXT: pand %xmm12, %xmm0 ; SSE41-NEXT: por %xmm11, %xmm0 +; SSE41-NEXT: movapd %xmm7, %xmm11 +; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm11 +; SSE41-NEXT: packssdw %xmm9, %xmm11 +; SSE41-NEXT: packssdw %xmm11, %xmm1 +; SSE41-NEXT: movapd %xmm6, %xmm9 +; SSE41-NEXT: xorpd %xmm2, %xmm9 +; SSE41-NEXT: movapd %xmm9, %xmm10 +; SSE41-NEXT: pcmpeqd %xmm8, %xmm10 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 +; SSE41-NEXT: por %xmm9, %xmm0 +; SSE41-NEXT: movapd %xmm7, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm9 +; SSE41-NEXT: movapd %xmm5, %xmm6 +; SSE41-NEXT: xorpd %xmm2, %xmm6 +; SSE41-NEXT: movapd %xmm6, %xmm10 +; SSE41-NEXT: pcmpeqd %xmm8, %xmm10 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pand %xmm10, %xmm0 +; SSE41-NEXT: por %xmm6, %xmm0 ; SSE41-NEXT: movapd %xmm7, %xmm6 ; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm6 -; SSE41-NEXT: packssdw %xmm10, %xmm6 -; SSE41-NEXT: movapd %xmm4, %xmm0 -; SSE41-NEXT: xorpd %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm8, %xmm5 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: pand %xmm5, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: por %xmm10, %xmm0 +; SSE41-NEXT: packssdw %xmm9, %xmm6 +; SSE41-NEXT: movapd %xmm4, %xmm5 +; SSE41-NEXT: xorpd %xmm2, %xmm5 +; SSE41-NEXT: movapd %xmm5, %xmm9 +; SSE41-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pand %xmm9, %xmm0 +; SSE41-NEXT: por %xmm5, %xmm0 ; SSE41-NEXT: movapd %xmm7, %xmm5 ; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm5 ; SSE41-NEXT: xorpd %xmm3, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm8, %xmm0 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2] -; SSE41-NEXT: pand %xmm0, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm8, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2] +; SSE41-NEXT: pand %xmm4, %xmm0 +; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7 ; SSE41-NEXT: packssdw %xmm5, %xmm7 ; SSE41-NEXT: packssdw %xmm7, %xmm6 diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll index 0a82827b78b6e..a5d83a86f295e 100644 --- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll +++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll @@ -40,11 +40,10 @@ define <2 x i32> @trunc_usat_v2i64_v2i32(<2 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] ; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] ; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: pandn %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 @@ -117,11 +116,10 @@ define void @trunc_usat_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] ; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] ; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: pandn %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 @@ -209,21 +207,20 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 ; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647] -; SSE41-NEXT: movdqa %xmm5, %xmm6 -; SSE41-NEXT: pcmpgtd %xmm3, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm3 -; SSE41-NEXT: pand %xmm6, %xmm3 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pxor %xmm4, %xmm5 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259455,9223372039002259455] +; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483647,2147483647,2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpgtd %xmm7, %xmm3 +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm4 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm4, %xmm0 +; SSE41-NEXT: pand %xmm6, %xmm0 ; SSE41-NEXT: movapd {{.*#+}} xmm4 = [4294967295,4294967295] ; SSE41-NEXT: movapd {{.*#+}} xmm5 = [4294967295,429496729] ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 @@ -404,54 +401,51 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(ptr %p0) { ; ; SSE41-LABEL: trunc_usat_v8i64_v8i32: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm3 -; SSE41-NEXT: movdqa 16(%rdi), %xmm6 -; SSE41-NEXT: movdqa 32(%rdi), %xmm7 +; SSE41-NEXT: movdqa (%rdi), %xmm4 +; SSE41-NEXT: movdqa 16(%rdi), %xmm7 +; SSE41-NEXT: movdqa 32(%rdi), %xmm8 ; SSE41-NEXT: movdqa 48(%rdi), %xmm1 -; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647] -; SSE41-NEXT: movdqa %xmm5, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm5, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm1, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm3 = [4294967295,4294967295] +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm9 +; SSE41-NEXT: pxor %xmm6, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455] +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647] +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 ; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 -; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm8[0,2] -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm5, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm9 +; SSE41-NEXT: movdqa %xmm8, %xmm1 +; SSE41-NEXT: pxor %xmm6, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm1 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm1, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1 +; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm9[0,2] +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm8 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 ; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm5, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 -; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm7[0,2] -; SSE41-NEXT: movaps %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pand %xmm5, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 +; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm8[0,2] +; SSE41-NEXT: movaps %xmm3, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v8i64_v8i32: @@ -544,11 +538,10 @@ define <2 x i16> @trunc_usat_v2i64_v2i16(<2 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] ; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] ; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: pandn %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 @@ -639,11 +632,10 @@ define void @trunc_usat_v2i64_v2i16_store(<2 x i64> %a0, ptr %p1) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] ; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] ; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: pandn %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 @@ -756,27 +748,26 @@ define <4 x i16> @trunc_usat_v4i64_v4i16(<4 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm3 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm7, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm1, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm5, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002324991,9223372039002324991] +; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm0 ; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pand %xmm7, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 -; SSE41-NEXT: packusdw %xmm5, %xmm2 +; SSE41-NEXT: packusdw %xmm6, %xmm2 ; SSE41-NEXT: packusdw %xmm2, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq @@ -889,30 +880,29 @@ define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) { ; SSE41-LABEL: trunc_usat_v4i64_v4i16_store: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movapd {{.*#+}} xmm3 = [65535,65535] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm7, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm1, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [65535,65535] +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm5, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002324991,9223372039002324991] +; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183] +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm0 ; SSE41-NEXT: pand %xmm6, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: packusdw %xmm5, %xmm3 -; SSE41-NEXT: packusdw %xmm3, %xmm3 -; SSE41-NEXT: movq %xmm3, (%rdi) +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pand %xmm7, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4 +; SSE41-NEXT: packusdw %xmm6, %xmm4 +; SSE41-NEXT: packusdw %xmm4, %xmm4 +; SSE41-NEXT: movq %xmm4, (%rdi) ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v4i64_v4i16_store: @@ -1055,55 +1045,52 @@ define <8 x i16> @trunc_usat_v8i64_v8i16(ptr %p0) { ; ; SSE41-LABEL: trunc_usat_v8i64_v8i16: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm7 -; SSE41-NEXT: movdqa 16(%rdi), %xmm1 -; SSE41-NEXT: movdqa 32(%rdi), %xmm3 -; SSE41-NEXT: movdqa 48(%rdi), %xmm6 -; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa (%rdi), %xmm8 +; SSE41-NEXT: movdqa 16(%rdi), %xmm2 +; SSE41-NEXT: movdqa 32(%rdi), %xmm4 +; SSE41-NEXT: movdqa 48(%rdi), %xmm7 +; SSE41-NEXT: movapd {{.*#+}} xmm3 = [65535,65535] +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm2, %xmm9 +; SSE41-NEXT: pxor %xmm6, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991] +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147549183,2147549183,2147549183,2147549183] ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183] -; SSE41-NEXT: movdqa %xmm5, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm5, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm1, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 ; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 -; SSE41-NEXT: packusdw %xmm8, %xmm1 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm5, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm9 +; SSE41-NEXT: movdqa %xmm8, %xmm2 +; SSE41-NEXT: pxor %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm2 +; SSE41-NEXT: packusdw %xmm9, %xmm2 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm8 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 ; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm5, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 -; SSE41-NEXT: packusdw %xmm7, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm3, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 +; SSE41-NEXT: packusdw %xmm8, %xmm3 +; SSE41-NEXT: packusdw %xmm3, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v8i64_v8i16: @@ -1647,11 +1634,10 @@ define <2 x i8> @trunc_usat_v2i64_v2i8(<2 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] ; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] ; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: pandn %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 @@ -1743,11 +1729,10 @@ define void @trunc_usat_v2i64_v2i8_store(<2 x i64> %a0, ptr %p1) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: pxor %xmm1, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2] ; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2] ; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: pandn %xmm3, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2 @@ -1837,27 +1822,26 @@ define <4 x i8> @trunc_usat_v4i64_v4i8(<4 x i64> %a0) { ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm3 ; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm7, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm1, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm5, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259711,9223372039002259711] +; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm0 ; SSE41-NEXT: pand %xmm6, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm6 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm4 +; SSE41-NEXT: pand %xmm7, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 -; SSE41-NEXT: packusdw %xmm5, %xmm2 +; SSE41-NEXT: packusdw %xmm6, %xmm2 ; SSE41-NEXT: packusdw %xmm2, %xmm2 ; SSE41-NEXT: packuswb %xmm2, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm0 @@ -1972,31 +1956,30 @@ define void @trunc_usat_v4i64_v4i8_store(<4 x i64> %a0, ptr%p1) { ; SSE41-LABEL: trunc_usat_v4i64_v4i8_store: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm6, %xmm7 -; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm7, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm5 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm1, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255] +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm5, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259711,9223372039002259711] +; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm7, %xmm6 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm8, %xmm0 ; SSE41-NEXT: pand %xmm6, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: packusdw %xmm5, %xmm3 -; SSE41-NEXT: packusdw %xmm3, %xmm3 -; SSE41-NEXT: packuswb %xmm3, %xmm3 -; SSE41-NEXT: movd %xmm3, (%rdi) +; SSE41-NEXT: movapd %xmm4, %xmm6 +; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE41-NEXT: pand %xmm7, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4 +; SSE41-NEXT: packusdw %xmm6, %xmm4 +; SSE41-NEXT: packusdw %xmm4, %xmm4 +; SSE41-NEXT: packuswb %xmm4, %xmm4 +; SSE41-NEXT: movd %xmm4, (%rdi) ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v4i64_v4i8_store: @@ -2138,56 +2121,53 @@ define <8 x i8> @trunc_usat_v8i64_v8i8(ptr %p0) { ; ; SSE41-LABEL: trunc_usat_v8i64_v8i8: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm7 -; SSE41-NEXT: movdqa 16(%rdi), %xmm1 -; SSE41-NEXT: movdqa 32(%rdi), %xmm3 -; SSE41-NEXT: movdqa 48(%rdi), %xmm6 -; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa (%rdi), %xmm8 +; SSE41-NEXT: movdqa 16(%rdi), %xmm2 +; SSE41-NEXT: movdqa 32(%rdi), %xmm4 +; SSE41-NEXT: movdqa 48(%rdi), %xmm7 +; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255] +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm2, %xmm9 +; SSE41-NEXT: pxor %xmm6, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711] +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903] ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm5, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 ; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm5, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm1, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1 -; SSE41-NEXT: packusdw %xmm8, %xmm1 -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm5, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm9 +; SSE41-NEXT: movdqa %xmm8, %xmm2 +; SSE41-NEXT: pxor %xmm6, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm2 +; SSE41-NEXT: packusdw %xmm9, %xmm2 +; SSE41-NEXT: movdqa %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm6, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm8 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 ; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: movapd %xmm2, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7 -; SSE41-NEXT: movdqa %xmm3, %xmm0 -; SSE41-NEXT: pxor %xmm4, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm6, %xmm5 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 -; SSE41-NEXT: pand %xmm5, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 -; SSE41-NEXT: packusdw %xmm7, %xmm2 -; SSE41-NEXT: packusdw %xmm2, %xmm1 -; SSE41-NEXT: packuswb %xmm1, %xmm1 +; SSE41-NEXT: movapd %xmm3, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 +; SSE41-NEXT: pxor %xmm4, %xmm6 +; SSE41-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm5, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3 +; SSE41-NEXT: packusdw %xmm8, %xmm3 +; SSE41-NEXT: packusdw %xmm3, %xmm2 +; SSE41-NEXT: packuswb %xmm2, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v8i64_v8i8: @@ -2323,56 +2303,53 @@ define void @trunc_usat_v8i64_v8i8_store(ptr %p0, ptr%p1) { ; ; SSE41-LABEL: trunc_usat_v8i64_v8i8_store: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa (%rdi), %xmm7 -; SSE41-NEXT: movdqa 16(%rdi), %xmm6 -; SSE41-NEXT: movdqa 32(%rdi), %xmm2 -; SSE41-NEXT: movdqa 48(%rdi), %xmm5 -; SSE41-NEXT: movapd {{.*#+}} xmm1 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456] -; SSE41-NEXT: movdqa %xmm6, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm4, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm4, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm6, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: movdqa (%rdi), %xmm8 +; SSE41-NEXT: movdqa 16(%rdi), %xmm7 +; SSE41-NEXT: movdqa 32(%rdi), %xmm3 +; SSE41-NEXT: movdqa 48(%rdi), %xmm6 +; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255] +; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm7, %xmm9 +; SSE41-NEXT: pxor %xmm5, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711] +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm9 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903] +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 ; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm6 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm6 -; SSE41-NEXT: packusdw %xmm8, %xmm6 -; SSE41-NEXT: movdqa %xmm5, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm4, %xmm8 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm8 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9 +; SSE41-NEXT: movdqa %xmm8, %xmm7 +; SSE41-NEXT: pxor %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm7 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm7, %xmm0 +; SSE41-NEXT: movapd %xmm2, %xmm7 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm7 +; SSE41-NEXT: packusdw %xmm9, %xmm7 +; SSE41-NEXT: movdqa %xmm6, %xmm8 +; SSE41-NEXT: pxor %xmm5, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm8 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm9, %xmm0 ; SSE41-NEXT: pand %xmm8, %xmm0 -; SSE41-NEXT: movapd %xmm1, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm3, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm5, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pand %xmm4, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1 -; SSE41-NEXT: packusdw %xmm7, %xmm1 -; SSE41-NEXT: packusdw %xmm1, %xmm6 -; SSE41-NEXT: packuswb %xmm6, %xmm6 -; SSE41-NEXT: movq %xmm6, (%rsi) +; SSE41-NEXT: movapd %xmm2, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8 +; SSE41-NEXT: pxor %xmm3, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm4, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2 +; SSE41-NEXT: packusdw %xmm8, %xmm2 +; SSE41-NEXT: packusdw %xmm2, %xmm7 +; SSE41-NEXT: packuswb %xmm7, %xmm7 +; SSE41-NEXT: movq %xmm7, (%rsi) ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v8i64_v8i8_store: @@ -2561,103 +2538,96 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(ptr %p0) { ; ; SSE41-LABEL: trunc_usat_v16i64_v16i8: ; SSE41: # %bb.0: -; SSE41-NEXT: movdqa 96(%rdi), %xmm2 -; SSE41-NEXT: movdqa 112(%rdi), %xmm4 -; SSE41-NEXT: movdqa 64(%rdi), %xmm7 -; SSE41-NEXT: movdqa 80(%rdi), %xmm8 -; SSE41-NEXT: movdqa (%rdi), %xmm11 -; SSE41-NEXT: movdqa 16(%rdi), %xmm1 -; SSE41-NEXT: movdqa 32(%rdi), %xmm9 -; SSE41-NEXT: movdqa 48(%rdi), %xmm10 -; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255] -; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa 96(%rdi), %xmm3 +; SSE41-NEXT: movdqa 112(%rdi), %xmm5 +; SSE41-NEXT: movdqa 64(%rdi), %xmm8 +; SSE41-NEXT: movdqa 80(%rdi), %xmm9 +; SSE41-NEXT: movdqa (%rdi), %xmm12 +; SSE41-NEXT: movdqa 16(%rdi), %xmm2 +; SSE41-NEXT: movdqa 32(%rdi), %xmm10 +; SSE41-NEXT: movdqa 48(%rdi), %xmm11 +; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255] +; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456] +; SSE41-NEXT: movdqa %xmm2, %xmm13 +; SSE41-NEXT: pxor %xmm7, %xmm13 +; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711] +; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm13 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903] ; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903] -; SSE41-NEXT: movdqa %xmm6, %xmm13 -; SSE41-NEXT: pcmpgtd %xmm12, %xmm13 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm14, %xmm0 ; SSE41-NEXT: pand %xmm13, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm12 -; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm12 -; SSE41-NEXT: movdqa %xmm11, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm6, %xmm13 -; SSE41-NEXT: pcmpgtd %xmm1, %xmm13 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 -; SSE41-NEXT: pand %xmm13, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm1 -; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm1 -; SSE41-NEXT: packusdw %xmm12, %xmm1 -; SSE41-NEXT: movdqa %xmm10, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm6, %xmm12 -; SSE41-NEXT: pcmpgtd %xmm11, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm13 +; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm13 +; SSE41-NEXT: movdqa %xmm12, %xmm2 +; SSE41-NEXT: pxor %xmm7, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm14, %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm2 +; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm2 +; SSE41-NEXT: packusdw %xmm13, %xmm2 +; SSE41-NEXT: movdqa %xmm11, %xmm12 +; SSE41-NEXT: pxor %xmm7, %xmm12 +; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm12 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm13, %xmm0 ; SSE41-NEXT: pand %xmm12, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm11 +; SSE41-NEXT: movapd %xmm4, %xmm12 +; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm12 +; SSE41-NEXT: movdqa %xmm10, %xmm11 +; SSE41-NEXT: pxor %xmm7, %xmm11 +; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm11 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm13, %xmm0 +; SSE41-NEXT: pand %xmm11, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm11 ; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm11 -; SSE41-NEXT: movdqa %xmm9, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm6, %xmm12 -; SSE41-NEXT: pcmpgtd %xmm10, %xmm12 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 -; SSE41-NEXT: pand %xmm12, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm10 -; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm10 -; SSE41-NEXT: packusdw %xmm11, %xmm10 -; SSE41-NEXT: packusdw %xmm10, %xmm1 -; SSE41-NEXT: movdqa %xmm8, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm6, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm9, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 -; SSE41-NEXT: pand %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm9 -; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm9 -; SSE41-NEXT: movdqa %xmm7, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm6, %xmm10 -; SSE41-NEXT: pcmpgtd %xmm8, %xmm10 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 +; SSE41-NEXT: packusdw %xmm12, %xmm11 +; SSE41-NEXT: packusdw %xmm11, %xmm2 +; SSE41-NEXT: movdqa %xmm9, %xmm10 +; SSE41-NEXT: pxor %xmm7, %xmm10 +; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm10 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm0 ; SSE41-NEXT: pand %xmm10, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm8 -; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8 -; SSE41-NEXT: packusdw %xmm9, %xmm8 -; SSE41-NEXT: movdqa %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2] -; SSE41-NEXT: movdqa %xmm6, %xmm9 -; SSE41-NEXT: pcmpgtd %xmm7, %xmm9 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm10 +; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm10 +; SSE41-NEXT: movdqa %xmm8, %xmm9 +; SSE41-NEXT: pxor %xmm7, %xmm9 +; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm9 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm11, %xmm0 ; SSE41-NEXT: pand %xmm9, %xmm0 -; SSE41-NEXT: movapd %xmm3, %xmm7 -; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm7 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: pxor %xmm5, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2] -; SSE41-NEXT: pcmpgtd %xmm4, %xmm6 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd %xmm5, %xmm0 -; SSE41-NEXT: pand %xmm6, %xmm0 -; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3 -; SSE41-NEXT: packusdw %xmm7, %xmm3 -; SSE41-NEXT: packusdw %xmm3, %xmm8 -; SSE41-NEXT: packuswb %xmm8, %xmm1 +; SSE41-NEXT: movapd %xmm4, %xmm9 +; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm9 +; SSE41-NEXT: packusdw %xmm10, %xmm9 +; SSE41-NEXT: movdqa %xmm5, %xmm8 +; SSE41-NEXT: pxor %xmm7, %xmm8 +; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm8 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pcmpgtd %xmm10, %xmm0 +; SSE41-NEXT: pand %xmm8, %xmm0 +; SSE41-NEXT: movapd %xmm4, %xmm8 +; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm8 +; SSE41-NEXT: pxor %xmm3, %xmm7 +; SSE41-NEXT: pcmpeqd %xmm7, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm6, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm4 +; SSE41-NEXT: packusdw %xmm8, %xmm4 +; SSE41-NEXT: packusdw %xmm4, %xmm9 +; SSE41-NEXT: packuswb %xmm9, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: trunc_usat_v16i64_v16i8: diff --git a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll index c3e9a2b6841ae..54dc107fd0c10 100644 --- a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll +++ b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll @@ -3176,10 +3176,11 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_32(<2 x i64> % ; ; X86-AVX2-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_32: ; X86-AVX2: # %bb.0: -; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 -; X86-AVX2-NEXT: vpsrad $31, %xmm0, %xmm1 -; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; X86-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4294967294,4294967294,4294967294,4294967294] +; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm1 +; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; X86-AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3] ; X86-AVX2-NEXT: retl ; ; X64-SSE2-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_32: diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll index dea2446ecc320..4837532aaf5c5 100644 --- a/llvm/test/CodeGen/X86/vselect.ll +++ b/llvm/test/CodeGen/X86/vselect.ll @@ -740,14 +740,24 @@ define i64 @vselect_any_extend_vector_inreg_crash(ptr %x) { ; SSE-NEXT: shll $15, %eax ; SSE-NEXT: retq ; -; AVX-LABEL: vselect_any_extend_vector_inreg_crash: -; AVX: # %bb.0: -; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; AVX-NEXT: vpcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: andl $1, %eax -; AVX-NEXT: shll $15, %eax -; AVX-NEXT: retq +; AVX1-LABEL: vselect_any_extend_vector_inreg_crash: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX1-NEXT: vpcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovd %xmm0, %eax +; AVX1-NEXT: andl $1, %eax +; AVX1-NEXT: shll $15, %eax +; AVX1-NEXT: retq +; +; AVX2-LABEL: vselect_any_extend_vector_inreg_crash: +; AVX2: # %bb.0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [49,49,49,49] +; AVX2-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, %eax +; AVX2-NEXT: andl $1, %eax +; AVX2-NEXT: shll $15, %eax +; AVX2-NEXT: retq 0: %1 = load <8 x i8>, ptr %x %2 = icmp eq <8 x i8> %1,