Skip to content

[Downstream Change] Revert ccef34 and related commits #966

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
john-brawn-arm wants to merge 1 commit into
base: release/arm-software/23.x
Choose a base branch
from
+289 −691
Open

[Downstream Change] Revert ccef34 and related commits #966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4100,6 +4100,18 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
// TODO: Using the original IR may not be accurate.
// Currently, ARM will use the underlying IR to calculate gather/scatter
// instruction cost.
[[maybe_unused]] auto IsReverseMask = [this, R]() {
VPValue *Mask = getMask();
if (!Mask)
return false;

if (isa<VPWidenLoadEVLRecipe, VPWidenStoreEVLRecipe>(R))
return match(Mask, m_Intrinsic<Intrinsic::experimental_vp_reverse>());

return match(Mask, m_Reverse(m_VPValue()));
};
assert(!IsReverseMask() &&
"Inconsecutive memory access should not have reverse order");
Type *PtrTy = getAddr()->getScalarType();
const Value *Ptr = getAddr()->getUnderlyingValue();

Expand Down
98 changes: 6 additions & 92 deletions llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1887,58 +1887,6 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
}
}

/// Removes the permutation pattern \p Perm from any elementwise operations
/// in the plan, by constructing a new permutation via \p Build.
/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
template <typename Match_t, typename Builder>
static void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !vputils::isElementwise(Def))
continue;

// At least one of the ops must be a permutation.
if (!any_of(Def->operands(), match_fn(Perm(m_VPValue()))))
continue;

// All operands must be permuted or a live in (splat).
if (!all_of(
Def->operands(),
match_fn(m_CombineOr(m_OneUse(Perm(m_VPValue())), m_LiveIn()))))
continue;

VPValue *X;
// Remove the inner permutations.
for (unsigned I = 0; I < Def->getNumOperands(); I++)
if (match(Def->getOperand(I), Perm(m_VPValue(X))))
Def->setOperand(I, X);

VPSingleDefRecipe *Res = Build(Def);
Res->insertAfter(Def);
Def->replaceUsesWithIf(
Res, [&Res](VPUser &U, unsigned _) { return &U != Res; });
}
}
}

void VPlanTransforms::simplifyReverses(VPlan &Plan) {
// Pull out reverses from any elementwise op.
// binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
pullOutPermutations(
Plan, [](const auto &X) { return m_Reverse(X); },
[](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });

// reverse(reverse(x)) -> x
VPValue *X;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry())))
for (VPRecipeBase &R : make_early_inc_range(*VPBB))
if (match(&R, m_Reverse(m_Reverse(m_VPValue(X)))))
R.getVPSingleValue()->replaceAllUsesWith(X);
}

/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
/// header mask to be simplified further when tail folding, e.g. in
/// optimizeEVLMasks.
Expand Down Expand Up @@ -2935,7 +2883,6 @@ void VPlanTransforms::optimize(VPlan &Plan) {
RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
RUN_VPLAN_PASS(simplifyRecipes, Plan);
RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
RUN_VPLAN_PASS(simplifyReverses, Plan);
RUN_VPLAN_PASS(removeDeadRecipes, Plan);

RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
Expand Down Expand Up @@ -3273,45 +3220,11 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
}
}

// Pull out left splices from any elementwise op.
// binop(splice.left(poison, x, evl), live-in)
// -> splice.left(poison, binop(x,live-in), evl)
pullOutPermutations(
Plan,
[&EVL](const auto &X) {
return m_Intrinsic<Intrinsic::vector_splice_left>(m_Poison(), X,
m_Specific(EVL));
},
[&Plan, &EVL](auto *X) {
return new VPWidenIntrinsicRecipe(
Intrinsic::vector_splice_left,
{Plan.getPoison(X->getScalarType()), X, EVL}, X->getScalarType(),
{}, {}, X->getDebugLoc());
});

// Fold the following splice patterns:
// splice.right(splice.left(poison, x, evl), poison, evl) -> x
// Fold the following splice patterns into vp.reverse for reverse accesses:
// vector.reverse(splice.left(poison, x, evl)) -> vp.reverse(x, true, evl)
// splice.right(vector.reverse(x), poison, evl) -> vp.reverse(x, true, evl)
for (VPUser *U : collectUsersRecursively(EVL)) {
auto *R = cast<VPRecipeBase>(U);
// Remove potentially dead left splices from the transform above.
if (match(U, m_Intrinsic<Intrinsic::vector_splice_left>()) &&
R->getVPSingleValue()->getNumUsers() == 0) {
OldRecipes.push_back(R);
continue;
}

VPValue *X;
if (match(U, m_Intrinsic<Intrinsic::vector_splice_right>(
m_Intrinsic<Intrinsic::vector_splice_left>(
m_Poison(), m_VPValue(X), m_Specific(EVL)),
m_Poison(), m_Specific(EVL)))) {
R->getVPSingleValue()->replaceAllUsesWith(X);
OldRecipes.push_back(R);
continue;
}

if (!match(U,
m_CombineOr(
m_Reverse(m_Intrinsic<Intrinsic::vector_splice_left>(
Expand All @@ -3320,12 +3233,13 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
m_Reverse(m_VPValue(X)), m_Poison(), m_Specific(EVL)))))
continue;

auto *Def = cast<VPSingleDefRecipe>(U);
auto *VPReverse = new VPWidenIntrinsicRecipe(
Intrinsic::experimental_vp_reverse, {X, Plan.getTrue(), EVL},
X->getScalarType(), {}, {}, R->getDebugLoc());
VPReverse->insertBefore(R);
R->getVPSingleValue()->replaceAllUsesWith(VPReverse);
OldRecipes.push_back(R);
X->getScalarType(), {}, {}, Def->getDebugLoc());
VPReverse->insertBefore(Def);
Def->replaceAllUsesWith(VPReverse);
OldRecipes.push_back(Def);
}

for (VPRecipeBase *R : reverse(OldRecipes)) {
Expand Down
3 changes: 0 additions & 3 deletions llvm/lib/Transforms/Vectorize/VPlanTransforms.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,9 +400,6 @@ struct VPlanTransforms {
/// Perform instcombine-like simplifications on recipes in \p Plan.
static void simplifyRecipes(VPlan &Plan);

/// Cancel out redundant reverses in \p Plan, e.g. reverse(reverse(x)) -> x.
static void simplifyReverses(VPlan &Plan);

/// Remove BranchOnCond recipes with true or false conditions together with
/// removing dead edges to their successors. If \p OnlyLatches is true, only
/// process loop latches. Returns true if incoming values from any phi-like
Expand Down
9 changes: 0 additions & 9 deletions llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,15 +382,6 @@ static bool preservesUniformity(unsigned Opcode) {
}
}

bool vputils::isElementwise(const VPValue *V) {
unsigned Opcode = TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe>(
[](auto *R) { return R->getOpcode(); })
.Default([](auto *) { return 0; });
// TODO: Handle more opcodes and recipes.
return Instruction::isUnaryOp(Opcode) || Instruction::isBinaryOp(Opcode);
}

bool vputils::isSingleScalar(const VPValue *VPV) {
// Live-in, symbolic and canonical-IV region values are single-scalar.
if (auto *RV = dyn_cast<VPRegionValue>(VPV))
Expand Down
3 changes: 0 additions & 3 deletions llvm/lib/Transforms/Vectorize/VPlanUtils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ bool isSingleScalar(const VPValue *VPV);
/// VPDerivedIV or the canonical IV).
bool isUniformAcrossVFsAndUFs(const VPValue *V);

/// Return true if \p V is elementwise, i.e. none of the lanes are permuted.
bool isElementwise(const VPValue *V);

/// Returns the header block of the first, top-level loop, or null if none
/// exist.
VPBasicBlock *getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT);
Expand Down
5 changes: 4 additions & 1 deletion llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,12 +286,15 @@ define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC,
; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP10]]
; CHECK-TF-NEXT: [[REVERSE:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[REVERSE]], <vscale x 16 x i8> poison)
; CHECK-TF-NEXT: [[REVERSE3:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_LOAD]])
; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[PTRB]], i64 [[OFFSET_IDX]]
; CHECK-TF-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP10]]
; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[REVERSE]], <vscale x 16 x i8> poison)
; CHECK-TF-NEXT: [[REVERSE7:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
; CHECK-TF-NEXT: [[REVERSE6:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_LOAD5]])
; CHECK-TF-NEXT: [[TMP14:%.*]] = add <vscale x 16 x i8> [[REVERSE6]], [[REVERSE3]]
; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[PTRC]], i64 [[OFFSET_IDX]]
; CHECK-TF-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 [[TMP10]]
; CHECK-TF-NEXT: [[REVERSE7:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[TMP14]])
; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[REVERSE7]], ptr align 1 [[TMP16]], <vscale x 16 x i1> [[REVERSE]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[IV_START]])
Expand Down
9 changes: 4 additions & 5 deletions llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,13 +350,13 @@ define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
; CHECK-NOTF: vector.body:
; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
; CHECK-NOTF: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
; CHECK-NOTF: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %[[LOAD]])

; CHECK-TF-NOREV-LABEL: @reverse(
; CHECK-TF-NOREV: vector.body:
; CHECK-TF-NOREV-NOT: %{{.*}} = phi <vscale x 4 x i1>
; CHECK-TF-NOREV: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
; CHECK-TF-NOREV: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
; CHECK-TF-NOREV: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %[[LOAD]])

; CHECK-TF-LABEL: @reverse(
; CHECK-TF: vector.body:
Expand All @@ -381,13 +381,12 @@ entry:

for.body:
%indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]
%indvars.ptr = phi ptr [ %dst, %entry ], [ %indvars.ptr.next, %for.body ]
%arrayidx = getelementptr inbounds double, ptr %src, i64 %indvars.iv
%0 = load double, ptr %arrayidx, align 8
%add = fadd double %0, 1.000000e+00
store double %add, ptr %indvars.ptr, align 8
%arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %indvars.iv
store double %add, ptr %arrayidx2, align 8
%indvars.iv.next = add nsw i64 %indvars.iv, -1
%indvars.ptr.next = getelementptr inbounds double, ptr %indvars.ptr, i64 1
%cmp.not = icmp eq i64 %indvars.iv, 0
br i1 %cmp.not, label %for.end, label %for.body

Expand Down
6 changes: 4 additions & 2 deletions llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,11 @@ define void @vector_reverse_mask_nxv4i1(ptr %a, ptr %cond, i64 %N) #0 {
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[TMP12]], i64 [[TMP9]]
; CHECK-NEXT: [[REVERSE2:%.*]] = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> [[TMP11]])
; CHECK-NEXT: [[REVERSE3:%.*]] = call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr align 8 [[TMP13]], <vscale x 4 x i1> [[REVERSE2]], <vscale x 4 x double> poison), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0(ptr align 8 [[TMP13]], <vscale x 4 x i1> [[REVERSE2]], <vscale x 4 x double> poison), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[REVERSE3:%.*]] = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> [[WIDE_MASKED_LOAD]])
; CHECK-NEXT: [[TMP14:%.*]] = fadd <vscale x 4 x double> [[REVERSE3]], splat (double 1.000000e+00)
; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0(<vscale x 4 x double> [[TMP14]], ptr align 8 [[TMP13]], <vscale x 4 x i1> [[REVERSE2]]), !alias.scope [[META3]], !noalias [[META0]]
; CHECK-NEXT: [[REVERSE4:%.*]] = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> [[TMP14]])
; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0(<vscale x 4 x double> [[REVERSE4]], ptr align 8 [[TMP13]], <vscale x 4 x i1> [[REVERSE2]]), !alias.scope [[META3]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
Expand Down
16 changes: 12 additions & 4 deletions llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,15 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i64 [[TMP22]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x double>, ptr [[TMP14]], align 8
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x double>, ptr [[TMP15]], align 8
; CHECK-NEXT: [[TMP16:%.*]] = fadd <vscale x 8 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP17:%.*]] = fadd <vscale x 8 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
; CHECK-NEXT: [[REVERSE:%.*]] = call <vscale x 8 x double> @llvm.vector.reverse.nxv8f64(<vscale x 8 x double> [[WIDE_LOAD]])
; CHECK-NEXT: [[REVERSE2:%.*]] = call <vscale x 8 x double> @llvm.vector.reverse.nxv8f64(<vscale x 8 x double> [[WIDE_LOAD1]])
; CHECK-NEXT: [[TMP18:%.*]] = fadd <vscale x 8 x double> [[REVERSE]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP19:%.*]] = fadd <vscale x 8 x double> [[REVERSE2]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, ptr [[TMP21]], i64 [[TMP12]]
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds double, ptr [[TMP21]], i64 [[TMP22]]
; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 8 x double> @llvm.vector.reverse.nxv8f64(<vscale x 8 x double> [[TMP18]])
; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 8 x double> @llvm.vector.reverse.nxv8f64(<vscale x 8 x double> [[TMP19]])
; CHECK-NEXT: store <vscale x 8 x double> [[TMP16]], ptr [[TMP20]], align 8
; CHECK-NEXT: store <vscale x 8 x double> [[TMP17]], ptr [[TMP24]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
Expand Down Expand Up @@ -112,11 +116,15 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP25]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP17]], align 8
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i64>, ptr [[TMP18]], align 8
; CHECK-NEXT: [[TMP19:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD]], splat (i64 1)
; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD3]], splat (i64 1)
; CHECK-NEXT: [[REVERSE:%.*]] = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD]])
; CHECK-NEXT: [[REVERSE4:%.*]] = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD3]])
; CHECK-NEXT: [[TMP21:%.*]] = add <vscale x 8 x i64> [[REVERSE]], splat (i64 1)
; CHECK-NEXT: [[TMP22:%.*]] = add <vscale x 8 x i64> [[REVERSE4]], splat (i64 1)
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 [[TMP25]]
; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> [[TMP21]])
; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> [[TMP22]])
; CHECK-NEXT: store <vscale x 8 x i64> [[TMP19]], ptr [[TMP23]], align 8
; CHECK-NEXT: store <vscale x 8 x i64> [[TMP20]], ptr [[TMP27]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
Expand Down
8 changes: 6 additions & 2 deletions llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,12 @@ define void @vector_reverse_mask_v4i1(ptr noalias %a, ptr noalias %cond, i64 %N)
; CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP8]], <4 x i1> [[REVERSE3]], <4 x double> poison)
; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP9]], <4 x i1> [[REVERSE5]], <4 x double> poison)
; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD6]], splat (double 1.000000e+00)
; CHECK-NEXT: [[REVERSE6:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>

@david-arm david-arm Aug 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's unfortunate that we're removing these folds, however this is more of a cost model problem since the instcombine pass would apply the same folds afterwards. Removing the shuffles or vector.reverse calls increases the likelihood of vectorisation by lowering the cost.

You can see the effects of this in the vector-reverse.ll test below. I find it hard to believe this doesn't introduce a regression somewhere, but if these reverts have been fully tested on various benchmarks then I guess it's ok.

; CHECK-NEXT: [[REVERSE7:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD6]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP16:%.*]] = fadd <4 x double> [[REVERSE6]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP17:%.*]] = fadd <4 x double> [[REVERSE7]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP10]], ptr align 8 [[TMP8]], <4 x i1> [[REVERSE3]])
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP11]], ptr align 8 [[TMP9]], <4 x i1> [[REVERSE5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
Expand Down
Loading