Merge branch 'main' into users/mcberg2021_LoopDistribute2

Add a boolean marker noting when loop distribution was sucessfully applied
in a loops meta data, then check for loops which are already distributed
to prevent reprocessing.

llvm/docs/LangRef.rst

@@ -7828,6 +7828,13 @@ The attributes in this metadata are added to all followup loops of the
 loop distribution pass. See
 :ref:`Transformation Metadata <transformation-metadata>` for details.
 
+'``llvm.loop.isdistributed``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If a loop was successfully processed by the loop distribution pass,
+this metadata is added (i.e. has been distributed).  See
+:ref:`Transformation Metadata <transformation-metadata>` for details.
+
 '``llvm.licm.disable``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

llvm/docs/TransformMetadata.rst

@@ -346,6 +346,18 @@ It is recommended to add ``llvm.loop.disable_nonforced`` to
 fallback version (which is likely never executed) is further optimized
 which would increase the code size.
 
+Attributes defined in ``llvm.loop.isdistributed`` are added to successfully
+distributed loops to prevent subsequent reprocessing.
+
+As an example, the following instructs a loop to be ignored during
+loop distribution.
+
+.. code-block:: llvm
+
+    !4 = distinct !{!4, !5, !6}
+    !5 = !{!"llvm.loop.mustprogress"}
+    !6 = !{!"llvm.loop.isdistributed", i32 1}
+
 Versioning LICM
 ---------------
 

llvm/lib/Transforms/Scalar/LoopDistribute.cpp

@@ -113,6 +113,8 @@ static cl::opt<bool> EnableLoopDistribute(
     cl::desc("Enable the new, experimental LoopDistribution Pass"),
     cl::init(false));
 
+static const char *DistributedMetaData = "llvm.loop.isdistributed";
+
 STATISTIC(NumLoopsDistributed, "Number of loops distributed");
 
 namespace {
@@ -826,6 +828,8 @@ public:
           {LLVMLoopDistributeFollowupAll, LLVMLoopDistributeFollowupFallback},
           "llvm.loop.distribute.", true);
       LVer.getNonVersionedLoop()->setLoopID(UnversionedLoopID);
+      addStringMetadataToLoop(LVer.getNonVersionedLoop(), DistributedMetaData,
+                              true);
     }
 
     // Create identical copies of the original loop for each partition and hook
@@ -986,6 +990,13 @@ static bool runImpl(Function &F, LoopInfo *LI, DominatorTree *DT,
   for (Loop *L : Worklist) {
     LoopDistributeForLoop LDL(L, &F, LI, DT, SE, LAIs, ORE);
 
+    // Do not reprocess loops we already distributed
+    if (getOptionalBoolLoopAttribute(L, DistributedMetaData).value_or(false)) {
+      LLVM_DEBUG(
+          dbgs() << "LDist: Distributed loop guarded for reprocessing\n");
+      continue;
+    }
+
     // If distribution was forced for the specific loop to be
     // enabled/disabled, follow that.  Otherwise use the global flag.
     if (LDL.isForced().value_or(EnableLoopDistribute))

llvm/test/Transforms/LoopDistribute/cross-partition-access.ll

@@ -64,30 +64,30 @@ define dso_local void @_Z13distribution3PiS_S_S_i(ptr nocapture noundef %a, ptr
 ; CHECK:       [[FOR_BODY_LDIST1]]:
 ; CHECK-NEXT:    [[IDXPROM_LDIST1:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LDIST1]] ], [ [[I6_LDIST1:%.*]], %[[FOR_BODY_LDIST1]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IDXPROM_LDIST1]]
-; CHECK-NEXT:    [[I2_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META6:![0-9]+]]
+; CHECK-NEXT:    [[I2_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
 ; CHECK-NEXT:    [[ADD4_LDIST1:%.*]] = add nsw i32 [[I2_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDX8_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM_LDIST1]]
-; CHECK-NEXT:    store i32 [[ADD4_LDIST1]], ptr [[ARRAYIDX8_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT:    store i32 [[ADD4_LDIST1]], ptr [[ARRAYIDX8_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META10:![0-9]+]], !noalias [[META12:![0-9]+]]
 ; CHECK-NEXT:    [[I3_LDIST1:%.*]] = getelementptr i32, ptr [[C]], i64 [[IDXPROM_LDIST1]]
 ; CHECK-NEXT:    [[ARRAYIDX17_LDIST1:%.*]] = getelementptr i8, ptr [[I3_LDIST1]], i64 -4
-; CHECK-NEXT:    [[I4_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX17_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META14:![0-9]+]], !noalias [[META15:![0-9]+]]
+; CHECK-NEXT:    [[I4_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX17_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]]
 ; CHECK-NEXT:    [[SUB18_LDIST1:%.*]] = sub nsw i32 [[ADD4_LDIST1]], [[I4_LDIST1]]
-; CHECK-NEXT:    store i32 [[SUB18_LDIST1]], ptr [[I3_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META14]], !noalias [[META15]]
+; CHECK-NEXT:    store i32 [[SUB18_LDIST1]], ptr [[I3_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15]], !noalias [[META16]]
 ; CHECK-NEXT:    [[I6_LDIST1]] = add i64 [[IDXPROM_LDIST1]], 1
 ; CHECK-NEXT:    [[CMP1_NOT_LDIST1:%.*]] = icmp eq i64 [[I6_LDIST1]], [[LEN]]
-; CHECK-NEXT:    br i1 [[CMP1_NOT_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       [[FOR_BODY_PH]]:
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH]] ], [ [[I6:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM]]
-; CHECK-NEXT:    [[I5:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9]], !noalias [[META11]]
+; CHECK-NEXT:    [[I5:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA0]], !alias.scope [[META10]], !noalias [[META12]]
 ; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[I5]], 2
 ; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IDXPROM]]
-; CHECK-NEXT:    store i32 [[ADD27]], ptr [[ARRAYIDX31]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15]], !noalias [[META6]]
+; CHECK-NEXT:    store i32 [[ADD27]], ptr [[ARRAYIDX31]], align 4, !tbaa [[TBAA0]], !alias.scope [[META16]], !noalias [[META7]]
 ; CHECK-NEXT:    [[I6]] = add i64 [[IDXPROM]], 1
 ; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i64 [[I6]], [[LEN]]
-; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[END_LOOPEXIT_LOOPEXIT20:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP16]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[END_LOOPEXIT_LOOPEXIT20:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP17]]
 ; CHECK:       [[END_LOOPEXIT_LOOPEXIT]]:
 ; CHECK-NEXT:    br label %[[END_LOOPEXIT:.*]]
 ; CHECK:       [[END_LOOPEXIT_LOOPEXIT20]]:
@@ -143,17 +143,18 @@ end:                                              ; preds = %end.loopexit, %entr
 ; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
 ; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
 ; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
 ; CHECK: [[META5]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.isdistributed", i32 1}
-; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
-; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
-; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]}
+; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
-; CHECK: [[META11]] = !{[[META12:![0-9]+]], [[META13:![0-9]+]], [[META7]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META8]]}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]], [[META14:![0-9]+]], [[META8]]}
-; CHECK: [[META13]] = distinct !{[[META13]], [[META8]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META9]]}
-; CHECK: [[META14]] = !{[[META12]]}
+; CHECK: [[META14]] = distinct !{[[META14]], [[META9]]}
 ; CHECK: [[META15]] = !{[[META13]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META5]]}
+; CHECK: [[META16]] = !{[[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
 ;.

llvm/test/Transforms/LoopDistribute/followup.ll

@@ -58,29 +58,29 @@ define void @f(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) {
 ; CHECK:       [[FOR_BODY_LDIST1]]:
 ; CHECK-NEXT:    [[IND_LDIST1:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LDIST1]] ], [ [[ADD_LDIST1:%.*]], %[[FOR_BODY_LDIST1]] ]
 ; CHECK-NEXT:    [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IND_LDIST1]]
-; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
+; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IND_LDIST1]]
-; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4, !alias.scope [[META10:![0-9]+]]
+; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4, !alias.scope [[META11:![0-9]+]]
 ; CHECK-NEXT:    [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]]
 ; CHECK-NEXT:    [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]]
-; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META3]], !noalias [[META6]]
+; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META4]], !noalias [[META7]]
 ; CHECK-NEXT:    [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], 20
-; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[FOR_BODY_PH]]:
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[IND:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[IND]], 1
 ; CHECK-NEXT:    [[ARRAYIDXD:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IND]]
-; CHECK-NEXT:    [[LOADD:%.*]] = load i32, ptr [[ARRAYIDXD]], align 4, !alias.scope [[META14:![0-9]+]]
+; CHECK-NEXT:    [[LOADD:%.*]] = load i32, ptr [[ARRAYIDXD]], align 4, !alias.scope [[META15:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXE:%.*]] = getelementptr inbounds i32, ptr [[E]], i64 [[IND]]
-; CHECK-NEXT:    [[LOADE:%.*]] = load i32, ptr [[ARRAYIDXE]], align 4, !alias.scope [[META15:![0-9]+]]
+; CHECK-NEXT:    [[LOADE:%.*]] = load i32, ptr [[ARRAYIDXE]], align 4, !alias.scope [[META16:![0-9]+]]
 ; CHECK-NEXT:    [[MULC:%.*]] = mul i32 [[LOADD]], [[LOADE]]
 ; CHECK-NEXT:    [[ARRAYIDXC:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IND]]
-; CHECK-NEXT:    store i32 [[MULC]], ptr [[ARRAYIDXC]], align 4, !alias.scope [[META16:![0-9]+]], !noalias [[META10]]
+; CHECK-NEXT:    store i32 [[MULC]], ptr [[ARRAYIDXC]], align 4, !alias.scope [[META17:![0-9]+]], !noalias [[META11]]
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[ADD]], 20
-; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT16:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT16:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[FOR_END_LOOPEXIT]]:
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[FOR_END_LOOPEXIT16]]:
@@ -131,23 +131,24 @@ for.end:
 !4 = !{!"llvm.loop.distribute.followup_sequential", !{!"FollowupSequential", i32 8}}
 !5 = !{!"llvm.loop.distribute.followup_fallback", !{!"FollowupFallback"}}
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"FollowupAll"}
 ; CHECK: [[META2]] = !{!"FollowupFallback"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.isdistributed", i32 1}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]]}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], !"LVerDomain"}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
-; CHECK: [[META6]] = !{[[META7:![0-9]+]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
-; CHECK: [[META7]] = distinct !{[[META7]], [[META5]]}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META5]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
-; CHECK: [[META9]] = distinct !{[[META9]], [[META5]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META6]]}
-; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
-; CHECK: [[META11]] = distinct !{[[META11]], [[META5]]}
+; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META13:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
-; CHECK: [[META13]] = !{!"FollowupSequential", i32 8}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META14:![0-9]+]]}
-; CHECK: [[META14]] = !{[[META8]]}
+; CHECK: [[META14]] = !{!"FollowupSequential", i32 8}
 ; CHECK: [[META15]] = !{[[META9]]}
-; CHECK: [[META16]] = !{[[META7]]}
+; CHECK: [[META16]] = !{[[META10]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META18:![0-9]+]]}
+; CHECK: [[META17]] = !{[[META8]]}
-; CHECK: [[META18]] = !{!"FollowupCoincident", i1 false}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META19:![0-9]+]]}
+; CHECK: [[META19]] = !{!"FollowupCoincident", i1 false}
 ;.

llvm/test/Transforms/LoopDistribute/laa-invalidation.ll

@@ -27,13 +27,13 @@ define void @test_pr50940(ptr %A, ptr %B) {
 ; CHECK-NEXT:    store i16 1, ptr [[B]], align 1
 ; CHECK-NEXT:    [[IV_NEXT_LVER_ORIG]] = add nuw nsw i16 [[IV_LVER_ORIG]], 1
 ; CHECK-NEXT:    [[C_1_LVER_ORIG:%.*]] = icmp ult i16 [[IV_LVER_ORIG]], 38
-; CHECK-NEXT:    br i1 [[C_1_LVER_ORIG]], label [[INNER_LVER_ORIG]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[C_1_LVER_ORIG]], label [[INNER_LVER_ORIG]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       inner.ph3.ldist1:
 ; CHECK-NEXT:    br label [[INNER_LDIST1:%.*]]
 ; CHECK:       inner.ldist1:
 ; CHECK-NEXT:    [[IV_LDIST1:%.*]] = phi i16 [ 0, [[INNER_PH3_LDIST1]] ], [ [[IV_NEXT_LDIST1:%.*]], [[INNER_LDIST1]] ]
-; CHECK-NEXT:    [[L_LDIST1:%.*]] = load <2 x i16>, ptr [[UGLYGEP]], align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[L_LDIST1:%.*]] = load <2 x i16>, ptr [[UGLYGEP]], align 1, !alias.scope [[META2:![0-9]+]], !noalias [[META5:![0-9]+]]
-; CHECK-NEXT:    store i16 0, ptr [[GEP_A_3]], align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    store i16 0, ptr [[GEP_A_3]], align 1, !alias.scope [[META2]], !noalias [[META5]]
 ; CHECK-NEXT:    [[IV_NEXT_LDIST1]] = add nuw nsw i16 [[IV_LDIST1]], 1
 ; CHECK-NEXT:    [[C_1_LDIST1:%.*]] = icmp ult i16 [[IV_LDIST1]], 38
 ; CHECK-NEXT:    br i1 [[C_1_LDIST1]], label [[INNER_LDIST1]], label [[INNER_PH3:%.*]]
@@ -41,7 +41,7 @@ define void @test_pr50940(ptr %A, ptr %B) {
 ; CHECK-NEXT:    br label [[INNER:%.*]]
 ; CHECK:       inner:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[INNER_PH3]] ], [ [[IV_NEXT:%.*]], [[INNER]] ]
-; CHECK-NEXT:    store i16 1, ptr [[B]], align 1, !alias.scope !3
+; CHECK-NEXT:    store i16 1, ptr [[B]], align 1, !alias.scope [[META5]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i16 [[IV]], 38
 ; CHECK-NEXT:    br i1 [[C_1]], label [[INNER]], label [[EXIT_LOOPEXIT4:%.*]]

llvm/test/Transforms/LoopDistribute/no-reprocess.ll

@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -aa-pipeline=basic-aa -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -debug-only=loop-distribute --disable-output -S < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Test if a loop already distributed will not reprocess because of metadata
+; information marking it as processed.
+
+; CHECK: LDist: Distributed loop guarded for reprocessing
+; CHECK: LDist: Skipping;
+
+define dso_local void @_Z13distribution3PiS_S_S_i(ptr noundef captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef captures(none) %c, ptr noundef writeonly captures(none) %d, i64 noundef signext %len) {
+entry:
+  %cmp = icmp sgt i64 %len, 0
+  br i1 %cmp, label %end, label %for.body.lver.check
+
+for.body.lver.check:                              ; preds = %entry
+  %0 = shl i64 %len, 2
+  %scevgep = getelementptr i8, ptr %a, i64 %0
+  %scevgep1 = getelementptr i8, ptr %c, i64 -4
+  %scevgep2 = getelementptr i8, ptr %c, i64 %0
+  %scevgep3 = getelementptr i8, ptr %d, i64 %0
+  %scevgep4 = getelementptr i8, ptr %b, i64 %0
+  %bound0 = icmp ult ptr %a, %scevgep2
+  %bound1 = icmp ult ptr %scevgep1, %scevgep
+  %found.conflict = and i1 %bound0, %bound1
+  %bound05 = icmp ult ptr %a, %scevgep3
+  %bound16 = icmp ult ptr %d, %scevgep
+  %found.conflict7 = and i1 %bound05, %bound16
+  %conflict.rdx = or i1 %found.conflict, %found.conflict7
+  %bound08 = icmp ult ptr %a, %scevgep4
+  %bound19 = icmp ult ptr %b, %scevgep
+  %found.conflict10 = and i1 %bound08, %bound19
+  %conflict.rdx11 = or i1 %conflict.rdx, %found.conflict10
+  %bound012 = icmp ult ptr %scevgep1, %scevgep3
+  %bound113 = icmp ult ptr %d, %scevgep2
+  %found.conflict14 = and i1 %bound012, %bound113
+  %conflict.rdx15 = or i1 %conflict.rdx11, %found.conflict14
+  %bound016 = icmp ult ptr %d, %scevgep4
+  %bound117 = icmp ult ptr %b, %scevgep3
+  %found.conflict18 = and i1 %bound016, %bound117
+  %conflict.rdx19 = or i1 %conflict.rdx15, %found.conflict18
+  br i1 %conflict.rdx19, label %for.body.ph.lver.orig, label %for.body.ph.ldist1
+
+for.body.ph.lver.orig:                            ; preds = %for.body.lver.check
+  br label %for.body.lver.orig
+
+for.body.lver.orig:                               ; preds = %for.body.lver.orig, %for.body.ph.lver.orig
+  %indvars.iv.lver.orig = phi i64 [ 0, %for.body.ph.lver.orig ], [ %indvars.iv.next.lver.orig, %for.body.lver.orig ]
+  %arrayidx.lver.orig = getelementptr inbounds i32, ptr %b, i64 %indvars.iv.lver.orig
+  %i2.lver.orig = load i32, ptr %arrayidx.lver.orig, align 4, !tbaa !0
+  %add4.lver.orig = add nsw i32 %i2.lver.orig, 1
+  %arrayidx8.lver.orig = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.lver.orig
+  store i32 %add4.lver.orig, ptr %arrayidx8.lver.orig, align 4, !tbaa !0
+  %i3.lver.orig = getelementptr i32, ptr %c, i64 %indvars.iv.lver.orig
+  %arrayidx17.lver.orig = getelementptr i8, ptr %i3.lver.orig, i64 -4
+  %i4.lver.orig = load i32, ptr %arrayidx17.lver.orig, align 4, !tbaa !0
+  %sub18.lver.orig = sub nsw i32 %add4.lver.orig, %i4.lver.orig
+  store i32 %sub18.lver.orig, ptr %i3.lver.orig, align 4, !tbaa !0
+  %i5.lver.orig = load i32, ptr %arrayidx8.lver.orig, align 4, !tbaa !0
+  %add27.lver.orig = add nsw i32 %i5.lver.orig, 2
+  %arrayidx31.lver.orig = getelementptr inbounds i32, ptr %d, i64 %indvars.iv.lver.orig
+  store i32 %add27.lver.orig, ptr %arrayidx31.lver.orig, align 4, !tbaa !0
+  %indvars.iv.next.lver.orig = add i64 %indvars.iv.lver.orig, 1
+  %cmp1.not.lver.orig = icmp eq i64 %indvars.iv.next.lver.orig, %len
+  br i1 %cmp1.not.lver.orig, label %end.loopexit.loopexit, label %for.body.lver.orig, !llvm.loop !4
+
+for.body.ph.ldist1:                               ; preds = %for.body.lver.check
+  br label %for.body.ldist1
+
+for.body.ldist1:                                  ; preds = %for.body.ldist1, %for.body.ph.ldist1
+  %indvars.iv.ldist1 = phi i64 [ 0, %for.body.ph.ldist1 ], [ %indvars.iv.next.ldist1, %for.body.ldist1 ]
+  %arrayidx.ldist1 = getelementptr inbounds i32, ptr %b, i64 %indvars.iv.ldist1
+  %i2.ldist1 = load i32, ptr %arrayidx.ldist1, align 4, !tbaa !0, !alias.scope !7
+  %add4.ldist1 = add nsw i32 %i2.ldist1, 1
+  %arrayidx8.ldist1 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.ldist1
+  store i32 %add4.ldist1, ptr %arrayidx8.ldist1, align 4, !tbaa !0, !alias.scope !10, !noalias !12
+  %i3.ldist1 = getelementptr i32, ptr %c, i64 %indvars.iv.ldist1
+  %arrayidx17.ldist1 = getelementptr i8, ptr %i3.ldist1, i64 -4
+  %i4.ldist1 = load i32, ptr %arrayidx17.ldist1, align 4, !tbaa !0, !alias.scope !15, !noalias !16
+  %sub18.ldist1 = sub nsw i32 %add4.ldist1, %i4.ldist1
+  store i32 %sub18.ldist1, ptr %i3.ldist1, align 4, !tbaa !0, !alias.scope !15, !noalias !16
+  %indvars.iv.next.ldist1 = add i64 %indvars.iv.ldist1, 1
+  %cmp1.not.ldist1 = icmp eq i64 %indvars.iv.next.ldist1, %len
+  br i1 %cmp1.not.ldist1, label %for.body.ph, label %for.body.ldist1, !llvm.loop !17
+
+for.body.ph:                                      ; preds = %for.body.ldist1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.ph
+  %indvars.iv = phi i64 [ 0, %for.body.ph ], [ %indvars.iv.next, %for.body ]
+  %arrayidx8 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+  %i5 = load i32, ptr %arrayidx8, align 4, !tbaa !0, !alias.scope !10, !noalias !12
+  %add27 = add nsw i32 %i5, 2
+  %arrayidx31 = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
+  store i32 %add27, ptr %arrayidx31, align 4, !tbaa !0, !alias.scope !16, !noalias !7
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %cmp1.not = icmp eq i64 %indvars.iv.next, %len
+  br i1 %cmp1.not, label %end.loopexit.loopexit20, label %for.body, !llvm.loop !17
+
+end.loopexit.loopexit:                            ; preds = %for.body.lver.orig
+  br label %end.loopexit
+
+end.loopexit.loopexit20:                          ; preds = %for.body
+  br label %end.loopexit
+
+end.loopexit:                                     ; preds = %end.loopexit.loopexit20, %end.loopexit.loopexit
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C++ TBAA"}
+!4 = distinct !{!4, !5, !6}
+!5 = !{!"llvm.loop.mustprogress"}
+!6 = !{!"llvm.loop.isdistributed", i32 1}
+!7 = !{!8}
+!8 = distinct !{!8, !9}
+!9 = distinct !{!9, !"LVerDomain"}
+!10 = !{!11}
+!11 = distinct !{!11, !9}
+!12 = !{!13, !14, !8}
+!13 = distinct !{!13, !9}
+!14 = distinct !{!14, !9}
+!15 = !{!13}
+!16 = !{!14}
+!17 = distinct !{!17, !5}

llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll

@@ -36,7 +36,7 @@ define void @f(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d, p
 ; CHECK-NEXT:    [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]]
 ; CHECK-NEXT:    store i32 [[MULC_LVER_ORIG]], ptr [[ARRAYIDXC_LVER_ORIG]], align 4
 ; CHECK-NEXT:    [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
+; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.body.ph.ldist1:
 ; CHECK-NEXT:    br label [[FOR_BODY_LDIST1:%.*]]
 ; CHECK:       for.body.ldist1:
@@ -45,14 +45,14 @@ define void @f(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d, p
 ; CHECK-NEXT:    [[MUL_LDIST1:%.*]] = mul i32 [[IND1_LDIST1]], 2
 ; CHECK-NEXT:    [[MUL_EXT_LDIST1:%.*]] = zext i32 [[MUL_LDIST1]] to i64
 ; CHECK-NEXT:    [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_EXT_LDIST1]]
-; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META2:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[MUL_EXT_LDIST1]]
 ; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4
 ; CHECK-NEXT:    [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]]
 ; CHECK-NEXT:    [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1
 ; CHECK-NEXT:    [[INC1_LDIST1]] = add i32 [[IND1_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]]
-; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META5:![0-9]+]]
 ; CHECK-NEXT:    [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]]
 ; CHECK:       for.body.ph:
@@ -161,7 +161,7 @@ define void @f_with_offset(ptr noalias %b, ptr noalias %c, ptr noalias %d, ptr n
 ; CHECK-NEXT:    [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]]
 ; CHECK-NEXT:    store i32 [[MULC_LVER_ORIG]], ptr [[ARRAYIDXC_LVER_ORIG]], align 4
 ; CHECK-NEXT:    [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
+; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       for.body.ph.ldist1:
 ; CHECK-NEXT:    br label [[FOR_BODY_LDIST1:%.*]]
 ; CHECK:       for.body.ldist1:
@@ -170,14 +170,14 @@ define void @f_with_offset(ptr noalias %b, ptr noalias %c, ptr noalias %d, ptr n
 ; CHECK-NEXT:    [[MUL_LDIST1:%.*]] = mul i32 [[IND1_LDIST1]], 2
 ; CHECK-NEXT:    [[MUL_EXT_LDIST1:%.*]] = zext i32 [[MUL_LDIST1]] to i64
 ; CHECK-NEXT:    [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_EXT_LDIST1]]
-; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META5:![0-9]+]]
+; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META8:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[MUL_EXT_LDIST1]]
 ; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4
 ; CHECK-NEXT:    [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]]
 ; CHECK-NEXT:    [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1
 ; CHECK-NEXT:    [[INC1_LDIST1]] = add i32 [[IND1_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]]
-; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META8:![0-9]+]]
+; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META11:![0-9]+]]
 ; CHECK-NEXT:    [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]]
 ; CHECK:       for.body.ph:

llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll

@@ -42,8 +42,8 @@ for.body:                                         ; preds = %for.body, %entry
 
   %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
 
-; CHECK: %loadA.ldist1 = {{.*}} !noalias !25
+; CHECK: %loadA.ldist1 = {{.*}} !alias.scope !24, !noalias !27
-; A noalias C: !25 -> { 17(15), 18(15), 19(15), 26(24) }
+; A noalias C: !33 -> { 19(17), 20(17), 21(17), 28(26) }
 ;                       ^^^^^^
   %loadA = load i32, ptr %arrayidxA, align 4
 
@@ -60,15 +60,15 @@ for.body:                                         ; preds = %for.body, %entry
 
   %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
 
-; CHECK: %loadD = {{.*}} !alias.scope !31
+; CHECK: %loadD = {{.*}} !alias.scope !33
-; D's scope: !31 -> { 18(15), 32(33) }
+; D's scope: !33 -> { 20(17), 34(35) }
 ;                             ^^^^^^
   %loadD = load i32, ptr %arrayidxD, align 4
 
   %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
 
-; CHECK: %loadE = {{.*}} !alias.scope !34
+; CHECK: %loadE = {{.*}} !alias.scope !36
-; E's scope: !34 -> { 19(15), 35(33) }
+; E's scope: !36 -> { 21(17), 37(33) }
 ;                             ^^^^^^
   %loadE = load i32, ptr %arrayidxE, align 4
 
@@ -76,8 +76,8 @@ for.body:                                         ; preds = %for.body, %entry
 
   %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
 
-; CHECK: store i32 %mulC, {{.*}} !alias.scope !36, !noalias !38
+; CHECK: store i32 %mulC, {{.*}} !alias.scope !38, !noalias !40
-; C's scope: !36 -> { 17(15), 37(33) }
+; C's scope: !38 -> { 19(17), 39(35)
 ;                     ^^^^^^
 ; C noalias D and E: !38 -> { 21(15), 32(33), 35(33) }
 ;                                     ^^^^^^  ^^^^^^
@@ -92,15 +92,19 @@ for.end:                                          ; preds = %for.body
 
 ; Domain for the second loop versioning for the top loop after
 ; distribution.
-; CHECK: !15 = distinct !{!15, !"LVerDomain"}
+; CHECK: !17 = distinct !{!17, !"LVerDomain"}
-; CHECK: !17 = distinct !{!17, !15}
+; CHECK: !19 = distinct !{!19, !17}
-; CHECK: !25 = !{!17, !18, !19, !26}
+; CHECK: !20 = distinct !{!20, !17}
-; CHECK: !31 = !{!18, !32}
+; CHECK: !21 = distinct !{!21, !17}
-; CHECK: !32 = distinct !{!32, !33}
+; CHECK: !27 = !{!19, !20, !21, !28}
+; CHECK: !28 = distinct !{!28, !26}
+; CHECK: !33 = !{!20, !34}
+; CHECK: !34 = distinct !{!34, !35}
 ; Domain for the second loop versioning for the bottom loop after
 ; distribution.
-; CHECK: !33 = distinct !{!33, !"LVerDomain"}
+; CHECK: !35 = distinct !{!35, !"LVerDomain"}
-; CHECK: !34 = !{!19, !35}
+; CHECK: !36 = !{!21, !37}
-; CHECK: !35 = distinct !{!35, !33}
+; CHECK: !37 = distinct !{!37, !35}
-; CHECK: !36 = !{!17, !37}
+; CHECK: !38 = !{!19, !39}
-; CHECK: !38 = !{!21, !32, !35}
+; CHECK: !39 = distinct !{!39, !35}
+; CHECK: !40 = !{!23, !34, !37}

llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected

@@ -72,12 +72,12 @@ define void @ldist(i1 %cond, ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:  LDist: Partition 0:
 ; CHECK-NEXT:  for.body.ldist1: ; preds = %if.end.ldist1, %for.body.ph.ldist1
 ; CHECK-NEXT:    %iv.ldist1 = phi i16 [ 0, %for.body.ph.ldist1 ], [ %iv.next.ldist1, %if.end.ldist1 ]
-; CHECK-NEXT:    %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !2, !noalias !5
-; CHECK-NEXT:    store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !2, !noalias !5
 ; CHECK-NEXT:    br i1 %cond, label %if.then.ldist1, label %if.end.ldist1
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.then.ldist1: ; preds = %for.body.ldist1
-; CHECK-NEXT:    %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !2, !noalias !5
 ; CHECK-NEXT:    br label %if.end.ldist1
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.end.ldist1: ; preds = %if.then.ldist1, %for.body.ldist1