AtomicExpand: Refactor atomic instruction handling (#102914)
Move the processing of an instruction into a helper function. Also avoid redundant checking for all types of atomic instructions. Including the assert, it was effectively performing the same check 3 times.
This commit is contained in:
Matt Arsenault
GitHub
parent
55323ca6c8
commit
2d7a2c1212
@ -119,6 +119,8 @@ private:
|
|||||||
llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
|
llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
|
||||||
CreateCmpXchgInstFun CreateCmpXchg);
|
CreateCmpXchgInstFun CreateCmpXchg);
|
||||||
|
|
||||||
|
bool processAtomicInstr(Instruction *I);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
bool run(Function &F, const TargetMachine *TM);
|
bool run(Function &F, const TargetMachine *TM);
|
||||||
};
|
};
|
||||||
@ -203,6 +205,143 @@ static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
|
|||||||
Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
|
Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
|
||||||
|
auto *LI = dyn_cast<LoadInst>(I);
|
||||||
|
auto *SI = dyn_cast<StoreInst>(I);
|
||||||
|
auto *RMWI = dyn_cast<AtomicRMWInst>(I);
|
||||||
|
auto *CASI = dyn_cast<AtomicCmpXchgInst>(I);
|
||||||
|
|
||||||
|
bool MadeChange = false;
|
||||||
|
|
||||||
|
// If the Size/Alignment is not supported, replace with a libcall.
|
||||||
|
if (LI) {
|
||||||
|
if (!LI->isAtomic())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (!atomicSizeSupported(TLI, LI)) {
|
||||||
|
expandAtomicLoadToLibcall(LI);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (TLI->shouldCastAtomicLoadInIR(LI) ==
|
||||||
|
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
|
||||||
|
I = LI = convertAtomicLoadToIntegerType(LI);
|
||||||
|
MadeChange = true;
|
||||||
|
}
|
||||||
|
} else if (SI) {
|
||||||
|
if (!SI->isAtomic())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (!atomicSizeSupported(TLI, SI)) {
|
||||||
|
expandAtomicStoreToLibcall(SI);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (TLI->shouldCastAtomicStoreInIR(SI) ==
|
||||||
|
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
|
||||||
|
I = SI = convertAtomicStoreToIntegerType(SI);
|
||||||
|
MadeChange = true;
|
||||||
|
}
|
||||||
|
} else if (RMWI) {
|
||||||
|
if (!atomicSizeSupported(TLI, RMWI)) {
|
||||||
|
expandAtomicRMWToLibcall(RMWI);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
|
||||||
|
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
|
||||||
|
I = RMWI = convertAtomicXchgToIntegerType(RMWI);
|
||||||
|
MadeChange = true;
|
||||||
|
}
|
||||||
|
} else if (CASI) {
|
||||||
|
if (!atomicSizeSupported(TLI, CASI)) {
|
||||||
|
expandAtomicCASToLibcall(CASI);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: when we're ready to make the change at the IR level, we can
|
||||||
|
// extend convertCmpXchgToInteger for floating point too.
|
||||||
|
if (CASI->getCompareOperand()->getType()->isPointerTy()) {
|
||||||
|
// TODO: add a TLI hook to control this so that each target can
|
||||||
|
// convert to lowering the original type one at a time.
|
||||||
|
I = CASI = convertCmpXchgToIntegerType(CASI);
|
||||||
|
MadeChange = true;
|
||||||
|
}
|
||||||
|
} else
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (TLI->shouldInsertFencesForAtomic(I)) {
|
||||||
|
auto FenceOrdering = AtomicOrdering::Monotonic;
|
||||||
|
if (LI && isAcquireOrStronger(LI->getOrdering())) {
|
||||||
|
FenceOrdering = LI->getOrdering();
|
||||||
|
LI->setOrdering(AtomicOrdering::Monotonic);
|
||||||
|
} else if (SI && isReleaseOrStronger(SI->getOrdering())) {
|
||||||
|
FenceOrdering = SI->getOrdering();
|
||||||
|
SI->setOrdering(AtomicOrdering::Monotonic);
|
||||||
|
} else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
|
||||||
|
isAcquireOrStronger(RMWI->getOrdering()))) {
|
||||||
|
FenceOrdering = RMWI->getOrdering();
|
||||||
|
RMWI->setOrdering(AtomicOrdering::Monotonic);
|
||||||
|
} else if (CASI &&
|
||||||
|
TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
|
||||||
|
TargetLoweringBase::AtomicExpansionKind::None &&
|
||||||
|
(isReleaseOrStronger(CASI->getSuccessOrdering()) ||
|
||||||
|
isAcquireOrStronger(CASI->getSuccessOrdering()) ||
|
||||||
|
isAcquireOrStronger(CASI->getFailureOrdering()))) {
|
||||||
|
// If a compare and swap is lowered to LL/SC, we can do smarter fence
|
||||||
|
// insertion, with a stronger one on the success path than on the
|
||||||
|
// failure path. As a result, fence insertion is directly done by
|
||||||
|
// expandAtomicCmpXchg in that case.
|
||||||
|
FenceOrdering = CASI->getMergedOrdering();
|
||||||
|
CASI->setSuccessOrdering(AtomicOrdering::Monotonic);
|
||||||
|
CASI->setFailureOrdering(AtomicOrdering::Monotonic);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (FenceOrdering != AtomicOrdering::Monotonic) {
|
||||||
|
MadeChange |= bracketInstWithFences(I, FenceOrdering);
|
||||||
|
}
|
||||||
|
} else if (I->hasAtomicStore() &&
|
||||||
|
TLI->shouldInsertTrailingFenceForAtomicStore(I)) {
|
||||||
|
auto FenceOrdering = AtomicOrdering::Monotonic;
|
||||||
|
if (SI)
|
||||||
|
FenceOrdering = SI->getOrdering();
|
||||||
|
else if (RMWI)
|
||||||
|
FenceOrdering = RMWI->getOrdering();
|
||||||
|
else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=
|
||||||
|
TargetLoweringBase::AtomicExpansionKind::LLSC)
|
||||||
|
// LLSC is handled in expandAtomicCmpXchg().
|
||||||
|
FenceOrdering = CASI->getSuccessOrdering();
|
||||||
|
|
||||||
|
IRBuilder Builder(I);
|
||||||
|
if (auto TrailingFence =
|
||||||
|
TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
|
||||||
|
TrailingFence->moveAfter(I);
|
||||||
|
MadeChange = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (LI)
|
||||||
|
MadeChange |= tryExpandAtomicLoad(LI);
|
||||||
|
else if (SI)
|
||||||
|
MadeChange |= tryExpandAtomicStore(SI);
|
||||||
|
else if (RMWI) {
|
||||||
|
// There are two different ways of expanding RMW instructions:
|
||||||
|
// - into a load if it is idempotent
|
||||||
|
// - into a Cmpxchg/LL-SC loop otherwise
|
||||||
|
// we try them in that order.
|
||||||
|
|
||||||
|
if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
|
||||||
|
MadeChange = true;
|
||||||
|
|
||||||
|
} else {
|
||||||
|
MadeChange |= tryExpandAtomicRMW(RMWI);
|
||||||
|
}
|
||||||
|
} else if (CASI)
|
||||||
|
MadeChange |= tryExpandAtomicCmpXchg(CASI);
|
||||||
|
|
||||||
|
return MadeChange;
|
||||||
|
}
|
||||||
|
|
||||||
bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
|
bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
|
||||||
const auto *Subtarget = TM->getSubtargetImpl(F);
|
const auto *Subtarget = TM->getSubtargetImpl(F);
|
||||||
if (!Subtarget->enableAtomicExpand())
|
if (!Subtarget->enableAtomicExpand())
|
||||||
@ -210,6 +349,8 @@ bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
|
|||||||
TLI = Subtarget->getTargetLowering();
|
TLI = Subtarget->getTargetLowering();
|
||||||
DL = &F.getDataLayout();
|
DL = &F.getDataLayout();
|
||||||
|
|
||||||
|
bool MadeChange = false;
|
||||||
|
|
||||||
SmallVector<Instruction *, 1> AtomicInsts;
|
SmallVector<Instruction *, 1> AtomicInsts;
|
||||||
|
|
||||||
// Changing control-flow while iterating through it is a bad idea, so gather a
|
// Changing control-flow while iterating through it is a bad idea, so gather a
|
||||||
@ -218,134 +359,11 @@ bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
|
|||||||
if (I.isAtomic() && !isa<FenceInst>(&I))
|
if (I.isAtomic() && !isa<FenceInst>(&I))
|
||||||
AtomicInsts.push_back(&I);
|
AtomicInsts.push_back(&I);
|
||||||
|
|
||||||
bool MadeChange = false;
|
|
||||||
for (auto *I : AtomicInsts) {
|
for (auto *I : AtomicInsts) {
|
||||||
auto LI = dyn_cast<LoadInst>(I);
|
if (processAtomicInstr(I))
|
||||||
auto SI = dyn_cast<StoreInst>(I);
|
|
||||||
auto RMWI = dyn_cast<AtomicRMWInst>(I);
|
|
||||||
auto CASI = dyn_cast<AtomicCmpXchgInst>(I);
|
|
||||||
assert((LI || SI || RMWI || CASI) && "Unknown atomic instruction");
|
|
||||||
|
|
||||||
// If the Size/Alignment is not supported, replace with a libcall.
|
|
||||||
if (LI) {
|
|
||||||
if (!atomicSizeSupported(TLI, LI)) {
|
|
||||||
expandAtomicLoadToLibcall(LI);
|
|
||||||
MadeChange = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} else if (SI) {
|
|
||||||
if (!atomicSizeSupported(TLI, SI)) {
|
|
||||||
expandAtomicStoreToLibcall(SI);
|
|
||||||
MadeChange = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} else if (RMWI) {
|
|
||||||
if (!atomicSizeSupported(TLI, RMWI)) {
|
|
||||||
expandAtomicRMWToLibcall(RMWI);
|
|
||||||
MadeChange = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} else if (CASI) {
|
|
||||||
if (!atomicSizeSupported(TLI, CASI)) {
|
|
||||||
expandAtomicCASToLibcall(CASI);
|
|
||||||
MadeChange = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (LI && TLI->shouldCastAtomicLoadInIR(LI) ==
|
|
||||||
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
|
|
||||||
I = LI = convertAtomicLoadToIntegerType(LI);
|
|
||||||
MadeChange = true;
|
MadeChange = true;
|
||||||
} else if (SI &&
|
|
||||||
TLI->shouldCastAtomicStoreInIR(SI) ==
|
|
||||||
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
|
|
||||||
I = SI = convertAtomicStoreToIntegerType(SI);
|
|
||||||
MadeChange = true;
|
|
||||||
} else if (RMWI &&
|
|
||||||
TLI->shouldCastAtomicRMWIInIR(RMWI) ==
|
|
||||||
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
|
|
||||||
I = RMWI = convertAtomicXchgToIntegerType(RMWI);
|
|
||||||
MadeChange = true;
|
|
||||||
} else if (CASI) {
|
|
||||||
// TODO: when we're ready to make the change at the IR level, we can
|
|
||||||
// extend convertCmpXchgToInteger for floating point too.
|
|
||||||
if (CASI->getCompareOperand()->getType()->isPointerTy()) {
|
|
||||||
// TODO: add a TLI hook to control this so that each target can
|
|
||||||
// convert to lowering the original type one at a time.
|
|
||||||
I = CASI = convertCmpXchgToIntegerType(CASI);
|
|
||||||
MadeChange = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (TLI->shouldInsertFencesForAtomic(I)) {
|
|
||||||
auto FenceOrdering = AtomicOrdering::Monotonic;
|
|
||||||
if (LI && isAcquireOrStronger(LI->getOrdering())) {
|
|
||||||
FenceOrdering = LI->getOrdering();
|
|
||||||
LI->setOrdering(AtomicOrdering::Monotonic);
|
|
||||||
} else if (SI && isReleaseOrStronger(SI->getOrdering())) {
|
|
||||||
FenceOrdering = SI->getOrdering();
|
|
||||||
SI->setOrdering(AtomicOrdering::Monotonic);
|
|
||||||
} else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
|
|
||||||
isAcquireOrStronger(RMWI->getOrdering()))) {
|
|
||||||
FenceOrdering = RMWI->getOrdering();
|
|
||||||
RMWI->setOrdering(AtomicOrdering::Monotonic);
|
|
||||||
} else if (CASI &&
|
|
||||||
TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
|
|
||||||
TargetLoweringBase::AtomicExpansionKind::None &&
|
|
||||||
(isReleaseOrStronger(CASI->getSuccessOrdering()) ||
|
|
||||||
isAcquireOrStronger(CASI->getSuccessOrdering()) ||
|
|
||||||
isAcquireOrStronger(CASI->getFailureOrdering()))) {
|
|
||||||
// If a compare and swap is lowered to LL/SC, we can do smarter fence
|
|
||||||
// insertion, with a stronger one on the success path than on the
|
|
||||||
// failure path. As a result, fence insertion is directly done by
|
|
||||||
// expandAtomicCmpXchg in that case.
|
|
||||||
FenceOrdering = CASI->getMergedOrdering();
|
|
||||||
CASI->setSuccessOrdering(AtomicOrdering::Monotonic);
|
|
||||||
CASI->setFailureOrdering(AtomicOrdering::Monotonic);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (FenceOrdering != AtomicOrdering::Monotonic) {
|
|
||||||
MadeChange |= bracketInstWithFences(I, FenceOrdering);
|
|
||||||
}
|
|
||||||
} else if (I->hasAtomicStore() &&
|
|
||||||
TLI->shouldInsertTrailingFenceForAtomicStore(I)) {
|
|
||||||
auto FenceOrdering = AtomicOrdering::Monotonic;
|
|
||||||
if (SI)
|
|
||||||
FenceOrdering = SI->getOrdering();
|
|
||||||
else if (RMWI)
|
|
||||||
FenceOrdering = RMWI->getOrdering();
|
|
||||||
else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=
|
|
||||||
TargetLoweringBase::AtomicExpansionKind::LLSC)
|
|
||||||
// LLSC is handled in expandAtomicCmpXchg().
|
|
||||||
FenceOrdering = CASI->getSuccessOrdering();
|
|
||||||
|
|
||||||
IRBuilder Builder(I);
|
|
||||||
if (auto TrailingFence =
|
|
||||||
TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
|
|
||||||
TrailingFence->moveAfter(I);
|
|
||||||
MadeChange = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (LI)
|
|
||||||
MadeChange |= tryExpandAtomicLoad(LI);
|
|
||||||
else if (SI)
|
|
||||||
MadeChange |= tryExpandAtomicStore(SI);
|
|
||||||
else if (RMWI) {
|
|
||||||
// There are two different ways of expanding RMW instructions:
|
|
||||||
// - into a load if it is idempotent
|
|
||||||
// - into a Cmpxchg/LL-SC loop otherwise
|
|
||||||
// we try them in that order.
|
|
||||||
|
|
||||||
if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
|
|
||||||
MadeChange = true;
|
|
||||||
} else {
|
|
||||||
MadeChange |= tryExpandAtomicRMW(RMWI);
|
|
||||||
}
|
|
||||||
} else if (CASI)
|
|
||||||
MadeChange |= tryExpandAtomicCmpXchg(CASI);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return MadeChange;
|
return MadeChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user