shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp
Pierre van Houtryve fa3d789df1
[RFC][TableGen] Restructure TableGen Source (#80847) 
Refactor of the llvm-tblgen source into:
- a "Basic" library, which contains the bare minimum utilities to build
`llvm-min-tablegen`
- a "Common" library which contains all of the helpers for TableGen
backends. Such helpers can be shared by more than one backend, and even
unit tested (e.g. CodeExpander is, maybe we can add more over time)

Fixes #80647
2024-03-25 09:40:35 +01:00

2515 lines
97 KiB
C++
Raw Blame History

//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// This tablegen backend emits code for use by the GlobalISel instruction
/// selector. See include/llvm/Target/GlobalISel/Target.td.
///
/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
/// backend, filters out the ones that are unsupported, maps
/// SelectionDAG-specific constructs to their GlobalISel counterpart
/// (when applicable: MVT to LLT; SDNode to generic Instruction).
///
/// Not all patterns are supported: pass the tablegen invocation
/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,
/// as well as why.
///
/// The generated file defines a single method:
/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;
/// intended to be used in InstructionSelector::select as the first-step
/// selector for the patterns that don't require complex C++.
///
/// FIXME: We'll probably want to eventually define a base
/// "TargetGenInstructionSelector" class.
///
//===----------------------------------------------------------------------===//
#include "Basic/CodeGenIntrinsics.h"
#include "Common/CodeGenDAGPatterns.h"
#include "Common/CodeGenInstruction.h"
#include "Common/CodeGenRegisters.h"
#include "Common/CodeGenTarget.h"
#include "Common/GlobalISel/GlobalISelMatchTable.h"
#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
#include "Common/InfoByHwMode.h"
#include "Common/SubtargetFeatureInfo.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGenTypes/LowLevelType.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/Support/CodeGenCoverage.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/ScopedPrinter.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Record.h"
#include "llvm/TableGen/TableGenBackend.h"
#include <numeric>
#include <string>
using namespace llvm;
using namespace llvm::gi;
using action_iterator = RuleMatcher::action_iterator;
#define DEBUG_TYPE "gisel-emitter"
STATISTIC(NumPatternTotal, "Total number of patterns");
STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
STATISTIC(NumPatternsTested,
"Number of patterns executed according to coverage information");
cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
static cl::opt<bool> WarnOnSkippedPatterns(
"warn-on-skipped-patterns",
cl::desc("Explain why a pattern was skipped for inclusion "
"in the GlobalISel selector"),
cl::init(false), cl::cat(GlobalISelEmitterCat));
static cl::opt<bool> GenerateCoverage(
"instrument-gisel-coverage",
cl::desc("Generate coverage instrumentation for GlobalISel"),
cl::init(false), cl::cat(GlobalISelEmitterCat));
static cl::opt<std::string> UseCoverageFile(
"gisel-coverage-file", cl::init(""),
cl::desc("Specify file to retrieve coverage information from"),
cl::cat(GlobalISelEmitterCat));
static cl::opt<bool> OptimizeMatchTable(
"optimize-match-table",
cl::desc("Generate an optimized version of the match table"),
cl::init(true), cl::cat(GlobalISelEmitterCat));
namespace {
static std::string explainPredicates(const TreePatternNode &N) {
std::string Explanation;
StringRef Separator = "";
for (const TreePredicateCall &Call : N.getPredicateCalls()) {
const TreePredicateFn &P = Call.Fn;
Explanation +=
(Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();
Separator = ", ";
if (P.isAlwaysTrue())
Explanation += " always-true";
if (P.isImmediatePattern())
Explanation += " immediate";
if (P.isUnindexed())
Explanation += " unindexed";
if (P.isNonExtLoad())
Explanation += " non-extload";
if (P.isAnyExtLoad())
Explanation += " extload";
if (P.isSignExtLoad())
Explanation += " sextload";
if (P.isZeroExtLoad())
Explanation += " zextload";
if (P.isNonTruncStore())
Explanation += " non-truncstore";
if (P.isTruncStore())
Explanation += " truncstore";
if (Record *VT = P.getMemoryVT())
Explanation += (" MemVT=" + VT->getName()).str();
if (Record *VT = P.getScalarMemoryVT())
Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();
if (ListInit *AddrSpaces = P.getAddressSpaces()) {
raw_string_ostream OS(Explanation);
OS << " AddressSpaces=[";
StringRef AddrSpaceSeparator;
for (Init *Val : AddrSpaces->getValues()) {
IntInit *IntVal = dyn_cast<IntInit>(Val);
if (!IntVal)
continue;
OS << AddrSpaceSeparator << IntVal->getValue();
AddrSpaceSeparator = ", ";
}
OS << ']';
}
int64_t MinAlign = P.getMinAlignment();
if (MinAlign > 0)
Explanation += " MinAlign=" + utostr(MinAlign);
if (P.isAtomicOrderingMonotonic())
Explanation += " monotonic";
if (P.isAtomicOrderingAcquire())
Explanation += " acquire";
if (P.isAtomicOrderingRelease())
Explanation += " release";
if (P.isAtomicOrderingAcquireRelease())
Explanation += " acq_rel";
if (P.isAtomicOrderingSequentiallyConsistent())
Explanation += " seq_cst";
if (P.isAtomicOrderingAcquireOrStronger())
Explanation += " >=acquire";
if (P.isAtomicOrderingWeakerThanAcquire())
Explanation += " <acquire";
if (P.isAtomicOrderingReleaseOrStronger())
Explanation += " >=release";
if (P.isAtomicOrderingWeakerThanRelease())
Explanation += " <release";
}
return Explanation;
}
std::string explainOperator(Record *Operator) {
if (Operator->isSubClassOf("SDNode"))
return (" (" + Operator->getValueAsString("Opcode") + ")").str();
if (Operator->isSubClassOf("Intrinsic"))
return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();
if (Operator->isSubClassOf("ComplexPattern"))
return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +
")")
.str();
if (Operator->isSubClassOf("SDNodeXForm"))
return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +
")")
.str();
return (" (Operator " + Operator->getName() + " not understood)").str();
}
/// Helper function to let the emitter report skip reason error messages.
static Error failedImport(const Twine &Reason) {
return make_error<StringError>(Reason, inconvertibleErrorCode());
}
static Error isTrivialOperatorNode(const TreePatternNode &N) {
std::string Explanation;
std::string Separator;
bool HasUnsupportedPredicate = false;
for (const TreePredicateCall &Call : N.getPredicateCalls()) {
const TreePredicateFn &Predicate = Call.Fn;
if (Predicate.isAlwaysTrue())
continue;
if (Predicate.isImmediatePattern())
continue;
if (Predicate.hasNoUse())
continue;
if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||
Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())
continue;
if (Predicate.isNonTruncStore() || Predicate.isTruncStore())
continue;
if (Predicate.isLoad() && Predicate.getMemoryVT())
continue;
if (Predicate.isLoad() || Predicate.isStore()) {
if (Predicate.isUnindexed())
continue;
}
if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
const ListInit *AddrSpaces = Predicate.getAddressSpaces();
if (AddrSpaces && !AddrSpaces->empty())
continue;
if (Predicate.getMinAlignment() > 0)
continue;
}
if (Predicate.isAtomic() && Predicate.getMemoryVT())
continue;
if (Predicate.isAtomic() &&
(Predicate.isAtomicOrderingMonotonic() ||
Predicate.isAtomicOrderingAcquire() ||
Predicate.isAtomicOrderingRelease() ||
Predicate.isAtomicOrderingAcquireRelease() ||
Predicate.isAtomicOrderingSequentiallyConsistent() ||
Predicate.isAtomicOrderingAcquireOrStronger() ||
Predicate.isAtomicOrderingWeakerThanAcquire() ||
Predicate.isAtomicOrderingReleaseOrStronger() ||
Predicate.isAtomicOrderingWeakerThanRelease()))
continue;
if (Predicate.hasGISelPredicateCode())
continue;
HasUnsupportedPredicate = true;
Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
Separator = ", ";
Explanation += (Separator + "first-failing:" +
Predicate.getOrigPatFragRecord()->getRecord()->getName())
.str();
break;
}
if (!HasUnsupportedPredicate)
return Error::success();
return failedImport(Explanation);
}
static Record *getInitValueAsRegClass(Init *V) {
if (DefInit *VDefInit = dyn_cast<DefInit>(V)) {
if (VDefInit->getDef()->isSubClassOf("RegisterOperand"))
return VDefInit->getDef()->getValueAsDef("RegClass");
if (VDefInit->getDef()->isSubClassOf("RegisterClass"))
return VDefInit->getDef();
}
return nullptr;
}
static std::string getScopedName(unsigned Scope, const std::string &Name) {
return ("pred:" + Twine(Scope) + ":" + Name).str();
}
static std::string getMangledRootDefName(StringRef DefOperandName) {
return ("DstI[" + DefOperandName + "]").str();
}
//===- GlobalISelEmitter class --------------------------------------------===//
static Expected<LLTCodeGen> getInstResultType(const TreePatternNode &Dst,
const CodeGenTarget &Target) {
// While we allow more than one output (both implicit and explicit defs)
// below, we only expect one explicit def here.
assert(Dst.getOperator()->isSubClassOf("Instruction"));
CodeGenInstruction &InstInfo = Target.getInstruction(Dst.getOperator());
if (!InstInfo.Operands.NumDefs)
return failedImport("Dst pattern child needs a def");
ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();
if (ChildTypes.size() < 1)
return failedImport("Dst pattern child has no result");
// If there are multiple results, just take the first one (this is how
// SelectionDAG does it).
std::optional<LLTCodeGen> MaybeOpTy;
if (ChildTypes.front().isMachineValueType()) {
MaybeOpTy = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);
}
if (!MaybeOpTy)
return failedImport("Dst operand has an unsupported type");
return *MaybeOpTy;
}
class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
public:
explicit GlobalISelEmitter(RecordKeeper &RK);
void emitAdditionalImpl(raw_ostream &OS) override;
void emitMIPredicateFns(raw_ostream &OS) override;
void emitI64ImmPredicateFns(raw_ostream &OS) override;
void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
void emitAPIntImmPredicateFns(raw_ostream &OS) override;
void emitTestSimplePredicate(raw_ostream &OS) override;
void emitRunCustomAction(raw_ostream &OS) override;
void postProcessRule(RuleMatcher &M);
const CodeGenTarget &getTarget() const override { return Target; }
StringRef getClassName() const override { return ClassName; }
void run(raw_ostream &OS);
private:
std::string ClassName;
const RecordKeeper &RK;
const CodeGenDAGPatterns CGP;
const CodeGenTarget &Target;
CodeGenRegBank &CGRegs;
std::vector<Record *> AllPatFrags;
/// Keep track of the equivalence between SDNodes and Instruction by mapping
/// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
/// check for attributes on the relation such as CheckMMOIsNonAtomic.
/// This is defined using 'GINodeEquiv' in the target description.
DenseMap<Record *, Record *> NodeEquivs;
/// Keep track of the equivalence between ComplexPattern's and
/// GIComplexOperandMatcher. Map entries are specified by subclassing
/// GIComplexPatternEquiv.
DenseMap<const Record *, const Record *> ComplexPatternEquivs;
/// Keep track of the equivalence between SDNodeXForm's and
/// GICustomOperandRenderer. Map entries are specified by subclassing
/// GISDNodeXFormEquiv.
DenseMap<const Record *, const Record *> SDNodeXFormEquivs;
/// Keep track of Scores of PatternsToMatch similar to how the DAG does.
/// This adds compatibility for RuleMatchers to use this for ordering rules.
DenseMap<uint64_t, int> RuleMatcherScores;
// Rule coverage information.
std::optional<CodeGenCoverage> RuleCoverage;
/// Variables used to help with collecting of named operands for predicates
/// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set
/// to the number of named operands that predicate expects. Store locations in
/// StoreIdxForName correspond to the order in which operand names appear in
/// predicate's argument list.
/// When we visit named operand and WaitingForNamedOperands is not zero, add
/// matcher that will record operand and decrease counter.
unsigned WaitingForNamedOperands = 0;
StringMap<unsigned> StoreIdxForName;
void gatherOpcodeValues();
void gatherTypeIDValues();
void gatherNodeEquivs();
Record *findNodeEquiv(Record *N) const;
const CodeGenInstruction *getEquivNode(Record &Equiv,
const TreePatternNode &N) const;
Error importRulePredicates(RuleMatcher &M, ArrayRef<Record *> Predicates);
Expected<InstructionMatcher &>
createAndImportSelDAGMatcher(RuleMatcher &Rule,
InstructionMatcher &InsnMatcher,
const TreePatternNode &Src, unsigned &TempOpIdx);
Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R,
unsigned &TempOpIdx) const;
Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
const TreePatternNode &SrcChild,
bool OperandIsAPointer, bool OperandIsImmArg,
unsigned OpIdx, unsigned &TempOpIdx);
Expected<BuildMIAction &> createAndImportInstructionRenderer(
RuleMatcher &M, InstructionMatcher &InsnMatcher,
const TreePatternNode &Src, const TreePatternNode &Dst);
Expected<action_iterator> createAndImportSubInstructionRenderer(
action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
const TreePatternNode &Src, unsigned TempReg);
Expected<action_iterator>
createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
const TreePatternNode &Dst);
Expected<action_iterator>
importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
BuildMIAction &DstMIBuilder,
const TreePatternNode &Src,
const TreePatternNode &Dst, unsigned Start = 0);
Expected<action_iterator> importExplicitUseRenderers(
action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
const llvm::TreePatternNode &Dst, const TreePatternNode &Src);
Expected<action_iterator> importExplicitUseRenderer(
action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder,
const TreePatternNode &DstChild, const TreePatternNode &Src);
Error importDefaultOperandRenderers(action_iterator InsertPt, RuleMatcher &M,
BuildMIAction &DstMIBuilder,
const DAGDefaultOperand &DefaultOp) const;
Error
importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
const std::vector<Record *> &ImplicitDefs) const;
/// Analyze pattern \p P, returning a matcher for it if possible.
/// Otherwise, return an Error explaining why we don't support it.
Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
void declareSubtargetFeature(Record *Predicate);
unsigned declareHwModeCheck(StringRef HwModeFeatures);
MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
bool WithCoverage);
/// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
/// CodeGenRegisterClass will support the CodeGenRegisterClass of
/// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
/// If no register class is found, return std::nullopt.
std::optional<const CodeGenRegisterClass *>
inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
const TreePatternNode &SuperRegNode,
const TreePatternNode &SubRegIdxNode);
std::optional<CodeGenSubRegIndex *>
inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode);
/// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
/// Return std::nullopt if no such class exists.
std::optional<const CodeGenRegisterClass *>
inferSuperRegisterClass(const TypeSetByHwMode &Ty,
const TreePatternNode &SubRegIdxNode);
/// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
std::optional<const CodeGenRegisterClass *>
getRegClassFromLeaf(const TreePatternNode &Leaf);
/// Return a CodeGenRegisterClass for \p N if one can be found. Return
/// std::nullopt otherwise.
std::optional<const CodeGenRegisterClass *>
inferRegClassFromPattern(const TreePatternNode &N);
/// Return the size of the MemoryVT in this predicate, if possible.
std::optional<unsigned>
getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
// Add builtin predicates.
Expected<InstructionMatcher &>
addBuiltinPredicates(const Record *SrcGIEquivOrNull,
const TreePredicateFn &Predicate,
InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
};
StringRef getPatFragPredicateEnumName(Record *R) { return R->getName(); }
void GlobalISelEmitter::gatherOpcodeValues() {
InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
}
void GlobalISelEmitter::gatherTypeIDValues() {
LLTOperandMatcher::initTypeIDValuesMap();
}
void GlobalISelEmitter::gatherNodeEquivs() {
assert(NodeEquivs.empty());
for (Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv"))
NodeEquivs[Equiv->getValueAsDef("Node")] = Equiv;
assert(ComplexPatternEquivs.empty());
for (Record *Equiv : RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) {
Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent");
if (!SelDAGEquiv)
continue;
ComplexPatternEquivs[SelDAGEquiv] = Equiv;
}
assert(SDNodeXFormEquivs.empty());
for (Record *Equiv : RK.getAllDerivedDefinitions("GISDNodeXFormEquiv")) {
Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent");
if (!SelDAGEquiv)
continue;
SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
}
}
Record *GlobalISelEmitter::findNodeEquiv(Record *N) const {
return NodeEquivs.lookup(N);
}
const CodeGenInstruction *
GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode &N) const {
if (N.getNumChildren() >= 1) {
// setcc operation maps to two different G_* instructions based on the type.
if (!Equiv.isValueUnset("IfFloatingPoint") &&
MVT(N.getChild(0).getSimpleType(0)).isFloatingPoint())
return &Target.getInstruction(Equiv.getValueAsDef("IfFloatingPoint"));
}
if (!Equiv.isValueUnset("IfConvergent") &&
N.getIntrinsicInfo(CGP)->isConvergent)
return &Target.getInstruction(Equiv.getValueAsDef("IfConvergent"));
for (const TreePredicateCall &Call : N.getPredicateCalls()) {
const TreePredicateFn &Predicate = Call.Fn;
if (!Equiv.isValueUnset("IfSignExtend") &&
(Predicate.isLoad() || Predicate.isAtomic()) &&
Predicate.isSignExtLoad())
return &Target.getInstruction(Equiv.getValueAsDef("IfSignExtend"));
if (!Equiv.isValueUnset("IfZeroExtend") &&
(Predicate.isLoad() || Predicate.isAtomic()) &&
Predicate.isZeroExtLoad())
return &Target.getInstruction(Equiv.getValueAsDef("IfZeroExtend"));
}
return &Target.getInstruction(Equiv.getValueAsDef("I"));
}
GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK)
: GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
ClassName = Target.getName().str() + "InstructionSelector";
}
//===- Emitter ------------------------------------------------------------===//
Error GlobalISelEmitter::importRulePredicates(RuleMatcher &M,
ArrayRef<Record *> Predicates) {
for (Record *Pred : Predicates) {
if (Pred->getValueAsString("CondString").empty())
continue;
declareSubtargetFeature(Pred);
M.addRequiredFeature(Pred);
}
return Error::success();
}
std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(
const TreePredicateFn &Predicate) {
std::optional<LLTCodeGen> MemTyOrNone =
MVTToLLT(getValueType(Predicate.getMemoryVT()));
if (!MemTyOrNone)
return std::nullopt;
// Align so unusual types like i1 don't get rounded down.
return llvm::alignTo(
static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), 8);
}
Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
SmallVector<unsigned, 4> ParsedAddrSpaces;
for (Init *Val : AddrSpaces->getValues()) {
IntInit *IntVal = dyn_cast<IntInit>(Val);
if (!IntVal)
return failedImport("Address space is not an integer");
ParsedAddrSpaces.push_back(IntVal->getValue());
}
if (!ParsedAddrSpaces.empty()) {
InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(
0, ParsedAddrSpaces);
return InsnMatcher;
}
}
int64_t MinAlign = Predicate.getMinAlignment();
if (MinAlign > 0) {
InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(0, MinAlign);
return InsnMatcher;
}
}
// G_LOAD is used for both non-extending and any-extending loads.
if (Predicate.isLoad() && Predicate.isNonExtLoad()) {
InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);
return InsnMatcher;
}
if (Predicate.isLoad() && Predicate.isAnyExtLoad()) {
InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
return InsnMatcher;
}
if (Predicate.isStore()) {
if (Predicate.isTruncStore()) {
if (Predicate.getMemoryVT() != nullptr) {
// FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
if (!MemSizeInBits)
return failedImport("MemVT could not be converted to LLT");
InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0, *MemSizeInBits /
8);
} else {
InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
}
return InsnMatcher;
}
if (Predicate.isNonTruncStore()) {
// We need to check the sizes match here otherwise we could incorrectly
// match truncating stores with non-truncating ones.
InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);
}
}
assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
// No check required. We already did it by swapping the opcode.
if (!SrcGIEquivOrNull->isValueUnset("IfSignExtend") &&
Predicate.isSignExtLoad())
return InsnMatcher;
// No check required. We already did it by swapping the opcode.
if (!SrcGIEquivOrNull->isValueUnset("IfZeroExtend") &&
Predicate.isZeroExtLoad())
return InsnMatcher;
// No check required. G_STORE by itself is a non-extending store.
if (Predicate.isNonTruncStore())
return InsnMatcher;
if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
if (Predicate.getMemoryVT() != nullptr) {
auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
if (!MemSizeInBits)
return failedImport("MemVT could not be converted to LLT");
InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0,
*MemSizeInBits / 8);
return InsnMatcher;
}
}
if (Predicate.isLoad() || Predicate.isStore()) {
// No check required. A G_LOAD/G_STORE is an unindexed load.
if (Predicate.isUnindexed())
return InsnMatcher;
}
if (Predicate.isAtomic()) {
if (Predicate.isAtomicOrderingMonotonic()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Monotonic");
return InsnMatcher;
}
if (Predicate.isAtomicOrderingAcquire()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Acquire");
return InsnMatcher;
}
if (Predicate.isAtomicOrderingRelease()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Release");
return InsnMatcher;
}
if (Predicate.isAtomicOrderingAcquireRelease()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"AcquireRelease");
return InsnMatcher;
}
if (Predicate.isAtomicOrderingSequentiallyConsistent()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"SequentiallyConsistent");
return InsnMatcher;
}
}
if (Predicate.isAtomicOrderingAcquireOrStronger()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"Acquire", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
return InsnMatcher;
}
if (Predicate.isAtomicOrderingWeakerThanAcquire()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"Acquire", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
return InsnMatcher;
}
if (Predicate.isAtomicOrderingReleaseOrStronger()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"Release", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
return InsnMatcher;
}
if (Predicate.isAtomicOrderingWeakerThanRelease()) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"Release", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
return InsnMatcher;
}
HasAddedMatcher = false;
return InsnMatcher;
}
Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
const TreePatternNode &Src, unsigned &TempOpIdx) {
const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());
Record *SrcGIEquivOrNull = nullptr;
const CodeGenInstruction *SrcGIOrNull = nullptr;
// Start with the defined operands (i.e., the results of the root operator).
if (Src.isLeaf()) {
Init *SrcInit = Src.getLeafValue();
if (isa<IntInit>(SrcInit)) {
InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
&Target.getInstruction(RK.getDef("G_CONSTANT")));
} else
return failedImport(
"Unable to deduce gMIR opcode to handle Src (which is a leaf)");
} else {
SrcGIEquivOrNull = findNodeEquiv(Src.getOperator());
if (!SrcGIEquivOrNull)
return failedImport("Pattern operator lacks an equivalent Instruction" +
explainOperator(Src.getOperator()));
SrcGIOrNull = getEquivNode(*SrcGIEquivOrNull, Src);
// The operators look good: match the opcode
InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull);
}
unsigned OpIdx = 0;
for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {
// Results don't have a name unless they are the root node. The caller will
// set the name if appropriate.
const bool OperandIsAPointer =
SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(OpIdx);
OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);
if (auto Error = OM.addTypeCheckPredicate(VTy, OperandIsAPointer))
return failedImport(toString(std::move(Error)) +
" for result of Src pattern operator");
}
for (const TreePredicateCall &Call : Src.getPredicateCalls()) {
const TreePredicateFn &Predicate = Call.Fn;
bool HasAddedBuiltinMatcher = true;
if (Predicate.isAlwaysTrue())
continue;
if (Predicate.isImmediatePattern()) {
InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(Predicate);
continue;
}
auto InsnMatcherOrError = addBuiltinPredicates(
SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedBuiltinMatcher);
if (auto Error = InsnMatcherOrError.takeError())
return std::move(Error);
// FIXME: This should be part of addBuiltinPredicates(). If we add this at
// the start of addBuiltinPredicates() without returning, then there might
// be cases where we hit the last return before which the
// HasAddedBuiltinMatcher will be set to false. The predicate could be
// missed if we add it in the middle or at the end due to return statements
// after the addPredicate<>() calls.
if (Predicate.hasNoUse()) {
InsnMatcher.addPredicate<NoUsePredicateMatcher>();
HasAddedBuiltinMatcher = true;
}
if (Predicate.hasGISelPredicateCode()) {
if (Predicate.usesOperands()) {
assert(WaitingForNamedOperands == 0 &&
"previous predicate didn't find all operands or "
"nested predicate that uses operands");
TreePattern *TP = Predicate.getOrigPatFragRecord();
WaitingForNamedOperands = TP->getNumArgs();
for (unsigned i = 0; i < WaitingForNamedOperands; ++i)
StoreIdxForName[getScopedName(Call.Scope, TP->getArgName(i))] = i;
}
InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(Predicate);
continue;
}
if (!HasAddedBuiltinMatcher) {
return failedImport("Src pattern child has predicate (" +
explainPredicates(Src) + ")");
}
}
if (SrcGIEquivOrNull &&
SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic"))
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic");
else if (SrcGIEquivOrNull &&
SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) {
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
}
if (Src.isLeaf()) {
Init *SrcInit = Src.getLeafValue();
if (IntInit *SrcIntInit = dyn_cast<IntInit>(SrcInit)) {
OperandMatcher &OM =
InsnMatcher.addOperand(OpIdx++, Src.getName(), TempOpIdx);
OM.addPredicate<LiteralIntOperandMatcher>(SrcIntInit->getValue());
} else
return failedImport(
"Unable to deduce gMIR opcode to handle Src (which is a leaf)");
} else {
assert(SrcGIOrNull &&
"Expected to have already found an equivalent Instruction");
if (SrcGIOrNull->TheDef->getName() == "G_CONSTANT" ||
SrcGIOrNull->TheDef->getName() == "G_FCONSTANT") {
// imm/fpimm still have operands but we don't need to do anything with it
// here since we don't support ImmLeaf predicates yet. However, we still
// need to note the hidden operand to get GIM_CheckNumOperands correct.
InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);
return InsnMatcher;
}
// Special case because the operand order is changed from setcc. The
// predicate operand needs to be swapped from the last operand to the first
// source.
unsigned NumChildren = Src.getNumChildren();
bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP";
if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") {
const TreePatternNode &SrcChild = Src.getChild(NumChildren - 1);
if (SrcChild.isLeaf()) {
DefInit *DI = dyn_cast<DefInit>(SrcChild.getLeafValue());
Record *CCDef = DI ? DI->getDef() : nullptr;
if (!CCDef || !CCDef->isSubClassOf("CondCode"))
return failedImport("Unable to handle CondCode");
OperandMatcher &OM =
InsnMatcher.addOperand(OpIdx++, SrcChild.getName(), TempOpIdx);
StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate")
: CCDef->getValueAsString("ICmpPredicate");
if (!PredType.empty()) {
OM.addPredicate<CmpPredicateOperandMatcher>(std::string(PredType));
// Process the other 2 operands normally.
--NumChildren;
}
}
}
// Match the used operands (i.e. the children of the operator).
bool IsIntrinsic =
SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||
SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||
SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_CONVERGENT" ||
SrcGIOrNull->TheDef->getName() ==
"G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";
const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CGP);
if (IsIntrinsic && !II)
return failedImport("Expected IntInit containing intrinsic ID)");
for (unsigned i = 0; i != NumChildren; ++i) {
const TreePatternNode &SrcChild = Src.getChild(i);
// We need to determine the meaning of a literal integer based on the
// context. If this is a field required to be an immediate (such as an
// immarg intrinsic argument), the required predicates are different than
// a constant which may be materialized in a register. If we have an
// argument that is required to be an immediate, we should not emit an LLT
// type check, and should not be looking for a G_CONSTANT defined
// register.
bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i);
// SelectionDAG allows pointers to be represented with iN since it doesn't
// distinguish between pointers and integers but they are different types
// in GlobalISel. Coerce integers to pointers to address space 0 if the
// context indicates a pointer.
//
bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i);
if (IsIntrinsic) {
// For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
// following the defs is an intrinsic ID.
if (i == 0) {
OperandMatcher &OM =
InsnMatcher.addOperand(OpIdx++, SrcChild.getName(), TempOpIdx);
OM.addPredicate<IntrinsicIDOperandMatcher>(II);
continue;
}
// We have to check intrinsics for llvm_anyptr_ty and immarg parameters.
//
// Note that we have to look at the i-1th parameter, because we don't
// have the intrinsic ID in the intrinsic's parameter list.
OperandIsAPointer |= II->isParamAPointer(i - 1);
OperandIsImmArg |= II->isParamImmArg(i - 1);
}
if (auto Error =
importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer,
OperandIsImmArg, OpIdx++, TempOpIdx))
return std::move(Error);
}
}
return InsnMatcher;
}
Error GlobalISelEmitter::importComplexPatternOperandMatcher(
OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const {
const auto &ComplexPattern = ComplexPatternEquivs.find(R);
if (ComplexPattern == ComplexPatternEquivs.end())
return failedImport("SelectionDAG ComplexPattern (" + R->getName() +
") not mapped to GlobalISel");
OM.addPredicate<ComplexPatternOperandMatcher>(OM, *ComplexPattern->second);
TempOpIdx++;
return Error::success();
}
// Get the name to use for a pattern operand. For an anonymous physical register
// input, this should use the register name.
static StringRef getSrcChildName(const TreePatternNode &SrcChild,
Record *&PhysReg) {
StringRef SrcChildName = SrcChild.getName();
if (SrcChildName.empty() && SrcChild.isLeaf()) {
if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild.getLeafValue())) {
auto *ChildRec = ChildDefInit->getDef();
if (ChildRec->isSubClassOf("Register")) {
SrcChildName = ChildRec->getName();
PhysReg = ChildRec;
}
}
}
return SrcChildName;
}
Error GlobalISelEmitter::importChildMatcher(
RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
const TreePatternNode &SrcChild, bool OperandIsAPointer,
bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) {
Record *PhysReg = nullptr;
std::string SrcChildName = std::string(getSrcChildName(SrcChild, PhysReg));
if (!SrcChild.isLeaf() &&
SrcChild.getOperator()->isSubClassOf("ComplexPattern")) {
// The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
// "MY_PAT:op1:op2" and the ones with same "name" represent same operand.
std::string PatternName = std::string(SrcChild.getOperator()->getName());
for (unsigned i = 0; i < SrcChild.getNumChildren(); ++i) {
PatternName += ":";
PatternName += SrcChild.getChild(i).getName();
}
SrcChildName = PatternName;
}
OperandMatcher &OM =
PhysReg ? InsnMatcher.addPhysRegInput(PhysReg, OpIdx, TempOpIdx)
: InsnMatcher.addOperand(OpIdx, SrcChildName, TempOpIdx);
if (OM.isSameAsAnotherOperand())
return Error::success();
ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();
if (ChildTypes.size() != 1)
return failedImport("Src pattern child has multiple results");
// Check MBB's before the type check since they are not a known type.
if (!SrcChild.isLeaf()) {
if (SrcChild.getOperator()->isSubClassOf("SDNode")) {
auto &ChildSDNI = CGP.getSDNodeInfo(SrcChild.getOperator());
if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") {
OM.addPredicate<MBBOperandMatcher>();
return Error::success();
}
if (SrcChild.getOperator()->getName() == "timm") {
OM.addPredicate<ImmOperandMatcher>();
// Add predicates, if any
for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
const TreePredicateFn &Predicate = Call.Fn;
// Only handle immediate patterns for now
if (Predicate.isImmediatePattern()) {
OM.addPredicate<OperandImmPredicateMatcher>(Predicate);
}
}
return Error::success();
}
}
}
// Immediate arguments have no meaningful type to check as they don't have
// registers.
if (!OperandIsImmArg) {
if (auto Error =
OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer))
return failedImport(toString(std::move(Error)) + " for Src operand (" +
to_string(SrcChild) + ")");
}
// Try look up SrcChild for a (named) predicate operand if there is any.
if (WaitingForNamedOperands) {
auto &ScopedNames = SrcChild.getNamesAsPredicateArg();
if (!ScopedNames.empty()) {
auto PA = ScopedNames.begin();
std::string Name = getScopedName(PA->getScope(), PA->getIdentifier());
OM.addPredicate<RecordNamedOperandMatcher>(StoreIdxForName[Name], Name);
--WaitingForNamedOperands;
}
}
// Check for nested instructions.
if (!SrcChild.isLeaf()) {
if (SrcChild.getOperator()->isSubClassOf("ComplexPattern")) {
// When a ComplexPattern is used as an operator, it should do the same
// thing as when used as a leaf. However, the children of the operator
// name the sub-operands that make up the complex operand and we must
// prepare to reference them in the renderer too.
unsigned RendererID = TempOpIdx;
if (auto Error = importComplexPatternOperandMatcher(
OM, SrcChild.getOperator(), TempOpIdx))
return Error;
for (unsigned i = 0, e = SrcChild.getNumChildren(); i != e; ++i) {
auto &SubOperand = SrcChild.getChild(i);
if (!SubOperand.getName().empty()) {
if (auto Error = Rule.defineComplexSubOperand(
SubOperand.getName(), SrcChild.getOperator(), RendererID, i,
SrcChildName))
return Error;
}
}
return Error::success();
}
auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
InsnMatcher.getRuleMatcher(), SrcChild.getName());
if (!MaybeInsnOperand) {
// This isn't strictly true. If the user were to provide exactly the same
// matchers as the original operand then we could allow it. However, it's
// simpler to not permit the redundant specification.
return failedImport(
"Nested instruction cannot be the same as another operand");
}
// Map the node to a gMIR instruction.
InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
auto InsnMatcherOrError = createAndImportSelDAGMatcher(
Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx);
if (auto Error = InsnMatcherOrError.takeError())
return Error;
return Error::success();
}
if (SrcChild.hasAnyPredicate())
return failedImport("Src pattern child has unsupported predicate");
// Check for constant immediates.
if (auto *ChildInt = dyn_cast<IntInit>(SrcChild.getLeafValue())) {
if (OperandIsImmArg) {
// Checks for argument directly in operand list
OM.addPredicate<LiteralIntOperandMatcher>(ChildInt->getValue());
} else {
// Checks for materialized constant
OM.addPredicate<ConstantIntOperandMatcher>(ChildInt->getValue());
}
return Error::success();
}
// Check for def's like register classes or ComplexPattern's.
if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild.getLeafValue())) {
auto *ChildRec = ChildDefInit->getDef();
// Check for register classes.
if (ChildRec->isSubClassOf("RegisterClass") ||
ChildRec->isSubClassOf("RegisterOperand")) {
OM.addPredicate<RegisterBankOperandMatcher>(
Target.getRegisterClass(getInitValueAsRegClass(ChildDefInit)));
return Error::success();
}
if (ChildRec->isSubClassOf("Register")) {
// This just be emitted as a copy to the specific register.
ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();
const CodeGenRegisterClass *RC =
CGRegs.getMinimalPhysRegClass(ChildRec, &VT);
if (!RC) {
return failedImport(
"Could not determine physical register class of pattern source");
}
OM.addPredicate<RegisterBankOperandMatcher>(*RC);
return Error::success();
}
// Check for ValueType.
if (ChildRec->isSubClassOf("ValueType")) {
// We already added a type check as standard practice so this doesn't need
// to do anything.
return Error::success();
}
// Check for ComplexPattern's.
if (ChildRec->isSubClassOf("ComplexPattern"))
return importComplexPatternOperandMatcher(OM, ChildRec, TempOpIdx);
if (ChildRec->isSubClassOf("ImmLeaf")) {
return failedImport(
"Src pattern child def is an unsupported tablegen class (ImmLeaf)");
}
// Place holder for SRCVALUE nodes. Nothing to do here.
if (ChildRec->getName() == "srcvalue")
return Error::success();
const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
InsnMatcher.getRuleMatcher(), SrcChild.getName(), false);
InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
const CodeGenInstruction &BuildVector =
Target.getInstruction(RK.getDef("G_BUILD_VECTOR"));
const CodeGenInstruction &BuildVectorTrunc =
Target.getInstruction(RK.getDef("G_BUILD_VECTOR_TRUNC"));
// Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
// as an alternative.
InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
ArrayRef({&BuildVector, &BuildVectorTrunc}));
// TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
// theoretically not emit any opcode check, but getOpcodeMatcher currently
// has to succeed.
OperandMatcher &OM =
InsnOperand.getInsnMatcher().addOperand(0, "", TempOpIdx);
if (auto Error =
OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */))
return failedImport(toString(std::move(Error)) +
" for result of Src pattern operator");
InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
: VectorSplatImmPredicateMatcher::AllZeros);
return Error::success();
}
return failedImport(
"Src pattern child def is an unsupported tablegen class");
}
return failedImport("Src pattern child is an unsupported kind");
}
Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder,
const TreePatternNode &DstChild, const TreePatternNode &Src) {
const auto &SubOperand = Rule.getComplexSubOperand(DstChild.getName());
if (SubOperand) {
DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
*std::get<0>(*SubOperand), DstChild.getName(), std::get<1>(*SubOperand),
std::get<2>(*SubOperand));
return InsertPt;
}
if (!DstChild.isLeaf()) {
if (DstChild.getOperator()->isSubClassOf("SDNodeXForm")) {
auto &Child = DstChild.getChild(0);
auto I = SDNodeXFormEquivs.find(DstChild.getOperator());
if (I != SDNodeXFormEquivs.end()) {
Record *XFormOpc = DstChild.getOperator()->getValueAsDef("Opcode");
if (XFormOpc->getName() == "timm") {
// If this is a TargetConstant, there won't be a corresponding
// instruction to transform. Instead, this will refer directly to an
// operand in an instruction's operand list.
DstMIBuilder.addRenderer<CustomOperandRenderer>(*I->second,
Child.getName());
} else {
DstMIBuilder.addRenderer<CustomRenderer>(*I->second, Child.getName());
}
return InsertPt;
}
return failedImport("SDNodeXForm " + Child.getName() +
" has no custom renderer");
}
// We accept 'bb' here. It's an operator because BasicBlockSDNode isn't
// inline, but in MI it's just another operand.
if (DstChild.getOperator()->isSubClassOf("SDNode")) {
auto &ChildSDNI = CGP.getSDNodeInfo(DstChild.getOperator());
if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") {
DstMIBuilder.addRenderer<CopyRenderer>(DstChild.getName());
return InsertPt;
}
}
// Similarly, imm is an operator in TreePatternNode's view but must be
// rendered as operands.
// FIXME: The target should be able to choose sign-extended when appropriate
// (e.g. on Mips).
if (DstChild.getOperator()->getName() == "timm") {
DstMIBuilder.addRenderer<CopyRenderer>(DstChild.getName());
return InsertPt;
} else if (DstChild.getOperator()->getName() == "imm") {
DstMIBuilder.addRenderer<CopyConstantAsImmRenderer>(DstChild.getName());
return InsertPt;
} else if (DstChild.getOperator()->getName() == "fpimm") {
DstMIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(
DstChild.getName());
return InsertPt;
}
if (DstChild.getOperator()->isSubClassOf("Instruction")) {
auto OpTy = getInstResultType(DstChild, Target);
if (!OpTy)
return OpTy.takeError();
unsigned TempRegID = Rule.allocateTempRegID();
InsertPt =
Rule.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);
DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
auto InsertPtOrError = createAndImportSubInstructionRenderer(
++InsertPt, Rule, DstChild, Src, TempRegID);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
return InsertPtOrError.get();
}
return failedImport("Dst pattern child isn't a leaf node or an MBB" +
llvm::to_string(DstChild));
}
// It could be a specific immediate in which case we should just check for
// that immediate.
if (const IntInit *ChildIntInit =
dyn_cast<IntInit>(DstChild.getLeafValue())) {
DstMIBuilder.addRenderer<ImmRenderer>(ChildIntInit->getValue());
return InsertPt;
}
// Otherwise, we're looking for a bog-standard RegisterClass operand.
if (auto *ChildDefInit = dyn_cast<DefInit>(DstChild.getLeafValue())) {
auto *ChildRec = ChildDefInit->getDef();
ArrayRef<TypeSetByHwMode> ChildTypes = DstChild.getExtTypes();
if (ChildTypes.size() != 1)
return failedImport("Dst pattern child has multiple results");
std::optional<LLTCodeGen> OpTyOrNone;
if (ChildTypes.front().isMachineValueType())
OpTyOrNone = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);
if (!OpTyOrNone)
return failedImport("Dst operand has an unsupported type");
if (ChildRec->isSubClassOf("Register")) {
DstMIBuilder.addRenderer<AddRegisterRenderer>(Target, ChildRec);
return InsertPt;
}
if (ChildRec->isSubClassOf("RegisterClass") ||
ChildRec->isSubClassOf("RegisterOperand") ||
ChildRec->isSubClassOf("ValueType")) {
if (ChildRec->isSubClassOf("RegisterOperand") &&
!ChildRec->isValueUnset("GIZeroRegister")) {
DstMIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(
DstChild.getName(), ChildRec->getValueAsDef("GIZeroRegister"));
return InsertPt;
}
DstMIBuilder.addRenderer<CopyRenderer>(DstChild.getName());
return InsertPt;
}
if (ChildRec->isSubClassOf("SubRegIndex")) {
CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(ChildRec);
DstMIBuilder.addRenderer<ImmRenderer>(SubIdx->EnumValue);
return InsertPt;
}
if (ChildRec->isSubClassOf("ComplexPattern")) {
const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec);
if (ComplexPattern == ComplexPatternEquivs.end())
return failedImport(
"SelectionDAG ComplexPattern not mapped to GlobalISel");
const OperandMatcher &OM = Rule.getOperandMatcher(DstChild.getName());
DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
*ComplexPattern->second, DstChild.getName(),
OM.getAllocatedTemporariesBaseID());
return InsertPt;
}
return failedImport(
"Dst pattern child def is an unsupported tablegen class");
}
// Handle the case where the MVT/register class is omitted in the dest pattern
// but MVT exists in the source pattern.
if (isa<UnsetInit>(DstChild.getLeafValue())) {
for (unsigned NumOp = 0; NumOp < Src.getNumChildren(); NumOp++)
if (Src.getChild(NumOp).getName() == DstChild.getName()) {
DstMIBuilder.addRenderer<CopyRenderer>(Src.getChild(NumOp).getName());
return InsertPt;
}
}
return failedImport("Dst pattern child is an unsupported kind");
}
Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
RuleMatcher &M, InstructionMatcher &InsnMatcher, const TreePatternNode &Src,
const TreePatternNode &Dst) {
auto InsertPtOrError = createInstructionRenderer(M.actions_end(), M, Dst);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
action_iterator InsertPt = InsertPtOrError.get();
BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());
for (auto PhysInput : InsnMatcher.getPhysRegInputs()) {
InsertPt = M.insertAction<BuildMIAction>(
InsertPt, M.allocateOutputInsnID(),
&Target.getInstruction(RK.getDef("COPY")));
BuildMIAction &CopyToPhysRegMIBuilder =
*static_cast<BuildMIAction *>(InsertPt->get());
CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(
Target, PhysInput.first, true);
CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysInput.first);
}
if (auto Error =
importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Src, Dst)
.takeError())
return std::move(Error);
if (auto Error =
importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst, Src)
.takeError())
return std::move(Error);
return DstMIBuilder;
}
Expected<action_iterator>
GlobalISelEmitter::createAndImportSubInstructionRenderer(
const action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
const TreePatternNode &Src, unsigned TempRegID) {
auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
// TODO: Assert there's exactly one result.
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
BuildMIAction &DstMIBuilder =
*static_cast<BuildMIAction *>(InsertPtOrError.get()->get());
// Assign the result to TempReg.
DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true);
// Handle additional (ignored) results.
if (DstMIBuilder.getCGI()->Operands.NumDefs > 1) {
InsertPtOrError = importExplicitDefRenderers(
std::prev(*InsertPtOrError), M, DstMIBuilder, Src, Dst, /*Start=*/1);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
}
InsertPtOrError = importExplicitUseRenderers(InsertPtOrError.get(), M,
DstMIBuilder, Dst, Src);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
// We need to make sure that when we import an INSERT_SUBREG as a
// subinstruction that it ends up being constrained to the correct super
// register and subregister classes.
auto OpName = Target.getInstruction(Dst.getOperator()).TheDef->getName();
if (OpName == "INSERT_SUBREG") {
auto SubClass = inferRegClassFromPattern(Dst.getChild(1));
if (!SubClass)
return failedImport(
"Cannot infer register class from INSERT_SUBREG operand #1");
std::optional<const CodeGenRegisterClass *> SuperClass =
inferSuperRegisterClassForNode(Dst.getExtType(0), Dst.getChild(0),
Dst.getChild(2));
if (!SuperClass)
return failedImport(
"Cannot infer register class for INSERT_SUBREG operand #0");
// The destination and the super register source of an INSERT_SUBREG must
// be the same register class.
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 1, **SuperClass);
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass);
return InsertPtOrError.get();
}
if (OpName == "EXTRACT_SUBREG") {
// EXTRACT_SUBREG selects into a subregister COPY but unlike most
// instructions, the result register class is controlled by the
// subregisters of the operand. As a result, we must constrain the result
// class rather than check that it's already the right one.
auto SuperClass = inferRegClassFromPattern(Dst.getChild(0));
if (!SuperClass)
return failedImport(
"Cannot infer register class from EXTRACT_SUBREG operand #0");
auto SubIdx = inferSubRegIndexForNode(Dst.getChild(1));
if (!SubIdx)
return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
const auto SrcRCDstRCPair =
(*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 0, *SrcRCDstRCPair->second);
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 1, *SrcRCDstRCPair->first);
// We're done with this pattern! It's eligible for GISel emission; return
// it.
return InsertPtOrError.get();
}
// Similar to INSERT_SUBREG, we also have to handle SUBREG_TO_REG as a
// subinstruction.
if (OpName == "SUBREG_TO_REG") {
auto SubClass = inferRegClassFromPattern(Dst.getChild(1));
if (!SubClass)
return failedImport(
"Cannot infer register class from SUBREG_TO_REG child #1");
auto SuperClass =
inferSuperRegisterClass(Dst.getExtType(0), Dst.getChild(2));
if (!SuperClass)
return failedImport(
"Cannot infer register class for SUBREG_TO_REG operand #0");
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass);
return InsertPtOrError.get();
}
if (OpName == "REG_SEQUENCE") {
auto SuperClass = inferRegClassFromPattern(Dst.getChild(0));
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
unsigned Num = Dst.getNumChildren();
for (unsigned I = 1; I != Num; I += 2) {
const TreePatternNode &SubRegChild = Dst.getChild(I + 1);
auto SubIdx = inferSubRegIndexForNode(SubRegChild);
if (!SubIdx)
return failedImport("REG_SEQUENCE child is not a subreg index");
const auto SrcRCDstRCPair =
(*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
M.insertAction<ConstrainOperandToRegClassAction>(
InsertPt, DstMIBuilder.getInsnID(), I, *SrcRCDstRCPair->second);
}
return InsertPtOrError.get();
}
M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt,
DstMIBuilder.getInsnID());
return InsertPtOrError.get();
}
Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer(
action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst) {
Record *DstOp = Dst.getOperator();
if (!DstOp->isSubClassOf("Instruction")) {
if (DstOp->isSubClassOf("ValueType"))
return failedImport(
"Pattern operator isn't an instruction (it's a ValueType)");
return failedImport("Pattern operator isn't an instruction");
}
CodeGenInstruction *DstI = &Target.getInstruction(DstOp);
// COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
// attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
StringRef Name = DstI->TheDef->getName();
if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")
DstI = &Target.getInstruction(RK.getDef("COPY"));
return M.insertAction<BuildMIAction>(InsertPt, M.allocateOutputInsnID(),
DstI);
}
Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
const TreePatternNode &Src, const TreePatternNode &Dst, unsigned Start) {
const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
const unsigned SrcNumDefs = Src.getExtTypes().size();
const unsigned DstNumDefs = DstI->Operands.NumDefs;
if (DstNumDefs == 0)
return InsertPt;
for (unsigned I = Start; I < SrcNumDefs; ++I) {
std::string OpName = getMangledRootDefName(DstI->Operands[I].Name);
// CopyRenderer saves a StringRef, so cannot pass OpName itself -
// let's use a string with an appropriate lifetime.
StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName();
DstMIBuilder.addRenderer<CopyRenderer>(PermanentRef);
}
// Some instructions have multiple defs, but are missing a type entry
// (e.g. s_cc_out operands).
if (Dst.getExtTypes().size() < DstNumDefs)
return failedImport("unhandled discarded def");
for (unsigned I = SrcNumDefs; I < DstNumDefs; ++I) {
const TypeSetByHwMode &ExtTy = Dst.getExtType(I);
if (!ExtTy.isMachineValueType())
return failedImport("unsupported typeset");
auto OpTy = MVTToLLT(ExtTy.getMachineValueType().SimpleTy);
if (!OpTy)
return failedImport("unsupported type");
unsigned TempRegID = M.allocateTempRegID();
InsertPt =
M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);
DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true, nullptr, true);
}
return InsertPt;
}
Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
const llvm::TreePatternNode &Dst, const llvm::TreePatternNode &Src) {
const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
CodeGenInstruction *OrigDstI = &Target.getInstruction(Dst.getOperator());
StringRef Name = OrigDstI->TheDef->getName();
unsigned ExpectedDstINumUses = Dst.getNumChildren();
// EXTRACT_SUBREG needs to use a subregister COPY.
if (Name == "EXTRACT_SUBREG") {
if (!Dst.getChild(1).isLeaf())
return failedImport("EXTRACT_SUBREG child #1 is not a leaf");
DefInit *SubRegInit = dyn_cast<DefInit>(Dst.getChild(1).getLeafValue());
if (!SubRegInit)
return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
const TreePatternNode &ValChild = Dst.getChild(0);
if (!ValChild.isLeaf()) {
// We really have to handle the source instruction, and then insert a
// copy from the subregister.
auto ExtractSrcTy = getInstResultType(ValChild, Target);
if (!ExtractSrcTy)
return ExtractSrcTy.takeError();
unsigned TempRegID = M.allocateTempRegID();
InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, *ExtractSrcTy,
TempRegID);
auto InsertPtOrError = createAndImportSubInstructionRenderer(
++InsertPt, M, ValChild, Src, TempRegID);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, false, SubIdx);
return InsertPt;
}
// If this is a source operand, this is just a subregister copy.
Record *RCDef = getInitValueAsRegClass(ValChild.getLeafValue());
if (!RCDef)
return failedImport("EXTRACT_SUBREG child #0 could not "
"be coerced to a register class");
CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
const auto SrcRCDstRCPair =
RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);
if (SrcRCDstRCPair) {
assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
if (SrcRCDstRCPair->first != RC)
return failedImport("EXTRACT_SUBREG requires an additional COPY");
}
StringRef RegOperandName = Dst.getChild(0).getName();
if (const auto &SubOperand = M.getComplexSubOperand(RegOperandName)) {
DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
*std::get<0>(*SubOperand), RegOperandName, std::get<1>(*SubOperand),
std::get<2>(*SubOperand), SubIdx);
return InsertPt;
}
DstMIBuilder.addRenderer<CopySubRegRenderer>(RegOperandName, SubIdx);
return InsertPt;
}
if (Name == "REG_SEQUENCE") {
if (!Dst.getChild(0).isLeaf())
return failedImport("REG_SEQUENCE child #0 is not a leaf");
Record *RCDef = getInitValueAsRegClass(Dst.getChild(0).getLeafValue());
if (!RCDef)
return failedImport("REG_SEQUENCE child #0 could not "
"be coerced to a register class");
if ((ExpectedDstINumUses - 1) % 2 != 0)
return failedImport("Malformed REG_SEQUENCE");
for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
const TreePatternNode &ValChild = Dst.getChild(I);
const TreePatternNode &SubRegChild = Dst.getChild(I + 1);
if (DefInit *SubRegInit = dyn_cast<DefInit>(SubRegChild.getLeafValue())) {
CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
auto InsertPtOrError =
importExplicitUseRenderer(InsertPt, M, DstMIBuilder, ValChild, Src);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
InsertPt = InsertPtOrError.get();
DstMIBuilder.addRenderer<SubRegIndexRenderer>(SubIdx);
}
}
return InsertPt;
}
// Render the explicit uses.
unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;
if (Name == "COPY_TO_REGCLASS") {
DstINumUses--; // Ignore the class constraint.
ExpectedDstINumUses--;
}
// NumResults - This is the number of results produced by the instruction in
// the "outs" list.
unsigned NumResults = OrigDstI->Operands.NumDefs;
// Number of operands we know the output instruction must have. If it is
// variadic, we could have more operands.
unsigned NumFixedOperands = DstI->Operands.size();
// Loop over all of the fixed operands of the instruction pattern, emitting
// code to fill them all in. The node 'N' usually has number children equal to
// the number of input operands of the instruction. However, in cases where
// there are predicate operands for an instruction, we need to fill in the
// 'execute always' values. Match up the node operands to the instruction
// operands to do this.
unsigned Child = 0;
// Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
// number of operands at the end of the list which have default values.
// Those can come from the pattern if it provides enough arguments, or be
// filled in with the default if the pattern hasn't provided them. But any
// operand with a default value _before_ the last mandatory one will be
// filled in with their defaults unconditionally.
unsigned NonOverridableOperands = NumFixedOperands;
while (NonOverridableOperands > NumResults &&
CGP.operandHasDefault(DstI->Operands[NonOverridableOperands - 1].Rec))
--NonOverridableOperands;
unsigned NumDefaultOps = 0;
for (unsigned I = 0; I != DstINumUses; ++I) {
unsigned InstOpNo = DstI->Operands.NumDefs + I;
// Determine what to emit for this operand.
Record *OperandNode = DstI->Operands[InstOpNo].Rec;
// If the operand has default values, introduce them now.
if (CGP.operandHasDefault(OperandNode) &&
(InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {
// This is a predicate or optional def operand which the pattern has not
// overridden, or which we aren't letting it override; emit the 'default
// ops' operands.
Record *OperandNode = DstI->Operands[InstOpNo].Rec;
if (auto Error = importDefaultOperandRenderers(
InsertPt, M, DstMIBuilder, CGP.getDefaultOperand(OperandNode)))
return std::move(Error);
++NumDefaultOps;
continue;
}
auto InsertPtOrError = importExplicitUseRenderer(InsertPt, M, DstMIBuilder,
Dst.getChild(Child), Src);
if (auto Error = InsertPtOrError.takeError())
return std::move(Error);
InsertPt = InsertPtOrError.get();
++Child;
}
if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)
return failedImport("Expected " + llvm::to_string(DstINumUses) +
" used operands but found " +
llvm::to_string(ExpectedDstINumUses) +
" explicit ones and " + llvm::to_string(NumDefaultOps) +
" default ones");
return InsertPt;
}
Error GlobalISelEmitter::importDefaultOperandRenderers(
action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
const DAGDefaultOperand &DefaultOp) const {
for (const auto &Op : DefaultOp.DefaultOps) {
const auto &N = *Op;
if (!N.isLeaf())
return failedImport("Could not add default op");
const auto *DefaultOp = N.getLeafValue();
if (const DefInit *DefaultDefOp = dyn_cast<DefInit>(DefaultOp)) {
std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(N.getSimpleType(0));
auto Def = DefaultDefOp->getDef();
if (Def->getName() == "undef_tied_input") {
unsigned TempRegID = M.allocateTempRegID();
M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTyOrNone,
TempRegID);
InsertPt = M.insertAction<BuildMIAction>(
InsertPt, M.allocateOutputInsnID(),
&Target.getInstruction(RK.getDef("IMPLICIT_DEF")));
BuildMIAction &IDMIBuilder =
*static_cast<BuildMIAction *>(InsertPt->get());
IDMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
} else {
DstMIBuilder.addRenderer<AddRegisterRenderer>(Target, Def);
}
continue;
}
if (const IntInit *DefaultIntOp = dyn_cast<IntInit>(DefaultOp)) {
DstMIBuilder.addRenderer<ImmRenderer>(DefaultIntOp->getValue());
continue;
}
return failedImport("Could not add default op");
}
return Error::success();
}
Error GlobalISelEmitter::importImplicitDefRenderers(
BuildMIAction &DstMIBuilder,
const std::vector<Record *> &ImplicitDefs) const {
if (!ImplicitDefs.empty())
return failedImport("Pattern defines a physical register");
return Error::success();
}
std::optional<const CodeGenRegisterClass *>
GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) {
assert(Leaf.isLeaf() && "Expected leaf?");
Record *RCRec = getInitValueAsRegClass(Leaf.getLeafValue());
if (!RCRec)
return std::nullopt;
CodeGenRegisterClass *RC = CGRegs.getRegClass(RCRec);
if (!RC)
return std::nullopt;
return RC;
}
std::optional<const CodeGenRegisterClass *>
GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) {
if (N.isLeaf())
return getRegClassFromLeaf(N);
// We don't have a leaf node, so we have to try and infer something. Check
// that we have an instruction that we can infer something from.
// Only handle things that produce at least one value (if multiple values,
// just take the first one).
if (N.getNumTypes() < 1)
return std::nullopt;
Record *OpRec = N.getOperator();
// We only want instructions.
if (!OpRec->isSubClassOf("Instruction"))
return std::nullopt;
// Don't want to try and infer things when there could potentially be more
// than one candidate register class.
auto &Inst = Target.getInstruction(OpRec);
// Handle any special-case instructions which we can safely infer register
// classes from.
StringRef InstName = Inst.TheDef->getName();
bool IsRegSequence = InstName == "REG_SEQUENCE";
if (IsRegSequence || InstName == "COPY_TO_REGCLASS") {
// If we have a COPY_TO_REGCLASS, then we need to handle it specially. It
// has the desired register class as the first child.
const TreePatternNode &RCChild = N.getChild(IsRegSequence ? 0 : 1);
if (!RCChild.isLeaf())
return std::nullopt;
return getRegClassFromLeaf(RCChild);
}
if (InstName == "INSERT_SUBREG") {
const TreePatternNode &Child0 = N.getChild(0);
assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");
const TypeSetByHwMode &VTy = Child0.getExtType(0);
return inferSuperRegisterClassForNode(VTy, Child0, N.getChild(2));
}
if (InstName == "EXTRACT_SUBREG") {
assert(N.getNumTypes() == 1 && "Unexpected number of types!");
const TypeSetByHwMode &VTy = N.getExtType(0);
return inferSuperRegisterClass(VTy, N.getChild(1));
}
// Handle destination record types that we can safely infer a register class
// from.
const auto &DstIOperand = Inst.Operands[0];
Record *DstIOpRec = DstIOperand.Rec;
if (DstIOpRec->isSubClassOf("RegisterOperand")) {
DstIOpRec = DstIOpRec->getValueAsDef("RegClass");
const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec);
return &RC;
}
if (DstIOpRec->isSubClassOf("RegisterClass")) {
const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec);
return &RC;
}
return std::nullopt;
}
std::optional<const CodeGenRegisterClass *>
GlobalISelEmitter::inferSuperRegisterClass(
const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) {
// We need a ValueTypeByHwMode for getSuperRegForSubReg.
if (!Ty.isValueTypeByHwMode(false))
return std::nullopt;
if (!SubRegIdxNode.isLeaf())
return std::nullopt;
DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode.getLeafValue());
if (!SubRegInit)
return std::nullopt;
CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
// Use the information we found above to find a minimal register class which
// supports the subregister and type we want.
auto RC =
Target.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), CGRegs, SubIdx,
/* MustBeAllocatable */ true);
if (!RC)
return std::nullopt;
return *RC;
}
std::optional<const CodeGenRegisterClass *>
GlobalISelEmitter::inferSuperRegisterClassForNode(
const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,
const TreePatternNode &SubRegIdxNode) {
// Check if we already have a defined register class for the super register
// node. If we do, then we should preserve that rather than inferring anything
// from the subregister index node. We can assume that whoever wrote the
// pattern in the first place made sure that the super register and
// subregister are compatible.
if (std::optional<const CodeGenRegisterClass *> SuperRegisterClass =
inferRegClassFromPattern(SuperRegNode))
return *SuperRegisterClass;
return inferSuperRegisterClass(Ty, SubRegIdxNode);
}
std::optional<CodeGenSubRegIndex *> GlobalISelEmitter::inferSubRegIndexForNode(
const TreePatternNode &SubRegIdxNode) {
if (!SubRegIdxNode.isLeaf())
return std::nullopt;
DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode.getLeafValue());
if (!SubRegInit)
return std::nullopt;
return CGRegs.getSubRegIdx(SubRegInit->getDef());
}
Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
// Keep track of the matchers and actions to emit.
int Score = P.getPatternComplexity(CGP);
RuleMatcher M(P.getSrcRecord()->getLoc());
RuleMatcherScores[M.getRuleID()] = Score;
M.addAction<DebugCommentAction>(llvm::to_string(P.getSrcPattern()) +
" => " +
llvm::to_string(P.getDstPattern()));
SmallVector<Record *, 4> Predicates;
P.getPredicateRecords(Predicates);
if (auto Error = importRulePredicates(M, Predicates))
return std::move(Error);
if (!P.getHwModeFeatures().empty())
M.addHwModeIdx(declareHwModeCheck(P.getHwModeFeatures()));
// Next, analyze the pattern operators.
TreePatternNode &Src = P.getSrcPattern();
TreePatternNode &Dst = P.getDstPattern();
// If the root of either pattern isn't a simple operator, ignore it.
if (auto Err = isTrivialOperatorNode(Dst))
return failedImport("Dst pattern root isn't a trivial operator (" +
toString(std::move(Err)) + ")");
if (auto Err = isTrivialOperatorNode(Src))
return failedImport("Src pattern root isn't a trivial operator (" +
toString(std::move(Err)) + ")");
// The different predicates and matchers created during
// addInstructionMatcher use the RuleMatcher M to set up their
// instruction ID (InsnVarID) that are going to be used when
// M is going to be emitted.
// However, the code doing the emission still relies on the IDs
// returned during that process by the RuleMatcher when issuing
// the recordInsn opcodes.
// Because of that:
// 1. The order in which we created the predicates
// and such must be the same as the order in which we emit them,
// and
// 2. We need to reset the generation of the IDs in M somewhere between
// addInstructionMatcher and emit
//
// FIXME: Long term, we don't want to have to rely on this implicit
// naming being the same. One possible solution would be to have
// explicit operator for operation capture and reference those.
// The plus side is that it would expose opportunities to share
// the capture accross rules. The downside is that it would
// introduce a dependency between predicates (captures must happen
// before their first use.)
InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src.getName());
unsigned TempOpIdx = 0;
const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
auto InsnMatcherOrError =
createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx);
if (auto Error = InsnMatcherOrError.takeError())
return std::move(Error);
InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
if (Dst.isLeaf()) {
Record *RCDef = getInitValueAsRegClass(Dst.getLeafValue());
if (RCDef) {
const CodeGenRegisterClass &RC = Target.getRegisterClass(RCDef);
// We need to replace the def and all its uses with the specified
// operand. However, we must also insert COPY's wherever needed.
// For now, emit a copy and let the register allocator clean up.
auto &DstI = Target.getInstruction(RK.getDef("COPY"));
const auto &DstIOperand = DstI.Operands[0];
OperandMatcher &OM0 = InsnMatcher.getOperand(0);
OM0.setSymbolicName(DstIOperand.Name);
M.defineOperand(OM0.getSymbolicName(), OM0);
OM0.addPredicate<RegisterBankOperandMatcher>(RC);
auto &DstMIBuilder =
M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &DstI);
DstMIBuilder.addRenderer<CopyRenderer>(DstIOperand.Name);
DstMIBuilder.addRenderer<CopyRenderer>(Dst.getName());
M.addAction<ConstrainOperandToRegClassAction>(0, 0, RC);
// Erase the root.
unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
M.addAction<EraseInstAction>(RootInsnID);
// We're done with this pattern! It's eligible for GISel emission; return
// it.
++NumPatternImported;
return std::move(M);
}
return failedImport("Dst pattern root isn't a known leaf");
}
// Start with the defined operands (i.e., the results of the root operator).
Record *DstOp = Dst.getOperator();
if (!DstOp->isSubClassOf("Instruction"))
return failedImport("Pattern operator isn't an instruction");
auto &DstI = Target.getInstruction(DstOp);
StringRef DstIName = DstI.TheDef->getName();
unsigned DstNumDefs = DstI.Operands.NumDefs,
SrcNumDefs = Src.getExtTypes().size();
if (DstNumDefs < SrcNumDefs) {
if (DstNumDefs != 0)
return failedImport("Src pattern result has more defs than dst MI (" +
to_string(SrcNumDefs) + " def(s) vs " +
to_string(DstNumDefs) + " def(s))");
bool FoundNoUsePred = false;
for (const auto &Pred : InsnMatcher.predicates()) {
if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Pred.get())))
break;
}
if (!FoundNoUsePred)
return failedImport("Src pattern result has " + to_string(SrcNumDefs) +
" def(s) without the HasNoUse predicate set to true "
"but Dst MI has no def");
}
// The root of the match also has constraints on the register bank so that it
// matches the result instruction.
unsigned OpIdx = 0;
unsigned N = std::min(DstNumDefs, SrcNumDefs);
for (unsigned I = 0; I < N; ++I) {
const TypeSetByHwMode &VTy = Src.getExtType(I);
const auto &DstIOperand = DstI.Operands[OpIdx];
PointerUnion<Record *, const CodeGenRegisterClass *> MatchedRC =
DstIOperand.Rec;
if (DstIName == "COPY_TO_REGCLASS") {
MatchedRC = getInitValueAsRegClass(Dst.getChild(1).getLeafValue());
if (MatchedRC.isNull())
return failedImport(
"COPY_TO_REGCLASS operand #1 isn't a register class");
} else if (DstIName == "REG_SEQUENCE") {
MatchedRC = getInitValueAsRegClass(Dst.getChild(0).getLeafValue());
if (MatchedRC.isNull())
return failedImport("REG_SEQUENCE operand #0 isn't a register class");
} else if (DstIName == "EXTRACT_SUBREG") {
auto InferredClass = inferRegClassFromPattern(Dst.getChild(0));
if (!InferredClass)
return failedImport(
"Could not infer class for EXTRACT_SUBREG operand #0");
// We can assume that a subregister is in the same bank as it's super
// register.
MatchedRC = (*InferredClass)->getDef();
} else if (DstIName == "INSERT_SUBREG") {
auto MaybeSuperClass =
inferSuperRegisterClassForNode(VTy, Dst.getChild(0), Dst.getChild(2));
if (!MaybeSuperClass)
return failedImport(
"Cannot infer register class for INSERT_SUBREG operand #0");
// Move to the next pattern here, because the register class we found
// doesn't necessarily have a record associated with it. So, we can't
// set DstIOpRec using this.
MatchedRC = *MaybeSuperClass;
} else if (DstIName == "SUBREG_TO_REG") {
auto MaybeRegClass = inferSuperRegisterClass(VTy, Dst.getChild(2));
if (!MaybeRegClass)
return failedImport(
"Cannot infer register class for SUBREG_TO_REG operand #0");
MatchedRC = *MaybeRegClass;
} else if (MatchedRC.get<Record *>()->isSubClassOf("RegisterOperand"))
MatchedRC = MatchedRC.get<Record *>()->getValueAsDef("RegClass");
else if (!MatchedRC.get<Record *>()->isSubClassOf("RegisterClass"))
return failedImport("Dst MI def isn't a register class" + to_string(Dst));
OperandMatcher &OM = InsnMatcher.getOperand(OpIdx);
// The operand names declared in the DstI instruction are unrelated to
// those used in pattern's source and destination DAGs, so mangle the
// former to prevent implicitly adding unexpected
// GIM_CheckIsSameOperand predicates by the defineOperand method.
OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name));
M.defineOperand(OM.getSymbolicName(), OM);
if (MatchedRC.is<Record *>())
MatchedRC = &Target.getRegisterClass(MatchedRC.get<Record *>());
OM.addPredicate<RegisterBankOperandMatcher>(
*MatchedRC.get<const CodeGenRegisterClass *>());
++OpIdx;
}
auto DstMIBuilderOrError =
createAndImportInstructionRenderer(M, InsnMatcher, Src, Dst);
if (auto Error = DstMIBuilderOrError.takeError())
return std::move(Error);
BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
// Render the implicit defs.
// These are only added to the root of the result.
if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs()))
return std::move(Error);
DstMIBuilder.chooseInsnToMutate(M);
// Constrain the registers to classes. This is normally derived from the
// emitted instruction but a few instructions require special handling.
if (DstIName == "COPY_TO_REGCLASS") {
// COPY_TO_REGCLASS does not provide operand constraints itself but the
// result is constrained to the class given by the second child.
Record *DstIOpRec = getInitValueAsRegClass(Dst.getChild(1).getLeafValue());
if (DstIOpRec == nullptr)
return failedImport("COPY_TO_REGCLASS operand #1 isn't a register class");
M.addAction<ConstrainOperandToRegClassAction>(
0, 0, Target.getRegisterClass(DstIOpRec));
} else if (DstIName == "EXTRACT_SUBREG") {
auto SuperClass = inferRegClassFromPattern(Dst.getChild(0));
if (!SuperClass)
return failedImport(
"Cannot infer register class from EXTRACT_SUBREG operand #0");
auto SubIdx = inferSubRegIndexForNode(Dst.getChild(1));
if (!SubIdx)
return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
// It would be nice to leave this constraint implicit but we're required
// to pick a register class so constrain the result to a register class
// that can hold the correct MVT.
//
// FIXME: This may introduce an extra copy if the chosen class doesn't
// actually contain the subregisters.
assert(Src.getExtTypes().size() == 1 &&
"Expected Src of EXTRACT_SUBREG to have one result type");
const auto SrcRCDstRCPair =
(*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
if (!SrcRCDstRCPair) {
return failedImport("subreg index is incompatible "
"with inferred reg class");
}
assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
M.addAction<ConstrainOperandToRegClassAction>(0, 0,
*SrcRCDstRCPair->second);
M.addAction<ConstrainOperandToRegClassAction>(0, 1, *SrcRCDstRCPair->first);
} else if (DstIName == "INSERT_SUBREG") {
assert(Src.getExtTypes().size() == 1 &&
"Expected Src of INSERT_SUBREG to have one result type");
// We need to constrain the destination, a super regsister source, and a
// subregister source.
auto SubClass = inferRegClassFromPattern(Dst.getChild(1));
if (!SubClass)
return failedImport(
"Cannot infer register class from INSERT_SUBREG operand #1");
auto SuperClass = inferSuperRegisterClassForNode(
Src.getExtType(0), Dst.getChild(0), Dst.getChild(2));
if (!SuperClass)
return failedImport(
"Cannot infer register class for INSERT_SUBREG operand #0");
M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass);
M.addAction<ConstrainOperandToRegClassAction>(0, 1, **SuperClass);
M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass);
} else if (DstIName == "SUBREG_TO_REG") {
// We need to constrain the destination and subregister source.
assert(Src.getExtTypes().size() == 1 &&
"Expected Src of SUBREG_TO_REG to have one result type");
// Attempt to infer the subregister source from the first child. If it has
// an explicitly given register class, we'll use that. Otherwise, we will
// fail.
auto SubClass = inferRegClassFromPattern(Dst.getChild(1));
if (!SubClass)
return failedImport(
"Cannot infer register class from SUBREG_TO_REG child #1");
// We don't have a child to look at that might have a super register node.
auto SuperClass =
inferSuperRegisterClass(Src.getExtType(0), Dst.getChild(2));
if (!SuperClass)
return failedImport(
"Cannot infer register class for SUBREG_TO_REG operand #0");
M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass);
M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass);
} else if (DstIName == "REG_SEQUENCE") {
auto SuperClass = inferRegClassFromPattern(Dst.getChild(0));
M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass);
unsigned Num = Dst.getNumChildren();
for (unsigned I = 1; I != Num; I += 2) {
TreePatternNode &SubRegChild = Dst.getChild(I + 1);
auto SubIdx = inferSubRegIndexForNode(SubRegChild);
if (!SubIdx)
return failedImport("REG_SEQUENCE child is not a subreg index");
const auto SrcRCDstRCPair =
(*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
M.addAction<ConstrainOperandToRegClassAction>(0, I,
*SrcRCDstRCPair->second);
}
} else {
M.addAction<ConstrainOperandsToDefinitionAction>(0);
}
// Erase the root.
unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
M.addAction<EraseInstAction>(RootInsnID);
// We're done with this pattern! It's eligible for GISel emission; return it.
++NumPatternImported;
return std::move(M);
}
MatchTable
GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
bool Optimize, bool WithCoverage) {
std::vector<Matcher *> InputRules;
for (Matcher &Rule : Rules)
InputRules.push_back(&Rule);
if (!Optimize)
return MatchTable::buildTable(InputRules, WithCoverage);
unsigned CurrentOrdering = 0;
StringMap<unsigned> OpcodeOrder;
for (RuleMatcher &Rule : Rules) {
const StringRef Opcode = Rule.getOpcode();
assert(!Opcode.empty() && "Didn't expect an undefined opcode");
if (OpcodeOrder.count(Opcode) == 0)
OpcodeOrder[Opcode] = CurrentOrdering++;
}
llvm::stable_sort(InputRules, [&OpcodeOrder](const Matcher *A,
const Matcher *B) {
auto *L = static_cast<const RuleMatcher *>(A);
auto *R = static_cast<const RuleMatcher *>(B);
return std::tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) <
std::tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands());
});
for (Matcher *Rule : InputRules)
Rule->optimize();
std::vector<std::unique_ptr<Matcher>> MatcherStorage;
std::vector<Matcher *> OptRules =
optimizeRules<GroupMatcher>(InputRules, MatcherStorage);
for (Matcher *Rule : OptRules)
Rule->optimize();
OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage);
return MatchTable::buildTable(OptRules, WithCoverage);
}
void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
OS << "bool " << getClassName()
<< "::selectImpl(MachineInstr &I, CodeGenCoverage "
"&CoverageInfo) const {\n"
<< " const PredicateBitset AvailableFeatures = "
"getAvailableFeatures();\n"
<< " MachineIRBuilder B(I);\n"
<< " State.MIs.clear();\n"
<< " State.MIs.push_back(&I);\n\n"
<< " if (executeMatchTable(*this, State, ExecInfo, B"
<< ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
<< ", &CoverageInfo)) {\n"
<< " return true;\n"
<< " }\n\n"
<< " return false;\n"
<< "}\n\n";
}
void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
std::vector<Record *> MatchedRecords;
std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
std::back_inserter(MatchedRecords), [&](Record *R) {
return !R->getValueAsString("GISelPredicateCode").empty();
});
emitMIPredicateFnsImpl<Record *>(
OS,
" const MachineFunction &MF = *MI.getParent()->getParent();\n"
" const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
" const auto &Operands = State.RecordedOperands;\n"
" (void)Operands;\n"
" (void)MRI;",
ArrayRef<Record *>(MatchedRecords), &getPatFragPredicateEnumName,
[&](Record *R) { return R->getValueAsString("GISelPredicateCode"); },
"PatFrag predicates.");
}
void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
std::vector<Record *> MatchedRecords;
std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
std::back_inserter(MatchedRecords), [&](Record *R) {
bool Unset;
return !R->getValueAsString("ImmediateCode").empty() &&
!R->getValueAsBitOrUnset("IsAPFloat", Unset) &&
!R->getValueAsBit("IsAPInt");
});
emitImmPredicateFnsImpl<Record *>(
OS, "I64", "int64_t", ArrayRef<Record *>(MatchedRecords),
&getPatFragPredicateEnumName,
[&](Record *R) { return R->getValueAsString("ImmediateCode"); },
"PatFrag predicates.");
}
void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
std::vector<Record *> MatchedRecords;
std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
std::back_inserter(MatchedRecords), [&](Record *R) {
bool Unset;
return !R->getValueAsString("ImmediateCode").empty() &&
R->getValueAsBitOrUnset("IsAPFloat", Unset);
});
emitImmPredicateFnsImpl<Record *>(
OS, "APFloat", "const APFloat &", ArrayRef<Record *>(MatchedRecords),
&getPatFragPredicateEnumName,
[&](Record *R) { return R->getValueAsString("ImmediateCode"); },
"PatFrag predicates.");
}
void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
std::vector<Record *> MatchedRecords;
std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
std::back_inserter(MatchedRecords), [&](Record *R) {
return !R->getValueAsString("ImmediateCode").empty() &&
R->getValueAsBit("IsAPInt");
});
emitImmPredicateFnsImpl<Record *>(
OS, "APInt", "const APInt &", ArrayRef<Record *>(MatchedRecords),
&getPatFragPredicateEnumName,
[&](Record *R) { return R->getValueAsString("ImmediateCode"); },
"PatFrag predicates.");
}
void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
<< " llvm_unreachable(\"" + getClassName() +
" does not support simple predicates!\");\n"
<< " return false;\n"
<< "}\n";
}
void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
OS << "void " << getClassName()
<< "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
"{\n"
<< " llvm_unreachable(\"" + getClassName() +
" does not support custom C++ actions!\");\n"
<< "}\n";
}
void GlobalISelEmitter::postProcessRule(RuleMatcher &M) {
SmallPtrSet<Record *, 16> UsedRegs;
// TODO: deal with subregs?
for (auto &A : M.actions()) {
auto *MI = dyn_cast<BuildMIAction>(A.get());
if (!MI)
continue;
for (auto *Use : MI->getCGI()->ImplicitUses)
UsedRegs.insert(Use);
}
for (auto &A : M.actions()) {
auto *MI = dyn_cast<BuildMIAction>(A.get());
if (!MI)
continue;
for (auto *Def : MI->getCGI()->ImplicitDefs) {
if (!UsedRegs.contains(Def))
MI->setDeadImplicitDef(Def);
}
}
}
void GlobalISelEmitter::run(raw_ostream &OS) {
if (!UseCoverageFile.empty()) {
RuleCoverage = CodeGenCoverage();
auto RuleCoverageBufOrErr = MemoryBuffer::getFile(UseCoverageFile);
if (!RuleCoverageBufOrErr) {
PrintWarning(SMLoc(), "Missing rule coverage data");
RuleCoverage = std::nullopt;
} else {
if (!RuleCoverage->parse(*RuleCoverageBufOrErr.get(), Target.getName())) {
PrintWarning(SMLoc(), "Ignoring invalid or missing rule coverage data");
RuleCoverage = std::nullopt;
}
}
}
// Track the run-time opcode values
gatherOpcodeValues();
// Track the run-time LLT ID values
gatherTypeIDValues();
// Track the GINodeEquiv definitions.
gatherNodeEquivs();
AllPatFrags = RK.getAllDerivedDefinitions("PatFrags");
emitSourceFileHeader(
("Global Instruction Selector for the " + Target.getName() + " target")
.str(),
OS);
std::vector<RuleMatcher> Rules;
// Look through the SelectionDAG patterns we found, possibly emitting some.
for (const PatternToMatch &Pat : CGP.ptms()) {
++NumPatternTotal;
auto MatcherOrErr = runOnPattern(Pat);
// The pattern analysis can fail, indicating an unsupported pattern.
// Report that if we've been asked to do so.
if (auto Err = MatcherOrErr.takeError()) {
if (WarnOnSkippedPatterns) {
PrintWarning(Pat.getSrcRecord()->getLoc(),
"Skipped pattern: " + toString(std::move(Err)));
} else {
consumeError(std::move(Err));
}
++NumPatternImportsSkipped;
continue;
}
if (RuleCoverage) {
if (RuleCoverage->isCovered(MatcherOrErr->getRuleID()))
++NumPatternsTested;
else
PrintWarning(Pat.getSrcRecord()->getLoc(),
"Pattern is not covered by a test");
}
Rules.push_back(std::move(MatcherOrErr.get()));
postProcessRule(Rules.back());
}
// Comparison function to order records by name.
auto orderByName = [](const Record *A, const Record *B) {
return A->getName() < B->getName();
};
std::vector<Record *> ComplexPredicates =
RK.getAllDerivedDefinitions("GIComplexOperandMatcher");
llvm::sort(ComplexPredicates, orderByName);
std::vector<StringRef> CustomRendererFns;
transform(RK.getAllDerivedDefinitions("GICustomOperandRenderer"),
std::back_inserter(CustomRendererFns), [](const auto &Record) {
return Record->getValueAsString("RendererFn");
});
// Sort and remove duplicates to get a list of unique renderer functions, in
// case some were mentioned more than once.
llvm::sort(CustomRendererFns);
CustomRendererFns.erase(
std::unique(CustomRendererFns.begin(), CustomRendererFns.end()),
CustomRendererFns.end());
// Create a table containing the LLT objects needed by the matcher and an enum
// for the matcher to reference them with.
std::vector<LLTCodeGen> TypeObjects;
append_range(TypeObjects, KnownTypes);
llvm::sort(TypeObjects);
// Sort rules.
llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) {
int ScoreA = RuleMatcherScores[A.getRuleID()];
int ScoreB = RuleMatcherScores[B.getRuleID()];
if (ScoreA > ScoreB)
return true;
if (ScoreB > ScoreA)
return false;
if (A.isHigherPriorityThan(B)) {
assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
"and less important at "
"the same time");
return true;
}
return false;
});
unsigned MaxTemporaries = 0;
for (const auto &Rule : Rules)
MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());
// Build match table
const MatchTable Table =
buildMatchTable(Rules, OptimizeMatchTable, GenerateCoverage);
emitPredicateBitset(OS, "GET_GLOBALISEL_PREDICATE_BITSET");
emitTemporariesDecl(OS, "GET_GLOBALISEL_TEMPORARIES_DECL");
emitTemporariesInit(OS, MaxTemporaries, "GET_GLOBALISEL_TEMPORARIES_INIT");
emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates,
CustomRendererFns, "GET_GLOBALISEL_IMPL");
emitPredicatesDecl(OS, "GET_GLOBALISEL_PREDICATES_DECL");
emitPredicatesInit(OS, "GET_GLOBALISEL_PREDICATES_INIT");
}
void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) {
SubtargetFeatures.try_emplace(Predicate, Predicate, SubtargetFeatures.size());
}
unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
return HwModes.emplace(HwModeFeatures.str(), HwModes.size()).first->second;
}
} // end anonymous namespace
//===----------------------------------------------------------------------===//
static TableGen::Emitter::OptClass<GlobalISelEmitter>
X("gen-global-isel", "Generate GlobalISel selector");
Reference in New Issue View Git Blame Copy Permalink