89c8c80bd5
If f32 denormals were enabled pre-gfx9, we would still try to implement this with v_max_f32. Pre-gfx9, these instructions ignored the denormal mode and did not flush. Switch to the multiply form for f32 as a workaround which should always work in any case. This fixes conformance failures when the library implementation of fmin/fmax were accidentally not inlined, forcing the assumption of no flushing on targets where denormals are not enabled by default. This is a workaround, since really we should not be mixing code with different FP mode expectations, but prefer the lowering that will work in any mode. Now this will always use max to implement canonicalize on gfx9+. This is only really beneficial for f64. For f32/f16 it's a neutral choice (and worse in terms of code size in 1 case), but possibly worse for the compiler since it does add an extra register use operand. Leave this change for later.
818 lines
25 KiB
TableGen
818 lines
25 KiB
TableGen
//===-- AMDGPUInstructions.td - Common instruction defs ---*- tablegen -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file contains instruction defs that are common to all hw codegen
|
|
// targets.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class AddressSpacesImpl {
|
|
int Flat = 0;
|
|
int Global = 1;
|
|
int Region = 2;
|
|
int Local = 3;
|
|
int Constant = 4;
|
|
int Private = 5;
|
|
}
|
|
|
|
def AddrSpaces : AddressSpacesImpl;
|
|
|
|
|
|
class AMDGPUInst <dag outs, dag ins, string asm = "",
|
|
list<dag> pattern = []> : Instruction {
|
|
field bit isRegisterLoad = 0;
|
|
field bit isRegisterStore = 0;
|
|
|
|
let Namespace = "AMDGPU";
|
|
let OutOperandList = outs;
|
|
let InOperandList = ins;
|
|
let AsmString = asm;
|
|
let Pattern = pattern;
|
|
let Itinerary = NullALU;
|
|
|
|
// SoftFail is a field the disassembler can use to provide a way for
|
|
// instructions to not match without killing the whole decode process. It is
|
|
// mainly used for ARM, but Tablegen expects this field to exist or it fails
|
|
// to build the decode table.
|
|
field bits<64> SoftFail = 0;
|
|
|
|
let DecoderNamespace = Namespace;
|
|
|
|
let TSFlags{63} = isRegisterLoad;
|
|
let TSFlags{62} = isRegisterStore;
|
|
}
|
|
|
|
class AMDGPUShaderInst <dag outs, dag ins, string asm = "",
|
|
list<dag> pattern = []> : AMDGPUInst<outs, ins, asm, pattern> {
|
|
|
|
field bits<32> Inst = 0xffffffff;
|
|
}
|
|
|
|
//===---------------------------------------------------------------------===//
|
|
// Return instruction
|
|
//===---------------------------------------------------------------------===//
|
|
|
|
class ILFormat<dag outs, dag ins, string asmstr, list<dag> pattern>
|
|
: Instruction {
|
|
|
|
let Namespace = "AMDGPU";
|
|
dag OutOperandList = outs;
|
|
dag InOperandList = ins;
|
|
let Pattern = pattern;
|
|
let AsmString = !strconcat(asmstr, "\n");
|
|
let isPseudo = 1;
|
|
let Itinerary = NullALU;
|
|
bit hasIEEEFlag = 0;
|
|
bit hasZeroOpFlag = 0;
|
|
let mayLoad = 0;
|
|
let mayStore = 0;
|
|
let hasSideEffects = 0;
|
|
let isCodeGenOnly = 1;
|
|
}
|
|
|
|
def TruePredicate : Predicate<"">;
|
|
|
|
// FIXME: Tablegen should specially supports this
|
|
def FalsePredicate : Predicate<"false">;
|
|
|
|
// Add a predicate to the list if does not already exist to deduplicate it.
|
|
class PredConcat<list<Predicate> lst, Predicate pred> {
|
|
list<Predicate> ret =
|
|
!foldl([pred], lst, acc, cur,
|
|
!listconcat(acc, !if(!eq(!cast<string>(cur),!cast<string>(pred)),
|
|
[], [cur])));
|
|
}
|
|
|
|
class PredicateControl {
|
|
Predicate SubtargetPredicate = TruePredicate;
|
|
Predicate AssemblerPredicate = TruePredicate;
|
|
Predicate WaveSizePredicate = TruePredicate;
|
|
list<Predicate> OtherPredicates = [];
|
|
list<Predicate> Predicates = PredConcat<
|
|
PredConcat<PredConcat<OtherPredicates,
|
|
SubtargetPredicate>.ret,
|
|
AssemblerPredicate>.ret,
|
|
WaveSizePredicate>.ret;
|
|
}
|
|
|
|
class AMDGPUPat<dag pattern, dag result> : Pat<pattern, result>,
|
|
PredicateControl;
|
|
|
|
let RecomputePerFunction = 1 in {
|
|
def FP16Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().allFP64FP16Denormals()">;
|
|
def FP32Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().allFP32Denormals()">;
|
|
def FP64Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().allFP64FP16Denormals()">;
|
|
def NoFP16Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().allFP64FP16Denormals()">;
|
|
def NoFP32Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().allFP32Denormals()">;
|
|
def NoFP64Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().allFP64FP16Denormals()">;
|
|
def UnsafeFPMath : Predicate<"TM.Options.UnsafeFPMath">;
|
|
}
|
|
|
|
def FMA : Predicate<"Subtarget->hasFMA()">;
|
|
|
|
def InstFlag : OperandWithDefaultOps <i32, (ops (i32 0))>;
|
|
|
|
def u16ImmTarget : AsmOperandClass {
|
|
let Name = "U16Imm";
|
|
let RenderMethod = "addImmOperands";
|
|
}
|
|
|
|
def s16ImmTarget : AsmOperandClass {
|
|
let Name = "S16Imm";
|
|
let RenderMethod = "addImmOperands";
|
|
}
|
|
|
|
let OperandType = "OPERAND_IMMEDIATE" in {
|
|
|
|
def u32imm : Operand<i32> {
|
|
let PrintMethod = "printU32ImmOperand";
|
|
}
|
|
|
|
def u16imm : Operand<i16> {
|
|
let PrintMethod = "printU16ImmOperand";
|
|
let ParserMatchClass = u16ImmTarget;
|
|
}
|
|
|
|
def s16imm : Operand<i16> {
|
|
let PrintMethod = "printU16ImmOperand";
|
|
let ParserMatchClass = s16ImmTarget;
|
|
}
|
|
|
|
def u8imm : Operand<i8> {
|
|
let PrintMethod = "printU8ImmOperand";
|
|
}
|
|
|
|
} // End OperandType = "OPERAND_IMMEDIATE"
|
|
|
|
//===--------------------------------------------------------------------===//
|
|
// Custom Operands
|
|
//===--------------------------------------------------------------------===//
|
|
def brtarget : Operand<OtherVT>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Misc. PatFrags
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class HasOneUseUnaryOp<SDPatternOperator op> : PatFrag<
|
|
(ops node:$src0),
|
|
(op $src0),
|
|
[{ return N->hasOneUse(); }]> {
|
|
|
|
let GISelPredicateCode = [{
|
|
return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
|
|
}];
|
|
}
|
|
|
|
class HasOneUseBinOp<SDPatternOperator op> : PatFrag<
|
|
(ops node:$src0, node:$src1),
|
|
(op $src0, $src1),
|
|
[{ return N->hasOneUse(); }]> {
|
|
let GISelPredicateCode = [{
|
|
return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
|
|
}];
|
|
}
|
|
|
|
class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag<
|
|
(ops node:$src0, node:$src1, node:$src2),
|
|
(op $src0, $src1, $src2),
|
|
[{ return N->hasOneUse(); }]> {
|
|
let GISelPredicateCode = [{
|
|
return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
|
|
}];
|
|
}
|
|
|
|
let Properties = [SDNPCommutative, SDNPAssociative] in {
|
|
def smax_oneuse : HasOneUseBinOp<smax>;
|
|
def smin_oneuse : HasOneUseBinOp<smin>;
|
|
def umax_oneuse : HasOneUseBinOp<umax>;
|
|
def umin_oneuse : HasOneUseBinOp<umin>;
|
|
|
|
def fminnum_oneuse : HasOneUseBinOp<fminnum>;
|
|
def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;
|
|
|
|
def fminnum_ieee_oneuse : HasOneUseBinOp<fminnum_ieee>;
|
|
def fmaxnum_ieee_oneuse : HasOneUseBinOp<fmaxnum_ieee>;
|
|
|
|
|
|
def and_oneuse : HasOneUseBinOp<and>;
|
|
def or_oneuse : HasOneUseBinOp<or>;
|
|
def xor_oneuse : HasOneUseBinOp<xor>;
|
|
} // Properties = [SDNPCommutative, SDNPAssociative]
|
|
|
|
def not_oneuse : HasOneUseUnaryOp<not>;
|
|
|
|
def add_oneuse : HasOneUseBinOp<add>;
|
|
def sub_oneuse : HasOneUseBinOp<sub>;
|
|
|
|
def srl_oneuse : HasOneUseBinOp<srl>;
|
|
def shl_oneuse : HasOneUseBinOp<shl>;
|
|
|
|
def select_oneuse : HasOneUseTernaryOp<select>;
|
|
|
|
def AMDGPUmul_u24_oneuse : HasOneUseBinOp<AMDGPUmul_u24>;
|
|
def AMDGPUmul_i24_oneuse : HasOneUseBinOp<AMDGPUmul_i24>;
|
|
|
|
def srl_16 : PatFrag<
|
|
(ops node:$src0), (srl_oneuse node:$src0, (i32 16))
|
|
>;
|
|
|
|
|
|
def hi_i16_elt : PatFrag<
|
|
(ops node:$src0), (i16 (trunc (i32 (srl_16 node:$src0))))
|
|
>;
|
|
|
|
|
|
def hi_f16_elt : PatLeaf<
|
|
(vt), [{
|
|
if (N->getOpcode() != ISD::BITCAST)
|
|
return false;
|
|
SDValue Tmp = N->getOperand(0);
|
|
|
|
if (Tmp.getOpcode() != ISD::SRL)
|
|
return false;
|
|
if (const auto *RHS = dyn_cast<ConstantSDNode>(Tmp.getOperand(1))
|
|
return RHS->getZExtValue() == 16;
|
|
return false;
|
|
}]>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// PatLeafs for floating-point comparisons
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def COND_OEQ : PatFrags<(ops), [(OtherVT SETOEQ), (OtherVT SETEQ)]>;
|
|
def COND_ONE : PatFrags<(ops), [(OtherVT SETONE), (OtherVT SETNE)]>;
|
|
def COND_OGT : PatFrags<(ops), [(OtherVT SETOGT), (OtherVT SETGT)]>;
|
|
def COND_OGE : PatFrags<(ops), [(OtherVT SETOGE), (OtherVT SETGE)]>;
|
|
def COND_OLT : PatFrags<(ops), [(OtherVT SETOLT), (OtherVT SETLT)]>;
|
|
def COND_OLE : PatFrags<(ops), [(OtherVT SETOLE), (OtherVT SETLE)]>;
|
|
def COND_O : PatFrags<(ops), [(OtherVT SETO)]>;
|
|
def COND_UO : PatFrags<(ops), [(OtherVT SETUO)]>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// PatLeafs for unsigned / unordered comparisons
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def COND_UEQ : PatFrag<(ops), (OtherVT SETUEQ)>;
|
|
def COND_UNE : PatFrag<(ops), (OtherVT SETUNE)>;
|
|
def COND_UGT : PatFrag<(ops), (OtherVT SETUGT)>;
|
|
def COND_UGE : PatFrag<(ops), (OtherVT SETUGE)>;
|
|
def COND_ULT : PatFrag<(ops), (OtherVT SETULT)>;
|
|
def COND_ULE : PatFrag<(ops), (OtherVT SETULE)>;
|
|
|
|
// XXX - For some reason R600 version is preferring to use unordered
|
|
// for setne?
|
|
def COND_UNE_NE : PatFrags<(ops), [(OtherVT SETUNE), (OtherVT SETNE)]>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// PatLeafs for signed comparisons
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def COND_SGT : PatFrag<(ops), (OtherVT SETGT)>;
|
|
def COND_SGE : PatFrag<(ops), (OtherVT SETGE)>;
|
|
def COND_SLT : PatFrag<(ops), (OtherVT SETLT)>;
|
|
def COND_SLE : PatFrag<(ops), (OtherVT SETLE)>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// PatLeafs for integer equality
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def COND_EQ : PatFrags<(ops), [(OtherVT SETEQ), (OtherVT SETUEQ)]>;
|
|
def COND_NE : PatFrags<(ops), [(OtherVT SETNE), (OtherVT SETUNE)]>;
|
|
|
|
// FIXME: Should not need code predicate
|
|
//def COND_NULL : PatLeaf<(OtherVT null_frag)>;
|
|
def COND_NULL : PatLeaf <
|
|
(cond),
|
|
[{(void)N; return false;}]
|
|
>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// PatLeafs for Texture Constants
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def TEX_ARRAY : PatLeaf<
|
|
(imm),
|
|
[{uint32_t TType = (uint32_t)N->getZExtValue();
|
|
return TType == 9 || TType == 10 || TType == 16;
|
|
}]
|
|
>;
|
|
|
|
def TEX_RECT : PatLeaf<
|
|
(imm),
|
|
[{uint32_t TType = (uint32_t)N->getZExtValue();
|
|
return TType == 5;
|
|
}]
|
|
>;
|
|
|
|
def TEX_SHADOW : PatLeaf<
|
|
(imm),
|
|
[{uint32_t TType = (uint32_t)N->getZExtValue();
|
|
return (TType >= 6 && TType <= 8) || TType == 13;
|
|
}]
|
|
>;
|
|
|
|
def TEX_SHADOW_ARRAY : PatLeaf<
|
|
(imm),
|
|
[{uint32_t TType = (uint32_t)N->getZExtValue();
|
|
return TType == 11 || TType == 12 || TType == 17;
|
|
}]
|
|
>;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Load/Store Pattern Fragments
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3,
|
|
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
|
|
>;
|
|
|
|
class AddressSpaceList<list<int> AS> {
|
|
list<int> AddrSpaces = AS;
|
|
}
|
|
|
|
class Aligned<int Bytes> {
|
|
int MinAlignment = Bytes;
|
|
}
|
|
|
|
class StoreHi16<SDPatternOperator op> : PatFrag <
|
|
(ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr)> {
|
|
let IsStore = 1;
|
|
}
|
|
|
|
def LoadAddress_constant : AddressSpaceList<[ AddrSpaces.Constant ]>;
|
|
def LoadAddress_global : AddressSpaceList<[ AddrSpaces.Global, AddrSpaces.Constant ]>;
|
|
def StoreAddress_global : AddressSpaceList<[ AddrSpaces.Global ]>;
|
|
|
|
def LoadAddress_flat : AddressSpaceList<[ AddrSpaces.Flat,
|
|
AddrSpaces.Global,
|
|
AddrSpaces.Constant ]>;
|
|
def StoreAddress_flat : AddressSpaceList<[ AddrSpaces.Flat, AddrSpaces.Global ]>;
|
|
|
|
def LoadAddress_private : AddressSpaceList<[ AddrSpaces.Private ]>;
|
|
def StoreAddress_private : AddressSpaceList<[ AddrSpaces.Private ]>;
|
|
|
|
def LoadAddress_local : AddressSpaceList<[ AddrSpaces.Local ]>;
|
|
def StoreAddress_local : AddressSpaceList<[ AddrSpaces.Local ]>;
|
|
|
|
def LoadAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>;
|
|
def StoreAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>;
|
|
|
|
|
|
|
|
foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
|
|
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
|
|
|
|
def load_#as : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let IsNonExtLoad = 1;
|
|
}
|
|
|
|
def extloadi8_#as : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let MemoryVT = i8;
|
|
}
|
|
|
|
def extloadi16_#as : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let MemoryVT = i16;
|
|
}
|
|
|
|
def sextloadi8_#as : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let MemoryVT = i8;
|
|
}
|
|
|
|
def sextloadi16_#as : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let MemoryVT = i16;
|
|
}
|
|
|
|
def zextloadi8_#as : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let MemoryVT = i8;
|
|
}
|
|
|
|
def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let MemoryVT = i16;
|
|
}
|
|
|
|
def atomic_load_32_#as : PatFrag<(ops node:$ptr), (atomic_load_32 node:$ptr)> {
|
|
let IsAtomic = 1;
|
|
let MemoryVT = i32;
|
|
}
|
|
|
|
def atomic_load_64_#as : PatFrag<(ops node:$ptr), (atomic_load_64 node:$ptr)> {
|
|
let IsAtomic = 1;
|
|
let MemoryVT = i64;
|
|
}
|
|
} // End let AddressSpaces
|
|
} // End foreach as
|
|
|
|
|
|
foreach as = [ "global", "flat", "local", "private", "region" ] in {
|
|
let AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
|
|
def store_#as : PatFrag<(ops node:$val, node:$ptr),
|
|
(unindexedstore node:$val, node:$ptr)> {
|
|
let IsStore = 1;
|
|
let IsTruncStore = 0;
|
|
}
|
|
|
|
// truncstore fragments.
|
|
def truncstore_#as : PatFrag<(ops node:$val, node:$ptr),
|
|
(unindexedstore node:$val, node:$ptr)> {
|
|
let IsStore = 1;
|
|
let IsTruncStore = 1;
|
|
}
|
|
|
|
// TODO: We don't really need the truncstore here. We can use
|
|
// unindexedstore with MemoryVT directly, which will save an
|
|
// unnecessary check that the memory size is less than the value type
|
|
// in the generated matcher table.
|
|
def truncstorei8_#as : PatFrag<(ops node:$val, node:$ptr),
|
|
(truncstore node:$val, node:$ptr)> {
|
|
let IsStore = 1;
|
|
let MemoryVT = i8;
|
|
}
|
|
|
|
def truncstorei16_#as : PatFrag<(ops node:$val, node:$ptr),
|
|
(truncstore node:$val, node:$ptr)> {
|
|
let IsStore = 1;
|
|
let MemoryVT = i16;
|
|
}
|
|
|
|
def store_hi16_#as : StoreHi16 <truncstorei16>;
|
|
def truncstorei8_hi16_#as : StoreHi16<truncstorei8>;
|
|
def truncstorei16_hi16_#as : StoreHi16<truncstorei16>;
|
|
|
|
defm atomic_store_#as : binary_atomic_op<atomic_store>;
|
|
|
|
} // End let AddressSpaces
|
|
} // End foreach as
|
|
|
|
|
|
multiclass ret_noret_binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
|
|
foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
|
|
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
|
|
defm "_"#as : binary_atomic_op<atomic_op, IsInt>;
|
|
|
|
let PredicateCode = [{return (SDValue(N, 0).use_empty());}] in {
|
|
defm "_"#as#"_noret" : binary_atomic_op<atomic_op, IsInt>;
|
|
}
|
|
|
|
let PredicateCode = [{return !(SDValue(N, 0).use_empty());}] in {
|
|
defm "_"#as#"_ret" : binary_atomic_op<atomic_op, IsInt>;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
defm atomic_swap : ret_noret_binary_atomic_op<atomic_swap>;
|
|
defm atomic_load_add : ret_noret_binary_atomic_op<atomic_load_add>;
|
|
defm atomic_load_and : ret_noret_binary_atomic_op<atomic_load_and>;
|
|
defm atomic_load_max : ret_noret_binary_atomic_op<atomic_load_max>;
|
|
defm atomic_load_min : ret_noret_binary_atomic_op<atomic_load_min>;
|
|
defm atomic_load_or : ret_noret_binary_atomic_op<atomic_load_or>;
|
|
defm atomic_load_sub : ret_noret_binary_atomic_op<atomic_load_sub>;
|
|
defm atomic_load_umax : ret_noret_binary_atomic_op<atomic_load_umax>;
|
|
defm atomic_load_umin : ret_noret_binary_atomic_op<atomic_load_umin>;
|
|
defm atomic_load_xor : ret_noret_binary_atomic_op<atomic_load_xor>;
|
|
defm atomic_load_fadd : ret_noret_binary_atomic_op<atomic_load_fadd, 0>;
|
|
defm AMDGPUatomic_cmp_swap : ret_noret_binary_atomic_op<AMDGPUatomic_cmp_swap>;
|
|
|
|
|
|
def load_align8_local : PatFrag <(ops node:$ptr), (load_local node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let IsNonExtLoad = 1;
|
|
let MinAlignment = 8;
|
|
}
|
|
|
|
def load_align16_local : PatFrag <(ops node:$ptr), (load_local node:$ptr)> {
|
|
let IsLoad = 1;
|
|
let IsNonExtLoad = 1;
|
|
let MinAlignment = 16;
|
|
}
|
|
|
|
def store_align8_local: PatFrag<(ops node:$val, node:$ptr),
|
|
(store_local node:$val, node:$ptr)>, Aligned<8> {
|
|
let IsStore = 1;
|
|
let IsTruncStore = 0;
|
|
}
|
|
|
|
def store_align16_local: PatFrag<(ops node:$val, node:$ptr),
|
|
(store_local node:$val, node:$ptr)>, Aligned<16> {
|
|
let IsStore = 1;
|
|
let IsTruncStore = 0;
|
|
}
|
|
|
|
let AddressSpaces = StoreAddress_local.AddrSpaces in {
|
|
defm atomic_cmp_swap_local : ternary_atomic_op<atomic_cmp_swap>;
|
|
defm atomic_cmp_swap_local_m0 : ternary_atomic_op<atomic_cmp_swap_glue>;
|
|
}
|
|
|
|
let AddressSpaces = StoreAddress_region.AddrSpaces in {
|
|
defm atomic_cmp_swap_region : ternary_atomic_op<atomic_cmp_swap>;
|
|
defm atomic_cmp_swap_region_m0 : ternary_atomic_op<atomic_cmp_swap_glue>;
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Misc Pattern Fragments
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class Constants {
|
|
int TWO_PI = 0x40c90fdb;
|
|
int PI = 0x40490fdb;
|
|
int TWO_PI_INV = 0x3e22f983;
|
|
int FP_UINT_MAX_PLUS_1 = 0x4f800000; // 1 << 32 in floating point encoding
|
|
int FP16_ONE = 0x3C00;
|
|
int FP16_NEG_ONE = 0xBC00;
|
|
int FP32_ONE = 0x3f800000;
|
|
int FP32_NEG_ONE = 0xbf800000;
|
|
int FP64_ONE = 0x3ff0000000000000;
|
|
int FP64_NEG_ONE = 0xbff0000000000000;
|
|
}
|
|
def CONST : Constants;
|
|
|
|
def FP_ZERO : PatLeaf <
|
|
(fpimm),
|
|
[{return N->getValueAPF().isZero();}]
|
|
>;
|
|
|
|
def FP_ONE : PatLeaf <
|
|
(fpimm),
|
|
[{return N->isExactlyValue(1.0);}]
|
|
>;
|
|
|
|
def FP_HALF : PatLeaf <
|
|
(fpimm),
|
|
[{return N->isExactlyValue(0.5);}]
|
|
>;
|
|
|
|
/* Generic helper patterns for intrinsics */
|
|
/* -------------------------------------- */
|
|
|
|
class POW_Common <AMDGPUInst log_ieee, AMDGPUInst exp_ieee, AMDGPUInst mul>
|
|
: AMDGPUPat <
|
|
(fpow f32:$src0, f32:$src1),
|
|
(exp_ieee (mul f32:$src1, (log_ieee f32:$src0)))
|
|
>;
|
|
|
|
/* Other helper patterns */
|
|
/* --------------------- */
|
|
|
|
/* Extract element pattern */
|
|
class Extract_Element <ValueType sub_type, ValueType vec_type, int sub_idx,
|
|
SubRegIndex sub_reg>
|
|
: AMDGPUPat<
|
|
(sub_type (extractelt vec_type:$src, sub_idx)),
|
|
(EXTRACT_SUBREG $src, sub_reg)
|
|
>;
|
|
|
|
/* Insert element pattern */
|
|
class Insert_Element <ValueType elem_type, ValueType vec_type,
|
|
int sub_idx, SubRegIndex sub_reg>
|
|
: AMDGPUPat <
|
|
(insertelt vec_type:$vec, elem_type:$elem, sub_idx),
|
|
(INSERT_SUBREG $vec, $elem, sub_reg)
|
|
>;
|
|
|
|
// XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer
|
|
// can handle COPY instructions.
|
|
// bitconvert pattern
|
|
class BitConvert <ValueType dt, ValueType st, RegisterClass rc> : AMDGPUPat <
|
|
(dt (bitconvert (st rc:$src0))),
|
|
(dt rc:$src0)
|
|
>;
|
|
|
|
// XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer
|
|
// can handle COPY instructions.
|
|
class DwordAddrPat<ValueType vt, RegisterClass rc> : AMDGPUPat <
|
|
(vt (AMDGPUdwordaddr (vt rc:$addr))),
|
|
(vt rc:$addr)
|
|
>;
|
|
|
|
// BFI_INT patterns
|
|
|
|
multiclass BFIPatterns <Instruction BFI_INT,
|
|
Instruction LoadImm32,
|
|
RegisterClass RC64> {
|
|
// Definition from ISA doc:
|
|
// (y & x) | (z & ~x)
|
|
def : AMDGPUPat <
|
|
(or (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))),
|
|
(BFI_INT $x, $y, $z)
|
|
>;
|
|
|
|
// 64-bit version
|
|
def : AMDGPUPat <
|
|
(or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))),
|
|
(REG_SEQUENCE RC64,
|
|
(BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub0)),
|
|
(i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0,
|
|
(BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1)
|
|
>;
|
|
|
|
// SHA-256 Ch function
|
|
// z ^ (x & (y ^ z))
|
|
def : AMDGPUPat <
|
|
(xor i32:$z, (and i32:$x, (xor i32:$y, i32:$z))),
|
|
(BFI_INT $x, $y, $z)
|
|
>;
|
|
|
|
// 64-bit version
|
|
def : AMDGPUPat <
|
|
(xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))),
|
|
(REG_SEQUENCE RC64,
|
|
(BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub0)),
|
|
(i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0,
|
|
(BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1)
|
|
>;
|
|
|
|
def : AMDGPUPat <
|
|
(fcopysign f32:$src0, f32:$src1),
|
|
(BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, $src1)
|
|
>;
|
|
|
|
def : AMDGPUPat <
|
|
(f32 (fcopysign f32:$src0, f64:$src1)),
|
|
(BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0,
|
|
(i32 (EXTRACT_SUBREG RC64:$src1, sub1)))
|
|
>;
|
|
|
|
def : AMDGPUPat <
|
|
(f64 (fcopysign f64:$src0, f64:$src1)),
|
|
(REG_SEQUENCE RC64,
|
|
(i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
|
|
(BFI_INT (LoadImm32 (i32 0x7fffffff)),
|
|
(i32 (EXTRACT_SUBREG RC64:$src0, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$src1, sub1))), sub1)
|
|
>;
|
|
|
|
def : AMDGPUPat <
|
|
(f64 (fcopysign f64:$src0, f32:$src1)),
|
|
(REG_SEQUENCE RC64,
|
|
(i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
|
|
(BFI_INT (LoadImm32 (i32 0x7fffffff)),
|
|
(i32 (EXTRACT_SUBREG RC64:$src0, sub1)),
|
|
$src1), sub1)
|
|
>;
|
|
}
|
|
|
|
// SHA-256 Ma patterns
|
|
|
|
// ((x & z) | (y & (x | z))) -> BFI_INT (XOR x, y), z, y
|
|
multiclass SHA256MaPattern <Instruction BFI_INT, Instruction XOR, RegisterClass RC64> {
|
|
def : AMDGPUPat <
|
|
(or (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))),
|
|
(BFI_INT (XOR i32:$x, i32:$y), i32:$z, i32:$y)
|
|
>;
|
|
|
|
def : AMDGPUPat <
|
|
(or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))),
|
|
(REG_SEQUENCE RC64,
|
|
(BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub0))),
|
|
(i32 (EXTRACT_SUBREG RC64:$z, sub0)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub0))), sub0,
|
|
(BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub1))),
|
|
(i32 (EXTRACT_SUBREG RC64:$z, sub1)),
|
|
(i32 (EXTRACT_SUBREG RC64:$y, sub1))), sub1)
|
|
>;
|
|
}
|
|
|
|
// Bitfield extract patterns
|
|
|
|
def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{
|
|
return isMask_32(Imm);
|
|
}]>;
|
|
|
|
def IMMPopCount : SDNodeXForm<imm, [{
|
|
return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()), SDLoc(N),
|
|
MVT::i32);
|
|
}]>;
|
|
|
|
multiclass BFEPattern <Instruction UBFE, Instruction SBFE, Instruction MOV> {
|
|
def : AMDGPUPat <
|
|
(i32 (and (i32 (srl i32:$src, i32:$rshift)), IMMZeroBasedBitfieldMask:$mask)),
|
|
(UBFE $src, $rshift, (MOV (i32 (IMMPopCount $mask))))
|
|
>;
|
|
|
|
// x & ((1 << y) - 1)
|
|
def : AMDGPUPat <
|
|
(and i32:$src, (add_oneuse (shl_oneuse 1, i32:$width), -1)),
|
|
(UBFE $src, (MOV (i32 0)), $width)
|
|
>;
|
|
|
|
// x & ~(-1 << y)
|
|
def : AMDGPUPat <
|
|
(and i32:$src, (xor_oneuse (shl_oneuse -1, i32:$width), -1)),
|
|
(UBFE $src, (MOV (i32 0)), $width)
|
|
>;
|
|
|
|
// x & (-1 >> (bitwidth - y))
|
|
def : AMDGPUPat <
|
|
(and i32:$src, (srl_oneuse -1, (sub 32, i32:$width))),
|
|
(UBFE $src, (MOV (i32 0)), $width)
|
|
>;
|
|
|
|
// x << (bitwidth - y) >> (bitwidth - y)
|
|
def : AMDGPUPat <
|
|
(srl (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)),
|
|
(UBFE $src, (MOV (i32 0)), $width)
|
|
>;
|
|
|
|
def : AMDGPUPat <
|
|
(sra (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)),
|
|
(SBFE $src, (MOV (i32 0)), $width)
|
|
>;
|
|
}
|
|
|
|
// fshr pattern
|
|
class FSHRPattern <Instruction BIT_ALIGN> : AMDGPUPat <
|
|
(fshr i32:$src0, i32:$src1, i32:$src2),
|
|
(BIT_ALIGN $src0, $src1, $src2)
|
|
>;
|
|
|
|
// rotr pattern
|
|
class ROTRPattern <Instruction BIT_ALIGN> : AMDGPUPat <
|
|
(rotr i32:$src0, i32:$src1),
|
|
(BIT_ALIGN $src0, $src0, $src1)
|
|
>;
|
|
|
|
// Special conversion patterns
|
|
|
|
def cvt_rpi_i32_f32 : PatFrag <
|
|
(ops node:$src),
|
|
(fp_to_sint (ffloor (fadd $src, FP_HALF))),
|
|
[{ (void) N; return TM.Options.NoNaNsFPMath; }]
|
|
>;
|
|
|
|
def cvt_flr_i32_f32 : PatFrag <
|
|
(ops node:$src),
|
|
(fp_to_sint (ffloor $src)),
|
|
[{ (void)N; return TM.Options.NoNaNsFPMath; }]
|
|
>;
|
|
|
|
let AddedComplexity = 2 in {
|
|
class IMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <
|
|
(add (AMDGPUmul_i24 i32:$src0, i32:$src1), i32:$src2),
|
|
!if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),
|
|
(Inst $src0, $src1, $src2))
|
|
>;
|
|
|
|
class UMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <
|
|
(add (AMDGPUmul_u24 i32:$src0, i32:$src1), i32:$src2),
|
|
!if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),
|
|
(Inst $src0, $src1, $src2))
|
|
>;
|
|
} // AddedComplexity.
|
|
|
|
class RcpPat<Instruction RcpInst, ValueType vt> : AMDGPUPat <
|
|
(fdiv FP_ONE, vt:$src),
|
|
(RcpInst $src)
|
|
>;
|
|
|
|
class RsqPat<Instruction RsqInst, ValueType vt> : AMDGPUPat <
|
|
(AMDGPUrcp (fsqrt vt:$src)),
|
|
(RsqInst $src)
|
|
>;
|
|
|
|
// Instructions which select to the same v_min_f*
|
|
def fminnum_like : PatFrags<(ops node:$src0, node:$src1),
|
|
[(fminnum_ieee node:$src0, node:$src1),
|
|
(fminnum node:$src0, node:$src1)]
|
|
>;
|
|
|
|
// Instructions which select to the same v_max_f*
|
|
def fmaxnum_like : PatFrags<(ops node:$src0, node:$src1),
|
|
[(fmaxnum_ieee node:$src0, node:$src1),
|
|
(fmaxnum node:$src0, node:$src1)]
|
|
>;
|
|
|
|
def fminnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
|
|
[(fminnum_ieee_oneuse node:$src0, node:$src1),
|
|
(fminnum_oneuse node:$src0, node:$src1)]
|
|
>;
|
|
|
|
def fmaxnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
|
|
[(fmaxnum_ieee_oneuse node:$src0, node:$src1),
|
|
(fmaxnum_oneuse node:$src0, node:$src1)]
|
|
>;
|
|
|
|
def any_fmad : PatFrags<(ops node:$src0, node:$src1, node:$src2),
|
|
[(fmad node:$src0, node:$src1, node:$src2),
|
|
(AMDGPUfmad_ftz node:$src0, node:$src1, node:$src2)]
|
|
>;
|