diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index dd59520cafa4..73311c96b132 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -201,14 +201,13 @@ namespace Intrinsic { Kind == TruncArgument || Kind == SameVecWidthArgument || Kind == VecElementArgument || Kind == Subdivide2Argument || Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt); - return ArgumentInfo >> 3; + // Overload index is packed into lower 5 bits. + return ArgumentInfo & 0x1f; } ArgKind getArgumentKind() const { - assert(Kind == Argument || Kind == ExtendArgument || - Kind == TruncArgument || Kind == SameVecWidthArgument || - Kind == VecElementArgument || Kind == Subdivide2Argument || - Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt); - return (ArgKind)(ArgumentInfo & 7); + // Argument kind is packed into upper 3 bits. + assert(Kind == Argument); + return (ArgKind)((ArgumentInfo >> 5) & 0x7); } // VecOfAnyPtrsToElt uses both an overloaded argument (for address space) @@ -217,12 +216,14 @@ namespace Intrinsic { assert(Kind == VecOfAnyPtrsToElt); return ArgumentInfo >> 16; } + // OneNthEltsVecArguments uses both a divisor N and a reference argument for // the full-width vector to match unsigned getVectorDivisor() const { assert(Kind == OneNthEltsVecArgument); return ArgumentInfo >> 16; } + unsigned getRefArgNumber() const { assert(Kind == VecOfAnyPtrsToElt || Kind == OneNthEltsVecArgument); return ArgumentInfo & 0xFFFF; diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index e48c82ce89b5..abab8c3b5357 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -243,52 +243,40 @@ def ArgKind { int MatchType = 7; } -// Encode placeholder. -// [15:8] is the ID used how to resolve ArgCode. - -// (ACIdx << 3) | ArgCode -class EncAnyType { +// Placeholder to encode the overload index of the current type. We encode bit +// 8 = 1 to indicate that this entry needs to be patched up with the overload +// index (to prevent conflict with any valid not-to-be-patched IIT enccoding +// byte, whose value will be <= 255). The ArgKind itself is in the lower bits. +// Note that this is just a transient representation till its gets processed +// by `DoPatchOverloadIndex` below, so this is *not* the encoding of the +// final type signature. +class OverloadIndexPlaceholder { int ID = 0x100; - int ret = !or(ID, ArgCode); + int ret = !or(ID, ArgKindVal); } -// (Num << 3) | AK.MatchType -class EncMatchType { - int ID = 0x200; - int ret = !or(ID, Num); +// This class defines how the overload index and the arg kind are actually +// packed into a single byte for the final IIT table encoding. Overload index is +// packed in low 5 bits, argument kind is packed in upper 3 bits. This enables +// us to use the same packing for llvm_any* types, which use a argument kind +// and for partially dependent types like `LLVMVectorOfAnyPointersToElt` which +// do not use the argument kind and expect the overload index in the lower bits. +class PackOverloadIndex { + assert !lt(OverloadIndex, 32), "Cannot support more than 32 overload types"; + assert !lt(ArgKindVal, 8), "Cannot support more than 8 argument kinds"; + int ret = !or(!shl(ArgKindVal, 5), OverloadIndex); } -// (Num << 3) | ArgCodes[Num] -class EncSameWidth { - int ID = 0x300; - int ret = !or(ID, Num); -} - -// ACIdx -class EncNextArgA { - int ID = 0x400; - int ret = !or(ID, dummy); -} - -// Num -class EncNextArgN { - int ID = 0x500; - int ret = !or(ID, Num); -} - -class ResolveArgCode< - list ArgCodes, - int ACIdx, - int ax> { - int ah = !and(ax, 0xFF00); - int al = !and(ax, 0x00FF); +// This class handles the actual patching of the overload index into a component +// value `Sig` in the type signature. If the value is > 255, it's a placeholder +// value generated by OverloadIndexPlaceholder and patching is needed, else the +// value is left unchanged. +class PatchOverloadIndex { + int ArgKindVal = !and(Sig, 0x7); int ret = !cond( - !eq(ah, EncAnyType<>.ID) : !or(!shl(ACIdx, 3), al), - !eq(ah, EncMatchType<>.ID) : !or(!shl(al, 3), ArgKind.MatchType), - !eq(ah, EncSameWidth<>.ID) : !or(!shl(al, 3), ArgCodes[al]), - !eq(ah, EncNextArgA<>.ID) : ACIdx, - !eq(ah, EncNextArgN<>.ID) : al, - true : al); + // If the value is > 255, it indicates that patching is needed. + !gt(Sig, 255) : PackOverloadIndex.ret, + true: Sig); } //===----------------------------------------------------------------------===// @@ -384,10 +372,6 @@ defvar IIT_all_VectorTypes = !filter(iit, IIT_all, class LLVMType { ValueType VT = vt; - int isAny = vt.isOverloaded; - - int ArgCode = ?; - int Number = ?; list IITs = !filter(iit, IIT_all_FixedTypes, !not(!empty(!filter(iit_vt, iit.VTs, @@ -413,7 +397,7 @@ class LLVMType { } class LLVMAnyType : LLVMType { - let ArgCode = !cond( + int ArgCode = !cond( !eq(vt, Any) : ArgKind.Any, !eq(vt, iAny) : ArgKind.AnyInteger, !eq(vt, fAny) : ArgKind.AnyFloat, @@ -422,10 +406,10 @@ class LLVMAnyType : LLVMType { ); let Sig = [ IIT_ARG.Number, - EncAnyType.ret, + OverloadIndexPlaceholder .ret, ]; - assert isAny, "LLVMAnyType.VT should have isOverloaded"; + assert VT.isOverloaded, "LLVMAnyType.VT should have isOverloaded"; } class LLVMQualPointerType @@ -442,15 +426,54 @@ class LLVMQualPointerType ]); } -class LLVMAnyPointerType : LLVMAnyType { - assert isAny, "pAny should have isOverloaded"; +// Note: CodeGenIntrinsics.cpp seems to check this class to check pointers. +class LLVMAnyPointerType : LLVMAnyType; + +// Dependent types: These are types that depend on another LLVMAnyType overload +// type. There are 2 subclasses of dependent types: +// 1. Fully dependent types: dependent type can be completely derived from +// another overload type. +// 2. Partially dependent types: dependent type is constrained by another +// overload type, but cannot be fully derived from it. Such types get +// assigned an overload index and are a part of the overloaded types for an +// intrinsics. +class LLVMDependentType : LLVMType { + // Overload index of the overload type that this dependent type is dependent + // on. + int OverloadIndex = oidx; } -// Match the type of another intrinsic parameter. Number is an index into the -// list of overloaded types for the intrinsic (i.e, the overload index), -// excluding all the fixed types and all fully dependent types (i.e., all -// sub-classes of LLVMMatchType, except LLVMMatchTypeNextArg). The Number -// value must refer to a previously listed type. For example: +class LLVMFullyDependentType + : LLVMDependentType { + // For fully dependent overload types, the type signature is just the IIT code + // followed by the overload index of the overload type it depends on. + let Sig = [ + IIT_Info.Number, + PackOverloadIndex.ret + ]; +} + +class LLVMPartiallyDependentType + : LLVMDependentType { + // For partially dependent type, the type signature is the IIT code, followed + // by this type's oveerload index, followed by the overload index of the + // overload type its depends on. + let Sig = [ + IIT_Info.Number, + // This types overload index, arg kind ignored. + OverloadIndexPlaceholder <0>.ret, + // Overload index of the reference overload type, arg kind ignored. + PackOverloadIndex.ret, + ]; +} + +// ---------------------------------------------------------------------------- +// Various sub-classes of fully dependent types. + +// Match the type of another intrinsic parameter. `oidx` is the overload index +// of the overloaded type that this type is dependent on. Overload types are +// either LLVMAnyType or LLVMPartiallyDependentType. The `oidx` value must refer +// to a previously listed type. For example: // // Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_anyfloat_ty, LLVMMatchType<0>]> // @@ -467,54 +490,37 @@ class LLVMAnyPointerType : LLVMAnyType { // // LLVMMatchType<0> therefore will match the return type, and // LLVMMatchType<2> will match the 3rd argument. - -class LLVMMatchType - : LLVMType{ - let Number = num; - let Sig = [ - IIT_Info.Number, - EncMatchType.ret, - ]; -} - -class LLVMMatchTypeNextArg - : LLVMMatchType { - let Sig = [ - IIT_Info.Number, - EncNextArgA<>.ret, - EncNextArgN.ret, - ]; -} +class LLVMMatchType : LLVMFullyDependentType; // Match the type of another intrinsic parameter that is expected to be based on // an integral type (i.e. either iN or ), but change the scalar size to // be twice as wide or half as wide as the other type. This is only useful when // the intrinsic is overloaded, so the matched type should be declared as iAny. -class LLVMExtendedType : LLVMMatchType; -class LLVMTruncatedType : LLVMMatchType; +class LLVMExtendedType : LLVMFullyDependentType; +class LLVMTruncatedType : LLVMFullyDependentType; // Match the scalar/vector of another intrinsic parameter but with a different // element type. Either both are scalars or both are vectors with the same // number of elements. -class LLVMScalarOrSameVectorWidth - : LLVMMatchType { +class LLVMScalarOrSameVectorWidth + : LLVMFullyDependentType { let Sig = !listconcat([ IIT_SAME_VEC_WIDTH_ARG.Number, - EncSameWidth.ret, + // Overload index of the reference overload type, arg kind ignored. + PackOverloadIndex.ret, ], elty.Sig); } -class LLVMVectorOfAnyPointersToElt - : LLVMMatchTypeNextArg; -class LLVMVectorElementType : LLVMMatchType; +class LLVMVectorElementType + : LLVMFullyDependentType; // Match the type of another intrinsic parameter that is expected to be a // vector type, but change the element count to be 1/n of it. -class LLVMOneNthElementsVectorType - : LLVMMatchType { +class LLVMOneNthElementsVectorType + : LLVMFullyDependentType { let Sig = [ IIT_ONE_NTH_ELTS_VEC_ARG.Number, - EncNextArgN.ret, + PackOverloadIndex.ret, n, ]; } @@ -522,15 +528,22 @@ class LLVMOneNthElementsVectorType // Match the type of another intrinsic parameter that is expected to be a // vector type (i.e. ) but with each element subdivided to // form a vector with more elements that are smaller than the original. -class LLVMSubdivide2VectorType - : LLVMMatchType; -class LLVMSubdivide4VectorType - : LLVMMatchType; +class LLVMSubdivide2VectorType + : LLVMFullyDependentType; +class LLVMSubdivide4VectorType + : LLVMFullyDependentType; // Match the element count and bit width of another intrinsic parameter, but // change the element type to an integer. -class LLVMVectorOfBitcastsToInt - : LLVMMatchType; +class LLVMVectorOfBitcastsToInt + : LLVMFullyDependentType; + +// ---------------------------------------------------------------------------- +// Various sub-classes of partially dependent types. + +class LLVMVectorOfAnyPointersToElt + : LLVMPartiallyDependentType; + def llvm_void_ty : LLVMType; @@ -538,6 +551,7 @@ def llvm_any_ty : LLVMAnyType; def llvm_anyint_ty : LLVMAnyType; def llvm_anyfloat_ty : LLVMAnyType; def llvm_anyvector_ty : LLVMAnyType; +def llvm_anyptr_ty : LLVMAnyPointerType; // ptr addrspace(N) def llvm_i1_ty : LLVMType; def llvm_i8_ty : LLVMType; @@ -553,7 +567,6 @@ def llvm_f80_ty : LLVMType; def llvm_f128_ty : LLVMType; def llvm_ppcf128_ty : LLVMType; def llvm_ptr_ty : LLVMQualPointerType<0>; // ptr -def llvm_anyptr_ty : LLVMAnyPointerType; // ptr addrspace(N) def llvm_empty_ty : LLVMType; // { } def llvm_metadata_ty : LLVMType; // !{...} def llvm_token_ty : LLVMType; // token @@ -657,43 +670,49 @@ def llvm_exnref_ty : LLVMType; //===----------------------------------------------------------------------===// -class MakeIdx Set> { - list IdxsR = !foreach(i, !range(Set), - !if(Set[i], - !foldl(0, !range(0, i), m, j, !add(m, Set[j])), +// Computes the overload index for overloaded types used by an intrinsic. +// Input `IsOverloadedType` is a list of booleans, one for each type in the +// intrinsic's type signature. A 0 value indicate that that type is not an +// overloaded type, and 1 indicates that its an overloaded type. +// +// Assigns overload index by essentially computing a prefix sum on this list. +// The output list `ret` has value ? for non-overload types and the overload +// index for overloaded types. +class AssignOverloadIndex IsOverloadedType> { + list PrefixSum = !foreach(i, !range(IsOverloadedType), + !if(IsOverloadedType[i], + !foldl(0, !range(0, i), m, j, !add(m, IsOverloadedType[j])), -1)); - list Idxs = !foreach(a, IdxsR, !if(!ge(a, 0), a, ?)); + // Assign ? if the entry was -1, else its the expected prefix sum. + list ret = !foreach(a, PrefixSum, !if(!ge(a, 0), a, ?)); } -class TypeInfoGen< - list RetTypes, - list ParamTypes> { +class TypeInfoGen RetTypes, list ParamTypes> { list AllTypes = !listconcat(RetTypes, ParamTypes); - // ArgCodes for NextArg -- isAny or MatchTypeNextArg - list ACIdxs = MakeIdx< + // Assign overload index for all overloaded types + // (LLVMAnyType or LLVMPartiallyDependentType). + list OverloadIdxs = AssignOverloadIndex< !foreach(ty, AllTypes, - !or(ty.isAny, !isa(ty)))>.Idxs; + !or(!isa(ty), !isa(ty)))>.ret; - // ArgCodes (only for isAny or MatchTypeNextArg) - list ACTys = !filter(ty, AllTypes, - !or(ty.isAny, !isa(ty))); + // List of all overloaded types, in their overload-index order. + list OverloadTypes = !filter(ty, AllTypes, + !or(!isa(ty), !isa(ty))); - list ArgCodes = !foreach(ty, ACTys, ty.ArgCode); - - bit isOverloaded = !not(!empty(ACTys)); + bit isOverloaded = !not(!empty(OverloadTypes)); // Validate that the overload index referenced by dependent types always - // references an "isAny" type. + // references an LLVMAnyType type. list InvalidOverload = !foreach(ty, AllTypes, // an entry in the list will be 1 if its a dependent type and it // references another overload type that is not `Any`. !and( - !or(!isa(ty), !isa(ty)), - !if(!ge(ty.Number, !size(ACTys)), + !isa(ty), + !if(!ge(!cast(ty).OverloadIndex, !size(OverloadTypes)), 1, - !not(ACTys[ty.Number].isAny) + !not(!isa(OverloadTypes[!cast(ty).OverloadIndex])) ) ) ); @@ -701,7 +720,9 @@ class TypeInfoGen< "dependent types must reference an overload index of an \'llvm_any\' type"; list Types = !foreach(ty, AllTypes, - !if(!isa(ty), ACTys[ty.Number], ty)); + !if(!isa(ty), + OverloadTypes[!cast(ty).OverloadIndex], + ty)); list TypeSig = !listflatten(!listconcat( [!cond( @@ -710,10 +731,7 @@ class TypeInfoGen< true: [IIT_STRUCT.Number, !sub(!size(RetTypes), 2)])], !foreach(i, !range(AllTypes), !foreach(a, AllTypes[i].Sig, - ResolveArgCode< - ArgCodes, - ACIdxs[i], - a>.ret)))); + PatchOverloadIndex.ret)))); } //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 7b4b51a112e1..f576972183ec 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -928,7 +928,7 @@ class arglistmatchshift arglist, int shift> { list ret = !foreach(arg, arglist, !if(!isa(arg.Type), - AMDGPUArg(arg.Type).Number, shift)>, + AMDGPUArg(arg.Type).OverloadIndex, shift)>, arg.Name>, arg)); } @@ -942,7 +942,7 @@ class arglistconcat> arglists, int shift = 0> { lhs, arglistmatchshift.ret)); + !add(a, !isa(b.Type))))>.ret)); } // Represent texture/image types / dimensionality. @@ -1074,7 +1074,7 @@ class AMDGPUDimProfile(b))); list AddrArgs = arglistconcat<[ExtraAddrArgs, @@ -1087,12 +1087,12 @@ class AMDGPUDimProfile AddrTypes = !foreach(arg, AddrArgs, arg.Type); list AddrDefaultArgs = !foreach(arg, AddrArgs, - AMDGPUArg(arg.Type)), + AMDGPUArg(arg.Type), !isa(arg.Type)), !if(IsSample, llvm_float_ty, llvm_i32_ty), arg.Type), arg.Name>); list AddrA16Args = !foreach(arg, AddrArgs, - AMDGPUArg(arg.Type)), + AMDGPUArg(arg.Type), !isa(arg.Type)), !if(IsSample, llvm_half_ty, llvm_i16_ty), arg.Type), arg.Name>); } diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index 03159cf9398c..c8e698882ed2 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -1858,9 +1858,9 @@ class ImageDimIntrinsicInfo { bits<8> CachePolicyIndex = DimEval.CachePolicyArgIndex; bits<8> BiasTyArg = !add(I.P.NumRetAndDataAnyTypes, - !if(!eq(NumOffsetArgs, 0), 0, I.P.ExtraAddrArgs[0].Type.isAny)); + !if(!eq(NumOffsetArgs, 0), 0, !isa(I.P.ExtraAddrArgs[0].Type))); bits<8> GradientTyArg = !add(I.P.NumRetAndDataAnyTypes, - !foldl(0, I.P.ExtraAddrArgs, cnt, arg, !add(cnt, arg.Type.isAny))); + !foldl(0, I.P.ExtraAddrArgs, cnt, arg, !add(cnt, !isa(arg.Type)))); bits<8> CoordTyArg = !add(GradientTyArg, !if(I.P.Gradients, 1, 0)); } diff --git a/llvm/test/TableGen/intrinsic-struct.td b/llvm/test/TableGen/intrinsic-struct.td index 032cdc10e74e..8ecdcc854adc 100644 --- a/llvm/test/TableGen/intrinsic-struct.td +++ b/llvm/test/TableGen/intrinsic-struct.td @@ -17,42 +17,45 @@ include "llvm/IR/Intrinsics.td" // Make sure the encoding table is correctly generated. // CHECK-IMPL: IIT_LongEncodingTable // CHECK-IMPL-NEXT: 21, 255 -// CHECK-IMPL-SAME: 15, 1, 15, 9, 15, 17, 15, 25, 15, 33, 15, 41, 15, 49, 15, 57, 15, 65, 15, 73, 15, 81, +// There should be list of 257 '4's (IIT code for i32) followed by 0. We just +// match the suffix that ends with 0 (IIT_Done). +// CHECK-IMPL-SAME: 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0 +// CHECK-IMPL-NEXT: 15, 32, 0 // CHECK-IMPL-NEXT: 21, 0 -// CHECK-IMPL-SAME: 15, 1, 15, 9, 0 +// CHECK-IMPL-SAME: 15, 32, 15, 33, 0 // CHECK-IMPL-NEXT: 21, 7 -// CHECK-IMPL-SAME: 15, 1, 15, 9, 15, 17, 15, 25, 15, 33, 15, 41, 15, 49, 15, 57, 15, 65, 0 +// CHECK-IMPL-SAME: 15, 32, 15, 33, 15, 34, 15, 35, 15, 36, 15, 37, 15, 38, 15, 39, 15, 40, 0, // CHECK-IMPL-NEXT: 21, 8 -// CHECK-IMPL-SAME: 15, 1, 15, 9, 15, 17, 15, 25, 15, 33, 15, 41, 15, 49, 15, 57, 15, 65, 15, 73, 0 +// CHECK-IMPL-SAME: 15, 32, 15, 33, 15, 34, 15, 35, 15, 36, 15, 37, 15, 38, 15, 39, 15, 40, 15, 41, 0, def int_returns_a0_results : Intrinsic< [], - [], [], "llvm.returns.a0.results">; + [], []>; def int_returns_b1_results : Intrinsic< [llvm_anyint_ty], - [], [], "llvm.returns.b1.results">; + [], []>; def int_returns_c2_results : Intrinsic< !listsplat(llvm_anyint_ty, 2), - [], [], "llvm.returns.c2.results">; + [], []>; def int_returns_d9_results : Intrinsic< !listsplat(llvm_anyint_ty, 9), - [], [], "llvm.returns.d9.results">; + [], []>; def int_returns_e10_results : Intrinsic< !listsplat(llvm_anyint_ty, 10), - [], [], "llvm.returns.e10.results">; + [], []>; def int_returns_f257_results : Intrinsic< - !listsplat(llvm_anyint_ty, 257), - [], [], "llvm.returns.f257.results">; + !listsplat(llvm_i32_ty, 257), + [], []>; #ifdef ENABLE_ERROR // CHECK-ERROR: error: intrinsics can only return upto 257 values, 'int_returns_g258_results' returns 258 values // CHECK-ERROR-NEXT: def int_returns_g258_results : Intrinsic< def int_returns_g258_results : Intrinsic< - !listsplat(llvm_anyint_ty, 258), - [], [], "llvm.returns.g258.results">; + !listsplat(llvm_i32_ty, 258), + [], []>; #endif