From eaf04be3417c595ded2dd67180c8101425a281cc Mon Sep 17 00:00:00 2001 From: Viktoria Maximova Date: Thu, 19 Mar 2026 11:34:57 +0100 Subject: [PATCH] [SPIR-V] Complete SPV_INTEL_16bit_atomics extension support (#184312) Add missing capabilities to finalize SPV_INTEL_16bit_atomics extension: - AtomicInt16CompareExchangeINTEL (6260): for i16 load/store/exchange/cmpxchg - Int16AtomicsINTEL (6261): for i16 arithmetic atomics (add, sub, min, max, etc.) - AtomicBFloat16LoadStoreINTEL (6262): for bfloat16 load/store/exchange This completes the implementation started in 6ef3218. Specification: https://github.com/intel/llvm/blob/sycl/sycl/doc/design/spirv-extensions/SPV_INTEL_16bit_atomics.asciidoc --- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 61 ++++++++++++++----- .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 9 ++- .../atomic_bfloat16_load_store_xchg.ll | 42 +++++++++++++ .../atomic_int16_arithmetic.ll | 44 +++++++++++++ .../atomic_int16_load_store_xchg_cmpxchg.ll | 44 +++++++++++++ .../atomicrmw_faddfsub_bfloat16.ll | 7 ++- .../atomicrmw_fminfmax_bfloat16.ll | 5 +- 7 files changed, 190 insertions(+), 22 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_bfloat16_load_store_xchg.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_arithmetic.ll create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_load_store_xchg_cmpxchg.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 182a002ecbc1..3fb6810db4e9 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -1421,7 +1421,8 @@ void addInstrRequirements(const MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI, const SPIRVSubtarget &ST) { SPIRV::RequirementHandler &Reqs = MAI.Reqs; - switch (MI.getOpcode()) { + unsigned Op = MI.getOpcode(); + switch (Op) { case SPIRV::OpMemoryModel: { int64_t Addr = MI.getOperand(0).getImm(); Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand, @@ -1628,7 +1629,7 @@ void addInstrRequirements(const MachineInstr &MI, case SPIRV::OpAtomicXor: { const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); const MachineInstr *InstrPtr = &MI; - if (MI.getOpcode() == SPIRV::OpAtomicStore) { + if (Op == SPIRV::OpAtomicStore) { assert(MI.getOperand(3).isReg()); InstrPtr = MRI.getVRegDef(MI.getOperand(3).getReg()); assert(InstrPtr && "Unexpected type instruction for OpAtomicStore"); @@ -1636,10 +1637,44 @@ void addInstrRequirements(const MachineInstr &MI, assert(InstrPtr->getOperand(1).isReg() && "Unexpected operand in atomic"); Register TypeReg = InstrPtr->getOperand(1).getReg(); SPIRVTypeInst TypeDef = MRI.getVRegDef(TypeReg); + if (TypeDef->getOpcode() == SPIRV::OpTypeInt) { unsigned BitWidth = TypeDef->getOperand(1).getImm(); if (BitWidth == 64) Reqs.addCapability(SPIRV::Capability::Int64Atomics); + else if (BitWidth == 16) { + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics)) + report_fatal_error( + "16-bit integer atomic operations require the following SPIR-V " + "extension: SPV_INTEL_16bit_atomics", + false); + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics); + switch (Op) { + case SPIRV::OpAtomicLoad: + case SPIRV::OpAtomicStore: + case SPIRV::OpAtomicExchange: + case SPIRV::OpAtomicCompareExchange: + case SPIRV::OpAtomicCompareExchangeWeak: + Reqs.addCapability( + SPIRV::Capability::AtomicInt16CompareExchangeINTEL); + break; + default: + Reqs.addCapability(SPIRV::Capability::Int16AtomicsINTEL); + break; + } + } + } else if (isBFloat16Type(TypeDef)) { + if (is_contained({SPIRV::OpAtomicLoad, SPIRV::OpAtomicStore, + SPIRV::OpAtomicExchange}, + Op)) { + if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics)) + report_fatal_error( + "The atomic bfloat16 instruction requires the following SPIR-V " + "extension: SPV_INTEL_16bit_atomics", + false); + Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics); + Reqs.addCapability(SPIRV::Capability::AtomicBFloat16LoadStoreINTEL); + } } break; } @@ -2002,8 +2037,7 @@ void addInstrRequirements(const MachineInstr &MI, {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4}, {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}}; - const auto OpCode = MI.getOpcode(); - const unsigned LayoutNum = LayoutToInstMap[OpCode]; + const unsigned LayoutNum = LayoutToInstMap[Op]; Register RegLayout = MI.getOperand(LayoutNum).getReg(); const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout); @@ -2021,12 +2055,12 @@ void addInstrRequirements(const MachineInstr &MI, } // Nothing to do. - if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR || - OpCode == SPIRV::OpCooperativeMatrixStoreKHR) + if (Op == SPIRV::OpCooperativeMatrixLoadKHR || + Op == SPIRV::OpCooperativeMatrixStoreKHR) break; std::string InstName; - switch (OpCode) { + switch (Op) { case SPIRV::OpCooperativeMatrixPrefetchINTEL: InstName = "OpCooperativeMatrixPrefetchINTEL"; break; @@ -2045,7 +2079,7 @@ void addInstrRequirements(const MachineInstr &MI, report_fatal_error(ErrorMsg.c_str(), false); } Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix); - if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) { + if (Op == SPIRV::OpCooperativeMatrixPrefetchINTEL) { Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL); break; } @@ -2090,17 +2124,17 @@ void addInstrRequirements(const MachineInstr &MI, SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry(); SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr; SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg()); - if (MI.getOpcode() == SPIRV::OpConvertHandleToImageINTEL && + if (Op == SPIRV::OpConvertHandleToImageINTEL && TyDef->getOpcode() != SPIRV::OpTypeImage) { report_fatal_error("Incorrect return type for the instruction " "OpConvertHandleToImageINTEL", false); - } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSamplerINTEL && + } else if (Op == SPIRV::OpConvertHandleToSamplerINTEL && TyDef->getOpcode() != SPIRV::OpTypeSampler) { report_fatal_error("Incorrect return type for the instruction " "OpConvertHandleToSamplerINTEL", false); - } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSampledImageINTEL && + } else if (Op == SPIRV::OpConvertHandleToSampledImageINTEL && TyDef->getOpcode() != SPIRV::OpTypeSampledImage) { report_fatal_error("Incorrect return type for the instruction " "OpConvertHandleToSampledImageINTEL", @@ -2133,12 +2167,11 @@ void addInstrRequirements(const MachineInstr &MI, Reqs.addExtension(SPIRV::Extension::SPV_INTEL_2d_block_io); Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockIOINTEL); - const auto OpCode = MI.getOpcode(); - if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) { + if (Op == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) { Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransposeINTEL); break; } - if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) { + if (Op == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) { Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransformINTEL); break; } diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index f1d115b424c9..cf8ed17eaef5 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -518,6 +518,7 @@ defm FragmentMaskAMD : CapabilityOperand<5010, 0, 0, [], [Shader]>; defm StencilExportEXT : CapabilityOperand<5013, 0, 0, [], [Shader]>; defm ImageReadWriteLodAMD : CapabilityOperand<5015, 0, 0, [], [Shader]>; defm ShaderClockKHR : CapabilityOperand<5055, 0, 0, [SPV_KHR_shader_clock], []>; +defm BFloat16TypeKHR : CapabilityOperand<5116, 0, 0, [SPV_KHR_bfloat16], []>; defm SampleMaskOverrideCoverageNV : CapabilityOperand<5249, 0, 0, [], [SampleRateShading]>; defm GeometryShaderPassthroughNV : CapabilityOperand<5251, 0, 0, [], [Geometry]>; defm ShaderViewportIndexLayerEXT : CapabilityOperand<5254, 0, 0, [], [MultiViewport]>; @@ -576,12 +577,15 @@ defm AtomicFloat32AddEXT : CapabilityOperand<6033, 0, 0, [SPV_EXT_shader_atomic_ defm AtomicFloat64AddEXT : CapabilityOperand<6034, 0, 0, [SPV_EXT_shader_atomic_float_add], []>; defm FmaKHR : CapabilityOperand<6030, 0, 0, [SPV_KHR_fma], []>; defm AtomicFloat16AddEXT : CapabilityOperand<6095, 0, 0, [SPV_EXT_shader_atomic_float16_add], []>; -defm AtomicBFloat16AddINTEL : CapabilityOperand<6255, 0, 0, [SPV_INTEL_16bit_atomics], []>; +defm AtomicBFloat16AddINTEL : CapabilityOperand<6255, 0, 0, [SPV_INTEL_16bit_atomics], [BFloat16TypeKHR]>; defm AtomicFloat16MinMaxEXT : CapabilityOperand<5616, 0, 0, [SPV_EXT_shader_atomic_float_min_max], []>; defm AtomicFloat32MinMaxEXT : CapabilityOperand<5612, 0, 0, [SPV_EXT_shader_atomic_float_min_max], []>; defm AtomicFloat64MinMaxEXT : CapabilityOperand<5613, 0, 0, [SPV_EXT_shader_atomic_float_min_max], []>; -defm AtomicBFloat16MinMaxINTEL : CapabilityOperand<6256, 0, 0, [SPV_INTEL_16bit_atomics], []>; +defm AtomicBFloat16MinMaxINTEL : CapabilityOperand<6256, 0, 0, [SPV_INTEL_16bit_atomics], [BFloat16TypeKHR]>; defm AtomicFloat16VectorNV : CapabilityOperand<5404, 0, 0, [SPV_NV_shader_atomic_fp16_vector], []>; +defm AtomicInt16CompareExchangeINTEL : CapabilityOperand<6260, 0, 0, [SPV_INTEL_16bit_atomics], [Int16]>; +defm Int16AtomicsINTEL : CapabilityOperand<6261, 0, 0, [SPV_INTEL_16bit_atomics], [AtomicInt16CompareExchangeINTEL]>; +defm AtomicBFloat16LoadStoreINTEL : CapabilityOperand<6262, 0, 0, [SPV_INTEL_16bit_atomics], [BFloat16TypeKHR]>; defm VariableLengthArrayINTEL : CapabilityOperand<5817, 0, 0, [SPV_INTEL_variable_length_array], []>; defm GroupUniformArithmeticKHR : CapabilityOperand<6400, 0, 0, [SPV_KHR_uniform_group_instructions], []>; defm USMStorageClassesINTEL : CapabilityOperand<5935, 0, 0, [SPV_INTEL_usm_storage_classes], [Kernel]>; @@ -622,7 +626,6 @@ defm Int4TypeINTEL : CapabilityOperand<5112, 0, 0, [SPV_INTEL_int4], []>; defm Int4CooperativeMatrixINTEL : CapabilityOperand<5114, 0, 0, [SPV_INTEL_int4], [Int4TypeINTEL, CooperativeMatrixKHR]>; defm TensorFloat32RoundingINTEL : CapabilityOperand<6425, 0, 0, [SPV_INTEL_tensor_float32_conversion], []>; defm MaskedGatherScatterINTEL : CapabilityOperand<6427, 0, 0, [SPV_INTEL_masked_gather_scatter], []>; -defm BFloat16TypeKHR : CapabilityOperand<5116, 0, 0, [SPV_KHR_bfloat16], []>; defm BFloat16DotProductKHR : CapabilityOperand<5117, 0, 0, [SPV_KHR_bfloat16], [BFloat16TypeKHR]>; defm BFloat16CooperativeMatrixKHR : CapabilityOperand<5118, 0, 0, [SPV_KHR_bfloat16], [BFloat16TypeKHR, CooperativeMatrixKHR]>; defm BlockingPipesALTERA : CapabilityOperand<5945, 0, 0, [SPV_ALTERA_blocking_pipes], []>; diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_bfloat16_load_store_xchg.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_bfloat16_load_store_xchg.ll new file mode 100644 index 000000000000..788c1e67c0a9 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_bfloat16_load_store_xchg.ll @@ -0,0 +1,42 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_bfloat16 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_16bit_atomics,+SPV_KHR_bfloat16 %s -o - | FileCheck %s + +; CHECK-ERROR: LLVM ERROR: The atomic bfloat16 instruction requires the following SPIR-V extension: SPV_INTEL_16bit_atomics + +; CHECK-DAG: Capability BFloat16TypeKHR +; CHECK-DAG: Capability AtomicBFloat16LoadStoreINTEL +; CHECK-DAG: Extension "SPV_KHR_bfloat16" +; CHECK-DAG: Extension "SPV_INTEL_16bit_atomics" +; CHECK-DAG: %[[#TyBF16:]] = OpTypeFloat 16 0 +; CHECK-DAG: %[[#TyBF16Ptr:]] = OpTypePointer CrossWorkgroup %[[#TyBF16]] +; CHECK-DAG: %[[#TyInt32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#Const42:]] = OpConstant %[[#TyBF16]] 16936{{$}} +; CHECK-DAG: %[[#Const0:]] = OpConstantNull %[[#TyBF16]] +; CHECK-DAG: %[[#ScopeDevice:]] = OpConstant %[[#TyInt32]] 1{{$}} +; CHECK-DAG: %[[#ScopeAllSvmDevices:]] = OpConstantNull %[[#TyInt32]] +; CHECK-DAG: %[[#MemSemAcqRel:]] = OpConstant %[[#TyInt32]] 528{{$}} +; CHECK-DAG: %[[#MemSeqCst:]] = OpConstant %[[#TyInt32]] 16{{$}} + +; CHECK-DAG: %[[#Val:]] = OpVariable %[[#TyBF16Ptr]] CrossWorkgroup %[[#Const0]] + +; CHECK: OpAtomicLoad %[[#TyBF16]] %[[#Val]] %[[#ScopeDevice]] %[[#MemSemAcqRel]] +; CHECK: OpAtomicStore %[[#Val]] %[[#ScopeDevice]] %[[#MemSemAcqRel]] %[[#Const42]] +; CHECK: OpAtomicExchange %[[#TyBF16]] %[[#Val]] %[[#ScopeAllSvmDevices]] %[[#MemSeqCst]] %[[#Const42]] + + +@val = private addrspace(1) global bfloat 0.000000e+00 + +define spir_func void @test_atomic_bfloat16_load_store_xchg() { +entry: +; TODO: 'load atomic'/'store atomic' LLVM instructions are not yet lowered to +; OpAtomicLoad/OpAtomicStore by the SPIRV backend; use OCL builtins instead. +; TODO: test Vulkan path as well once we support atomic load/store. + %load = call spir_func bfloat @_Z11atomic_loadPU3AS1VU7_AtomicDF16b(ptr addrspace(1) @val) + call spir_func void @_Z12atomic_storePU3AS1VU7_AtomicDF16bDF16b(ptr addrspace(1) @val, bfloat 42.000000e+00) + %xchg = atomicrmw xchg ptr addrspace(1) @val, bfloat 42.000000e+00 seq_cst + ret void +} + +declare spir_func bfloat @_Z11atomic_loadPU3AS1VU7_AtomicDF16b(ptr addrspace(1)) +declare spir_func void @_Z12atomic_storePU3AS1VU7_AtomicDF16bDF16b(ptr addrspace(1), bfloat) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_arithmetic.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_arithmetic.ll new file mode 100644 index 000000000000..2a33a129d607 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_arithmetic.ll @@ -0,0 +1,44 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_16bit_atomics %s -o - | FileCheck %s + +; CHECK-ERROR: LLVM ERROR: 16-bit integer atomic operations require the following SPIR-V extension: SPV_INTEL_16bit_atomics + +; CHECK-DAG: Capability Int16 +; CHECK-DAG: Capability Int16AtomicsINTEL +; CHECK-DAG: Extension "SPV_INTEL_16bit_atomics" +; CHECK-DAG: %[[#TyInt16:]] = OpTypeInt 16 0 +; CHECK-DAG: %[[#TyInt32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#TyInt16Ptr:]] = OpTypePointer CrossWorkgroup %[[#TyInt16]] +; CHECK-DAG: %[[#Const5:]] = OpConstant %[[#TyInt16]] 5{{$}} +; CHECK-DAG: %[[#Const0:]] = OpConstantNull %[[#TyInt16]] +; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#TyInt32]] +; CHECK-DAG: %[[#MemSeqCst:]] = OpConstant %[[#TyInt32]] 16{{$}} +; CHECK-DAG: %[[#Val:]] = OpVariable %[[#TyInt16Ptr]] CrossWorkgroup %[[#Const0]] + +; CHECK: OpAtomicIAdd %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicISub %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicUMin %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicUMax %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicSMin %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicSMax %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicAnd %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicOr %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] +; CHECK: OpAtomicXor %[[#TyInt16]] %[[#Val]] %[[#Scope]] %[[#MemSeqCst]] %[[#Const5]] + + +@val = private addrspace(1) global i16 0 + +define spir_func void @test_atomic_int16_arithmetic() { +entry: + %add = atomicrmw add ptr addrspace(1) @val, i16 5 seq_cst + %sub = atomicrmw sub ptr addrspace(1) @val, i16 5 seq_cst + %umin = atomicrmw umin ptr addrspace(1) @val, i16 5 seq_cst + %umax = atomicrmw umax ptr addrspace(1) @val, i16 5 seq_cst + %smin = atomicrmw min ptr addrspace(1) @val, i16 5 seq_cst + %smax = atomicrmw max ptr addrspace(1) @val, i16 5 seq_cst + %and = atomicrmw and ptr addrspace(1) @val, i16 5 seq_cst + %or = atomicrmw or ptr addrspace(1) @val, i16 5 seq_cst + %xor = atomicrmw xor ptr addrspace(1) @val, i16 5 seq_cst + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_load_store_xchg_cmpxchg.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_load_store_xchg_cmpxchg.ll new file mode 100644 index 000000000000..d3396b27d697 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomic_int16_load_store_xchg_cmpxchg.ll @@ -0,0 +1,44 @@ +; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR + +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_16bit_atomics %s -o - | FileCheck %s + +; CHECK-ERROR: LLVM ERROR: 16-bit integer atomic operations require the following SPIR-V extension: SPV_INTEL_16bit_atomics + +; CHECK-DAG: Capability Int16 +; CHECK-DAG: Capability AtomicInt16CompareExchangeINTEL +; CHECK-DAG: Extension "SPV_INTEL_16bit_atomics" +; CHECK-DAG: %[[#TyInt16:]] = OpTypeInt 16 0 +; CHECK-DAG: %[[#TyInt16Ptr:]] = OpTypePointer CrossWorkgroup %[[#TyInt16]] +; CHECK-DAG: %[[#TyInt32:]] = OpTypeInt 32 0 + +; CHECK-DAG: %[[#Const0:]] = OpConstantNull %[[#TyInt16]] +; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#TyInt16]] 1{{$}} +; CHECK-DAG: %[[#Const42:]] = OpConstant %[[#TyInt16]] 42{{$}} +; CHECK-DAG: %[[#ScopeDevice:]] = OpConstant %[[#TyInt32]] 1{{$}} +; CHECK-DAG: %[[#ScopeAllSvmDevices:]] = OpConstantNull %[[#TyInt32]] +; CHECK-DAG: %[[#MemSemAcqRel:]] = OpConstant %[[#TyInt32]] 528{{$}} +; CHECK-DAG: %[[#MemSeqCst:]] = OpConstant %[[#TyInt32]] 16{{$}} + +; CHECK-DAG: %[[#Val:]] = OpVariable %[[#TyInt16Ptr]] CrossWorkgroup %[[#Const0]] + +; CHECK: OpAtomicLoad %[[#TyInt16]] %[[#Val]] %[[#ScopeDevice]] %[[#MemSemAcqRel]] +; CHECK: OpAtomicStore %[[#Val]] %[[#ScopeDevice]] %[[#MemSemAcqRel]] %[[#Const42]] +; CHECK: OpAtomicExchange %[[#TyInt16]] %[[#Val]] %[[#ScopeAllSvmDevices]] %[[#MemSeqCst]] %[[#Const42]] +; CHECK: OpAtomicCompareExchange %[[#TyInt16]] %[[#]] %[[#ScopeAllSvmDevices]] %[[#MemSeqCst]] %[[#MemSeqCst]] %[[#Const42]] %[[#Const1]] + + +@val = private addrspace(1) global i16 0 + +define spir_func void @test_atomic_int16_basic() { +entry: +; TODO: 'load atomic'/'store atomic' LLVM instructions are not yet lowered to +; OpAtomicLoad/OpAtomicStore by the SPIRV backend; use OCL builtins instead. + %load = call spir_func i16 @_Z11atomic_loadPU3AS1VU7_Atomics(ptr addrspace(1) @val) + call spir_func void @_Z12atomic_storePU3AS1VU7_Atomicss(ptr addrspace(1) @val, i16 42) + %xchg = atomicrmw xchg ptr addrspace(1) @val, i16 42 seq_cst + %cmpxchg = cmpxchg ptr addrspace(1) @val, i16 1, i16 42 seq_cst seq_cst + ret void +} + +declare spir_func i16 @_Z11atomic_loadPU3AS1VU7_Atomics(ptr addrspace(1)) +declare spir_func void @_Z12atomic_storePU3AS1VU7_Atomicss(ptr addrspace(1), i16) diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_faddfsub_bfloat16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_faddfsub_bfloat16.ll index a189b2a65558..e996f5ce170b 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_faddfsub_bfloat16.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_faddfsub_bfloat16.ll @@ -2,6 +2,7 @@ ; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_add,+SPV_KHR_bfloat16 %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR2 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_add,+SPV_INTEL_16bit_atomics,+SPV_KHR_bfloat16,+SPV_INTEL_bfloat16_arithmetic %s -o - | FileCheck %s +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-vulkan --spirv-ext=+SPV_EXT_shader_atomic_float_add,+SPV_INTEL_16bit_atomics,+SPV_KHR_bfloat16,+SPV_INTEL_bfloat16_arithmetic %s -o - | FileCheck %s ; CHECK-ERROR1: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_add ; CHECK-ERROR2: LLVM ERROR: The atomic bfloat16 instruction requires the following SPIR-V extension: SPV_INTEL_16bit_atomics @@ -15,7 +16,7 @@ ; CHECK-DAG: %[[TyBF16Ptr:[0-9]+]] = OpTypePointer {{[a-zA-Z]+}} %[[TyBF16]] ; CHECK-DAG: %[[TyInt32:[0-9]+]] = OpTypeInt 32 0 ; CHECK-DAG: %[[ConstBF16:[0-9]+]] = OpConstant %[[TyBF16]] 16936{{$}} -; CHECK-DAG: %[[Const0:[0-9]+]] = OpConstantNull %[[TyBF16]] +; CHECK-DAG: %[[Const0:[0-9]+]] = {{OpConstantNull|OpConstant}} %[[TyBF16]]{{( 0)?}} ; CHECK-DAG: %[[BF16Ptr:[0-9]+]] = OpVariable %[[TyBF16Ptr]] CrossWorkgroup %[[Const0]] ; CHECK-DAG: %[[ScopeAllSvmDevices:[0-9]+]] = OpConstantNull %[[TyInt32]] ; CHECK-DAG: %[[MemSeqCst:[0-9]+]] = OpConstant %[[TyInt32]] 16{{$}} @@ -24,9 +25,9 @@ ; CHECK: OpAtomicFAddEXT %[[TyBF16]] %[[BF16Ptr]] %[[ScopeAllSvmDevices]] %[[MemSeqCst]] %[[NegatedConstBF16]] -@f = common dso_local local_unnamed_addr addrspace(1) global bfloat 0.000000e+00, align 8 +@f = private addrspace(1) global bfloat 0.000000e+00 -define dso_local spir_func void @test1() local_unnamed_addr { +define spir_func void @test1() { entry: %addval = atomicrmw fadd ptr addrspace(1) @f, bfloat 42.000000e+00 seq_cst %subval = atomicrmw fsub ptr addrspace(1) @f, bfloat 42.000000e+00 seq_cst diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax_bfloat16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax_bfloat16.ll index dd8448039ec6..10958a410484 100644 --- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax_bfloat16.ll +++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax_bfloat16.ll @@ -1,5 +1,6 @@ ; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_min_max,+SPV_KHR_bfloat16 %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_min_max,+SPV_INTEL_16bit_atomics,+SPV_KHR_bfloat16 %s -o - | FileCheck %s +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-vulkan --spirv-ext=+SPV_EXT_shader_atomic_float_min_max,+SPV_INTEL_16bit_atomics,+SPV_KHR_bfloat16 %s -o - | FileCheck %s ; CHECK-ERROR: LLVM ERROR: The atomic bfloat16 instruction requires the following SPIR-V extension: SPV_INTEL_16bit_atomics @@ -11,14 +12,14 @@ ; CHECK-DAG: %[[TyBF16Ptr:[0-9]+]] = OpTypePointer {{[a-zA-Z]+}} %[[TyBF16]] ; CHECK-DAG: %[[TyInt32:[0-9]+]] = OpTypeInt 32 0 ; CHECK-DAG: %[[ConstBF16:[0-9]+]] = OpConstant %[[TyBF16]] 16936{{$}} -; CHECK-DAG: %[[Const0:[0-9]+]] = OpConstantNull %[[TyBF16]] +; CHECK-DAG: %[[Const0:[0-9]+]] = {{OpConstantNull|OpConstant}} %[[TyBF16]]{{( 0)?}} ; CHECK-DAG: %[[BF16Ptr:[0-9]+]] = OpVariable %[[TyBF16Ptr]] CrossWorkgroup %[[Const0]] ; CHECK-DAG: %[[ScopeAllSvmDevices:[0-9]+]] = OpConstantNull %[[TyInt32]] ; CHECK-DAG: %[[MemSeqCst:[0-9]+]] = OpConstant %[[TyInt32]] 16{{$}} ; CHECK: OpAtomicFMinEXT %[[TyBF16]] %[[BF16Ptr]] %[[ScopeAllSvmDevices]] %[[MemSeqCst]] %[[ConstBF16]] ; CHECK: OpAtomicFMaxEXT %[[TyBF16]] %[[BF16Ptr]] %[[ScopeAllSvmDevices]] %[[MemSeqCst]] %[[ConstBF16]] -@f = common dso_local local_unnamed_addr addrspace(1) global bfloat 0.000000e+00, align 8 +@f = private addrspace(1) global bfloat 0.000000e+00 define spir_func void @test1() { entry: