diff --git a/mlir/include/mlir/ExecutionEngine/Float16bits.h b/mlir/include/mlir/ExecutionEngine/Float16bits.h index 5eb1f2ce0763..ad409841b2a9 100644 --- a/mlir/include/mlir/ExecutionEngine/Float16bits.h +++ b/mlir/include/mlir/ExecutionEngine/Float16bits.h @@ -48,6 +48,9 @@ MLIR_FLOAT16_EXPORT std::ostream &operator<<(std::ostream &os, const f16 &f); // Outputs a bfloat value. MLIR_FLOAT16_EXPORT std::ostream &operator<<(std::ostream &os, const bf16 &d); +MLIR_FLOAT16_EXPORT bool operator==(const f16 &f1, const f16 &f2); +MLIR_FLOAT16_EXPORT bool operator==(const bf16 &bf1, const bf16 &bf2); + extern "C" MLIR_FLOAT16_EXPORT void printF16(uint16_t bits); extern "C" MLIR_FLOAT16_EXPORT void printBF16(uint16_t bits); diff --git a/mlir/include/mlir/ExecutionEngine/RunnerUtils.h b/mlir/include/mlir/ExecutionEngine/RunnerUtils.h index ebf95f90f374..9a748370c36e 100644 --- a/mlir/include/mlir/ExecutionEngine/RunnerUtils.h +++ b/mlir/include/mlir/ExecutionEngine/RunnerUtils.h @@ -331,9 +331,9 @@ int64_t verifyMemRef(const DynamicMemRefType &actual, } // Return the number of errors. int64_t printCounter = 0; - return MemRefDataVerifier::verify( - std::cerr, actual.basePtr, expected.basePtr, actual.rank, actual.offset, - actual.sizes, actual.strides, printCounter); + return MemRefDataVerifier::verify(std::cerr, actual.data, expected.data, + actual.rank, actual.offset, actual.sizes, + actual.strides, printCounter); } /// Verify the equivalence of two unranked memrefs and return the number of @@ -429,8 +429,18 @@ _mlir_ciface_printMemref1dC64(StridedMemRefType *m); extern "C" MLIR_RUNNERUTILS_EXPORT void _mlir_ciface_printMemrefVector4x4xf32( StridedMemRefType, 2> *m); +extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefI8( + UnrankedMemRefType *actual, UnrankedMemRefType *expected); +extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefI16( + UnrankedMemRefType *actual, UnrankedMemRefType *expected); extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefI32( UnrankedMemRefType *actual, UnrankedMemRefType *expected); +extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefI64( + UnrankedMemRefType *actual, UnrankedMemRefType *expected); +extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefBF16( + UnrankedMemRefType *actual, UnrankedMemRefType *expected); +extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefF16( + UnrankedMemRefType *actual, UnrankedMemRefType *expected); extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefF32( UnrankedMemRefType *actual, UnrankedMemRefType *expected); extern "C" MLIR_RUNNERUTILS_EXPORT int64_t _mlir_ciface_verifyMemRefF64( diff --git a/mlir/lib/ExecutionEngine/Float16bits.cpp b/mlir/lib/ExecutionEngine/Float16bits.cpp index 38a05fe86bbd..841610e3c161 100644 --- a/mlir/lib/ExecutionEngine/Float16bits.cpp +++ b/mlir/lib/ExecutionEngine/Float16bits.cpp @@ -150,6 +150,10 @@ std::ostream &operator<<(std::ostream &os, const bf16 &d) { return os; } +bool operator==(const f16 &f1, const f16 &f2) { return f1.bits == f2.bits; } + +bool operator==(const bf16 &f1, const bf16 &f2) { return f1.bits == f2.bits; } + // Mark these symbols as weak so they don't conflict when compiler-rt also // defines them. #define ATTR_WEAK diff --git a/mlir/lib/ExecutionEngine/RunnerUtils.cpp b/mlir/lib/ExecutionEngine/RunnerUtils.cpp index 378aa7ce35ef..eaa83bfc693f 100644 --- a/mlir/lib/ExecutionEngine/RunnerUtils.cpp +++ b/mlir/lib/ExecutionEngine/RunnerUtils.cpp @@ -219,12 +219,42 @@ _mlir_ciface_printMemref1dC64(StridedMemRefType *M) { impl::printMemRef(*M); } +extern "C" int64_t +_mlir_ciface_verifyMemRefI8(UnrankedMemRefType *actual, + UnrankedMemRefType *expected) { + return impl::verifyMemRef(*actual, *expected); +} + +extern "C" int64_t +_mlir_ciface_verifyMemRefI16(UnrankedMemRefType *actual, + UnrankedMemRefType *expected) { + return impl::verifyMemRef(*actual, *expected); +} + extern "C" int64_t _mlir_ciface_verifyMemRefI32(UnrankedMemRefType *actual, UnrankedMemRefType *expected) { return impl::verifyMemRef(*actual, *expected); } +extern "C" int64_t +_mlir_ciface_verifyMemRefI64(UnrankedMemRefType *actual, + UnrankedMemRefType *expected) { + return impl::verifyMemRef(*actual, *expected); +} + +extern "C" int64_t +_mlir_ciface_verifyMemRefF16(UnrankedMemRefType *actual, + UnrankedMemRefType *expected) { + return impl::verifyMemRef(*actual, *expected); +} + +extern "C" int64_t +_mlir_ciface_verifyMemRefBF16(UnrankedMemRefType *actual, + UnrankedMemRefType *expected) { + return impl::verifyMemRef(*actual, *expected); +} + extern "C" int64_t _mlir_ciface_verifyMemRefF32(UnrankedMemRefType *actual, UnrankedMemRefType *expected) { diff --git a/mlir/test/Integration/Dialect/Memref/verify-memref.mlir b/mlir/test/Integration/Dialect/Memref/verify-memref.mlir new file mode 100644 index 000000000000..b7e2a46688f4 --- /dev/null +++ b/mlir/test/Integration/Dialect/Memref/verify-memref.mlir @@ -0,0 +1,167 @@ +// RUN: mlir-opt %s \ +// RUN: -func-bufferize -arith-bufferize --canonicalize \ +// RUN: -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm\ +// RUN: -convert-func-to-llvm -reconcile-unrealized-casts |\ +// RUN: mlir-cpu-runner \ +// RUN: -e entry -entry-point-result=void \ +// RUN: -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils |\ +// RUN: FileCheck %s + +module { + func.func private @verifyMemRefI8(%a : tensor<*xi8>, %b : tensor<*xi8>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefI16(%a : tensor<*xi16>, %b : tensor<*xi16>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefI32(%a : tensor<*xi32>, %b : tensor<*xi32>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefI64(%a : tensor<*xi64>, %b : tensor<*xi64>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefBF16(%a : tensor<*xbf16>, %b : tensor<*xbf16>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefF16(%a : tensor<*xf16>, %b : tensor<*xf16>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefF32(%a : tensor<*xf32>, %b : tensor<*xf32>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefF64(%a : tensor<*xf64>, %b : tensor<*xf64>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefC32(%a : tensor<*xcomplex>, %b : tensor<*xcomplex>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefC64(%a : tensor<*xcomplex>, %b : tensor<*xcomplex>) -> i64 attributes { llvm.emit_c_interface } + func.func private @verifyMemRefInd(%a : tensor<*xindex>, %b : tensor<*xindex>) -> i64 attributes { llvm.emit_c_interface } + + func.func @entry() { + %i8 = arith.constant dense<90> : tensor<3x3xi8> + %i16 = arith.constant dense<1> : tensor<3x3xi16> + %i32 = arith.constant dense<2> : tensor<3x3xi32> + %i64 = arith.constant dense<3> : tensor<3x3xi64> + %f16 = arith.constant dense<1.5> : tensor<3x3xf16> + %bf16 = arith.constant dense<2.5> : tensor<3x3xbf16> + %f32 = arith.constant dense<3.5> : tensor<3x3xf32> + %f64 = arith.constant dense<4.5> : tensor<3x3xf64> + %c32 = arith.constant dense<(1.000000e+01,5.000000e+00)> : tensor<3x3xcomplex> + %c64 = arith.constant dense<(2.000000e+01,5.000000e+00)> : tensor<3x3xcomplex> + %ind = arith.constant dense<4> : tensor<3x3xindex> + + %1 = tensor.cast %i8 : tensor<3x3xi8> to tensor<*xi8> + %2 = tensor.cast %i16 : tensor<3x3xi16> to tensor<*xi16> + %3 = tensor.cast %i32 : tensor<3x3xi32> to tensor<*xi32> + %4 = tensor.cast %i64 : tensor<3x3xi64> to tensor<*xi64> + %5 = tensor.cast %f16 : tensor<3x3xf16> to tensor<*xf16> + %6 = tensor.cast %bf16 : tensor<3x3xbf16> to tensor<*xbf16> + %7 = tensor.cast %f32 : tensor<3x3xf32> to tensor<*xf32> + %8 = tensor.cast %f64 : tensor<3x3xf64> to tensor<*xf64> + %9 = tensor.cast %c32 : tensor<3x3xcomplex> to tensor<*xcomplex> + %10 = tensor.cast %c64 : tensor<3x3xcomplex> to tensor<*xcomplex> + %11 = tensor.cast %ind : tensor<3x3xindex> to tensor<*xindex> + + // + // Ensure that verifyMemRef could detect equal memrefs. + // + // CHECK: 0 + %res0 = call @verifyMemRefI8(%1, %1) : (tensor<*xi8>, tensor<*xi8>) -> (i64) + vector.print %res0 : i64 + + // CHECK-NEXT: 0 + %res1 = call @verifyMemRefI16(%2, %2) : (tensor<*xi16>, tensor<*xi16>) -> (i64) + vector.print %res1 : i64 + + // CHECK-NEXT: 0 + %res2 = call @verifyMemRefI32(%3, %3) : (tensor<*xi32>, tensor<*xi32>) -> (i64) + vector.print %res2 : i64 + + // CHECK-NEXT: 0 + %res3 = call @verifyMemRefI64(%4, %4) : (tensor<*xi64>, tensor<*xi64>) -> (i64) + vector.print %res3 : i64 + + // CHECK-NEXT: 0 + %res4 = call @verifyMemRefF16(%5, %5) : (tensor<*xf16>, tensor<*xf16>) -> (i64) + vector.print %res4 : i64 + + // CHECK-NEXT: 0 + %res5 = call @verifyMemRefBF16(%6, %6) : (tensor<*xbf16>, tensor<*xbf16>) -> (i64) + vector.print %res5 : i64 + + // CHECK-NEXT: 0 + %res6 = call @verifyMemRefF32(%7, %7) : (tensor<*xf32>, tensor<*xf32>) -> (i64) + vector.print %res6 : i64 + + // CHECK-NEXT: 0 + %res7 = call @verifyMemRefF64(%8, %8) : (tensor<*xf64>, tensor<*xf64>) -> (i64) + vector.print %res7 : i64 + + // CHECK-NEXT: 0 + %res8 = call @verifyMemRefC32(%9, %9) : (tensor<*xcomplex>, tensor<*xcomplex>) -> (i64) + vector.print %res8 : i64 + + // CHECK-NEXT: 0 + %res9 = call @verifyMemRefC64(%10, %10) : (tensor<*xcomplex>, tensor<*xcomplex>) -> (i64) + vector.print %res9 : i64 + + // CHECK-NEXT: 0 + %res10 = call @verifyMemRefInd(%11, %11) : (tensor<*xindex>, tensor<*xindex>) -> (i64) + vector.print %res10 : i64 + + // + // Ensure that verifyMemRef could detect the correct number of errors + // for unequal memrefs. + // + %m1 = arith.constant dense<100> : tensor<3x3xi8> + %f1 = tensor.cast %m1 : tensor<3x3xi8> to tensor<*xi8> + %fail_res1 = call @verifyMemRefI8(%1, %f1) : (tensor<*xi8>, tensor<*xi8>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res1 : i64 + + %m2 = arith.constant dense<100> : tensor<3x3xi16> + %f2 = tensor.cast %m2 : tensor<3x3xi16> to tensor<*xi16> + %fail_res2 = call @verifyMemRefI16(%2, %f2) : (tensor<*xi16>, tensor<*xi16>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res2 : i64 + + %m3 = arith.constant dense<100> : tensor<3x3xi32> + %f3 = tensor.cast %m3 : tensor<3x3xi32> to tensor<*xi32> + %fail_res3 = call @verifyMemRefI32(%3, %f3) : (tensor<*xi32>, tensor<*xi32>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res3 : i64 + + %m4 = arith.constant dense<100> : tensor<3x3xi64> + %f4 = tensor.cast %m4 : tensor<3x3xi64> to tensor<*xi64> + %fail_res4 = call @verifyMemRefI64(%4, %f4) : (tensor<*xi64>, tensor<*xi64>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res4 : i64 + + %m5 = arith.constant dense<100.0> : tensor<3x3xf16> + %f5 = tensor.cast %m5 : tensor<3x3xf16> to tensor<*xf16> + %fail_res5 = call @verifyMemRefF16(%5, %f5) : (tensor<*xf16>, tensor<*xf16>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res5 : i64 + + %m6 = arith.constant dense<100.0> : tensor<3x3xbf16> + %f6 = tensor.cast %m6 : tensor<3x3xbf16> to tensor<*xbf16> + %fail_res6 = call @verifyMemRefBF16(%6, %f6) : (tensor<*xbf16>, tensor<*xbf16>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res6 : i64 + + %m7 = arith.constant dense<100.0> : tensor<3x3xf32> + %f7 = tensor.cast %m7 : tensor<3x3xf32> to tensor<*xf32> + %fail_res7 = call @verifyMemRefF32(%7, %f7) : (tensor<*xf32>, tensor<*xf32>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res7 : i64 + + %m8 = arith.constant dense<100.0> : tensor<3x3xf64> + %f8 = tensor.cast %m8 : tensor<3x3xf64> to tensor<*xf64> + %fail_res8 = call @verifyMemRefF64(%8, %f8) : (tensor<*xf64>, tensor<*xf64>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res8 : i64 + + %m9 = arith.constant dense<(5.000000e+01,1.000000e+00)> : tensor<3x3xcomplex> + %f9 = tensor.cast %m9 : tensor<3x3xcomplex> to tensor<*xcomplex> + %fail_res9 = call @verifyMemRefC32(%9, %f9) : (tensor<*xcomplex>, tensor<*xcomplex>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res9 : i64 + + %m10 = arith.constant dense<(5.000000e+01,1.000000e+00)> : tensor<3x3xcomplex> + %f10 = tensor.cast %m10 : tensor<3x3xcomplex> to tensor<*xcomplex> + %fail_res10 = call @verifyMemRefC64(%10, %f10) : (tensor<*xcomplex>, tensor<*xcomplex>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res10 : i64 + + %m11 = arith.constant dense<100> : tensor<3x3xindex> + %f11 = tensor.cast %m11 : tensor<3x3xindex> to tensor<*xindex> + %fail_res11 = call @verifyMemRefInd(%11, %f11) : (tensor<*xindex>, tensor<*xindex>) -> (i64) + // CHECK-NEXT: 9 + vector.print %fail_res11 : i64 + + return + } +}