[libsycl] Add lit configuration files and basic test (#177407)

This PR brings initial version of lit configs for libsycl tests.
We plan to add more tests for libsycl integration with the compiler in
the future.

---------

Signed-off-by: Tikhomirova, Kseniya <kseniya.tikhomirova@intel.com>

libsycl/CMakeLists.txt

@@ -133,5 +133,14 @@ add_custom_target(libsycl-runtime-libraries
 )
 
 add_subdirectory(src)
-
 add_subdirectory(tools)
+
+if(LLVM_INCLUDE_TESTS)
+  add_subdirectory(test)
+endif()
+
+add_custom_target(libsycl-toolchain ALL
+  DEPENDS libsycl-runtime-libraries
+          sycl-ls
+  COMMENT "Building libsycl toolchain..."
+)

libsycl/test/CMakeLists.txt

@@ -0,0 +1,26 @@
+cmake_minimum_required(VERSION 3.20.0)
+
+message("Configuring libsycl tests")
+
+set(LIBSYCL_CXX_COMPILER "${LLVM_BINARY_DIR}/bin/clang++")
+set(LIBSYCL_TEST_COMPILER_OPTIONS "" CACHE STRING
+    "SYCL compiler options for building libsycl tests")
+
+configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+    MAIN_CONFIG
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+
+list(APPEND LIBSYCL_TEST_DEPS
+    libsycl-toolchain
+    FileCheck
+    not
+)
+
+add_lit_testsuite(check-sycl
+  "Running libsycl tests"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${LIBSYCL_TEST_DEPS})

libsycl/test/README.md

@@ -0,0 +1,83 @@
+## Getting Started
+
+This directory contains `libsycl` tests distributed in subdirectories based on
+testing scope. `libsycl` uses LIT to configure and run its tests.
+
+Please see the [Lit Command Guide](https://llvm.org/docs/CommandGuide/lit.html)
+for more information about LIT.
+
+## Prerequisites
+
+* Target runtime(s) to execute tests on devices.
+  TODO: add link to liboffload instruction once they add it.
+* Compiler & libsycl. Can be built following these
+  [instructions](/libsycl/docs/index.rst).
+
+## Run the tests
+
+`libsycl` is integrated via `LLVM_ENABLE_RUNTIMES` and is not visible as a top
+level target. The same is applicable to tests. To run `check-sycl` tests you need
+to prepend `<build>/runtimes/runtimes-bins/` to the paths of all tests.
+For example, to run all the `libsycl` tests you can do:
+```bash
+<build>/bin/llvm-lit <build>/runtimes/runtimes-bins/libsycl/test
+```
+
+To run an individual test, use the path to it instead.
+
+If you are using `ninja` as your build system, you can run all the tests in the
+`libsycl` testsuite as:
+
+```bash
+ ninja -C <build>/runtimes/runtimes-bins check-sycl
+ ```
+
+## CMake parameters
+
+These parameters can be used to configure tests:
+
+`LIBSYCL_CXX_COMPILER` - path to compiler to use it for building tests.
+
+`LIBSYCL_TEST_COMPILER_OPTIONS` - flags to be passed to `LIBSYCL_CXX_COMPILER` when
+    building libsycl tests.
+
+`LLVM_LIT` - path to llvm-lit tool.
+
+## Creating or modifying tests
+
+### LIT feature checks
+
+Following features can be passed to LIT via `REQUIRES`, `UNSUPPORTED`, etc.
+filters to limit test execution to the specific environment.
+
+#### Auto-detected features
+
+The following features are automatically detected by `llvm-lit` by scanning the
+environment:
+
+* `any-device` - presence of at least 1 supported device;
+* `any-device-is-gpu` - device type to be available;
+* `any-device-is-level_zero` - backend to be available;
+
+Note: the `sycl-ls` tool doesn't have an assigned feature since it is essential for
+test configuration and is always available.
+
+### llvm-lit parameters
+
+The following options can be passed to the `llvm-lit` tool with `--param`
+option to configure test execution:
+
+* `libsycl_compiler` - full path to compiler to use.
+* `extra_environment` - comma-separated list of variables with values to be
+  added to the test environment. Can be also set by the `LIT_EXTRA_ENVIRONMENT`
+  variable in CMake.
+* `extra_system_environment` - comma-separated list of variables to be
+  propagated from the host environment to the test environment. Can be also set
+  by `LIT_EXTRA_SYSTEM_ENVIRONMENT` variable in CMake.
+
+Example:
+
+```bash
+<build>/bin/llvm-lit  --param libsycl_compiler=path/to/clang++ \
+        <build>/runtimes/runtimes-bins/libsycl/test
+```

libsycl/test/basic/platform_get_devices.cpp

@@ -0,0 +1,125 @@
+// REQUIRES: any-device
+// RUN: %clangxx %sycl_options %s -o %t.out
+// RUN: %t.out
+//
+// Tests platform::get_devices for each device type.
+
+#include <sycl/sycl.hpp>
+
+#include <algorithm>
+#include <iostream>
+
+std::string BackendToString(sycl::backend Backend) {
+  switch (Backend) {
+  case sycl::backend::opencl:
+    return "opencl";
+  case sycl::backend::level_zero:
+    return "level_zero";
+  case sycl::backend::cuda:
+    return "cuda";
+  case sycl::backend::hip:
+    return "hip";
+  default:
+    return "unknown";
+  }
+}
+
+std::string DeviceTypeToString(sycl::info::device_type DevType) {
+  switch (DevType) {
+  case sycl::info::device_type::all:
+    return "device_type::all";
+  case sycl::info::device_type::cpu:
+    return "device_type::cpu";
+  case sycl::info::device_type::gpu:
+    return "device_type::gpu";
+  case sycl::info::device_type::accelerator:
+    return "device_type::accelerator";
+  case sycl::info::device_type::custom:
+    return "device_type::custom";
+  case sycl::info::device_type::automatic:
+    return "device_type::automatic";
+  case sycl::info::device_type::host:
+    return "device_type::host";
+  default:
+    return "unknown";
+  }
+}
+
+std::string GenerateDeviceDescription(sycl::info::device_type DevType,
+                                      const sycl::platform &Platform) {
+  return std::string(DeviceTypeToString(DevType)) + " (" +
+         BackendToString(Platform.get_backend()) + ")";
+}
+
+template <typename T1, typename T2>
+int Check(const T1 &LHS, const T2 &RHS, std::string TestName) {
+  if (LHS == RHS)
+    return 0;
+
+  std::cerr << "Failed check " << LHS << " != " << RHS << ": " << TestName
+            << std::endl;
+  return 1;
+}
+
+int CheckDeviceType(const sycl::platform &P, sycl::info::device_type DevType,
+                    std::vector<sycl::device> &AllDevices) {
+  // This check verifies data of device with specific device_type and if it is
+  // correctly chosen among all devices (device_type::all).
+  // Though there is no point to check device_type::all here.
+  assert(DevType != sycl::info::device_type::all);
+  int Failures = 0;
+
+  std::vector<sycl::device> Devices = P.get_devices(DevType);
+
+  if (DevType == sycl::info::device_type::automatic) {
+    if (AllDevices.empty()) {
+      Failures += Check(Devices.size(), 0,
+                        "No devices reported for device_type::all query, but "
+                        "device_type::automatic returns a device.");
+    } else {
+      Failures += Check(Devices.size(), 1,
+                        "Number of devices for device_type::automatic query.");
+      if (Devices.size())
+        Failures += Check(
+            std::count(AllDevices.begin(), AllDevices.end(), Devices[0]), 1,
+            "Device is in the set of device_type::all devices in the "
+            "platform.");
+    }
+    return Failures;
+  }
+
+  // Count devices with the type.
+  size_t DevCount = 0;
+  for (sycl::device Device : Devices)
+    DevCount += (Device.get_info<sycl::info::device::device_type>() == DevType);
+
+  Failures += Check(Devices.size(), DevCount,
+                    "Unexpected number of devices for " +
+                        GenerateDeviceDescription(DevType, P));
+
+  Failures +=
+      Check(std::all_of(Devices.begin(), Devices.end(),
+                        [&](const auto &Dev) {
+                          return std::count(AllDevices.begin(),
+                                            AllDevices.end(), Dev) == 1;
+                        }),
+            true,
+            "Not all devices for " + GenerateDeviceDescription(DevType, P) +
+                " appear in the list of all devices");
+
+  return Failures;
+}
+
+int main() {
+  int Failures = 0;
+  for (sycl::platform P : sycl::platform::get_platforms()) {
+    std::vector<sycl::device> Devices = P.get_devices();
+
+    for (sycl::info::device_type DevType :
+         {sycl::info::device_type::cpu, sycl::info::device_type::gpu,
+          sycl::info::device_type::accelerator, sycl::info::device_type::custom,
+          sycl::info::device_type::automatic, sycl::info::device_type::host})
+      Failures += CheckDeviceType(P, DevType, Devices);
+  }
+  return Failures;
+}

libsycl/test/lit.cfg.py

@@ -0,0 +1,211 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+import subprocess
+
+from lit.llvm import llvm_config
+import lit.formats
+from lit.llvm.subst import ToolSubst, FindTool
+
+# name: The name of this test suite.
+config.name = "libsycl"
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = [".cpp"]
+
+config.excludes = ["Inputs"]
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# allow expanding substitutions that are based on other substitutions
+config.recursiveExpansionLimit = 10
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = config.libsycl_obj_root
+
+# To be filled by lit.local.cfg files.
+config.required_features = []
+config.unsupported_features = []
+
+# Cleanup environment variables which may affect tests
+possibly_dangerous_env_vars = [
+    "COMPILER_PATH",
+    "RC_DEBUG_OPTIONS",
+    "CINDEXTEST_PREAMBLE_FILE",
+    "LIBRARY_PATH",
+    "CPATH",
+    "C_INCLUDE_PATH",
+    "CPLUS_INCLUDE_PATH",
+    "OBJC_INCLUDE_PATH",
+    "OBJCPLUS_INCLUDE_PATH",
+    "LIBCLANG_TIMING",
+    "LIBCLANG_OBJTRACKING",
+    "LIBCLANG_LOGGING",
+    "LIBCLANG_BGPRIO_INDEX",
+    "LIBCLANG_BGPRIO_EDIT",
+    "LIBCLANG_NOTHREADS",
+    "LIBCLANG_RESOURCE_USAGE",
+    "LIBCLANG_CODE_COMPLETION_LOGGING",
+    "INCLUDE",
+]
+
+for name in possibly_dangerous_env_vars:
+    if name in llvm_config.config.environment:
+        del llvm_config.config.environment[name]
+
+# Propagate some variables from the host environment.
+llvm_config.with_system_environment(
+    [
+        "PATH",
+    ]
+)
+
+# Take into account extra system environment variables if provided via parameter.
+if config.extra_system_environment:
+    lit_config.note(
+        "Extra system variables to propagate value from: "
+        + config.extra_system_environment
+    )
+    extra_env_vars = config.extra_system_environment.split(",")
+    for var in extra_env_vars:
+        if var in os.environ:
+            llvm_config.with_system_environment(var)
+
+llvm_config.with_environment("PATH", config.lit_tools_dir, append_path=True)
+
+# Configure LD_LIBRARY_PATH
+llvm_config.with_system_environment(
+    ["LD_LIBRARY_PATH", "LIBRARY_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH"]
+)
+llvm_config.with_environment(
+    "LD_LIBRARY_PATH", config.libsycl_libs_dir, append_path=True
+)
+
+llvm_config.with_environment("PATH", config.libsycl_tools_dir, append_path=True)
+
+if config.extra_environment:
+    lit_config.note("Extra environment variables")
+    for env_pair in config.extra_environment.split(","):
+        [var, val] = env_pair.split("=", 1)
+        if val:
+            llvm_config.with_environment(var, val)
+            lit_config.note("\t" + var + "=" + val)
+        else:
+            lit_config.note("\tUnset " + var)
+            llvm_config.with_environment(var, "")
+
+
+# Temporarily modify environment to be the same that we use when running tests
+class test_env:
+    def __enter__(self):
+        self.old_environ = dict(os.environ)
+        os.environ.clear()
+        os.environ.update(config.environment)
+        self.old_dir = os.getcwd()
+        os.chdir(config.libsycl_obj_root)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        os.environ.clear()
+        os.environ.update(self.old_environ)
+        os.chdir(self.old_dir)
+
+
+# General substritutions
+config.substitutions.append(
+    (
+        "%sycl_options",
+        " -lsycl"
+        + " -isystem "
+        + config.libsycl_include
+        + " -isystem "
+        + os.path.join(config.libsycl_include, "sycl")
+        + " -L"
+        + config.libsycl_libs_dir,
+    )
+)
+config.substitutions.append(("%sycl_libs_dir", config.libsycl_libs_dir))
+config.substitutions.append(("%sycl_static_libs_dir", config.libsycl_libs_dir))
+config.substitutions.append(("%obj_ext", ".o"))
+config.substitutions.append(("%sycl_include", "-isystem " + config.libsycl_include))
+config.substitutions.append(("%include_option", "-include"))
+config.substitutions.append(("%debug_option", "-g"))
+config.substitutions.append(("%cxx_std_option", "-std="))
+config.substitutions.append(("%fPIC", "-fPIC"))
+config.substitutions.append(("%shared_lib", "-shared"))
+config.substitutions.append(("%O0", "-O0"))
+
+sycl_ls = FindTool("sycl-ls").resolve(
+    llvm_config, os.pathsep.join([config.libsycl_bin_dir, config.llvm_tools_dir])
+)
+if not sycl_ls:
+    lit_config.fatal("can't find `sycl-ls`")
+
+tools = [
+    ToolSubst("FileCheck", unresolved="ignore"),
+    # not is only substituted in certain circumstances; this is lit's default
+    # behaviour.
+    ToolSubst(
+        r"\| \bnot\b", command=FindTool("not"), verbatim=True, unresolved="ignore"
+    ),
+    ToolSubst("sycl-ls", command=sycl_ls, unresolved="fatal"),
+]
+
+# Try and find each of these tools in the libsycl bin directory, in the llvm tools directory
+# or the PATH, in that order. If found, they will be added as substitutions with the full path
+# to the tool.
+llvm_config.add_tool_substitutions(
+    tools, [config.libsycl_bin_dir, config.llvm_tools_dir, os.environ.get("PATH", "")]
+)
+
+lit_config.note("Targeted devices: all")
+with test_env():
+    sycl_ls_output = subprocess.check_output(sycl_ls, text=True, shell=True)
+
+    devices = set()
+    for line in sycl_ls_output.splitlines():
+        if not line.startswith("["):
+            continue
+        backend, device = line[1:].split("]")[0].split(":")
+        devices.add("{}:{}".format(backend, device))
+    libsycl_devices = list(devices)
+
+available_devices = {
+    "level_zero": "gpu",
+}
+for d in libsycl_devices:
+    be, dev = d.split(":")
+    if be not in available_devices:
+        lit_config.error("Unsupported device {}".format(d))
+    if dev not in available_devices[be]:
+        lit_config.error("Unsupported device {}".format(d))
+
+if len(libsycl_devices) > 0:
+    config.available_features.add("any-device")
+
+for sycl_device in libsycl_devices:
+    be, dev = sycl_device.split(":")
+    config.available_features.add("any-device-is-" + dev)
+    config.available_features.add("any-device-is-" + be)
+
+# Check if user passed verbose-print parameter, if yes, add VERBOSE_PRINT macro
+if "verbose-print" in lit_config.params:
+    verbose_print = "-DVERBOSE_PRINT"
+else:
+    verbose_print = ""
+
+clangxx = " " + config.libsycl_compiler + " -Werror " + config.cxx_flags + verbose_print
+config.substitutions.append(("%clangxx", clangxx))
+
+config.test_format = lit.formats.ShTest()
+
+try:
+    import psutil
+
+    # Set timeout for a single test
+    lit_config.maxIndividualTestTime = 600
+
+except ImportError:
+    pass

libsycl/test/lit.site.cfg.py.in

@@ -0,0 +1,34 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+import subprocess
+import site
+
+site.addsitedir("@CMAKE_CURRENT_SOURCE_DIR@")
+
+config.libsycl_compiler = lit_config.params.get("libsycl_compiler", "@LIBSYCL_CXX_COMPILER@")
+config.libsycl_bin_dir  = os.path.dirname(config.libsycl_compiler)
+config.libsycl_root_dir = os.path.dirname(config.libsycl_bin_dir)
+
+config.cxx_flags = lit_config.params.get("cxx_flags", "@LIBSYCL_TEST_COMPILER_OPTIONS@")
+
+config.extra_environment = lit_config.params.get("extra_environment", "@LIT_EXTRA_ENVIRONMENT@")
+config.extra_system_environment = lit_config.params.get("extra_system_environment", "@LIT_EXTRA_SYSTEM_ENVIRONMENT@")
+
+def get_libsycl_tool_path(name):
+    try:
+        return subprocess.check_output([config.libsycl_compiler, "-print-prog-name=" + name], text=True)
+    except subprocess.CalledProcessError:
+        return os.path.join(config.libsycl_bin_dir, name)
+
+config.llvm_tools_dir = os.path.dirname(get_libsycl_tool_path("llvm-config"))
+config.lit_tools_dir = os.path.dirname("@TEST_SUITE_LIT@")
+
+config.libsycl_tools_dir = config.llvm_tools_dir
+config.libsycl_include = os.path.join(config.libsycl_root_dir, 'include')
+config.libsycl_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
+config.libsycl_libs_dir =  os.path.join(config.libsycl_root_dir, 'lib/x86_64-unknown-linux-gnu')
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")