Nikolas Klauser b173c12b54
[libc++] Make libcxx/selftest a top-level test directory (#144852)
These tests aren't actually libc++-specific for the most part. They test
the testsuite, so they aren't tied to any implementation. That means
anybody using the testsuite should be able to run these tests
successfully.

The most notable changes are that
- `test/libcxx/selftest/modules` moved to `test/libcxx/modules`, since
the assumptions these tests check are libc++-specific
- there is a new `lit.local.cfg` for `test/selftest`
- `selftest/dsl/dsl.sh.py` was modified since the path to the monorepo
root changed
2025-07-02 21:47:26 +02:00

630 lines
22 KiB
Python

# ===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===----------------------------------------------------------------------===##
# With picolibc, test_program_stderr_is_not_conflated_with_stdout fails
# because stdout & stderr are treated as the same.
# XFAIL: LIBCXX-PICOLIBC-FIXME
# Note: We prepend arguments with 'x' to avoid thinking there are too few
# arguments in case an argument is an empty string.
# RUN: %{python} %s x%S x%T x%{substitutions}
import base64
import copy
import os
import pickle
import platform
import subprocess
import sys
import unittest
from os.path import dirname
# Allow importing 'lit' and the 'libcxx' module. Make sure we put the lit
# path first so we don't find any system-installed version.
monorepoRoot = dirname(dirname(dirname(dirname(dirname(__file__)))))
sys.path = [
os.path.join(monorepoRoot, "libcxx", "utils"),
os.path.join(monorepoRoot, "llvm", "utils", "lit"),
] + sys.path
import libcxx.test.dsl as dsl
import lit.LitConfig
import lit.util
# Steal some parameters from the config running this test so that we can
# bootstrap our own TestingConfig.
args = list(map(lambda s: s[1:], sys.argv[1:8])) # Remove the leading 'x'
SOURCE_ROOT, EXEC_PATH, SUBSTITUTIONS = args
sys.argv[1:8] = []
# Decode the substitutions.
SUBSTITUTIONS = pickle.loads(base64.b64decode(SUBSTITUTIONS))
for s, sub in SUBSTITUTIONS:
print("Substitution '{}' is '{}'".format(s, sub))
class SetupConfigs(unittest.TestCase):
"""
Base class for the tests below -- it creates a fake TestingConfig.
"""
def setUp(self):
"""
Create a fake TestingConfig that can be populated however we wish for
the purpose of running unit tests below. We pre-populate it with the
minimum required substitutions.
"""
self.litConfig = lit.LitConfig.LitConfig(
progname="lit",
path=[],
quiet=False,
useValgrind=False,
valgrindLeakCheck=False,
valgrindArgs=[],
noExecute=False,
debug=False,
isWindows=platform.system() == "Windows",
order="smart",
params={},
)
self.config = lit.TestingConfig.TestingConfig.fromdefaults(self.litConfig)
self.config.environment = dict(os.environ)
self.config.test_source_root = SOURCE_ROOT
self.config.test_exec_root = EXEC_PATH
self.config.recursiveExpansionLimit = 10
self.config.substitutions = copy.deepcopy(SUBSTITUTIONS)
def getSubstitution(self, substitution):
"""
Return a given substitution from the TestingConfig. It is an error if
there is no such substitution.
"""
found = [x for (s, x) in self.config.substitutions if s == substitution]
assert len(found) == 1
return found[0]
def findIndex(list, pred):
"""Finds the index of the first element satisfying 'pred' in a list, or
'len(list)' if there is no such element."""
index = 0
for x in list:
if pred(x):
break
else:
index += 1
return index
class TestHasCompileFlag(SetupConfigs):
"""
Tests for libcxx.test.dsl.hasCompileFlag
"""
def test_no_flag_should_work(self):
self.assertTrue(dsl.hasCompileFlag(self.config, ""))
def test_flag_exists(self):
self.assertTrue(dsl.hasCompileFlag(self.config, "-O1"))
def test_nonexistent_flag(self):
self.assertFalse(
dsl.hasCompileFlag(self.config, "-this_is_not_a_flag_any_compiler_has")
)
def test_multiple_flags(self):
self.assertTrue(dsl.hasCompileFlag(self.config, "-O1 -Dhello"))
class TestSourceBuilds(SetupConfigs):
"""
Tests for libcxx.test.dsl.sourceBuilds
"""
def test_valid_program_builds(self):
source = """int main(int, char**) { return 0; }"""
self.assertTrue(dsl.sourceBuilds(self.config, source))
def test_compilation_error_fails(self):
source = """int main(int, char**) { this does not compile }"""
self.assertFalse(dsl.sourceBuilds(self.config, source))
def test_link_error_fails(self):
source = """extern void this_isnt_defined_anywhere();
int main(int, char**) { this_isnt_defined_anywhere(); return 0; }"""
self.assertFalse(dsl.sourceBuilds(self.config, source))
class TestProgramOutput(SetupConfigs):
"""
Tests for libcxx.test.dsl.programOutput
"""
def test_valid_program_returns_output(self):
source = """
#include <cstdio>
int main(int, char**) { std::printf("FOOBAR"); return 0; }
"""
self.assertEqual(dsl.programOutput(self.config, source), "FOOBAR")
def test_valid_program_returns_output_newline_handling(self):
source = """
#include <cstdio>
int main(int, char**) { std::printf("FOOBAR\\n"); return 0; }
"""
self.assertEqual(dsl.programOutput(self.config, source), "FOOBAR\n")
def test_valid_program_returns_no_output(self):
source = """
int main(int, char**) { return 0; }
"""
self.assertEqual(dsl.programOutput(self.config, source), "")
def test_program_that_fails_to_run_raises_runtime_error(self):
# The program compiles, but exits with an error
source = """
int main(int, char**) { return 1; }
"""
self.assertRaises(
dsl.ConfigurationRuntimeError,
lambda: dsl.programOutput(self.config, source),
)
def test_program_that_fails_to_compile_raises_compilation_error(self):
# The program doesn't compile
source = """
int main(int, char**) { this doesnt compile }
"""
self.assertRaises(
dsl.ConfigurationCompilationError,
lambda: dsl.programOutput(self.config, source),
)
def test_pass_arguments_to_program(self):
source = """
#include <cassert>
#include <string>
int main(int argc, char** argv) {
assert(argc == 3);
assert(argv[1] == std::string("first-argument"));
assert(argv[2] == std::string("second-argument"));
return 0;
}
"""
args = ["first-argument", "second-argument"]
self.assertEqual(dsl.programOutput(self.config, source, args=args), "")
def test_caching_is_not_too_aggressive(self):
# Run a program, then change the substitutions and run it again.
# Make sure the program is run the second time and the right result
# is given, to ensure we're not incorrectly caching the result of the
# first program run.
source = """
#include <cstdio>
int main(int, char**) {
std::printf("MACRO=%u\\n", MACRO);
return 0;
}
"""
compileFlagsIndex = findIndex(
self.config.substitutions, lambda x: x[0] == "%{compile_flags}"
)
compileFlags = self.config.substitutions[compileFlagsIndex][1]
self.config.substitutions[compileFlagsIndex] = (
"%{compile_flags}",
compileFlags + " -DMACRO=1",
)
output1 = dsl.programOutput(self.config, source)
self.assertEqual(output1, "MACRO=1\n")
self.config.substitutions[compileFlagsIndex] = (
"%{compile_flags}",
compileFlags + " -DMACRO=2",
)
output2 = dsl.programOutput(self.config, source)
self.assertEqual(output2, "MACRO=2\n")
def test_program_stderr_is_not_conflated_with_stdout(self):
# Run a program that produces stdout output and stderr output too, making
# sure the stderr output does not pollute the stdout output.
source = """
#include <cstdio>
int main(int, char**) {
std::fprintf(stdout, "STDOUT-OUTPUT");
std::fprintf(stderr, "STDERR-OUTPUT");
return 0;
}
"""
self.assertEqual(dsl.programOutput(self.config, source), "STDOUT-OUTPUT")
class TestProgramSucceeds(SetupConfigs):
"""
Tests for libcxx.test.dsl.programSucceeds
"""
def test_success(self):
source = """
int main(int, char**) { return 0; }
"""
self.assertTrue(dsl.programSucceeds(self.config, source))
def test_failure(self):
source = """
int main(int, char**) { return 1; }
"""
self.assertFalse(dsl.programSucceeds(self.config, source))
def test_compile_failure(self):
source = """
this does not compile
"""
self.assertRaises(
dsl.ConfigurationCompilationError,
lambda: dsl.programSucceeds(self.config, source),
)
class TestHasLocale(SetupConfigs):
"""
Tests for libcxx.test.dsl.hasLocale
"""
def test_doesnt_explode(self):
# It's really hard to test that a system has a given locale, so at least
# make sure we don't explode when we try to check it.
try:
dsl.hasAnyLocale(self.config, ["en_US.UTF-8"])
except subprocess.CalledProcessError:
self.fail("checking for hasLocale should not explode")
def test_nonexistent_locale(self):
self.assertFalse(
dsl.hasAnyLocale(self.config, ["forsurethisisnotanexistinglocale"])
)
def test_localization_program_doesnt_compile(self):
compilerIndex = findIndex(self.config.substitutions, lambda x: x[0] == "%{cxx}")
self.config.substitutions[compilerIndex] = (
"%{cxx}",
"this-is-certainly-not-a-valid-compiler!!",
)
self.assertRaises(
dsl.ConfigurationCompilationError,
lambda: dsl.hasAnyLocale(self.config, ["en_US.UTF-8"]),
)
class TestCompilerMacros(SetupConfigs):
"""
Tests for libcxx.test.dsl.compilerMacros
"""
def test_basic(self):
macros = dsl.compilerMacros(self.config)
self.assertIsInstance(macros, dict)
self.assertGreater(len(macros), 0)
for (k, v) in macros.items():
self.assertIsInstance(k, str)
self.assertIsInstance(v, str)
def test_no_flag(self):
macros = dsl.compilerMacros(self.config)
self.assertIn("__cplusplus", macros.keys())
def test_empty_flag(self):
macros = dsl.compilerMacros(self.config, "")
self.assertIn("__cplusplus", macros.keys())
def test_with_flag(self):
macros = dsl.compilerMacros(self.config, "-DFOO=3")
self.assertIn("__cplusplus", macros.keys())
self.assertEqual(macros["FOO"], "3")
def test_with_flags(self):
macros = dsl.compilerMacros(self.config, "-DFOO=3 -DBAR=hello")
self.assertIn("__cplusplus", macros.keys())
self.assertEqual(macros["FOO"], "3")
self.assertEqual(macros["BAR"], "hello")
class TestFeatureTestMacros(SetupConfigs):
"""
Tests for libcxx.test.dsl.featureTestMacros
"""
def test_basic(self):
macros = dsl.featureTestMacros(self.config)
self.assertIsInstance(macros, dict)
self.assertGreater(len(macros), 0)
for (k, v) in macros.items():
self.assertIsInstance(k, str)
self.assertIsInstance(v, int)
class TestFeature(SetupConfigs):
"""
Tests for libcxx.test.dsl.Feature
"""
def test_trivial(self):
feature = dsl.Feature(name="name")
origSubstitutions = copy.deepcopy(self.config.substitutions)
actions = feature.getActions(self.config)
self.assertTrue(len(actions) == 1)
for a in actions:
a.applyTo(self.config)
self.assertEqual(origSubstitutions, self.config.substitutions)
self.assertIn("name", self.config.available_features)
def test_name_can_be_a_callable(self):
feature = dsl.Feature(name=lambda cfg: "name")
for a in feature.getActions(self.config):
a.applyTo(self.config)
self.assertIn("name", self.config.available_features)
def test_name_is_not_a_string_1(self):
feature = dsl.Feature(name=None)
self.assertRaises(ValueError, lambda: feature.getActions(self.config))
self.assertRaises(ValueError, lambda: feature.pretty(self.config))
def test_name_is_not_a_string_2(self):
feature = dsl.Feature(name=lambda cfg: None)
self.assertRaises(ValueError, lambda: feature.getActions(self.config))
self.assertRaises(ValueError, lambda: feature.pretty(self.config))
def test_adding_action(self):
feature = dsl.Feature(name="name", actions=[dsl.AddCompileFlag("-std=c++03")])
origLinkFlags = copy.deepcopy(self.getSubstitution("%{link_flags}"))
for a in feature.getActions(self.config):
a.applyTo(self.config)
self.assertIn("name", self.config.available_features)
self.assertIn("-std=c++03", self.getSubstitution("%{compile_flags}"))
self.assertEqual(origLinkFlags, self.getSubstitution("%{link_flags}"))
def test_actions_can_be_a_callable(self):
feature = dsl.Feature(
name="name",
actions=lambda cfg: (
self.assertIs(self.config, cfg),
[dsl.AddCompileFlag("-std=c++03")],
)[1],
)
for a in feature.getActions(self.config):
a.applyTo(self.config)
self.assertIn("-std=c++03", self.getSubstitution("%{compile_flags}"))
def test_unsupported_feature(self):
feature = dsl.Feature(name="name", when=lambda _: False)
self.assertEqual(feature.getActions(self.config), [])
def test_is_supported_gets_passed_the_config(self):
feature = dsl.Feature(
name="name", when=lambda cfg: (self.assertIs(self.config, cfg), True)[1]
)
self.assertEqual(len(feature.getActions(self.config)), 1)
def _throw():
raise ValueError()
class TestParameter(SetupConfigs):
"""
Tests for libcxx.test.dsl.Parameter
"""
def test_empty_name_should_blow_up(self):
self.assertRaises(
ValueError,
lambda: dsl.Parameter(
name="", choices=["c++03"], type=str, help="", actions=lambda _: []
),
)
def test_empty_choices_should_blow_up(self):
self.assertRaises(
ValueError,
lambda: dsl.Parameter(
name="std", choices=[], type=str, help="", actions=lambda _: []
),
)
def test_no_choices_is_ok(self):
param = dsl.Parameter(name="triple", type=str, help="", actions=lambda _: [])
self.assertEqual(param.name, "triple")
def test_name_is_set_correctly(self):
param = dsl.Parameter(
name="std", choices=["c++03"], type=str, help="", actions=lambda _: []
)
self.assertEqual(param.name, "std")
def test_no_value_provided_and_no_default_value(self):
param = dsl.Parameter(
name="std", choices=["c++03"], type=str, help="", actions=lambda _: []
)
self.assertRaises(
ValueError, lambda: param.getActions(self.config, self.litConfig.params)
)
def test_no_value_provided_and_default_value(self):
param = dsl.Parameter(
name="std",
choices=["c++03"],
type=str,
help="",
default="c++03",
actions=lambda std: [dsl.AddFeature(std)],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("c++03", self.config.available_features)
def test_value_provided_on_command_line_and_no_default_value(self):
self.litConfig.params["std"] = "c++03"
param = dsl.Parameter(
name="std",
choices=["c++03"],
type=str,
help="",
actions=lambda std: [dsl.AddFeature(std)],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("c++03", self.config.available_features)
def test_value_provided_on_command_line_and_default_value(self):
"""The value provided on the command line should override the default value"""
self.litConfig.params["std"] = "c++11"
param = dsl.Parameter(
name="std",
choices=["c++03", "c++11"],
type=str,
default="c++03",
help="",
actions=lambda std: [dsl.AddFeature(std)],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("c++11", self.config.available_features)
self.assertNotIn("c++03", self.config.available_features)
def test_value_provided_in_config_and_default_value(self):
"""The value provided in the config should override the default value"""
self.config.std = "c++11"
param = dsl.Parameter(
name="std",
choices=["c++03", "c++11"],
type=str,
default="c++03",
help="",
actions=lambda std: [dsl.AddFeature(std)],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("c++11", self.config.available_features)
self.assertNotIn("c++03", self.config.available_features)
def test_value_provided_in_config_and_on_command_line(self):
"""The value on the command line should override the one in the config"""
self.config.std = "c++11"
self.litConfig.params["std"] = "c++03"
param = dsl.Parameter(
name="std",
choices=["c++03", "c++11"],
type=str,
help="",
actions=lambda std: [dsl.AddFeature(std)],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("c++03", self.config.available_features)
self.assertNotIn("c++11", self.config.available_features)
def test_no_actions(self):
self.litConfig.params["std"] = "c++03"
param = dsl.Parameter(
name="std", choices=["c++03"], type=str, help="", actions=lambda _: []
)
actions = param.getActions(self.config, self.litConfig.params)
self.assertEqual(actions, [])
def test_boolean_value_parsed_from_trueish_string_parameter(self):
self.litConfig.params["enable_exceptions"] = "True"
param = dsl.Parameter(
name="enable_exceptions",
choices=[True, False],
type=bool,
help="",
actions=lambda exceptions: [] if exceptions else _throw(),
)
self.assertEqual(param.getActions(self.config, self.litConfig.params), [])
def test_boolean_value_from_true_boolean_parameter(self):
self.litConfig.params["enable_exceptions"] = True
param = dsl.Parameter(
name="enable_exceptions",
choices=[True, False],
type=bool,
help="",
actions=lambda exceptions: [] if exceptions else _throw(),
)
self.assertEqual(param.getActions(self.config, self.litConfig.params), [])
def test_boolean_value_parsed_from_falseish_string_parameter(self):
self.litConfig.params["enable_exceptions"] = "False"
param = dsl.Parameter(
name="enable_exceptions",
choices=[True, False],
type=bool,
help="",
actions=lambda exceptions: []
if exceptions
else [dsl.AddFeature("-fno-exceptions")],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("-fno-exceptions", self.config.available_features)
def test_boolean_value_from_false_boolean_parameter(self):
self.litConfig.params["enable_exceptions"] = False
param = dsl.Parameter(
name="enable_exceptions",
choices=[True, False],
type=bool,
help="",
actions=lambda exceptions: []
if exceptions
else [dsl.AddFeature("-fno-exceptions")],
)
for a in param.getActions(self.config, self.litConfig.params):
a.applyTo(self.config)
self.assertIn("-fno-exceptions", self.config.available_features)
def test_list_parsed_from_comma_delimited_string_empty(self):
self.litConfig.params["additional_features"] = ""
param = dsl.Parameter(
name="additional_features", type=list, help="", actions=lambda f: f
)
self.assertEqual(param.getActions(self.config, self.litConfig.params), [])
def test_list_parsed_from_comma_delimited_string_1(self):
self.litConfig.params["additional_features"] = "feature1"
param = dsl.Parameter(
name="additional_features", type=list, help="", actions=lambda f: f
)
self.assertEqual(
param.getActions(self.config, self.litConfig.params), ["feature1"]
)
def test_list_parsed_from_comma_delimited_string_2(self):
self.litConfig.params["additional_features"] = "feature1,feature2"
param = dsl.Parameter(
name="additional_features", type=list, help="", actions=lambda f: f
)
self.assertEqual(
param.getActions(self.config, self.litConfig.params),
["feature1", "feature2"],
)
def test_list_parsed_from_comma_delimited_string_3(self):
self.litConfig.params["additional_features"] = "feature1,feature2, feature3"
param = dsl.Parameter(
name="additional_features", type=list, help="", actions=lambda f: f
)
self.assertEqual(
param.getActions(self.config, self.litConfig.params),
["feature1", "feature2", "feature3"],
)
if __name__ == "__main__":
unittest.main(verbosity=2)