shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[NFC][Py Reformat] Reformat python files in clang and clang-tools-extra

Browse Source 
This is an ongoing series of commits that are reformatting our
Python code.

Reformatting is done with `black`.

If you end up having problems merging this commit because you
have made changes to a python file, the best way to handle that
is to run git checkout --ours <yourfile> and then reformat it
with black.

If you run into any problems, post to discourse about it and
we will try to help.

RFC Thread below:

https://discourse.llvm.org/t/rfc-document-and-standardize-python-code-style

Reviewed By: MatzeB

Differential Revision: https://reviews.llvm.org/D150761
This commit is contained in:
Tobias Hieta 2023-05-17 10:56:49 +02:00
parent 9a515d8142
commit dd3c26a045
No known key found for this signature in database
GPG Key ID: 44F2485E45D59042
146 changed files with 13520 additions and 10910 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
clang-tools-extra/clang-include-fixer/find-all-symbols/tool/run-find-all-symbols.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python
#
#=- run-find-all-symbols.py - Parallel find-all-symbols runner -*- python -*-=#
# =- run-find-all-symbols.py - Parallel find-all-symbols runner -*- python -*-=#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
"""
Parallel find-all-symbols runner
@ -35,92 +35,99 @@ import threading
def find_compilation_database(path):
"""Adjusts the directory until a compilation database is found."""
result = './'
while not os.path.isfile(os.path.join(result, path)):
if os.path.realpath(result) == '/':
print('Error: could not find compilation database.')
sys.exit(1)
result += '../'
return os.path.realpath(result)
"""Adjusts the directory until a compilation database is found."""
result = "./"
while not os.path.isfile(os.path.join(result, path)):
if os.path.realpath(result) == "/":
print("Error: could not find compilation database.")
sys.exit(1)
result += "../"
return os.path.realpath(result)
def MergeSymbols(directory, args):
"""Merge all symbol files (yaml) in a given directory into a single file."""
invocation = [args.binary, '-merge-dir='+directory, args.saving_path]
subprocess.call(invocation)
print('Merge is finished. Saving results in ' + args.saving_path)
"""Merge all symbol files (yaml) in a given directory into a single file."""
invocation = [args.binary, "-merge-dir=" + directory, args.saving_path]
subprocess.call(invocation)
print("Merge is finished. Saving results in " + args.saving_path)
def run_find_all_symbols(args, tmpdir, build_path, queue):
"""Takes filenames out of queue and runs find-all-symbols on them."""
while True:
name = queue.get()
invocation = [args.binary, name, '-output-dir='+tmpdir, '-p='+build_path]
sys.stdout.write(' '.join(invocation) + '\n')
subprocess.call(invocation)
queue.task_done()
"""Takes filenames out of queue and runs find-all-symbols on them."""
while True:
name = queue.get()
invocation = [args.binary, name, "-output-dir=" + tmpdir, "-p=" + build_path]
sys.stdout.write(" ".join(invocation) + "\n")
subprocess.call(invocation)
queue.task_done()
def main():
parser = argparse.ArgumentParser(description='Runs find-all-symbols over all'
'files in a compilation database.')
parser.add_argument('-binary', metavar='PATH',
default='./bin/find-all-symbols',
help='path to find-all-symbols binary')
parser.add_argument('-j', type=int, default=0,
help='number of instances to be run in parallel.')
parser.add_argument('-p', dest='build_path',
help='path used to read a compilation database.')
parser.add_argument('-saving-path', default='./find_all_symbols_db.yaml',
help='result saving path')
args = parser.parse_args()
parser = argparse.ArgumentParser(
description="Runs find-all-symbols over all" "files in a compilation database."
)
parser.add_argument(
"-binary",
metavar="PATH",
default="./bin/find-all-symbols",
help="path to find-all-symbols binary",
)
parser.add_argument(
"-j", type=int, default=0, help="number of instances to be run in parallel."
)
parser.add_argument(
"-p", dest="build_path", help="path used to read a compilation database."
)
parser.add_argument(
"-saving-path", default="./find_all_symbols_db.yaml", help="result saving path"
)
args = parser.parse_args()
db_path = 'compile_commands.json'
db_path = "compile_commands.json"
if args.build_path is not None:
build_path = args.build_path
else:
build_path = find_compilation_database(db_path)
if args.build_path is not None:
build_path = args.build_path
else:
build_path = find_compilation_database(db_path)
tmpdir = tempfile.mkdtemp()
tmpdir = tempfile.mkdtemp()
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
files = [entry['file'] for entry in database]
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
files = [entry["file"] for entry in database]
# Filter out .rc files on Windows. CMake includes them for some reason.
files = [f for f in files if not f.endswith('.rc')]
# Filter out .rc files on Windows. CMake includes them for some reason.
files = [f for f in files if not f.endswith(".rc")]
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
try:
# Spin up a bunch of tidy-launching threads.
queue = Queue.Queue(max_task)
for _ in range(max_task):
t = threading.Thread(target=run_find_all_symbols,
args=(args, tmpdir, build_path, queue))
t.daemon = True
t.start()
try:
# Spin up a bunch of tidy-launching threads.
queue = Queue.Queue(max_task)
for _ in range(max_task):
t = threading.Thread(
target=run_find_all_symbols, args=(args, tmpdir, build_path, queue)
)
t.daemon = True
t.start()
# Fill the queue with files.
for name in files:
queue.put(name)
# Fill the queue with files.
for name in files:
queue.put(name)
# Wait for all threads to be done.
queue.join()
# Wait for all threads to be done.
queue.join()
MergeSymbols(tmpdir, args)
MergeSymbols(tmpdir, args)
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print("\nCtrl-C detected, goodbye.")
os.kill(0, 9)
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print('\nCtrl-C detected, goodbye.')
os.kill(0, 9)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

324
clang-tools-extra/clang-include-fixer/tool/clang-include-fixer.py
View File

@ -28,192 +28,216 @@ import vim
# set g:clang_include_fixer_path to the path to clang-include-fixer if it is not
# on the path.
# Change this to the full path if clang-include-fixer is not on the path.
binary = 'clang-include-fixer'
binary = "clang-include-fixer"
if vim.eval('exists("g:clang_include_fixer_path")') == "1":
binary = vim.eval('g:clang_include_fixer_path')
binary = vim.eval("g:clang_include_fixer_path")
maximum_suggested_headers = 3
if vim.eval('exists("g:clang_include_fixer_maximum_suggested_headers")') == "1":
maximum_suggested_headers = max(
1,
vim.eval('g:clang_include_fixer_maximum_suggested_headers'))
maximum_suggested_headers = max(
1, vim.eval("g:clang_include_fixer_maximum_suggested_headers")
)
increment_num = 5
if vim.eval('exists("g:clang_include_fixer_increment_num")') == "1":
increment_num = max(
1,
vim.eval('g:clang_include_fixer_increment_num'))
increment_num = max(1, vim.eval("g:clang_include_fixer_increment_num"))
jump_to_include = False
if vim.eval('exists("g:clang_include_fixer_jump_to_include")') == "1":
jump_to_include = vim.eval('g:clang_include_fixer_jump_to_include') != "0"
jump_to_include = vim.eval("g:clang_include_fixer_jump_to_include") != "0"
query_mode = False
if vim.eval('exists("g:clang_include_fixer_query_mode")') == "1":
query_mode = vim.eval('g:clang_include_fixer_query_mode') != "0"
query_mode = vim.eval("g:clang_include_fixer_query_mode") != "0"
def GetUserSelection(message, headers, maximum_suggested_headers):
eval_message = message + '\n'
for idx, header in enumerate(headers[0:maximum_suggested_headers]):
eval_message += "({0}). {1}\n".format(idx + 1, header)
eval_message += "Enter (q) to quit;"
if maximum_suggested_headers < len(headers):
eval_message += " (m) to show {0} more candidates.".format(
min(increment_num, len(headers) - maximum_suggested_headers))
eval_message = message + "\n"
for idx, header in enumerate(headers[0:maximum_suggested_headers]):
eval_message += "({0}). {1}\n".format(idx + 1, header)
eval_message += "Enter (q) to quit;"
if maximum_suggested_headers < len(headers):
eval_message += " (m) to show {0} more candidates.".format(
min(increment_num, len(headers) - maximum_suggested_headers)
)
eval_message += "\nSelect (default 1): "
res = vim.eval("input('{0}')".format(eval_message))
if res == '':
# choose the top ranked header by default
idx = 1
elif res == 'q':
raise Exception(' Insertion cancelled...')
elif res == 'm':
return GetUserSelection(message,
headers, maximum_suggested_headers + increment_num)
else:
try:
idx = int(res)
if idx <= 0 or idx > len(headers):
raise Exception()
except Exception:
# Show a new prompt on invalid option instead of aborting so that users
# don't need to wait for another clang-include-fixer run.
print("Invalid option: {}".format(res), file=sys.stderr)
return GetUserSelection(message, headers, maximum_suggested_headers)
return headers[idx - 1]
eval_message += "\nSelect (default 1): "
res = vim.eval("input('{0}')".format(eval_message))
if res == "":
# choose the top ranked header by default
idx = 1
elif res == "q":
raise Exception(" Insertion cancelled...")
elif res == "m":
return GetUserSelection(
message, headers, maximum_suggested_headers + increment_num
)
else:
try:
idx = int(res)
if idx <= 0 or idx > len(headers):
raise Exception()
except Exception:
# Show a new prompt on invalid option instead of aborting so that users
# don't need to wait for another clang-include-fixer run.
print("Invalid option: {}".format(res), file=sys.stderr)
return GetUserSelection(message, headers, maximum_suggested_headers)
return headers[idx - 1]
def execute(command, text):
# Avoid flashing a cmd prompt on Windows.
startupinfo = None
if sys.platform.startswith('win32'):
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
# Avoid flashing a cmd prompt on Windows.
startupinfo = None
if sys.platform.startswith("win32"):
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
p = subprocess.Popen(command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE, startupinfo=startupinfo)
return p.communicate(input=text.encode('utf-8'))
p = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
startupinfo=startupinfo,
)
return p.communicate(input=text.encode("utf-8"))
def InsertHeaderToVimBuffer(header, text):
command = [binary, "-stdin", "-insert-header=" + json.dumps(header),
vim.current.buffer.name]
stdout, stderr = execute(command, text)
if stderr:
raise Exception(stderr)
if stdout:
lines = stdout.splitlines()
sequence = difflib.SequenceMatcher(None, vim.current.buffer, lines)
line_num = None
for op in reversed(sequence.get_opcodes()):
if op[0] != 'equal':
vim.current.buffer[op[1]:op[2]] = lines[op[3]:op[4]]
if op[0] == 'insert':
# line_num in vim is 1-based.
line_num = op[1] + 1
command = [
binary,
"-stdin",
"-insert-header=" + json.dumps(header),
vim.current.buffer.name,
]
stdout, stderr = execute(command, text)
if stderr:
raise Exception(stderr)
if stdout:
lines = stdout.splitlines()
sequence = difflib.SequenceMatcher(None, vim.current.buffer, lines)
line_num = None
for op in reversed(sequence.get_opcodes()):
if op[0] != "equal":
vim.current.buffer[op[1] : op[2]] = lines[op[3] : op[4]]
if op[0] == "insert":
# line_num in vim is 1-based.
line_num = op[1] + 1
if jump_to_include and line_num:
vim.current.window.cursor = (line_num, 0)
if jump_to_include and line_num:
vim.current.window.cursor = (line_num, 0)
# The vim internal implementation (expand("cword"/"cWORD")) doesn't support
# our use case very well, we re-implement our own one.
def get_symbol_under_cursor():
line = vim.eval("line(\".\")")
# column number in vim is 1-based.
col = int(vim.eval("col(\".\")")) - 1
line_text = vim.eval("getline({0})".format(line))
if len(line_text) == 0: return ""
symbol_pos_begin = col
p = re.compile('[a-zA-Z0-9:_]')
while symbol_pos_begin >= 0 and p.match(line_text[symbol_pos_begin]):
symbol_pos_begin -= 1
line = vim.eval('line(".")')
# column number in vim is 1-based.
col = int(vim.eval('col(".")')) - 1
line_text = vim.eval("getline({0})".format(line))
if len(line_text) == 0:
return ""
symbol_pos_begin = col
p = re.compile("[a-zA-Z0-9:_]")
while symbol_pos_begin >= 0 and p.match(line_text[symbol_pos_begin]):
symbol_pos_begin -= 1
symbol_pos_end = col
while symbol_pos_end < len(line_text) and p.match(line_text[symbol_pos_end]):
symbol_pos_end += 1
return line_text[symbol_pos_begin+1:symbol_pos_end]
symbol_pos_end = col
while symbol_pos_end < len(line_text) and p.match(line_text[symbol_pos_end]):
symbol_pos_end += 1
return line_text[symbol_pos_begin + 1 : symbol_pos_end]
def main():
parser = argparse.ArgumentParser(
description='Vim integration for clang-include-fixer')
parser.add_argument('-db', default='yaml',
help='clang-include-fixer input format.')
parser.add_argument('-input', default='',
help='String to initialize the database.')
# Don't throw exception when parsing unknown arguments to make the script
# work in neovim.
# Neovim (at least v0.2.1) somehow mangles the sys.argv in a weird way: it
# will pass additional arguments (e.g. "-c script_host.py") to sys.argv,
# which makes the script fail.
args, _ = parser.parse_known_args()
parser = argparse.ArgumentParser(
description="Vim integration for clang-include-fixer"
)
parser.add_argument("-db", default="yaml", help="clang-include-fixer input format.")
parser.add_argument("-input", default="", help="String to initialize the database.")
# Don't throw exception when parsing unknown arguments to make the script
# work in neovim.
# Neovim (at least v0.2.1) somehow mangles the sys.argv in a weird way: it
# will pass additional arguments (e.g. "-c script_host.py") to sys.argv,
# which makes the script fail.
args, _ = parser.parse_known_args()
# Get the current text.
buf = vim.current.buffer
text = '\n'.join(buf)
# Get the current text.
buf = vim.current.buffer
text = "\n".join(buf)
if query_mode:
symbol = get_symbol_under_cursor()
if len(symbol) == 0:
print("Skip querying empty symbol.")
return
command = [binary, "-stdin", "-query-symbol="+get_symbol_under_cursor(),
"-db=" + args.db, "-input=" + args.input,
vim.current.buffer.name]
else:
# Run command to get all headers.
command = [binary, "-stdin", "-output-headers", "-db=" + args.db,
"-input=" + args.input, vim.current.buffer.name]
stdout, stderr = execute(command, text)
if stderr:
print("Error while running clang-include-fixer: {}".format(stderr),
file=sys.stderr)
if query_mode:
symbol = get_symbol_under_cursor()
if len(symbol) == 0:
print("Skip querying empty symbol.")
return
command = [
binary,
"-stdin",
"-query-symbol=" + get_symbol_under_cursor(),
"-db=" + args.db,
"-input=" + args.input,
vim.current.buffer.name,
]
else:
# Run command to get all headers.
command = [
binary,
"-stdin",
"-output-headers",
"-db=" + args.db,
"-input=" + args.input,
vim.current.buffer.name,
]
stdout, stderr = execute(command, text)
if stderr:
print(
"Error while running clang-include-fixer: {}".format(stderr),
file=sys.stderr,
)
return
include_fixer_context = json.loads(stdout)
query_symbol_infos = include_fixer_context["QuerySymbolInfos"]
if not query_symbol_infos:
print("The file is fine, no need to add a header.")
return
symbol = query_symbol_infos[0]["RawIdentifier"]
# The header_infos is already sorted by clang-include-fixer.
header_infos = include_fixer_context["HeaderInfos"]
# Deduplicate headers while keeping the order, so that the same header would
# not be suggested twice.
unique_headers = []
seen = set()
for header_info in header_infos:
header = header_info["Header"]
if header not in seen:
seen.add(header)
unique_headers.append(header)
if not unique_headers:
print("Couldn't find a header for {0}.".format(symbol))
return
try:
selected = unique_headers[0]
inserted_header_infos = header_infos
if len(unique_headers) > 1:
selected = GetUserSelection(
"choose a header file for {0}.".format(symbol),
unique_headers,
maximum_suggested_headers,
)
inserted_header_infos = [
header for header in header_infos if header["Header"] == selected
]
include_fixer_context["HeaderInfos"] = inserted_header_infos
InsertHeaderToVimBuffer(include_fixer_context, text)
print("Added #include {0} for {1}.".format(selected, symbol))
except Exception as error:
print(error, file=sys.stderr)
return
include_fixer_context = json.loads(stdout)
query_symbol_infos = include_fixer_context["QuerySymbolInfos"]
if not query_symbol_infos:
print("The file is fine, no need to add a header.")
return
symbol = query_symbol_infos[0]["RawIdentifier"]
# The header_infos is already sorted by clang-include-fixer.
header_infos = include_fixer_context["HeaderInfos"]
# Deduplicate headers while keeping the order, so that the same header would
# not be suggested twice.
unique_headers = []
seen = set()
for header_info in header_infos:
header = header_info["Header"]
if header not in seen:
seen.add(header)
unique_headers.append(header)
if not unique_headers:
print("Couldn't find a header for {0}.".format(symbol))
return
try:
selected = unique_headers[0]
inserted_header_infos = header_infos
if len(unique_headers) > 1:
selected = GetUserSelection(
"choose a header file for {0}.".format(symbol),
unique_headers, maximum_suggested_headers)
inserted_header_infos = [
header for header in header_infos if header["Header"] == selected]
include_fixer_context["HeaderInfos"] = inserted_header_infos
InsertHeaderToVimBuffer(include_fixer_context, text)
print("Added #include {0} for {1}.".format(selected, symbol))
except Exception as error:
print(error, file=sys.stderr)
return
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

882
clang-tools-extra/clang-tidy/add_new_check.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
#===- add_new_check.py - clang-tidy check generator ---------*- python -*--===#
# ===- add_new_check.py - clang-tidy check generator ---------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
# ===-----------------------------------------------------------------------===#
from __future__ import print_function
from __future__ import unicode_literals
@ -20,51 +20,57 @@ import sys
# Adapts the module's CMakelist file. Returns 'True' if it could add a new
# entry and 'False' if the entry already existed.
def adapt_cmake(module_path, check_name_camel):
filename = os.path.join(module_path, 'CMakeLists.txt')
filename = os.path.join(module_path, "CMakeLists.txt")
# The documentation files are encoded using UTF-8, however on Windows the
# default encoding might be different (e.g. CP-1252). To make sure UTF-8 is
# always used, use `io.open(filename, mode, encoding='utf8')` for reading and
# writing files here and elsewhere.
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
# The documentation files are encoded using UTF-8, however on Windows the
# default encoding might be different (e.g. CP-1252). To make sure UTF-8 is
# always used, use `io.open(filename, mode, encoding='utf8')` for reading and
# writing files here and elsewhere.
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
cpp_file = check_name_camel + '.cpp'
cpp_file = check_name_camel + ".cpp"
# Figure out whether this check already exists.
for line in lines:
if line.strip() == cpp_file:
return False
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
cpp_found = False
file_added = False
# Figure out whether this check already exists.
for line in lines:
cpp_line = line.strip().endswith('.cpp')
if (not file_added) and (cpp_line or cpp_found):
cpp_found = True
if (line.strip() > cpp_file) or (not cpp_line):
f.write(' ' + cpp_file + '\n')
file_added = True
f.write(line)
if line.strip() == cpp_file:
return False
return True
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
cpp_found = False
file_added = False
for line in lines:
cpp_line = line.strip().endswith(".cpp")
if (not file_added) and (cpp_line or cpp_found):
cpp_found = True
if (line.strip() > cpp_file) or (not cpp_line):
f.write(" " + cpp_file + "\n")
file_added = True
f.write(line)
return True
# Adds a header for the new check.
def write_header(module_path, module, namespace, check_name, check_name_camel):
filename = os.path.join(module_path, check_name_camel) + '.h'
print('Creating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
header_guard = ('LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_' + module.upper() + '_'
+ check_name_camel.upper() + '_H')
f.write('//===--- ')
f.write(os.path.basename(filename))
f.write(' - clang-tidy ')
f.write('-' * max(0, 42 - len(os.path.basename(filename))))
f.write('*- C++ -*-===//')
f.write("""
filename = os.path.join(module_path, check_name_camel) + ".h"
print("Creating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
header_guard = (
"LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_"
+ module.upper()
+ "_"
+ check_name_camel.upper()
+ "_H"
)
f.write("//===--- ")
f.write(os.path.basename(filename))
f.write(" - clang-tidy ")
f.write("-" * max(0, 42 - len(os.path.basename(filename))))
f.write("*- C++ -*-===//")
f.write(
"""
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@ -94,24 +100,29 @@ public:
} // namespace clang::tidy::%(namespace)s
#endif // %(header_guard)s
""" % {'header_guard': header_guard,
'check_name_camel': check_name_camel,
'check_name': check_name,
'module': module,
'namespace': namespace})
"""
% {
"header_guard": header_guard,
"check_name_camel": check_name_camel,
"check_name": check_name,
"module": module,
"namespace": namespace,
}
)
# Adds the implementation of the new check.
def write_implementation(module_path, module, namespace, check_name_camel):
filename = os.path.join(module_path, check_name_camel) + '.cpp'
print('Creating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
f.write('//===--- ')
f.write(os.path.basename(filename))
f.write(' - clang-tidy ')
f.write('-' * max(0, 51 - len(os.path.basename(filename))))
f.write('-===//')
f.write("""
filename = os.path.join(module_path, check_name_camel) + ".cpp"
print("Creating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
f.write("//===--- ")
f.write(os.path.basename(filename))
f.write(" - clang-tidy ")
f.write("-" * max(0, 51 - len(os.path.basename(filename))))
f.write("-===//")
f.write(
"""
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@ -144,138 +155,162 @@ void %(check_name)s::check(const MatchFinder::MatchResult &Result) {
}
} // namespace clang::tidy::%(namespace)s
""" % {'check_name': check_name_camel,
'module': module,
'namespace': namespace})
"""
% {"check_name": check_name_camel, "module": module, "namespace": namespace}
)
# Returns the source filename that implements the module.
def get_module_filename(module_path, module):
modulecpp = list(filter(
lambda p: p.lower() == module.lower() + 'tidymodule.cpp',
os.listdir(module_path)))[0]
return os.path.join(module_path, modulecpp)
modulecpp = list(
filter(
lambda p: p.lower() == module.lower() + "tidymodule.cpp",
os.listdir(module_path),
)
)[0]
return os.path.join(module_path, modulecpp)
# Modifies the module to include the new check.
def adapt_module(module_path, module, check_name, check_name_camel):
filename = get_module_filename(module_path, module)
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
filename = get_module_filename(module_path, module)
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
header_added = False
header_found = False
check_added = False
check_fq_name = module + '-' + check_name
check_decl = (' CheckFactories.registerCheck<' + check_name_camel +
'>(\n "' + check_fq_name + '");\n')
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
header_added = False
header_found = False
check_added = False
check_fq_name = module + "-" + check_name
check_decl = (
" CheckFactories.registerCheck<"
+ check_name_camel
+ '>(\n "'
+ check_fq_name
+ '");\n'
)
lines = iter(lines)
try:
while True:
line = next(lines)
if not header_added:
match = re.search('#include "(.*)"', line)
if match:
header_found = True
if match.group(1) > check_name_camel:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
elif header_found:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
if not check_added:
if line.strip() == '}':
check_added = True
f.write(check_decl)
else:
match = re.search('registerCheck<(.*)> *\( *(?:"([^"]*)")?', line)
prev_line = None
if match:
current_check_name = match.group(2)
if current_check_name is None:
# If we didn't find the check name on this line, look on the
# next one.
prev_line = line
lines = iter(lines)
try:
while True:
line = next(lines)
match = re.search(' *"([^"]*)"', line)
if match:
current_check_name = match.group(1)
if current_check_name > check_fq_name:
check_added = True
f.write(check_decl)
if prev_line:
f.write(prev_line)
f.write(line)
except StopIteration:
pass
if not header_added:
match = re.search('#include "(.*)"', line)
if match:
header_found = True
if match.group(1) > check_name_camel:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
elif header_found:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
if not check_added:
if line.strip() == "}":
check_added = True
f.write(check_decl)
else:
match = re.search(
'registerCheck<(.*)> *\( *(?:"([^"]*)")?', line
)
prev_line = None
if match:
current_check_name = match.group(2)
if current_check_name is None:
# If we didn't find the check name on this line, look on the
# next one.
prev_line = line
line = next(lines)
match = re.search(' *"([^"]*)"', line)
if match:
current_check_name = match.group(1)
if current_check_name > check_fq_name:
check_added = True
f.write(check_decl)
if prev_line:
f.write(prev_line)
f.write(line)
except StopIteration:
pass
# Adds a release notes entry.
def add_release_notes(module_path, module, check_name):
check_name_dashes = module + '-' + check_name
filename = os.path.normpath(os.path.join(module_path,
'../../docs/ReleaseNotes.rst'))
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
check_name_dashes = module + "-" + check_name
filename = os.path.normpath(
os.path.join(module_path, "../../docs/ReleaseNotes.rst")
)
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
lineMatcher = re.compile('New checks')
nextSectionMatcher = re.compile('New check aliases')
checkMatcher = re.compile('- New :doc:`(.*)')
lineMatcher = re.compile("New checks")
nextSectionMatcher = re.compile("New check aliases")
checkMatcher = re.compile("- New :doc:`(.*)")
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
note_added = False
header_found = False
add_note_here = False
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
note_added = False
header_found = False
add_note_here = False
for line in lines:
if not note_added:
match = lineMatcher.match(line)
match_next = nextSectionMatcher.match(line)
match_check = checkMatcher.match(line)
if match_check:
last_check = match_check.group(1)
if last_check > check_name_dashes:
add_note_here = True
for line in lines:
if not note_added:
match = lineMatcher.match(line)
match_next = nextSectionMatcher.match(line)
match_check = checkMatcher.match(line)
if match_check:
last_check = match_check.group(1)
if last_check > check_name_dashes:
add_note_here = True
if match_next:
add_note_here = True
if match_next:
add_note_here = True
if match:
header_found = True
f.write(line)
continue
if match:
header_found = True
f.write(line)
continue
if line.startswith('^^^^'):
f.write(line)
continue
if line.startswith("^^^^"):
f.write(line)
continue
if header_found and add_note_here:
if not line.startswith('^^^^'):
f.write("""- New :doc:`%s
if header_found and add_note_here:
if not line.startswith("^^^^"):
f.write(
"""- New :doc:`%s
<clang-tidy/checks/%s/%s>` check.
FIXME: add release notes.
""" % (check_name_dashes, module, check_name))
note_added = True
"""
% (check_name_dashes, module, check_name)
)
note_added = True
f.write(line)
f.write(line)
# Adds a test for the check.
def write_test(module_path, module, check_name, test_extension):
check_name_dashes = module + '-' + check_name
filename = os.path.normpath(os.path.join(
module_path, '..', '..', 'test', 'clang-tidy', 'checkers',
module, check_name + '.' + test_extension))
print('Creating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
f.write("""// RUN: %%check_clang_tidy %%s %(check_name_dashes)s %%t
check_name_dashes = module + "-" + check_name
filename = os.path.normpath(
os.path.join(
module_path,
"..",
"..",
"test",
"clang-tidy",
"checkers",
module,
check_name + "." + test_extension,
)
)
print("Creating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
f.write(
"""// RUN: %%check_clang_tidy %%s %(check_name_dashes)s %%t
// FIXME: Add something that triggers the check here.
void f();
@ -289,298 +324,327 @@ void f();
// FIXME: Add something that doesn't trigger the check here.
void awesome_f2();
""" % {'check_name_dashes': check_name_dashes})
"""
% {"check_name_dashes": check_name_dashes}
)
def get_actual_filename(dirname, filename):
if not os.path.isdir(dirname):
return ''
name = os.path.join(dirname, filename)
if (os.path.isfile(name)):
return name
caselessname = filename.lower()
for file in os.listdir(dirname):
if (file.lower() == caselessname):
return os.path.join(dirname, file)
return ''
if not os.path.isdir(dirname):
return ""
name = os.path.join(dirname, filename)
if os.path.isfile(name):
return name
caselessname = filename.lower()
for file in os.listdir(dirname):
if file.lower() == caselessname:
return os.path.join(dirname, file)
return ""
# Recreates the list of checks in the docs/clang-tidy/checks directory.
def update_checks_list(clang_tidy_path):
docs_dir = os.path.join(clang_tidy_path, '../docs/clang-tidy/checks')
filename = os.path.normpath(os.path.join(docs_dir, 'list.rst'))
# Read the content of the current list.rst file
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
# Get all existing docs
doc_files = []
for subdir in filter(lambda s: os.path.isdir(os.path.join(docs_dir, s)), os.listdir(docs_dir)):
for file in filter(lambda s: s.endswith('.rst'), os.listdir(os.path.join(docs_dir, subdir))):
doc_files.append([subdir, file])
doc_files.sort()
docs_dir = os.path.join(clang_tidy_path, "../docs/clang-tidy/checks")
filename = os.path.normpath(os.path.join(docs_dir, "list.rst"))
# Read the content of the current list.rst file
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
# Get all existing docs
doc_files = []
for subdir in filter(
lambda s: os.path.isdir(os.path.join(docs_dir, s)), os.listdir(docs_dir)
):
for file in filter(
lambda s: s.endswith(".rst"), os.listdir(os.path.join(docs_dir, subdir))
):
doc_files.append([subdir, file])
doc_files.sort()
# We couldn't find the source file from the check name, so try to find the
# class name that corresponds to the check in the module file.
def filename_from_module(module_name, check_name):
module_path = os.path.join(clang_tidy_path, module_name)
if not os.path.isdir(module_path):
return ''
module_file = get_module_filename(module_path, module_name)
if not os.path.isfile(module_file):
return ''
with io.open(module_file, 'r') as f:
code = f.read()
full_check_name = module_name + '-' + check_name
name_pos = code.find('"' + full_check_name + '"')
if name_pos == -1:
return ''
stmt_end_pos = code.find(';', name_pos)
if stmt_end_pos == -1:
return ''
stmt_start_pos = code.rfind(';', 0, name_pos)
if stmt_start_pos == -1:
stmt_start_pos = code.rfind('{', 0, name_pos)
if stmt_start_pos == -1:
return ''
stmt = code[stmt_start_pos+1:stmt_end_pos]
matches = re.search('registerCheck<([^>:]*)>\(\s*"([^"]*)"\s*\)', stmt)
if matches and matches[2] == full_check_name:
class_name = matches[1]
if '::' in class_name:
parts = class_name.split('::')
class_name = parts[-1]
class_path = os.path.join(clang_tidy_path, module_name, '..', *parts[0:-1])
# We couldn't find the source file from the check name, so try to find the
# class name that corresponds to the check in the module file.
def filename_from_module(module_name, check_name):
module_path = os.path.join(clang_tidy_path, module_name)
if not os.path.isdir(module_path):
return ""
module_file = get_module_filename(module_path, module_name)
if not os.path.isfile(module_file):
return ""
with io.open(module_file, "r") as f:
code = f.read()
full_check_name = module_name + "-" + check_name
name_pos = code.find('"' + full_check_name + '"')
if name_pos == -1:
return ""
stmt_end_pos = code.find(";", name_pos)
if stmt_end_pos == -1:
return ""
stmt_start_pos = code.rfind(";", 0, name_pos)
if stmt_start_pos == -1:
stmt_start_pos = code.rfind("{", 0, name_pos)
if stmt_start_pos == -1:
return ""
stmt = code[stmt_start_pos + 1 : stmt_end_pos]
matches = re.search('registerCheck<([^>:]*)>\(\s*"([^"]*)"\s*\)', stmt)
if matches and matches[2] == full_check_name:
class_name = matches[1]
if "::" in class_name:
parts = class_name.split("::")
class_name = parts[-1]
class_path = os.path.join(
clang_tidy_path, module_name, "..", *parts[0:-1]
)
else:
class_path = os.path.join(clang_tidy_path, module_name)
return get_actual_filename(class_path, class_name + ".cpp")
return ""
# Examine code looking for a c'tor definition to get the base class name.
def get_base_class(code, check_file):
check_class_name = os.path.splitext(os.path.basename(check_file))[0]
ctor_pattern = check_class_name + "\([^:]*\)\s*:\s*([A-Z][A-Za-z0-9]*Check)\("
matches = re.search("\s+" + check_class_name + "::" + ctor_pattern, code)
# The constructor might be inline in the header.
if not matches:
header_file = os.path.splitext(check_file)[0] + ".h"
if not os.path.isfile(header_file):
return ""
with io.open(header_file, encoding="utf8") as f:
code = f.read()
matches = re.search(" " + ctor_pattern, code)
if matches and matches[1] != "ClangTidyCheck":
return matches[1]
return ""
# Some simple heuristics to figure out if a check has an autofix or not.
def has_fixits(code):
for needle in [
"FixItHint",
"ReplacementText",
"fixit",
"TransformerClangTidyCheck",
]:
if needle in code:
return True
return False
# Try to figure out of the check supports fixits.
def has_auto_fix(check_name):
dirname, _, check_name = check_name.partition("-")
check_file = get_actual_filename(
os.path.join(clang_tidy_path, dirname),
get_camel_check_name(check_name) + ".cpp",
)
if not os.path.isfile(check_file):
# Some older checks don't end with 'Check.cpp'
check_file = get_actual_filename(
os.path.join(clang_tidy_path, dirname),
get_camel_name(check_name) + ".cpp",
)
if not os.path.isfile(check_file):
# Some checks aren't in a file based on the check name.
check_file = filename_from_module(dirname, check_name)
if not check_file or not os.path.isfile(check_file):
return ""
with io.open(check_file, encoding="utf8") as f:
code = f.read()
if has_fixits(code):
return ' "Yes"'
base_class = get_base_class(code, check_file)
if base_class:
base_file = os.path.join(clang_tidy_path, dirname, base_class + ".cpp")
if os.path.isfile(base_file):
with io.open(base_file, encoding="utf8") as f:
code = f.read()
if has_fixits(code):
return ' "Yes"'
return ""
def process_doc(doc_file):
check_name = doc_file[0] + "-" + doc_file[1].replace(".rst", "")
with io.open(os.path.join(docs_dir, *doc_file), "r", encoding="utf8") as doc:
content = doc.read()
match = re.search(".*:orphan:.*", content)
if match:
# Orphan page, don't list it.
return "", ""
match = re.search(".*:http-equiv=refresh: \d+;URL=(.*).html(.*)", content)
# Is it a redirect?
return check_name, match
def format_link(doc_file):
check_name, match = process_doc(doc_file)
if not match and check_name:
return " `%(check_name)s <%(module)s/%(check)s.html>`_,%(autofix)s\n" % {
"check_name": check_name,
"module": doc_file[0],
"check": doc_file[1].replace(".rst", ""),
"autofix": has_auto_fix(check_name),
}
else:
class_path = os.path.join(clang_tidy_path, module_name)
return get_actual_filename(class_path, class_name + '.cpp')
return ""
return ''
def format_link_alias(doc_file):
check_name, match = process_doc(doc_file)
if match and check_name:
module = doc_file[0]
check_file = doc_file[1].replace(".rst", "")
if match.group(1) == "https://clang.llvm.org/docs/analyzer/checkers":
title = "Clang Static Analyzer " + check_file
# Preserve the anchor in checkers.html from group 2.
target = match.group(1) + ".html" + match.group(2)
autofix = ""
else:
redirect_parts = re.search("^\.\./([^/]*)/([^/]*)$", match.group(1))
title = redirect_parts[1] + "-" + redirect_parts[2]
target = redirect_parts[1] + "/" + redirect_parts[2] + ".html"
autofix = has_auto_fix(title)
# Examine code looking for a c'tor definition to get the base class name.
def get_base_class(code, check_file):
check_class_name = os.path.splitext(os.path.basename(check_file))[0]
ctor_pattern = check_class_name + '\([^:]*\)\s*:\s*([A-Z][A-Za-z0-9]*Check)\('
matches = re.search('\s+' + check_class_name + '::' + ctor_pattern, code)
# The checker is just a redirect.
return (
" `%(check_name)s <%(module)s/%(check_file)s.html>`_, `%(title)s <%(target)s>`_,%(autofix)s\n"
% {
"check_name": check_name,
"module": module,
"check_file": check_file,
"target": target,
"title": title,
"autofix": autofix,
}
)
return ""
# The constructor might be inline in the header.
if not matches:
header_file = os.path.splitext(check_file)[0] + '.h'
if not os.path.isfile(header_file):
return ''
with io.open(header_file, encoding='utf8') as f:
code = f.read()
matches = re.search(' ' + ctor_pattern, code)
checks = map(format_link, doc_files)
checks_alias = map(format_link_alias, doc_files)
if matches and matches[1] != 'ClangTidyCheck':
return matches[1]
return ''
# Some simple heuristics to figure out if a check has an autofix or not.
def has_fixits(code):
for needle in ['FixItHint', 'ReplacementText', 'fixit',
'TransformerClangTidyCheck']:
if needle in code:
return True
return False
# Try to figure out of the check supports fixits.
def has_auto_fix(check_name):
dirname, _, check_name = check_name.partition('-')
check_file = get_actual_filename(os.path.join(clang_tidy_path, dirname),
get_camel_check_name(check_name) + '.cpp')
if not os.path.isfile(check_file):
# Some older checks don't end with 'Check.cpp'
check_file = get_actual_filename(os.path.join(clang_tidy_path, dirname),
get_camel_name(check_name) + '.cpp')
if not os.path.isfile(check_file):
# Some checks aren't in a file based on the check name.
check_file = filename_from_module(dirname, check_name)
if not check_file or not os.path.isfile(check_file):
return ''
with io.open(check_file, encoding='utf8') as f:
code = f.read()
if has_fixits(code):
return ' "Yes"'
base_class = get_base_class(code, check_file)
if base_class:
base_file = os.path.join(clang_tidy_path, dirname, base_class + '.cpp')
if os.path.isfile(base_file):
with io.open(base_file, encoding='utf8') as f:
code = f.read()
if has_fixits(code):
return ' "Yes"'
return ''
def process_doc(doc_file):
check_name = doc_file[0] + '-' + doc_file[1].replace('.rst', '')
with io.open(os.path.join(docs_dir, *doc_file), 'r', encoding='utf8') as doc:
content = doc.read()
match = re.search('.*:orphan:.*', content)
if match:
# Orphan page, don't list it.
return '', ''
match = re.search('.*:http-equiv=refresh: \d+;URL=(.*).html(.*)',
content)
# Is it a redirect?
return check_name, match
def format_link(doc_file):
check_name, match = process_doc(doc_file)
if not match and check_name:
return ' `%(check_name)s <%(module)s/%(check)s.html>`_,%(autofix)s\n' % {
'check_name': check_name,
'module': doc_file[0],
'check': doc_file[1].replace('.rst', ''),
'autofix': has_auto_fix(check_name)
}
else:
return ''
def format_link_alias(doc_file):
check_name, match = process_doc(doc_file)
if match and check_name:
module = doc_file[0]
check_file = doc_file[1].replace('.rst', '')
if match.group(1) == 'https://clang.llvm.org/docs/analyzer/checkers':
title = 'Clang Static Analyzer ' + check_file
# Preserve the anchor in checkers.html from group 2.
target = match.group(1) + '.html' + match.group(2)
autofix = ''
else:
redirect_parts = re.search('^\.\./([^/]*)/([^/]*)$', match.group(1))
title = redirect_parts[1] + '-' + redirect_parts[2]
target = redirect_parts[1] + '/' + redirect_parts[2] + '.html'
autofix = has_auto_fix(title)
# The checker is just a redirect.
return ' `%(check_name)s <%(module)s/%(check_file)s.html>`_, `%(title)s <%(target)s>`_,%(autofix)s\n' % {
'check_name': check_name,
'module': module,
'check_file': check_file,
'target': target,
'title': title,
'autofix': autofix
}
return ''
checks = map(format_link, doc_files)
checks_alias = map(format_link_alias, doc_files)
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
for line in lines:
f.write(line)
if line.strip() == '.. csv-table::':
# We dump the checkers
f.write(' :header: "Name", "Offers fixes"\n\n')
f.writelines(checks)
# and the aliases
f.write('\n\n')
f.write('.. csv-table:: Aliases..\n')
f.write(' :header: "Name", "Redirect", "Offers fixes"\n\n')
f.writelines(checks_alias)
break
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
for line in lines:
f.write(line)
if line.strip() == ".. csv-table::":
# We dump the checkers
f.write(' :header: "Name", "Offers fixes"\n\n')
f.writelines(checks)
# and the aliases
f.write("\n\n")
f.write(".. csv-table:: Aliases..\n")
f.write(' :header: "Name", "Redirect", "Offers fixes"\n\n')
f.writelines(checks_alias)
break
# Adds a documentation for the check.
def write_docs(module_path, module, check_name):
check_name_dashes = module + '-' + check_name
filename = os.path.normpath(os.path.join(
module_path, '../../docs/clang-tidy/checks/', module, check_name + '.rst'))
print('Creating %s...' % filename)
with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
f.write(""".. title:: clang-tidy - %(check_name_dashes)s
check_name_dashes = module + "-" + check_name
filename = os.path.normpath(
os.path.join(
module_path, "../../docs/clang-tidy/checks/", module, check_name + ".rst"
)
)
print("Creating %s..." % filename)
with io.open(filename, "w", encoding="utf8", newline="\n") as f:
f.write(
""".. title:: clang-tidy - %(check_name_dashes)s
%(check_name_dashes)s
%(underline)s
FIXME: Describe what patterns does the check detect and why. Give examples.
""" % {'check_name_dashes': check_name_dashes,
'underline': '=' * len(check_name_dashes)})
"""
% {
"check_name_dashes": check_name_dashes,
"underline": "=" * len(check_name_dashes),
}
)
def get_camel_name(check_name):
return ''.join(map(lambda elem: elem.capitalize(),
check_name.split('-')))
return "".join(map(lambda elem: elem.capitalize(), check_name.split("-")))
def get_camel_check_name(check_name):
return get_camel_name(check_name) + 'Check'
return get_camel_name(check_name) + "Check"
def main():
language_to_extension = {
'c': 'c',
'c++': 'cpp',
'objc': 'm',
'objc++': 'mm',
}
parser = argparse.ArgumentParser()
parser.add_argument(
'--update-docs',
action='store_true',
help='just update the list of documentation files, then exit')
parser.add_argument(
'--language',
help='language to use for new check (defaults to c++)',
choices=language_to_extension.keys(),
default='c++',
metavar='LANG')
parser.add_argument(
'module',
nargs='?',
help='module directory under which to place the new tidy check (e.g., misc)')
parser.add_argument(
'check',
nargs='?',
help='name of new tidy check to add (e.g. foo-do-the-stuff)')
args = parser.parse_args()
language_to_extension = {
"c": "c",
"c++": "cpp",
"objc": "m",
"objc++": "mm",
}
parser = argparse.ArgumentParser()
parser.add_argument(
"--update-docs",
action="store_true",
help="just update the list of documentation files, then exit",
)
parser.add_argument(
"--language",
help="language to use for new check (defaults to c++)",
choices=language_to_extension.keys(),
default="c++",
metavar="LANG",
)
parser.add_argument(
"module",
nargs="?",
help="module directory under which to place the new tidy check (e.g., misc)",
)
parser.add_argument(
"check", nargs="?", help="name of new tidy check to add (e.g. foo-do-the-stuff)"
)
args = parser.parse_args()
if args.update_docs:
update_checks_list(os.path.dirname(sys.argv[0]))
return
if args.update_docs:
update_checks_list(os.path.dirname(sys.argv[0]))
return
if not args.module or not args.check:
print('Module and check must be specified.')
parser.print_usage()
return
if not args.module or not args.check:
print("Module and check must be specified.")
parser.print_usage()
return
module = args.module
check_name = args.check
check_name_camel = get_camel_check_name(check_name)
if check_name.startswith(module):
print('Check name "%s" must not start with the module "%s". Exiting.' % (
check_name, module))
return
clang_tidy_path = os.path.dirname(sys.argv[0])
module_path = os.path.join(clang_tidy_path, module)
module = args.module
check_name = args.check
check_name_camel = get_camel_check_name(check_name)
if check_name.startswith(module):
print(
'Check name "%s" must not start with the module "%s". Exiting.'
% (check_name, module)
)
return
clang_tidy_path = os.path.dirname(sys.argv[0])
module_path = os.path.join(clang_tidy_path, module)
if not adapt_cmake(module_path, check_name_camel):
return
if not adapt_cmake(module_path, check_name_camel):
return
# Map module names to namespace names that don't conflict with widely used top-level namespaces.
if module == 'llvm':
namespace = module + '_check'
else:
namespace = module
# Map module names to namespace names that don't conflict with widely used top-level namespaces.
if module == "llvm":
namespace = module + "_check"
else:
namespace = module
write_header(module_path, module, namespace, check_name, check_name_camel)
write_implementation(module_path, module, namespace, check_name_camel)
adapt_module(module_path, module, check_name, check_name_camel)
add_release_notes(module_path, module, check_name)
test_extension = language_to_extension.get(args.language)
write_test(module_path, module, check_name, test_extension)
write_docs(module_path, module, check_name)
update_checks_list(clang_tidy_path)
print('Done. Now it\'s your turn!')
write_header(module_path, module, namespace, check_name, check_name_camel)
write_implementation(module_path, module, namespace, check_name_camel)
adapt_module(module_path, module, check_name, check_name_camel)
add_release_notes(module_path, module, check_name)
test_extension = language_to_extension.get(args.language)
write_test(module_path, module, check_name, test_extension)
write_docs(module_path, module, check_name)
update_checks_list(clang_tidy_path)
print("Done. Now it's your turn!")
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

576
clang-tools-extra/clang-tidy/rename_check.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
#===- rename_check.py - clang-tidy check renamer ------------*- python -*--===#
# ===- rename_check.py - clang-tidy check renamer ------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
# ===-----------------------------------------------------------------------===#
from __future__ import unicode_literals
@ -16,321 +16,387 @@ import io
import os
import re
def replaceInFileRegex(fileName, sFrom, sTo):
if sFrom == sTo:
return
if sFrom == sTo:
return
# The documentation files are encoded using UTF-8, however on Windows the
# default encoding might be different (e.g. CP-1252). To make sure UTF-8 is
# always used, use `io.open(filename, mode, encoding='utf8')` for reading and
# writing files here and elsewhere.
txt = None
with io.open(fileName, 'r', encoding='utf8') as f:
txt = f.read()
# The documentation files are encoded using UTF-8, however on Windows the
# default encoding might be different (e.g. CP-1252). To make sure UTF-8 is
# always used, use `io.open(filename, mode, encoding='utf8')` for reading and
# writing files here and elsewhere.
txt = None
with io.open(fileName, "r", encoding="utf8") as f:
txt = f.read()
txt = re.sub(sFrom, sTo, txt)
print("Replacing '%s' -> '%s' in '%s'..." % (sFrom, sTo, fileName))
with io.open(fileName, 'w', encoding='utf8') as f:
f.write(txt)
txt = re.sub(sFrom, sTo, txt)
print("Replacing '%s' -> '%s' in '%s'..." % (sFrom, sTo, fileName))
with io.open(fileName, "w", encoding="utf8") as f:
f.write(txt)
def replaceInFile(fileName, sFrom, sTo):
if sFrom == sTo:
return
txt = None
with io.open(fileName, 'r', encoding='utf8') as f:
txt = f.read()
if sFrom == sTo:
return
txt = None
with io.open(fileName, "r", encoding="utf8") as f:
txt = f.read()
if sFrom not in txt:
return
if sFrom not in txt:
return
txt = txt.replace(sFrom, sTo)
print("Replacing '%s' -> '%s' in '%s'..." % (sFrom, sTo, fileName))
with io.open(fileName, 'w', encoding='utf8') as f:
f.write(txt)
txt = txt.replace(sFrom, sTo)
print("Replacing '%s' -> '%s' in '%s'..." % (sFrom, sTo, fileName))
with io.open(fileName, "w", encoding="utf8") as f:
f.write(txt)
def generateCommentLineHeader(filename):
return ''.join(['//===--- ',
os.path.basename(filename),
' - clang-tidy ',
'-' * max(0, 42 - len(os.path.basename(filename))),
'*- C++ -*-===//'])
return "".join(
[
"//===--- ",
os.path.basename(filename),
" - clang-tidy ",
"-" * max(0, 42 - len(os.path.basename(filename))),
"*- C++ -*-===//",
]
)
def generateCommentLineSource(filename):
return ''.join(['//===--- ',
os.path.basename(filename),
' - clang-tidy',
'-' * max(0, 52 - len(os.path.basename(filename))),
'-===//'])
return "".join(
[
"//===--- ",
os.path.basename(filename),
" - clang-tidy",
"-" * max(0, 52 - len(os.path.basename(filename))),
"-===//",
]
)
def fileRename(fileName, sFrom, sTo):
if sFrom not in fileName or sFrom == sTo:
return fileName
newFileName = fileName.replace(sFrom, sTo)
print("Renaming '%s' -> '%s'..." % (fileName, newFileName))
os.rename(fileName, newFileName)
return newFileName
if sFrom not in fileName or sFrom == sTo:
return fileName
newFileName = fileName.replace(sFrom, sTo)
print("Renaming '%s' -> '%s'..." % (fileName, newFileName))
os.rename(fileName, newFileName)
return newFileName
def deleteMatchingLines(fileName, pattern):
lines = None
with io.open(fileName, 'r', encoding='utf8') as f:
lines = f.readlines()
lines = None
with io.open(fileName, "r", encoding="utf8") as f:
lines = f.readlines()
not_matching_lines = [l for l in lines if not re.search(pattern, l)]
if len(not_matching_lines) == len(lines):
return False
not_matching_lines = [l for l in lines if not re.search(pattern, l)]
if len(not_matching_lines) == len(lines):
return False
print("Removing lines matching '%s' in '%s'..." % (pattern, fileName))
print(' ' + ' '.join([l for l in lines if re.search(pattern, l)]))
with io.open(fileName, 'w', encoding='utf8') as f:
f.writelines(not_matching_lines)
print("Removing lines matching '%s' in '%s'..." % (pattern, fileName))
print(" " + " ".join([l for l in lines if re.search(pattern, l)]))
with io.open(fileName, "w", encoding="utf8") as f:
f.writelines(not_matching_lines)
return True
return True
def getListOfFiles(clang_tidy_path):
files = glob.glob(os.path.join(clang_tidy_path, '**'), recursive=True)
files += [os.path.normpath(os.path.join(clang_tidy_path,
'../docs/ReleaseNotes.rst'))]
files += glob.glob(os.path.join(clang_tidy_path, '..', 'test',
'clang-tidy', 'checkers', '**'), recursive=True)
files += glob.glob(os.path.join(clang_tidy_path, '..', 'docs',
'clang-tidy', 'checks', '*.rst'))
files += glob.glob(os.path.join(clang_tidy_path, '..', 'docs',
'clang-tidy', 'checks', "*", "*.rst"), recursive=True)
return [filename for filename in files if os.path.isfile(filename)]
files = glob.glob(os.path.join(clang_tidy_path, "**"), recursive=True)
files += [
os.path.normpath(os.path.join(clang_tidy_path, "../docs/ReleaseNotes.rst"))
]
files += glob.glob(
os.path.join(clang_tidy_path, "..", "test", "clang-tidy", "checkers", "**"),
recursive=True,
)
files += glob.glob(
os.path.join(clang_tidy_path, "..", "docs", "clang-tidy", "checks", "*.rst")
)
files += glob.glob(
os.path.join(
clang_tidy_path, "..", "docs", "clang-tidy", "checks", "*", "*.rst"
),
recursive=True,
)
return [filename for filename in files if os.path.isfile(filename)]
# Adapts the module's CMakelist file. Returns 'True' if it could add a new
# entry and 'False' if the entry already existed.
def adapt_cmake(module_path, check_name_camel):
filename = os.path.join(module_path, 'CMakeLists.txt')
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
filename = os.path.join(module_path, "CMakeLists.txt")
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
cpp_file = check_name_camel + '.cpp'
cpp_file = check_name_camel + ".cpp"
# Figure out whether this check already exists.
for line in lines:
if line.strip() == cpp_file:
return False
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8') as f:
cpp_found = False
file_added = False
# Figure out whether this check already exists.
for line in lines:
cpp_line = line.strip().endswith('.cpp')
if (not file_added) and (cpp_line or cpp_found):
cpp_found = True
if (line.strip() > cpp_file) or (not cpp_line):
f.write(' ' + cpp_file + '\n')
file_added = True
f.write(line)
if line.strip() == cpp_file:
return False
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8") as f:
cpp_found = False
file_added = False
for line in lines:
cpp_line = line.strip().endswith(".cpp")
if (not file_added) and (cpp_line or cpp_found):
cpp_found = True
if (line.strip() > cpp_file) or (not cpp_line):
f.write(" " + cpp_file + "\n")
file_added = True
f.write(line)
return True
return True
# Modifies the module to include the new check.
def adapt_module(module_path, module, check_name, check_name_camel):
modulecpp = next(iter(filter(
lambda p: p.lower() == module.lower() + 'tidymodule.cpp',
os.listdir(module_path))))
filename = os.path.join(module_path, modulecpp)
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
modulecpp = next(
iter(
filter(
lambda p: p.lower() == module.lower() + "tidymodule.cpp",
os.listdir(module_path),
)
)
)
filename = os.path.join(module_path, modulecpp)
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8') as f:
header_added = False
header_found = False
check_added = False
check_decl = (' CheckFactories.registerCheck<' + check_name_camel +
'>(\n "' + check_name + '");\n')
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8") as f:
header_added = False
header_found = False
check_added = False
check_decl = (
" CheckFactories.registerCheck<"
+ check_name_camel
+ '>(\n "'
+ check_name
+ '");\n'
)
for line in lines:
if not header_added:
match = re.search('#include "(.*)"', line)
if match:
header_found = True
if match.group(1) > check_name_camel:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
elif header_found:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
for line in lines:
if not header_added:
match = re.search('#include "(.*)"', line)
if match:
header_found = True
if match.group(1) > check_name_camel:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
elif header_found:
header_added = True
f.write('#include "' + check_name_camel + '.h"\n')
if not check_added:
if line.strip() == '}':
check_added = True
f.write(check_decl)
else:
match = re.search('registerCheck<(.*)>', line)
if match and match.group(1) > check_name_camel:
check_added = True
f.write(check_decl)
f.write(line)
if not check_added:
if line.strip() == "}":
check_added = True
f.write(check_decl)
else:
match = re.search("registerCheck<(.*)>", line)
if match and match.group(1) > check_name_camel:
check_added = True
f.write(check_decl)
f.write(line)
# Adds a release notes entry.
def add_release_notes(clang_tidy_path, old_check_name, new_check_name):
filename = os.path.normpath(os.path.join(clang_tidy_path,
'../docs/ReleaseNotes.rst'))
with io.open(filename, 'r', encoding='utf8') as f:
lines = f.readlines()
filename = os.path.normpath(
os.path.join(clang_tidy_path, "../docs/ReleaseNotes.rst")
)
with io.open(filename, "r", encoding="utf8") as f:
lines = f.readlines()
lineMatcher = re.compile('Renamed checks')
nextSectionMatcher = re.compile('Improvements to include-fixer')
checkMatcher = re.compile('- The \'(.*)')
lineMatcher = re.compile("Renamed checks")
nextSectionMatcher = re.compile("Improvements to include-fixer")
checkMatcher = re.compile("- The '(.*)")
print('Updating %s...' % filename)
with io.open(filename, 'w', encoding='utf8') as f:
note_added = False
header_found = False
add_note_here = False
print("Updating %s..." % filename)
with io.open(filename, "w", encoding="utf8") as f:
note_added = False
header_found = False
add_note_here = False
for line in lines:
if not note_added:
match = lineMatcher.match(line)
match_next = nextSectionMatcher.match(line)
match_check = checkMatcher.match(line)
if match_check:
last_check = match_check.group(1)
if last_check > old_check_name:
add_note_here = True
for line in lines:
if not note_added:
match = lineMatcher.match(line)
match_next = nextSectionMatcher.match(line)
match_check = checkMatcher.match(line)
if match_check:
last_check = match_check.group(1)
if last_check > old_check_name:
add_note_here = True
if match_next:
add_note_here = True
if match_next:
add_note_here = True
if match:
header_found = True
f.write(line)
continue
if match:
header_found = True
f.write(line)
continue
if line.startswith('^^^^'):
f.write(line)
continue
if line.startswith("^^^^"):
f.write(line)
continue
if header_found and add_note_here:
if not line.startswith('^^^^'):
f.write("""- The '%s' check was renamed to :doc:`%s
if header_found and add_note_here:
if not line.startswith("^^^^"):
f.write(
"""- The '%s' check was renamed to :doc:`%s
<clang-tidy/checks/%s/%s>`
""" % (old_check_name, new_check_name,
new_check_name.split('-', 1)[0],
'-'.join(new_check_name.split('-')[1:])))
note_added = True
"""
% (
old_check_name,
new_check_name,
new_check_name.split("-", 1)[0],
"-".join(new_check_name.split("-")[1:]),
)
)
note_added = True
f.write(line)
f.write(line)
def main():
parser = argparse.ArgumentParser(description='Rename clang-tidy check.')
parser.add_argument('old_check_name', type=str,
help='Old check name.')
parser.add_argument('new_check_name', type=str,
help='New check name.')
parser.add_argument('--check_class_name', type=str,
help='Old name of the class implementing the check.')
args = parser.parse_args()
parser = argparse.ArgumentParser(description="Rename clang-tidy check.")
parser.add_argument("old_check_name", type=str, help="Old check name.")
parser.add_argument("new_check_name", type=str, help="New check name.")
parser.add_argument(
"--check_class_name",
type=str,
help="Old name of the class implementing the check.",
)
args = parser.parse_args()
old_module = args.old_check_name.split('-')[0]
new_module = args.new_check_name.split('-')[0]
old_name = '-'.join(args.old_check_name.split('-')[1:])
new_name = '-'.join(args.new_check_name.split('-')[1:])
old_module = args.old_check_name.split("-")[0]
new_module = args.new_check_name.split("-")[0]
old_name = "-".join(args.old_check_name.split("-")[1:])
new_name = "-".join(args.new_check_name.split("-")[1:])
if args.check_class_name:
check_name_camel = args.check_class_name
else:
check_name_camel = (''.join(map(lambda elem: elem.capitalize(),
old_name.split('-'))) +
'Check')
new_check_name_camel = (''.join(map(lambda elem: elem.capitalize(),
new_name.split('-'))) +
'Check')
clang_tidy_path = os.path.dirname(__file__)
header_guard_variants = [
(args.old_check_name.replace('-', '_')).upper() + '_CHECK',
(old_module + '_' + check_name_camel).upper(),
(old_module + '_' + new_check_name_camel).upper(),
args.old_check_name.replace('-', '_').upper()]
header_guard_new = (new_module + '_' + new_check_name_camel).upper()
old_module_path = os.path.join(clang_tidy_path, old_module)
new_module_path = os.path.join(clang_tidy_path, new_module)
if (old_module != new_module):
# Remove the check from the old module.
cmake_lists = os.path.join(old_module_path, 'CMakeLists.txt')
check_found = deleteMatchingLines(cmake_lists, '\\b' + check_name_camel)
if not check_found:
print("Check name '%s' not found in %s. Exiting." %
(check_name_camel, cmake_lists))
return 1
modulecpp = next(iter(filter(
lambda p: p.lower() == old_module.lower() + 'tidymodule.cpp',
os.listdir(old_module_path))))
deleteMatchingLines(os.path.join(old_module_path, modulecpp),
'\\b' + check_name_camel + '|\\b' + args.old_check_name)
for filename in getListOfFiles(clang_tidy_path):
originalName = filename
filename = fileRename(filename, old_module + "/" + old_name, new_module + "/" + new_name)
filename = fileRename(filename, args.old_check_name, args.new_check_name)
filename = fileRename(filename, check_name_camel, new_check_name_camel)
replaceInFile(filename, generateCommentLineHeader(originalName),
generateCommentLineHeader(filename))
replaceInFile(filename, generateCommentLineSource(originalName),
generateCommentLineSource(filename))
for header_guard in header_guard_variants:
replaceInFile(filename, header_guard, header_guard_new)
if new_module + '/'+ new_name + '.rst' in filename:
replaceInFile(
filename,
args.old_check_name + '\n' + '=' * len(args.old_check_name) + '\n',
args.new_check_name + '\n' + '=' * len(args.new_check_name) + '\n')
replaceInFile(filename, args.old_check_name, args.new_check_name)
replaceInFile(filename, old_module + '::' + check_name_camel,
new_module + '::' + new_check_name_camel)
replaceInFile(filename, old_module + '/' + check_name_camel,
new_module + '/' + new_check_name_camel)
replaceInFile(filename, old_module + '/' + old_name,
new_module + '/' + new_name)
replaceInFile(filename, check_name_camel, new_check_name_camel)
if old_module != new_module or new_module == 'llvm':
if new_module == 'llvm':
new_namespace = new_module + '_check'
if args.check_class_name:
check_name_camel = args.check_class_name
else:
new_namespace = new_module
check_implementation_files = glob.glob(
os.path.join(old_module_path, new_check_name_camel + '*'))
for filename in check_implementation_files:
# Move check implementation to the directory of the new module.
filename = fileRename(filename, old_module_path, new_module_path)
replaceInFileRegex(filename,
'namespace clang::tidy::' + old_module + '[^ \n]*',
'namespace clang::tidy::' + new_namespace)
check_name_camel = (
"".join(map(lambda elem: elem.capitalize(), old_name.split("-"))) + "Check"
)
if old_module != new_module:
new_check_name_camel = (
"".join(map(lambda elem: elem.capitalize(), new_name.split("-"))) + "Check"
)
# Add check to the new module.
adapt_cmake(new_module_path, new_check_name_camel)
adapt_module(new_module_path, new_module, args.new_check_name,
new_check_name_camel)
clang_tidy_path = os.path.dirname(__file__)
os.system(os.path.join(clang_tidy_path, 'add_new_check.py')
+ ' --update-docs')
add_release_notes(clang_tidy_path, args.old_check_name, args.new_check_name)
header_guard_variants = [
(args.old_check_name.replace("-", "_")).upper() + "_CHECK",
(old_module + "_" + check_name_camel).upper(),
(old_module + "_" + new_check_name_camel).upper(),
args.old_check_name.replace("-", "_").upper(),
]
header_guard_new = (new_module + "_" + new_check_name_camel).upper()
old_module_path = os.path.join(clang_tidy_path, old_module)
new_module_path = os.path.join(clang_tidy_path, new_module)
if old_module != new_module:
# Remove the check from the old module.
cmake_lists = os.path.join(old_module_path, "CMakeLists.txt")
check_found = deleteMatchingLines(cmake_lists, "\\b" + check_name_camel)
if not check_found:
print(
"Check name '%s' not found in %s. Exiting."
% (check_name_camel, cmake_lists)
)
return 1
modulecpp = next(
iter(
filter(
lambda p: p.lower() == old_module.lower() + "tidymodule.cpp",
os.listdir(old_module_path),
)
)
)
deleteMatchingLines(
os.path.join(old_module_path, modulecpp),
"\\b" + check_name_camel + "|\\b" + args.old_check_name,
)
for filename in getListOfFiles(clang_tidy_path):
originalName = filename
filename = fileRename(
filename, old_module + "/" + old_name, new_module + "/" + new_name
)
filename = fileRename(filename, args.old_check_name, args.new_check_name)
filename = fileRename(filename, check_name_camel, new_check_name_camel)
replaceInFile(
filename,
generateCommentLineHeader(originalName),
generateCommentLineHeader(filename),
)
replaceInFile(
filename,
generateCommentLineSource(originalName),
generateCommentLineSource(filename),
)
for header_guard in header_guard_variants:
replaceInFile(filename, header_guard, header_guard_new)
if new_module + "/" + new_name + ".rst" in filename:
replaceInFile(
filename,
args.old_check_name + "\n" + "=" * len(args.old_check_name) + "\n",
args.new_check_name + "\n" + "=" * len(args.new_check_name) + "\n",
)
replaceInFile(filename, args.old_check_name, args.new_check_name)
replaceInFile(
filename,
old_module + "::" + check_name_camel,
new_module + "::" + new_check_name_camel,
)
replaceInFile(
filename,
old_module + "/" + check_name_camel,
new_module + "/" + new_check_name_camel,
)
replaceInFile(
filename, old_module + "/" + old_name, new_module + "/" + new_name
)
replaceInFile(filename, check_name_camel, new_check_name_camel)
if old_module != new_module or new_module == "llvm":
if new_module == "llvm":
new_namespace = new_module + "_check"
else:
new_namespace = new_module
check_implementation_files = glob.glob(
os.path.join(old_module_path, new_check_name_camel + "*")
)
for filename in check_implementation_files:
# Move check implementation to the directory of the new module.
filename = fileRename(filename, old_module_path, new_module_path)
replaceInFileRegex(
filename,
"namespace clang::tidy::" + old_module + "[^ \n]*",
"namespace clang::tidy::" + new_namespace,
)
if old_module != new_module:
# Add check to the new module.
adapt_cmake(new_module_path, new_check_name_camel)
adapt_module(
new_module_path, new_module, args.new_check_name, new_check_name_camel
)
os.system(os.path.join(clang_tidy_path, "add_new_check.py") + " --update-docs")
add_release_notes(clang_tidy_path, args.old_check_name, args.new_check_name)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

449
clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python
#
#===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
# ===-----------------------------------------------------------------------===#
r"""
ClangTidy Diff Checker
@ -37,11 +37,11 @@ import threading
import traceback
try:
import yaml
import yaml
except ImportError:
yaml = None
yaml = None
is_py2 = sys.version[0] == '2'
is_py2 = sys.version[0] == "2"
if is_py2:
import Queue as queue
@ -50,230 +50,271 @@ else:
def run_tidy(task_queue, lock, timeout):
watchdog = None
while True:
command = task_queue.get()
try:
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
watchdog = None
while True:
command = task_queue.get()
try:
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
if timeout is not None:
watchdog = threading.Timer(timeout, proc.kill)
watchdog.start()
if timeout is not None:
watchdog = threading.Timer(timeout, proc.kill)
watchdog.start()
stdout, stderr = proc.communicate()
stdout, stderr = proc.communicate()
with lock:
sys.stdout.write(stdout.decode('utf-8') + '\n')
sys.stdout.flush()
if stderr:
sys.stderr.write(stderr.decode('utf-8') + '\n')
sys.stderr.flush()
except Exception as e:
with lock:
sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
finally:
with lock:
if not (timeout is None or watchdog is None):
if not watchdog.is_alive():
sys.stderr.write('Terminated by timeout: ' +
' '.join(command) + '\n')
watchdog.cancel()
task_queue.task_done()
with lock:
sys.stdout.write(stdout.decode("utf-8") + "\n")
sys.stdout.flush()
if stderr:
sys.stderr.write(stderr.decode("utf-8") + "\n")
sys.stderr.flush()
except Exception as e:
with lock:
sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")
finally:
with lock:
if not (timeout is None or watchdog is None):
if not watchdog.is_alive():
sys.stderr.write(
"Terminated by timeout: " + " ".join(command) + "\n"
)
watchdog.cancel()
task_queue.task_done()
def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
for _ in range(max_tasks):
t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
t.daemon = True
t.start()
for _ in range(max_tasks):
t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
t.daemon = True
t.start()
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged = []
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
content = yaml.safe_load(open(replacefile, 'r'))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged = []
for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
content = yaml.safe_load(open(replacefile, "r"))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {'MainSourceFile': '', mergekey: merged}
with open(mergefile, 'w') as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, 'w').close()
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {"MainSourceFile": "", mergekey: merged}
with open(mergefile, "w") as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, "w").close()
def main():
parser = argparse.ArgumentParser(description=
'Run clang-tidy against changed files, and '
'output diagnostics only for modified '
'lines.')
parser.add_argument('-clang-tidy-binary', metavar='PATH',
default='clang-tidy',
help='path to clang-tidy binary')
parser.add_argument('-p', metavar='NUM', default=0,
help='strip the smallest prefix containing P slashes')
parser.add_argument('-regex', metavar='PATTERN', default=None,
help='custom pattern selecting file paths to check '
'(case sensitive, overrides -iregex)')
parser.add_argument('-iregex', metavar='PATTERN', default=
r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
help='custom pattern selecting file paths to check '
'(case insensitive, overridden by -regex)')
parser.add_argument('-j', type=int, default=1,
help='number of tidy instances to be run in parallel.')
parser.add_argument('-timeout', type=int, default=None,
help='timeout per each file in seconds.')
parser.add_argument('-fix', action='store_true', default=False,
help='apply suggested fixes')
parser.add_argument('-checks',
help='checks filter, when not specified, use clang-tidy '
'default',
default='')
parser.add_argument('-use-color', action='store_true',
help='Use colors in output')
parser.add_argument('-path', dest='build_path',
help='Path used to read a compile command database.')
if yaml:
parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
help='Create a yaml file to store suggested fixes in, '
'which can be applied with clang-apply-replacements.')
parser.add_argument('-extra-arg', dest='extra_arg',
action='append', default=[],
help='Additional argument to append to the compiler '
'command line.')
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
action='append', default=[],
help='Additional argument to prepend to the compiler '
'command line.')
parser.add_argument('-quiet', action='store_true', default=False,
help='Run clang-tidy in quiet mode')
parser.add_argument('-load', dest='plugins',
action='append', default=[],
help='Load the specified plugin in clang-tidy.')
parser = argparse.ArgumentParser(
description="Run clang-tidy against changed files, and "
"output diagnostics only for modified "
"lines."
)
parser.add_argument(
"-clang-tidy-binary",
metavar="PATH",
default="clang-tidy",
help="path to clang-tidy binary",
)
parser.add_argument(
"-p",
metavar="NUM",
default=0,
help="strip the smallest prefix containing P slashes",
)
parser.add_argument(
"-regex",
metavar="PATTERN",
default=None,
help="custom pattern selecting file paths to check "
"(case sensitive, overrides -iregex)",
)
parser.add_argument(
"-iregex",
metavar="PATTERN",
default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",
help="custom pattern selecting file paths to check "
"(case insensitive, overridden by -regex)",
)
parser.add_argument(
"-j",
type=int,
default=1,
help="number of tidy instances to be run in parallel.",
)
parser.add_argument(
"-timeout", type=int, default=None, help="timeout per each file in seconds."
)
parser.add_argument(
"-fix", action="store_true", default=False, help="apply suggested fixes"
)
parser.add_argument(
"-checks",
help="checks filter, when not specified, use clang-tidy " "default",
default="",
)
parser.add_argument("-use-color", action="store_true", help="Use colors in output")
parser.add_argument(
"-path", dest="build_path", help="Path used to read a compile command database."
)
if yaml:
parser.add_argument(
"-export-fixes",
metavar="FILE",
dest="export_fixes",
help="Create a yaml file to store suggested fixes in, "
"which can be applied with clang-apply-replacements.",
)
parser.add_argument(
"-extra-arg",
dest="extra_arg",
action="append",
default=[],
help="Additional argument to append to the compiler " "command line.",
)
parser.add_argument(
"-extra-arg-before",
dest="extra_arg_before",
action="append",
default=[],
help="Additional argument to prepend to the compiler " "command line.",
)
parser.add_argument(
"-quiet",
action="store_true",
default=False,
help="Run clang-tidy in quiet mode",
)
parser.add_argument(
"-load",
dest="plugins",
action="append",
default=[],
help="Load the specified plugin in clang-tidy.",
)
clang_tidy_args = []
argv = sys.argv[1:]
if '--' in argv:
clang_tidy_args.extend(argv[argv.index('--'):])
argv = argv[:argv.index('--')]
clang_tidy_args = []
argv = sys.argv[1:]
if "--" in argv:
clang_tidy_args.extend(argv[argv.index("--") :])
argv = argv[: argv.index("--")]
args = parser.parse_args(argv)
args = parser.parse_args(argv)
# Extract changed lines for each file.
filename = None
lines_by_file = {}
for line in sys.stdin:
match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
if match:
filename = match.group(2)
if filename is None:
continue
# Extract changed lines for each file.
filename = None
lines_by_file = {}
for line in sys.stdin:
match = re.search('^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)
if match:
filename = match.group(2)
if filename is None:
continue
if args.regex is not None:
if not re.match('^%s$' % args.regex, filename):
continue
else:
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
continue
if args.regex is not None:
if not re.match("^%s$" % args.regex, filename):
continue
else:
if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
continue
match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
if match:
start_line = int(match.group(1))
line_count = 1
if match.group(3):
line_count = int(match.group(3))
if line_count == 0:
continue
end_line = start_line + line_count - 1
lines_by_file.setdefault(filename, []).append([start_line, end_line])
match = re.search("^@@.*\+(\d+)(,(\d+))?", line)
if match:
start_line = int(match.group(1))
line_count = 1
if match.group(3):
line_count = int(match.group(3))
if line_count == 0:
continue
end_line = start_line + line_count - 1
lines_by_file.setdefault(filename, []).append([start_line, end_line])
if not any(lines_by_file):
print("No relevant changes found.")
sys.exit(0)
if not any(lines_by_file):
print("No relevant changes found.")
sys.exit(0)
max_task_count = args.j
if max_task_count == 0:
max_task_count = multiprocessing.cpu_count()
max_task_count = min(len(lines_by_file), max_task_count)
max_task_count = args.j
if max_task_count == 0:
max_task_count = multiprocessing.cpu_count()
max_task_count = min(len(lines_by_file), max_task_count)
tmpdir = None
if yaml and args.export_fixes:
tmpdir = tempfile.mkdtemp()
# Tasks for clang-tidy.
task_queue = queue.Queue(max_task_count)
# A lock for console output.
lock = threading.Lock()
# Run a pool of clang-tidy workers.
start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
# Form the common args list.
common_clang_tidy_args = []
if args.fix:
common_clang_tidy_args.append('-fix')
if args.checks != '':
common_clang_tidy_args.append('-checks=' + args.checks)
if args.quiet:
common_clang_tidy_args.append('-quiet')
if args.build_path is not None:
common_clang_tidy_args.append('-p=%s' % args.build_path)
if args.use_color:
common_clang_tidy_args.append('--use-color')
for arg in args.extra_arg:
common_clang_tidy_args.append('-extra-arg=%s' % arg)
for arg in args.extra_arg_before:
common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
for plugin in args.plugins:
common_clang_tidy_args.append('-load=%s' % plugin)
for name in lines_by_file:
line_filter_json = json.dumps(
[{"name": name, "lines": lines_by_file[name]}],
separators=(',', ':'))
# Run clang-tidy on files containing changes.
command = [args.clang_tidy_binary]
command.append('-line-filter=' + line_filter_json)
tmpdir = None
if yaml and args.export_fixes:
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
command.append('-export-fixes=' + tmp_name)
command.extend(common_clang_tidy_args)
command.append(name)
command.extend(clang_tidy_args)
tmpdir = tempfile.mkdtemp()
task_queue.put(command)
# Tasks for clang-tidy.
task_queue = queue.Queue(max_task_count)
# A lock for console output.
lock = threading.Lock()
# Wait for all threads to be done.
task_queue.join()
# Run a pool of clang-tidy workers.
start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
if yaml and args.export_fixes:
print('Writing fixes to ' + args.export_fixes + ' ...')
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
sys.stderr.write('Error exporting fixes.\n')
traceback.print_exc()
# Form the common args list.
common_clang_tidy_args = []
if args.fix:
common_clang_tidy_args.append("-fix")
if args.checks != "":
common_clang_tidy_args.append("-checks=" + args.checks)
if args.quiet:
common_clang_tidy_args.append("-quiet")
if args.build_path is not None:
common_clang_tidy_args.append("-p=%s" % args.build_path)
if args.use_color:
common_clang_tidy_args.append("--use-color")
for arg in args.extra_arg:
common_clang_tidy_args.append("-extra-arg=%s" % arg)
for arg in args.extra_arg_before:
common_clang_tidy_args.append("-extra-arg-before=%s" % arg)
for plugin in args.plugins:
common_clang_tidy_args.append("-load=%s" % plugin)
if tmpdir:
shutil.rmtree(tmpdir)
for name in lines_by_file:
line_filter_json = json.dumps(
[{"name": name, "lines": lines_by_file[name]}], separators=(",", ":")
)
# Run clang-tidy on files containing changes.
command = [args.clang_tidy_binary]
command.append("-line-filter=" + line_filter_json)
if yaml and args.export_fixes:
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir)
os.close(handle)
command.append("-export-fixes=" + tmp_name)
command.extend(common_clang_tidy_args)
command.append(name)
command.extend(clang_tidy_args)
task_queue.put(command)
# Wait for all threads to be done.
task_queue.join()
if yaml and args.export_fixes:
print("Writing fixes to " + args.export_fixes + " ...")
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
sys.stderr.write("Error exporting fixes.\n")
traceback.print_exc()
if tmpdir:
shutil.rmtree(tmpdir)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

700
clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
#===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
# ===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
# ===-----------------------------------------------------------------------===#
# FIXME: Integrate with clang-tidy-diff.py
@ -51,347 +51,443 @@ import threading
import traceback
try:
import yaml
import yaml
except ImportError:
yaml = None
yaml = None
def strtobool(val):
"""Convert a string representation of truth to a bool following LLVM's CLI argument parsing."""
"""Convert a string representation of truth to a bool following LLVM's CLI argument parsing."""
val = val.lower()
if val in ['', 'true', '1']:
return True
elif val in ['false', '0']:
return False
val = val.lower()
if val in ["", "true", "1"]:
return True
elif val in ["false", "0"]:
return False
# Return ArgumentTypeError so that argparse does not substitute its own error message
raise argparse.ArgumentTypeError(
"'{}' is invalid value for boolean argument! Try 0 or 1.".format(val)
)
# Return ArgumentTypeError so that argparse does not substitute its own error message
raise argparse.ArgumentTypeError(
"'{}' is invalid value for boolean argument! Try 0 or 1.".format(val)
)
def find_compilation_database(path):
"""Adjusts the directory until a compilation database is found."""
result = os.path.realpath('./')
while not os.path.isfile(os.path.join(result, path)):
parent = os.path.dirname(result)
if result == parent:
print('Error: could not find compilation database.')
sys.exit(1)
result = parent
return result
"""Adjusts the directory until a compilation database is found."""
result = os.path.realpath("./")
while not os.path.isfile(os.path.join(result, path)):
parent = os.path.dirname(result)
if result == parent:
print("Error: could not find compilation database.")
sys.exit(1)
result = parent
return result
def make_absolute(f, directory):
if os.path.isabs(f):
return f
return os.path.normpath(os.path.join(directory, f))
if os.path.isabs(f):
return f
return os.path.normpath(os.path.join(directory, f))
def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
header_filter, allow_enabling_alpha_checkers,
extra_arg, extra_arg_before, quiet, config_file_path,
config, line_filter, use_color, plugins):
"""Gets a command line for clang-tidy."""
start = [clang_tidy_binary]
if allow_enabling_alpha_checkers:
start.append('-allow-enabling-analyzer-alpha-checkers')
if header_filter is not None:
start.append('-header-filter=' + header_filter)
if line_filter is not None:
start.append('-line-filter=' + line_filter)
if use_color is not None:
if use_color:
start.append('--use-color')
else:
start.append('--use-color=false')
if checks:
start.append('-checks=' + checks)
if tmpdir is not None:
start.append('-export-fixes')
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
start.append(name)
for arg in extra_arg:
start.append('-extra-arg=%s' % arg)
for arg in extra_arg_before:
start.append('-extra-arg-before=%s' % arg)
start.append('-p=' + build_path)
if quiet:
start.append('-quiet')
if config_file_path:
start.append('--config-file=' + config_file_path)
elif config:
start.append('-config=' + config)
for plugin in plugins:
start.append('-load=' + plugin)
start.append(f)
return start
def get_tidy_invocation(
f,
clang_tidy_binary,
checks,
tmpdir,
build_path,
header_filter,
allow_enabling_alpha_checkers,
extra_arg,
extra_arg_before,
quiet,
config_file_path,
config,
line_filter,
use_color,
plugins,
):
"""Gets a command line for clang-tidy."""
start = [clang_tidy_binary]
if allow_enabling_alpha_checkers:
start.append("-allow-enabling-analyzer-alpha-checkers")
if header_filter is not None:
start.append("-header-filter=" + header_filter)
if line_filter is not None:
start.append("-line-filter=" + line_filter)
if use_color is not None:
if use_color:
start.append("--use-color")
else:
start.append("--use-color=false")
if checks:
start.append("-checks=" + checks)
if tmpdir is not None:
start.append("-export-fixes")
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir)
os.close(handle)
start.append(name)
for arg in extra_arg:
start.append("-extra-arg=%s" % arg)
for arg in extra_arg_before:
start.append("-extra-arg-before=%s" % arg)
start.append("-p=" + build_path)
if quiet:
start.append("-quiet")
if config_file_path:
start.append("--config-file=" + config_file_path)
elif config:
start.append("-config=" + config)
for plugin in plugins:
start.append("-load=" + plugin)
start.append(f)
return start
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged=[]
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
content = yaml.safe_load(open(replacefile, 'r'))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged = []
for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
content = yaml.safe_load(open(replacefile, "r"))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {'MainSourceFile': '', mergekey: merged}
with open(mergefile, 'w') as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, 'w').close()
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {"MainSourceFile": "", mergekey: merged}
with open(mergefile, "w") as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, "w").close()
def find_binary(arg, name, build_path):
"""Get the path for a binary or exit"""
if arg:
if shutil.which(arg):
return arg
else:
raise SystemExit(
"error: passed binary '{}' was not found or is not executable"
.format(arg))
"""Get the path for a binary or exit"""
if arg:
if shutil.which(arg):
return arg
else:
raise SystemExit(
"error: passed binary '{}' was not found or is not executable".format(
arg
)
)
built_path = os.path.join(build_path, "bin", name)
binary = shutil.which(name) or shutil.which(built_path)
if binary:
return binary
else:
raise SystemExit(
"error: failed to find {} in $PATH or at {}"
.format(name, built_path))
built_path = os.path.join(build_path, "bin", name)
binary = shutil.which(name) or shutil.which(built_path)
if binary:
return binary
else:
raise SystemExit(
"error: failed to find {} in $PATH or at {}".format(name, built_path)
)
def apply_fixes(args, clang_apply_replacements_binary, tmpdir):
"""Calls clang-apply-fixes on a given directory."""
invocation = [clang_apply_replacements_binary]
invocation.append('-ignore-insert-conflict')
if args.format:
invocation.append('-format')
if args.style:
invocation.append('-style=' + args.style)
invocation.append(tmpdir)
subprocess.call(invocation)
"""Calls clang-apply-fixes on a given directory."""
invocation = [clang_apply_replacements_binary]
invocation.append("-ignore-insert-conflict")
if args.format:
invocation.append("-format")
if args.style:
invocation.append("-style=" + args.style)
invocation.append(tmpdir)
subprocess.call(invocation)
def run_tidy(args, clang_tidy_binary, tmpdir, build_path, queue, lock,
failed_files):
"""Takes filenames out of queue and runs clang-tidy on them."""
while True:
name = queue.get()
invocation = get_tidy_invocation(name, clang_tidy_binary, args.checks,
tmpdir, build_path, args.header_filter,
args.allow_enabling_alpha_checkers,
args.extra_arg, args.extra_arg_before,
args.quiet, args.config_file, args.config,
args.line_filter, args.use_color,
args.plugins)
def run_tidy(args, clang_tidy_binary, tmpdir, build_path, queue, lock, failed_files):
"""Takes filenames out of queue and runs clang-tidy on them."""
while True:
name = queue.get()
invocation = get_tidy_invocation(
name,
clang_tidy_binary,
args.checks,
tmpdir,
build_path,
args.header_filter,
args.allow_enabling_alpha_checkers,
args.extra_arg,
args.extra_arg_before,
args.quiet,
args.config_file,
args.config,
args.line_filter,
args.use_color,
args.plugins,
)
proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = proc.communicate()
if proc.returncode != 0:
if proc.returncode < 0:
msg = "%s: terminated by signal %d\n" % (name, -proc.returncode)
err += msg.encode('utf-8')
failed_files.append(name)
with lock:
sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
if len(err) > 0:
sys.stdout.flush()
sys.stderr.write(err.decode('utf-8'))
queue.task_done()
proc = subprocess.Popen(
invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
output, err = proc.communicate()
if proc.returncode != 0:
if proc.returncode < 0:
msg = "%s: terminated by signal %d\n" % (name, -proc.returncode)
err += msg.encode("utf-8")
failed_files.append(name)
with lock:
sys.stdout.write(" ".join(invocation) + "\n" + output.decode("utf-8"))
if len(err) > 0:
sys.stdout.flush()
sys.stderr.write(err.decode("utf-8"))
queue.task_done()
def main():
parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
'in a compilation database. Requires '
'clang-tidy and clang-apply-replacements in '
'$PATH or in your build directory.')
parser.add_argument('-allow-enabling-alpha-checkers',
action='store_true', help='allow alpha checkers from '
'clang-analyzer.')
parser.add_argument('-clang-tidy-binary', metavar='PATH',
help='path to clang-tidy binary')
parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
help='path to clang-apply-replacements binary')
parser.add_argument('-checks', default=None,
help='checks filter, when not specified, use clang-tidy '
'default')
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument('-config', default=None,
help='Specifies a configuration in YAML/JSON format: '
' -config="{Checks: \'*\', '
' CheckOptions: {x: y}}" '
'When the value is empty, clang-tidy will '
'attempt to find a file named .clang-tidy for '
'each source file in its parent directories.')
config_group.add_argument('-config-file', default=None,
help='Specify the path of .clang-tidy or custom config '
'file: e.g. -config-file=/some/path/myTidyConfigFile. '
'This option internally works exactly the same way as '
'-config option after reading specified config file. '
'Use either -config-file or -config, not both.')
parser.add_argument('-header-filter', default=None,
help='regular expression matching the names of the '
'headers to output diagnostics from. Diagnostics from '
'the main file of each translation unit are always '
'displayed.')
parser.add_argument('-line-filter', default=None,
help='List of files with line ranges to filter the'
'warnings.')
if yaml:
parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
help='Create a yaml file to store suggested fixes in, '
'which can be applied with clang-apply-replacements.')
parser.add_argument('-j', type=int, default=0,
help='number of tidy instances to be run in parallel.')
parser.add_argument('files', nargs='*', default=['.*'],
help='files to be processed (regex on path)')
parser.add_argument('-fix', action='store_true', help='apply fix-its')
parser.add_argument('-format', action='store_true', help='Reformat code '
'after applying fixes')
parser.add_argument('-style', default='file', help='The style of reformat '
'code after applying fixes')
parser.add_argument('-use-color', type=strtobool, nargs='?', const=True,
help='Use colors in diagnostics, overriding clang-tidy\'s'
' default behavior. This option overrides the \'UseColor'
'\' option in .clang-tidy file, if any.')
parser.add_argument('-p', dest='build_path',
help='Path used to read a compile command database.')
parser.add_argument('-extra-arg', dest='extra_arg',
action='append', default=[],
help='Additional argument to append to the compiler '
'command line.')
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
action='append', default=[],
help='Additional argument to prepend to the compiler '
'command line.')
parser.add_argument('-quiet', action='store_true',
help='Run clang-tidy in quiet mode')
parser.add_argument('-load', dest='plugins',
action='append', default=[],
help='Load the specified plugin in clang-tidy.')
args = parser.parse_args()
parser = argparse.ArgumentParser(
description="Runs clang-tidy over all files "
"in a compilation database. Requires "
"clang-tidy and clang-apply-replacements in "
"$PATH or in your build directory."
)
parser.add_argument(
"-allow-enabling-alpha-checkers",
action="store_true",
help="allow alpha checkers from " "clang-analyzer.",
)
parser.add_argument(
"-clang-tidy-binary", metavar="PATH", help="path to clang-tidy binary"
)
parser.add_argument(
"-clang-apply-replacements-binary",
metavar="PATH",
help="path to clang-apply-replacements binary",
)
parser.add_argument(
"-checks",
default=None,
help="checks filter, when not specified, use clang-tidy " "default",
)
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument(
"-config",
default=None,
help="Specifies a configuration in YAML/JSON format: "
" -config=\"{Checks: '*', "
' CheckOptions: {x: y}}" '
"When the value is empty, clang-tidy will "
"attempt to find a file named .clang-tidy for "
"each source file in its parent directories.",
)
config_group.add_argument(
"-config-file",
default=None,
help="Specify the path of .clang-tidy or custom config "
"file: e.g. -config-file=/some/path/myTidyConfigFile. "
"This option internally works exactly the same way as "
"-config option after reading specified config file. "
"Use either -config-file or -config, not both.",
)
parser.add_argument(
"-header-filter",
default=None,
help="regular expression matching the names of the "
"headers to output diagnostics from. Diagnostics from "
"the main file of each translation unit are always "
"displayed.",
)
parser.add_argument(
"-line-filter",
default=None,
help="List of files with line ranges to filter the" "warnings.",
)
if yaml:
parser.add_argument(
"-export-fixes",
metavar="filename",
dest="export_fixes",
help="Create a yaml file to store suggested fixes in, "
"which can be applied with clang-apply-replacements.",
)
parser.add_argument(
"-j",
type=int,
default=0,
help="number of tidy instances to be run in parallel.",
)
parser.add_argument(
"files", nargs="*", default=[".*"], help="files to be processed (regex on path)"
)
parser.add_argument("-fix", action="store_true", help="apply fix-its")
parser.add_argument(
"-format", action="store_true", help="Reformat code " "after applying fixes"
)
parser.add_argument(
"-style",
default="file",
help="The style of reformat " "code after applying fixes",
)
parser.add_argument(
"-use-color",
type=strtobool,
nargs="?",
const=True,
help="Use colors in diagnostics, overriding clang-tidy's"
" default behavior. This option overrides the 'UseColor"
"' option in .clang-tidy file, if any.",
)
parser.add_argument(
"-p", dest="build_path", help="Path used to read a compile command database."
)
parser.add_argument(
"-extra-arg",
dest="extra_arg",
action="append",
default=[],
help="Additional argument to append to the compiler " "command line.",
)
parser.add_argument(
"-extra-arg-before",
dest="extra_arg_before",
action="append",
default=[],
help="Additional argument to prepend to the compiler " "command line.",
)
parser.add_argument(
"-quiet", action="store_true", help="Run clang-tidy in quiet mode"
)
parser.add_argument(
"-load",
dest="plugins",
action="append",
default=[],
help="Load the specified plugin in clang-tidy.",
)
args = parser.parse_args()
db_path = 'compile_commands.json'
db_path = "compile_commands.json"
if args.build_path is not None:
build_path = args.build_path
else:
# Find our database
build_path = find_compilation_database(db_path)
clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy",
build_path)
tmpdir = None
if args.fix:
clang_apply_replacements_binary = find_binary(
args.clang_apply_replacements_binary, "clang-apply-replacements",
build_path)
tmpdir = tempfile.mkdtemp()
try:
invocation = get_tidy_invocation("", clang_tidy_binary, args.checks,
None, build_path, args.header_filter,
args.allow_enabling_alpha_checkers,
args.extra_arg, args.extra_arg_before,
args.quiet, args.config_file, args.config,
args.line_filter, args.use_color,
args.plugins)
invocation.append('-list-checks')
invocation.append('-')
if args.quiet:
# Even with -quiet we still want to check if we can call clang-tidy.
with open(os.devnull, 'w') as dev_null:
subprocess.check_call(invocation, stdout=dev_null)
if args.build_path is not None:
build_path = args.build_path
else:
subprocess.check_call(invocation)
except:
print("Unable to run clang-tidy.", file=sys.stderr)
sys.exit(1)
# Find our database
build_path = find_compilation_database(db_path)
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
files = set([make_absolute(entry['file'], entry['directory'])
for entry in database])
clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy", build_path)
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
tmpdir = None
if args.fix:
clang_apply_replacements_binary = find_binary(
args.clang_apply_replacements_binary, "clang-apply-replacements", build_path
)
tmpdir = tempfile.mkdtemp()
# Build up a big regexy filter from all command line arguments.
file_name_re = re.compile('|'.join(args.files))
try:
invocation = get_tidy_invocation(
"",
clang_tidy_binary,
args.checks,
None,
build_path,
args.header_filter,
args.allow_enabling_alpha_checkers,
args.extra_arg,
args.extra_arg_before,
args.quiet,
args.config_file,
args.config,
args.line_filter,
args.use_color,
args.plugins,
)
invocation.append("-list-checks")
invocation.append("-")
if args.quiet:
# Even with -quiet we still want to check if we can call clang-tidy.
with open(os.devnull, "w") as dev_null:
subprocess.check_call(invocation, stdout=dev_null)
else:
subprocess.check_call(invocation)
except:
print("Unable to run clang-tidy.", file=sys.stderr)
sys.exit(1)
return_code = 0
try:
# Spin up a bunch of tidy-launching threads.
task_queue = queue.Queue(max_task)
# List of files with a non-zero return code.
failed_files = []
lock = threading.Lock()
for _ in range(max_task):
t = threading.Thread(target=run_tidy,
args=(args, clang_tidy_binary, tmpdir, build_path,
task_queue, lock, failed_files))
t.daemon = True
t.start()
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
files = set(
[make_absolute(entry["file"], entry["directory"]) for entry in database]
)
# Fill the queue with files.
for name in files:
if file_name_re.search(name):
task_queue.put(name)
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
# Wait for all threads to be done.
task_queue.join()
if len(failed_files):
return_code = 1
# Build up a big regexy filter from all command line arguments.
file_name_re = re.compile("|".join(args.files))
return_code = 0
try:
# Spin up a bunch of tidy-launching threads.
task_queue = queue.Queue(max_task)
# List of files with a non-zero return code.
failed_files = []
lock = threading.Lock()
for _ in range(max_task):
t = threading.Thread(
target=run_tidy,
args=(
args,
clang_tidy_binary,
tmpdir,
build_path,
task_queue,
lock,
failed_files,
),
)
t.daemon = True
t.start()
# Fill the queue with files.
for name in files:
if file_name_re.search(name):
task_queue.put(name)
# Wait for all threads to be done.
task_queue.join()
if len(failed_files):
return_code = 1
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print("\nCtrl-C detected, goodbye.")
if tmpdir:
shutil.rmtree(tmpdir)
os.kill(0, 9)
if yaml and args.export_fixes:
print("Writing fixes to " + args.export_fixes + " ...")
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
print("Error exporting fixes.\n", file=sys.stderr)
traceback.print_exc()
return_code = 1
if args.fix:
print("Applying fixes ...")
try:
apply_fixes(args, clang_apply_replacements_binary, tmpdir)
except:
print("Error applying fixes.\n", file=sys.stderr)
traceback.print_exc()
return_code = 1
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print('\nCtrl-C detected, goodbye.')
if tmpdir:
shutil.rmtree(tmpdir)
os.kill(0, 9)
if yaml and args.export_fixes:
print('Writing fixes to ' + args.export_fixes + ' ...')
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
print('Error exporting fixes.\n', file=sys.stderr)
traceback.print_exc()
return_code=1
if args.fix:
print('Applying fixes ...')
try:
apply_fixes(args, clang_apply_replacements_binary, tmpdir)
except:
print('Error applying fixes.\n', file=sys.stderr)
traceback.print_exc()
return_code = 1
if tmpdir:
shutil.rmtree(tmpdir)
sys.exit(return_code)
shutil.rmtree(tmpdir)
sys.exit(return_code)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

86
clang-tools-extra/clangd/TidyFastChecks.py
View File

@ -21,59 +21,78 @@ import sys
# Checks faster than FAST_THRESHOLD are fast, slower than SLOW_THRESHOLD slow.
# If a check is in between, we stick with our previous decision. This avoids
# enabling/disabling checks between releases due to random measurement jitter.
FAST_THRESHOLD = 8 # percent
FAST_THRESHOLD = 8 # percent
SLOW_THRESHOLD = 15
parser = argparse.ArgumentParser()
parser.add_argument('--target', help='X-macro output file. '
'If it exists, existing contents will be used for hysteresis',
default='clang-tools-extra/clangd/TidyFastChecks.inc')
parser.add_argument('--source', help='Source file to benchmark tidy checks',
default='clang/lib/Sema/Sema.cpp')
parser.add_argument('--clangd', help='clangd binary to invoke',
default='build/bin/clangd')
parser.add_argument('--checks', help='check glob to run', default='*')
parser.add_argument('--verbose', help='log clangd output', action='store_true')
parser.add_argument(
"--target",
help="X-macro output file. "
"If it exists, existing contents will be used for hysteresis",
default="clang-tools-extra/clangd/TidyFastChecks.inc",
)
parser.add_argument(
"--source",
help="Source file to benchmark tidy checks",
default="clang/lib/Sema/Sema.cpp",
)
parser.add_argument(
"--clangd", help="clangd binary to invoke", default="build/bin/clangd"
)
parser.add_argument("--checks", help="check glob to run", default="*")
parser.add_argument("--verbose", help="log clangd output", action="store_true")
args = parser.parse_args()
# Use the preprocessor to extract the list of previously-fast checks.
def read_old_fast(path):
text = subprocess.check_output(["cpp",
"-P", # Omit GNU line markers
"-nostdinc", # Don't include stdc-predef.h
"-DFAST(C,T)=C", # Print fast checks only
path])
text = subprocess.check_output(
[
"cpp",
"-P", # Omit GNU line markers
"-nostdinc", # Don't include stdc-predef.h
"-DFAST(C,T)=C", # Print fast checks only
path,
]
)
for line in text.splitlines():
if line.strip():
yield line.strip().decode('utf-8')
yield line.strip().decode("utf-8")
old_fast = list(read_old_fast(args.target)) if os.path.exists(args.target) else []
print(f"Old fast checks: {old_fast}", file=sys.stderr)
# Runs clangd --check --check-tidy-time.
# Yields (check, percent-overhead) pairs.
def measure():
process = subprocess.Popen([args.clangd,
"--check=" + args.source,
"--check-locations=0", # Skip useless slow steps.
"--check-tidy-time=" + args.checks],
stderr=subprocess.PIPE)
process = subprocess.Popen(
[
args.clangd,
"--check=" + args.source,
"--check-locations=0", # Skip useless slow steps.
"--check-tidy-time=" + args.checks,
],
stderr=subprocess.PIPE,
)
recording = False
for line in iter(process.stderr.readline, b""):
if args.verbose:
print("clangd> ", line, file=sys.stderr)
if not recording:
if b'Timing AST build with individual clang-tidy checks' in line:
recording = True
if b"Timing AST build with individual clang-tidy checks" in line:
recording = True
continue
if b'Finished individual clang-tidy checks' in line:
if b"Finished individual clang-tidy checks" in line:
return
match = re.search(rb'(\S+) = (\S+)%', line)
match = re.search(rb"(\S+) = (\S+)%", line)
if match:
yield (match.group(1).decode('utf-8'), float(match.group(2)))
yield (match.group(1).decode("utf-8"), float(match.group(2)))
with open(args.target, 'w', buffering=1) as target:
with open(args.target, "w", buffering=1) as target:
# Produce an includable X-macros fragment with our decisions.
print(f"""// This file is generated, do not edit it directly!
print(
f"""// This file is generated, do not edit it directly!
// Deltas are percentage regression in parsing {args.source}
#ifndef FAST
#define FAST(CHECK, DELTA)
@ -81,7 +100,9 @@ with open(args.target, 'w', buffering=1) as target:
#ifndef SLOW
#define SLOW(CHECK, DELTA)
#endif
""", file=target)
""",
file=target,
)
for check, time in measure():
threshold = SLOW_THRESHOLD if check in old_fast else FAST_THRESHOLD
@ -89,7 +110,10 @@ with open(args.target, 'w', buffering=1) as target:
print(f"{decision} {check} {time}% <= {threshold}%", file=sys.stderr)
print(f"{decision}({check}, {time})", file=target)
print("""
print(
"""
#undef FAST
#undef SLOW
""", file=target)
""",
file=target,
)

152
clang-tools-extra/clangd/quality/CompletionModelCodegen.py
View File

@ -28,19 +28,18 @@ class CppClass:
def ns_end(self):
"""Returns snippet for closing namespace declarations."""
close_ns = [
"} // namespace %s" % ns for ns in reversed(self.ns)]
close_ns = ["} // namespace %s" % ns for ns in reversed(self.ns)]
return "\n".join(close_ns)
def header_guard(filename):
'''Returns the header guard for the generated header.'''
"""Returns the header guard for the generated header."""
return "GENERATED_DECISION_FOREST_MODEL_%s_H" % filename.upper()
def boost_node(n, label, next_label):
"""Returns code snippet for a leaf/boost node."""
return "%s: return %sf;" % (label, n['score'])
return "%s: return %sf;" % (label, n["score"])
def if_greater_node(n, label, next_label):
@ -51,7 +50,12 @@ def if_greater_node(n, label, next_label):
Control falls through if condition is evaluated to false."""
threshold = n["threshold"]
return "%s: if (E.get%s() >= %s /*%s*/) goto %s;" % (
label, n['feature'], order_encode(threshold), threshold, next_label)
label,
n["feature"],
order_encode(threshold),
threshold,
next_label,
)
def if_member_node(n, label, next_label):
@ -59,21 +63,24 @@ def if_member_node(n, label, next_label):
Jumps to true_label if the Example feature (ENUM) is present in the set of enum values
described in the node.
Control falls through if condition is evaluated to false."""
members = '|'.join([
"BIT(%s_type::%s)" % (n['feature'], member)
for member in n["set"]
])
members = "|".join(
["BIT(%s_type::%s)" % (n["feature"], member) for member in n["set"]]
)
return "%s: if (E.get%s() & (%s)) goto %s;" % (
label, n['feature'], members, next_label)
label,
n["feature"],
members,
next_label,
)
def node(n, label, next_label):
"""Returns code snippet for the node."""
return {
'boost': boost_node,
'if_greater': if_greater_node,
'if_member': if_member_node,
}[n['operation']](n, label, next_label)
"boost": boost_node,
"if_greater": if_greater_node,
"if_member": if_member_node,
}[n["operation"]](n, label, next_label)
def tree(t, tree_num, node_num):
@ -97,18 +104,16 @@ def tree(t, tree_num, node_num):
code.append(node(t, label=label, next_label="t%d" % (tree_num + 1)))
return code, 1
false_code, false_size = tree(
t['else'], tree_num=tree_num, node_num=node_num+1)
false_code, false_size = tree(t["else"], tree_num=tree_num, node_num=node_num + 1)
true_node_num = node_num+false_size+1
true_node_num = node_num + false_size + 1
true_label = "t%d_n%d" % (tree_num, true_node_num)
true_code, true_size = tree(
t['then'], tree_num=tree_num, node_num=true_node_num)
true_code, true_size = tree(t["then"], tree_num=tree_num, node_num=true_node_num)
code.append(node(t, label=label, next_label=true_label))
return code+false_code+true_code, 1+false_size+true_size
return code + false_code + true_code, 1 + false_size + true_size
def gen_header_code(features_json, cpp_class, filename):
@ -127,23 +132,23 @@ def gen_header_code(features_json, cpp_class, filename):
if f["kind"] == "NUMBER":
# Floats are order-encoded to integers for faster comparison.
setters.append(
"void set%s(float V) { %s = OrderEncode(V); }" % (
feature, feature))
"void set%s(float V) { %s = OrderEncode(V); }" % (feature, feature)
)
elif f["kind"] == "ENUM":
setters.append(
"void set%s(unsigned V) { %s = 1LL << V; }" % (feature, feature))
"void set%s(unsigned V) { %s = 1LL << V; }" % (feature, feature)
)
else:
raise ValueError("Unhandled feature type.", f["kind"])
# Class members represent all the features of the Example.
class_members = [
"uint%d_t %s = 0;"
% (64 if f["kind"] == "ENUM" else 32, f['name'])
"uint%d_t %s = 0;" % (64 if f["kind"] == "ENUM" else 32, f["name"])
for f in features_json
]
getters = [
"LLVM_ATTRIBUTE_ALWAYS_INLINE uint%d_t get%s() const { return %s; }"
% (64 if f["kind"] == "ENUM" else 32, f['name'], f['name'])
% (64 if f["kind"] == "ENUM" else 32, f["name"], f["name"])
for f in features_json
]
nline = "\n "
@ -173,25 +178,32 @@ private:
float Evaluate(const %s&);
%s
#endif // %s
""" % (guard, guard, cpp_class.ns_begin(), cpp_class.name,
""" % (
guard,
guard,
cpp_class.ns_begin(),
cpp_class.name,
nline.join(setters),
nline.join(getters),
nline.join(class_members),
cpp_class.name, cpp_class.ns_end(), guard)
cpp_class.name,
cpp_class.ns_end(),
guard,
)
def order_encode(v):
i = struct.unpack('<I', struct.pack('<f', v))[0]
i = struct.unpack("<I", struct.pack("<f", v))[0]
TopBit = 1 << 31
# IEEE 754 floats compare like sign-magnitude integers.
if (i & TopBit): # Negative float
if i & TopBit: # Negative float
return (1 << 32) - i # low half of integers, order reversed.
return TopBit + i # top half of integers
def evaluate_func(forest_json, cpp_class):
"""Generates evaluation functions for each tree and combines them in
`float Evaluate(const {Example}&)` function. This function can be
`float Evaluate(const {Example}&)` function. This function can be
used to score an Example."""
code = ""
@ -200,10 +212,13 @@ def evaluate_func(forest_json, cpp_class):
code += "namespace {\n"
tree_num = 0
for tree_json in forest_json:
code += "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name)
code += " " + \
"\n ".join(
tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n"
code += "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (
tree_num,
cpp_class.name,
)
code += (
" " + "\n ".join(tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n"
)
code += "}\n\n"
tree_num += 1
code += "} // namespace\n\n"
@ -224,23 +239,20 @@ def gen_cpp_code(forest_json, features_json, filename, cpp_class):
"""Generates code for the .cpp file."""
# Headers
# Required by OrderEncode(float F).
angled_include = [
'#include <%s>' % h
for h in ["cstring", "limits"]
]
angled_include = ["#include <%s>" % h for h in ["cstring", "limits"]]
# Include generated header.
qouted_headers = {filename + '.h', 'llvm/ADT/bit.h'}
qouted_headers = {filename + ".h", "llvm/ADT/bit.h"}
# Headers required by ENUM features used by the model.
qouted_headers |= {f["header"]
for f in features_json if f["kind"] == "ENUM"}
qouted_headers |= {f["header"] for f in features_json if f["kind"] == "ENUM"}
quoted_include = ['#include "%s"' % h for h in sorted(qouted_headers)]
# using-decl for ENUM features.
using_decls = "\n".join("using %s_type = %s;" % (
feature['name'], feature['type'])
using_decls = "\n".join(
"using %s_type = %s;" % (feature["name"], feature["type"])
for feature in features_json
if feature["kind"] == "ENUM")
if feature["kind"] == "ENUM"
)
nl = "\n"
return """%s
@ -267,19 +279,25 @@ uint32_t %s::OrderEncode(float F) {
%s
%s
""" % (nl.join(angled_include), nl.join(quoted_include), cpp_class.ns_begin(),
using_decls, cpp_class.name, evaluate_func(forest_json, cpp_class),
cpp_class.ns_end())
""" % (
nl.join(angled_include),
nl.join(quoted_include),
cpp_class.ns_begin(),
using_decls,
cpp_class.name,
evaluate_func(forest_json, cpp_class),
cpp_class.ns_end(),
)
def main():
parser = argparse.ArgumentParser('DecisionForestCodegen')
parser.add_argument('--filename', help='output file name.')
parser.add_argument('--output_dir', help='output directory.')
parser.add_argument('--model', help='path to model directory.')
parser = argparse.ArgumentParser("DecisionForestCodegen")
parser.add_argument("--filename", help="output file name.")
parser.add_argument("--output_dir", help="output directory.")
parser.add_argument("--model", help="path to model directory.")
parser.add_argument(
'--cpp_class',
help='The name of the class (which may be a namespace-qualified) created in generated header.'
"--cpp_class",
help="The name of the class (which may be a namespace-qualified) created in generated header.",
)
ns = parser.parse_args()
@ -298,19 +316,23 @@ def main():
with open(model_file) as m:
forest_json = json.load(m)
with open(cpp_file, 'w+t') as output_cc:
with open(cpp_file, "w+t") as output_cc:
output_cc.write(
gen_cpp_code(forest_json=forest_json,
features_json=features_json,
filename=filename,
cpp_class=cpp_class))
gen_cpp_code(
forest_json=forest_json,
features_json=features_json,
filename=filename,
cpp_class=cpp_class,
)
)
with open(header_file, 'w+t') as output_h:
output_h.write(gen_header_code(
features_json=features_json,
cpp_class=cpp_class,
filename=filename))
with open(header_file, "w+t") as output_h:
output_h.write(
gen_header_code(
features_json=features_json, cpp_class=cpp_class, filename=filename
)
)
if __name__ == '__main__':
if __name__ == "__main__":
main()

24
clang-tools-extra/clangd/test/lit.cfg.py
View File

@ -4,9 +4,9 @@ lit.llvm.initialize(lit_config, config)
lit.llvm.llvm_config.use_clang([], [], required=False)
lit.llvm.llvm_config.use_default_substitutions()
config.name = 'Clangd'
config.suffixes = ['.test']
config.excludes = ['Inputs']
config.name = "Clangd"
config.suffixes = [".test"]
config.excludes = ["Inputs"]
config.test_format = lit.formats.ShTest(not lit.llvm.llvm_config.use_lit_shell)
config.test_source_root = config.clangd_source_dir + "/test"
config.test_exec_root = config.clangd_binary_dir + "/test"
@ -15,27 +15,27 @@ config.test_exec_root = config.clangd_binary_dir + "/test"
# Used to enable tests based on the required targets. Can be queried with e.g.
# REQUIRES: x86-registered-target
def calculate_arch_features(arch_string):
return [arch.lower() + '-registered-target' for arch in arch_string.split()]
return [arch.lower() + "-registered-target" for arch in arch_string.split()]
lit.llvm.llvm_config.feature_config([('--targets-built',
calculate_arch_features)])
lit.llvm.llvm_config.feature_config([("--targets-built", calculate_arch_features)])
# Clangd-specific lit environment.
config.substitutions.append(('%clangd-benchmark-dir',
config.clangd_binary_dir + "/benchmarks"))
config.substitutions.append(
("%clangd-benchmark-dir", config.clangd_binary_dir + "/benchmarks")
)
if config.clangd_build_xpc:
config.available_features.add('clangd-xpc-support')
config.available_features.add("clangd-xpc-support")
if config.clangd_enable_remote:
config.available_features.add('clangd-remote-index')
config.available_features.add("clangd-remote-index")
if config.clangd_tidy_checks:
config.available_features.add('clangd-tidy-checks')
config.available_features.add("clangd-tidy-checks")
if config.have_zlib:
config.available_features.add('zlib')
config.available_features.add("zlib")
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

5
clang-tools-extra/clangd/test/lit.local.cfg
View File

@ -1,6 +1,7 @@
import re
# We rely on the default -std being derived from the filetype.
# PS4 sets a different -std, and many tests break.
# FIXME: make our tests less brittle instead.
if re.match(r'.*-scei-ps4', config.target_triple):
config.unsupported = True
if re.match(r".*-scei-ps4", config.target_triple):
config.unsupported = True

149
clang-tools-extra/clangd/test/remote-index/pipeline_helper.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python
#
#===- pipeline_helper.py - Remote Index pipeline Helper *- python -------*--===#
# ===- pipeline_helper.py - Remote Index pipeline Helper *- python -------*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
import argparse
import os
@ -18,83 +18,100 @@ import threading
def kill_process_after_delay(server_process):
time.sleep(10)
if server_process.poll() is None:
server_process.kill()
time.sleep(10)
if server_process.poll() is None:
server_process.kill()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--input-file-name', required=True)
parser.add_argument('--project-root', required=True)
parser.add_argument('--index-file', required=True)
parser.add_argument('--server-arg', action='append', default=[])
parser.add_argument('--server-log', nargs='?', type=argparse.FileType('wb'), default=os.devnull)
parser = argparse.ArgumentParser()
parser.add_argument("--input-file-name", required=True)
parser.add_argument("--project-root", required=True)
parser.add_argument("--index-file", required=True)
parser.add_argument("--server-arg", action="append", default=[])
parser.add_argument(
"--server-log", nargs="?", type=argparse.FileType("wb"), default=os.devnull
)
args = parser.parse_args()
args = parser.parse_args()
# Grab an available port.
with socket() as s:
s.bind(('localhost', 0))
server_address = 'localhost:' + str(s.getsockname()[1])
# Grab an available port.
with socket() as s:
s.bind(("localhost", 0))
server_address = "localhost:" + str(s.getsockname()[1])
print('Initializing clangd-index-server...', file=sys.stderr)
index_server_process = subprocess.Popen([
'clangd-index-server', '--server-address=' + server_address,
args.index_file, args.project_root
] + args.server_arg,
stderr=subprocess.PIPE)
print("Initializing clangd-index-server...", file=sys.stderr)
index_server_process = subprocess.Popen(
[
"clangd-index-server",
"--server-address=" + server_address,
args.index_file,
args.project_root,
]
+ args.server_arg,
stderr=subprocess.PIPE,
)
# This will kill index_server_process if it hangs without printing init
# message.
shutdown_thread = threading.Thread(
target=kill_process_after_delay, args=(index_server_process,))
shutdown_thread.daemon = True
shutdown_thread.start()
# This will kill index_server_process if it hangs without printing init
# message.
shutdown_thread = threading.Thread(
target=kill_process_after_delay, args=(index_server_process,)
)
shutdown_thread.daemon = True
shutdown_thread.start()
# Wait for the server to warm-up.
found_init_message = False
while index_server_process.poll() is None:
line = index_server_process.stderr.readline()
args.server_log.write(line)
args.server_log.flush()
if b'Server listening' in line:
print('Server initialization complete.', file=sys.stderr)
found_init_message = True
break
# Wait for the server to warm-up.
found_init_message = False
while index_server_process.poll() is None:
line = index_server_process.stderr.readline()
args.server_log.write(line)
args.server_log.flush()
if b"Server listening" in line:
print("Server initialization complete.", file=sys.stderr)
found_init_message = True
break
if not found_init_message:
print('Server initialization failed. Shutting down.', file=sys.stderr)
sys.exit(1)
if not found_init_message:
print("Server initialization failed. Shutting down.", file=sys.stderr)
sys.exit(1)
print('Running clangd-index-server-monitor...', file=sys.stderr)
index_server_monitor_process = subprocess.Popen([
'clangd-index-server-monitor', server_address,
], stderr=subprocess.PIPE)
print("Running clangd-index-server-monitor...", file=sys.stderr)
index_server_monitor_process = subprocess.Popen(
[
"clangd-index-server-monitor",
server_address,
],
stderr=subprocess.PIPE,
)
index_server_monitor_process.wait()
index_server_monitor_process.wait()
in_file = open(args.input_file_name)
in_file = open(args.input_file_name)
print('Staring clangd...', file=sys.stderr)
clangd_process = subprocess.Popen([
'clangd', '--remote-index-address=' + server_address,
'--project-root=' + args.project_root, '--lit-test', '--sync'
],
stdin=in_file)
clangd_process.wait()
print(
'Clangd executed successfully, shutting down child processes.',
file=sys.stderr)
index_server_process.kill()
for line in index_server_process.stderr:
args.server_log.write(line)
args.server_log.flush()
print("Staring clangd...", file=sys.stderr)
clangd_process = subprocess.Popen(
[
"clangd",
"--remote-index-address=" + server_address,
"--project-root=" + args.project_root,
"--lit-test",
"--sync",
],
stdin=in_file,
)
clangd_process.wait()
print(
"Clangd executed successfully, shutting down child processes.", file=sys.stderr
)
index_server_process.kill()
for line in index_server_process.stderr:
args.server_log.write(line)
args.server_log.flush()
for line in index_server_monitor_process.stderr:
args.server_log.write(line)
args.server_log.flush()
for line in index_server_monitor_process.stderr:
args.server_log.write(line)
args.server_log.flush()
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

20
clang-tools-extra/clangd/unittests/lit.cfg.py
View File

@ -1,21 +1,23 @@
import lit.formats
config.name = "Clangd Unit Tests"
config.test_format = lit.formats.GoogleTest('.', 'Tests')
config.test_format = lit.formats.GoogleTest(".", "Tests")
config.test_source_root = config.clangd_binary_dir + "/unittests"
config.test_exec_root = config.clangd_binary_dir + "/unittests"
# Point the dynamic loader at dynamic libraries in 'lib'.
# FIXME: it seems every project has a copy of this logic. Move it somewhere.
import platform
if platform.system() == 'Darwin':
shlibpath_var = 'DYLD_LIBRARY_PATH'
elif platform.system() == 'Windows':
shlibpath_var = 'PATH'
if platform.system() == "Darwin":
shlibpath_var = "DYLD_LIBRARY_PATH"
elif platform.system() == "Windows":
shlibpath_var = "PATH"
else:
shlibpath_var = 'LD_LIBRARY_PATH'
config.environment[shlibpath_var] = os.path.pathsep.join((
"@SHLIBDIR@", "@LLVM_LIBS_DIR@",
config.environment.get(shlibpath_var,'')))
shlibpath_var = "LD_LIBRARY_PATH"
config.environment[shlibpath_var] = os.path.pathsep.join(
("@SHLIBDIR@", "@LLVM_LIBS_DIR@", config.environment.get(shlibpath_var, ""))
)
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

223
clang-tools-extra/docs/clang-tidy/checks/gen-static-analyzer-docs.py
View File

@ -10,49 +10,64 @@ import os
import re
"""Get path of script so files are always in correct directory"""
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
"""Get dict of checker related info and parse for full check names
Returns:
checkers: dict of checker info
"""
def get_checkers(checkers_td_directory):
p = subprocess.Popen(["llvm-tblgen", "--dump-json", "-I",
checkers_td_directory, checkers_td_directory+"Checkers.td"],
stdout=subprocess.PIPE)
table_entries = json.loads(p.communicate()[0])
documentable_checkers = []
checkers = table_entries["!instanceof"]["Checker"]
packages = table_entries["!instanceof"]["Package"]
p = subprocess.Popen(
[
"llvm-tblgen",
"--dump-json",
"-I",
checkers_td_directory,
checkers_td_directory + "Checkers.td",
],
stdout=subprocess.PIPE,
)
table_entries = json.loads(p.communicate()[0])
documentable_checkers = []
checkers = table_entries["!instanceof"]["Checker"]
packages = table_entries["!instanceof"]["Package"]
for checker_ in checkers:
checker = table_entries[checker_]
checker_name = checker["CheckerName"]
package_ = checker["ParentPackage"]["def"]
package = table_entries[package_]
package_name = package["PackageName"]
checker_package_prefix = package_name
parent_package_ = package["ParentPackage"]
hidden = (checker["Hidden"] != 0) or (package["Hidden"] != 0)
for checker_ in checkers:
checker = table_entries[checker_]
checker_name = checker["CheckerName"]
package_ = checker["ParentPackage"]["def"]
package = table_entries[package_]
package_name = package["PackageName"]
checker_package_prefix = package_name
parent_package_ = package["ParentPackage"]
hidden = (checker["Hidden"] != 0) or (package["Hidden"] != 0)
while(parent_package_ != None):
parent_package = table_entries[parent_package_["def"]]
checker_package_prefix = parent_package["PackageName"] + "." + checker_package_prefix
hidden = hidden or parent_package["Hidden"] != 0
parent_package_ = parent_package["ParentPackage"]
while parent_package_ != None:
parent_package = table_entries[parent_package_["def"]]
checker_package_prefix = (
parent_package["PackageName"] + "." + checker_package_prefix
)
hidden = hidden or parent_package["Hidden"] != 0
parent_package_ = parent_package["ParentPackage"]
full_package_name = "clang-analyzer-" + checker_package_prefix + "." + checker_name
anchor_url = re.sub("\.", "-", checker_package_prefix + "." + checker_name).lower()
full_package_name = (
"clang-analyzer-" + checker_package_prefix + "." + checker_name
)
anchor_url = re.sub(
"\.", "-", checker_package_prefix + "." + checker_name
).lower()
if(not hidden and "alpha" not in full_package_name.lower()):
checker["FullPackageName"] = full_package_name
checker["AnchorUrl"] = anchor_url
documentable_checkers.append(checker)
if not hidden and "alpha" not in full_package_name.lower():
checker["FullPackageName"] = full_package_name
checker["AnchorUrl"] = anchor_url
documentable_checkers.append(checker)
documentable_checkers.sort(key=lambda x: x["FullPackageName"])
return documentable_checkers
documentable_checkers.sort(key=lambda x: x["FullPackageName"])
return documentable_checkers
"""Generate documentation for checker
@ -61,88 +76,108 @@ Args:
only_help_text: Generate documentation based off the checker description.
Used when there is no other documentation to link to.
"""
def generate_documentation(checker, only_help_text=False):
with open(os.path.join(__location__, checker["FullPackageName"]+".rst"),"w") as f:
f.write(".. title:: clang-tidy - %s\n" % checker["FullPackageName"])
if(not only_help_text):
f.write(".. meta::\n")
f.write(" :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#%s\n" % checker["AnchorUrl"])
f.write("\n")
f.write("%s\n" % checker["FullPackageName"])
f.write("=" * len(checker["FullPackageName"]) + "\n")
f.write("\n")
if(only_help_text):
f.write("%s\n" % checker["HelpText"])
else:
f.write("The %s check is an alias, please see\n" % checker["FullPackageName"])
f.write("`Clang Static Analyzer Available Checkers <https://clang.llvm.org/docs/analyzer/checkers.html#%s>`_\n" % checker["AnchorUrl"])
f.write("for more information.\n")
f.close()
with open(
os.path.join(__location__, checker["FullPackageName"] + ".rst"), "w"
) as f:
f.write(".. title:: clang-tidy - %s\n" % checker["FullPackageName"])
if not only_help_text:
f.write(".. meta::\n")
f.write(
" :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#%s\n"
% checker["AnchorUrl"]
)
f.write("\n")
f.write("%s\n" % checker["FullPackageName"])
f.write("=" * len(checker["FullPackageName"]) + "\n")
f.write("\n")
if only_help_text:
f.write("%s\n" % checker["HelpText"])
else:
f.write(
"The %s check is an alias, please see\n" % checker["FullPackageName"]
)
f.write(
"`Clang Static Analyzer Available Checkers <https://clang.llvm.org/docs/analyzer/checkers.html#%s>`_\n"
% checker["AnchorUrl"]
)
f.write("for more information.\n")
f.close()
"""Update list.rst to include the new checks
Args:
checkers: dict acquired from get_checkers()
"""
def update_documentation_list(checkers):
with open(os.path.join(__location__, "list.rst"), "r+") as f:
f_text = f.read()
header, check_text= f_text.split(".. toctree::\n")
checks = check_text.split("\n")
for checker in checkers:
if((" %s" % checker["FullPackageName"]) not in checks):
checks.append(" %s" % checker["FullPackageName"])
checks.sort()
with open(os.path.join(__location__, "list.rst"), "r+") as f:
f_text = f.read()
header, check_text = f_text.split(".. toctree::\n")
checks = check_text.split("\n")
for checker in checkers:
if (" %s" % checker["FullPackageName"]) not in checks:
checks.append(" %s" % checker["FullPackageName"])
checks.sort()
#Overwrite file with new data
f.seek(0)
f.write(header)
f.write(".. toctree::")
for check in checks:
f.write("%s\n" % check)
f.close()
# Overwrite file with new data
f.seek(0)
f.write(header)
f.write(".. toctree::")
for check in checks:
f.write("%s\n" % check)
f.close()
default_path_monorepo = "../../../../clang/include/clang/StaticAnalyzer/Checkers/"
default_path_in_tree = "../../../../../include/clang/StaticAnalyzer/Checkers/"
default_path_monorepo = '../../../../clang/include/clang/StaticAnalyzer/Checkers/'
default_path_in_tree = '../../../../../include/clang/StaticAnalyzer/Checkers/'
def parse_arguments():
"""Set up and parse command-line arguments
Returns:
file_path: Path to Checkers.td"""
usage = """Parse Checkers.td to generate documentation for static analyzer checks"""
parse = argparse.ArgumentParser(description=usage)
"""Set up and parse command-line arguments
Returns:
file_path: Path to Checkers.td"""
usage = """Parse Checkers.td to generate documentation for static analyzer checks"""
parse = argparse.ArgumentParser(description=usage)
file_path_help = ("""Path to Checkers directory
file_path_help = """Path to Checkers directory
defaults to ../../../../clang/include/clang/StaticAnalyzer/Checkers/ if it exists
then to ../../../../../include/clang/StaticAnalyzer/Checkers/""")
then to ../../../../../include/clang/StaticAnalyzer/Checkers/"""
default_path=None
if(os.path.exists(default_path_monorepo)):
default_path = default_path_monorepo
elif(os.path.exists(default_path_in_tree)):
default_path = default_path_in_tree
default_path = None
if os.path.exists(default_path_monorepo):
default_path = default_path_monorepo
elif os.path.exists(default_path_in_tree):
default_path = default_path_in_tree
parse.add_argument("file", type=str, help=file_path_help, nargs='?', default=default_path)
args = parse.parse_args()
parse.add_argument(
"file", type=str, help=file_path_help, nargs="?", default=default_path
)
args = parse.parse_args()
if(args.file is None):
print("Could not find Checkers directory. Please see -h")
exit(1)
if args.file is None:
print("Could not find Checkers directory. Please see -h")
exit(1)
return args.file
return args.file
def main():
file_path = parse_arguments()
checkers = get_checkers(file_path)
for checker in checkers:
#No documentation nor alpha documentation
if(checker["Documentation"][1] == 0 and checker["Documentation"][0] == 0):
generate_documentation(checker, True)
else:
generate_documentation(checker)
print("Generated documentation for: %s" % checker["FullPackageName"])
update_documentation_list(checkers)
file_path = parse_arguments()
checkers = get_checkers(file_path)
for checker in checkers:
# No documentation nor alpha documentation
if checker["Documentation"][1] == 0 and checker["Documentation"][0] == 0:
generate_documentation(checker, True)
else:
generate_documentation(checker)
print("Generated documentation for: %s" % checker["FullPackageName"])
update_documentation_list(checkers)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

144
clang-tools-extra/docs/conf.py
View File

@ -17,66 +17,66 @@ from datetime import date
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo', 'sphinx.ext.mathjax']
extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = u'Extra Clang Tools'
copyright = u'2007-%d, The Clang Team' % date.today().year
project = "Extra Clang Tools"
copyright = "2007-%d, The Clang Team" % date.today().year
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'friendly'
pygments_style = "friendly"
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# modindex_common_prefix = []
in_progress_title = "(In-Progress) " if tags.has("PreRelease") else ""
@ -88,121 +88,124 @@ rst_epilog = f"""
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'haiku'
html_theme = "haiku"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'ExtraClangToolsdoc'
htmlhelp_basename = "ExtraClangToolsdoc"
# -- Options for LaTeX output --------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'ExtraClangTools.tex', u'Extra Clang Tools Documentation',
u'The Clang Team', 'manual'),
(
"index",
"ExtraClangTools.tex",
"Extra Clang Tools Documentation",
"The Clang Team",
"manual",
),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
@ -210,12 +213,17 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'extraclangtools', u'Extra Clang Tools Documentation',
[u'The Clang Team'], 1)
(
"index",
"extraclangtools",
"Extra Clang Tools Documentation",
["The Clang Team"],
1,
)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# man_show_urls = False
# -- Options for Texinfo output ------------------------------------------------
@ -224,16 +232,22 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'ExtraClangTools', u'Extra Clang Tools Documentation',
u'The Clang Team', 'ExtraClangTools', 'One line description of project.',
'Miscellaneous'),
(
"index",
"ExtraClangTools",
"Extra Clang Tools Documentation",
"The Clang Team",
"ExtraClangTools",
"One line description of project.",
"Miscellaneous",
),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# texinfo_show_urls = 'footnote'

20
clang-tools-extra/include-cleaner/test/Unit/lit.cfg.py
View File

@ -1,21 +1,23 @@
import lit.formats
config.name = "clangIncludeCleaner Unit Tests"
config.test_format = lit.formats.GoogleTest('.', 'Tests')
config.test_format = lit.formats.GoogleTest(".", "Tests")
config.test_source_root = config.clang_include_cleaner_binary_dir + "/unittests"
config.test_exec_root = config.clang_include_cleaner_binary_dir + "/unittests"
# Point the dynamic loader at dynamic libraries in 'lib'.
# FIXME: it seems every project has a copy of this logic. Move it somewhere.
import platform
if platform.system() == 'Darwin':
shlibpath_var = 'DYLD_LIBRARY_PATH'
elif platform.system() == 'Windows':
shlibpath_var = 'PATH'
if platform.system() == "Darwin":
shlibpath_var = "DYLD_LIBRARY_PATH"
elif platform.system() == "Windows":
shlibpath_var = "PATH"
else:
shlibpath_var = 'LD_LIBRARY_PATH'
config.environment[shlibpath_var] = os.path.pathsep.join((
"@SHLIBDIR@", "@LLVM_LIBS_DIR@",
config.environment.get(shlibpath_var,'')))
shlibpath_var = "LD_LIBRARY_PATH"
config.environment[shlibpath_var] = os.path.pathsep.join(
("@SHLIBDIR@", "@LLVM_LIBS_DIR@", config.environment.get(shlibpath_var, ""))
)
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

13
clang-tools-extra/include-cleaner/test/lit.cfg.py
View File

@ -3,17 +3,16 @@ import lit.llvm
lit.llvm.initialize(lit_config, config)
lit.llvm.llvm_config.use_default_substitutions()
config.name = 'ClangIncludeCleaner'
config.suffixes = ['.test', '.c', '.cpp']
config.excludes = ['Inputs']
config.name = "ClangIncludeCleaner"
config.suffixes = [".test", ".c", ".cpp"]
config.excludes = ["Inputs"]
config.test_format = lit.formats.ShTest(not lit.llvm.llvm_config.use_lit_shell)
config.test_source_root = config.clang_include_cleaner_source_dir + "/test"
config.test_exec_root = config.clang_include_cleaner_binary_dir + "/test"
config.environment['PATH'] = os.path.pathsep.join((
config.clang_tools_dir,
config.llvm_tools_dir,
config.environment['PATH']))
config.environment["PATH"] = os.path.pathsep.join(
(config.clang_tools_dir, config.llvm_tools_dir, config.environment["PATH"])
)
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

20
clang-tools-extra/pseudo/test/Unit/lit.cfg.py
View File

@ -1,21 +1,23 @@
import lit.formats
config.name = "clangPseudo Unit Tests"
config.test_format = lit.formats.GoogleTest('.', 'Tests')
config.test_format = lit.formats.GoogleTest(".", "Tests")
config.test_source_root = config.clang_pseudo_binary_dir + "/unittests"
config.test_exec_root = config.clang_pseudo_binary_dir + "/unittests"
# Point the dynamic loader at dynamic libraries in 'lib'.
# FIXME: it seems every project has a copy of this logic. Move it somewhere.
import platform
if platform.system() == 'Darwin':
shlibpath_var = 'DYLD_LIBRARY_PATH'
elif platform.system() == 'Windows':
shlibpath_var = 'PATH'
if platform.system() == "Darwin":
shlibpath_var = "DYLD_LIBRARY_PATH"
elif platform.system() == "Windows":
shlibpath_var = "PATH"
else:
shlibpath_var = 'LD_LIBRARY_PATH'
config.environment[shlibpath_var] = os.path.pathsep.join((
"@SHLIBDIR@", "@LLVM_LIBS_DIR@",
config.environment.get(shlibpath_var,'')))
shlibpath_var = "LD_LIBRARY_PATH"
config.environment[shlibpath_var] = os.path.pathsep.join(
("@SHLIBDIR@", "@LLVM_LIBS_DIR@", config.environment.get(shlibpath_var, ""))
)
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

13
clang-tools-extra/pseudo/test/lit.cfg.py
View File

@ -3,17 +3,16 @@ import lit.llvm
lit.llvm.initialize(lit_config, config)
lit.llvm.llvm_config.use_default_substitutions()
config.name = 'ClangPseudo'
config.suffixes = ['.test', '.c', '.cpp']
config.excludes = ['Inputs']
config.name = "ClangPseudo"
config.suffixes = [".test", ".c", ".cpp"]
config.excludes = ["Inputs"]
config.test_format = lit.formats.ShTest(not lit.llvm.llvm_config.use_lit_shell)
config.test_source_root = config.clang_pseudo_source_dir + "/test"
config.test_exec_root = config.clang_pseudo_binary_dir + "/test"
config.environment['PATH'] = os.path.pathsep.join((
config.clang_tools_dir,
config.llvm_tools_dir,
config.environment['PATH']))
config.environment["PATH"] = os.path.pathsep.join(
(config.clang_tools_dir, config.llvm_tools_dir, config.environment["PATH"])
)
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

4
clang-tools-extra/pseudo/test/lit.local.cfg
View File

@ -1,2 +1,2 @@
cxx_bnf_file = os.path.join(config.clang_pseudo_source_dir, 'lib', 'cxx', 'cxx.bnf')
config.substitutions.append(('%cxx-bnf-file', cxx_bnf_file))
cxx_bnf_file = os.path.join(config.clang_pseudo_source_dir, "lib", "cxx", "cxx.bnf")
config.substitutions.append(("%cxx-bnf-file", cxx_bnf_file))

21
clang-tools-extra/test/Unit/lit.cfg.py
View File

@ -5,7 +5,7 @@ import platform
import lit.formats
config.name = "Extra Tools Unit Tests"
config.suffixes = [] # Seems not to matter for google tests?
config.suffixes = [] # Seems not to matter for google tests?
# Test Source and Exec root dirs both point to the same directory where google
# test binaries are built.
@ -17,21 +17,22 @@ config.test_exec_root = config.test_source_root
# a special value for GoogleTest indicating that it should look through the
# entire testsuite recursively for tests (alternatively, one could provide a
# ;-separated list of subdirectories).
config.test_format = lit.formats.GoogleTest('.', 'Tests')
config.test_format = lit.formats.GoogleTest(".", "Tests")
if platform.system() == 'Darwin':
shlibpath_var = 'DYLD_LIBRARY_PATH'
elif platform.system() == 'Windows':
shlibpath_var = 'PATH'
if platform.system() == "Darwin":
shlibpath_var = "DYLD_LIBRARY_PATH"
elif platform.system() == "Windows":
shlibpath_var = "PATH"
else:
shlibpath_var = 'LD_LIBRARY_PATH'
shlibpath_var = "LD_LIBRARY_PATH"
# Point the dynamic loader at dynamic libraries in 'lib'.
shlibpath = os.path.pathsep.join((config.shlibdir, config.llvm_libs_dir,
config.environment.get(shlibpath_var,'')))
shlibpath = os.path.pathsep.join(
(config.shlibdir, config.llvm_libs_dir, config.environment.get(shlibpath_var, ""))
)
# Win32 seeks DLLs along %PATH%.
if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir):
if sys.platform in ["win32", "cygwin"] and os.path.isdir(config.shlibdir):
shlibpath = os.path.pathsep.join((config.shlibdir, shlibpath))
config.environment[shlibpath_var] = shlibpath

403
clang-tools-extra/test/clang-tidy/check_clang_tidy.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
#===- check_clang_tidy.py - ClangTidy Test Helper ------------*- python -*--===#
# ===- check_clang_tidy.py - ClangTidy Test Helper ------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
r"""
ClangTidy Test Helper
@ -40,228 +40,275 @@ import sys
def write_file(file_name, text):
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
f.truncate()
with open(file_name, "w", encoding="utf-8") as f:
f.write(text)
f.truncate()
def try_run(args, raise_error=True):
try:
process_output = \
subprocess.check_output(args, stderr=subprocess.STDOUT).decode(errors='ignore')
except subprocess.CalledProcessError as e:
process_output = e.output.decode(errors='ignore')
print('%s failed:\n%s' % (' '.join(args), process_output))
if raise_error:
raise
return process_output
try:
process_output = subprocess.check_output(args, stderr=subprocess.STDOUT).decode(
errors="ignore"
)
except subprocess.CalledProcessError as e:
process_output = e.output.decode(errors="ignore")
print("%s failed:\n%s" % (" ".join(args), process_output))
if raise_error:
raise
return process_output
# This class represents the appearance of a message prefix in a file.
class MessagePrefix:
def __init__(self, label):
self.has_message = False
self.prefixes = []
self.label = label
def __init__(self, label):
self.has_message = False
self.prefixes = []
self.label = label
def check(self, file_check_suffix, input_text):
self.prefix = self.label + file_check_suffix
self.has_message = self.prefix in input_text
if self.has_message:
self.prefixes.append(self.prefix)
return self.has_message
def check(self, file_check_suffix, input_text):
self.prefix = self.label + file_check_suffix
self.has_message = self.prefix in input_text
if self.has_message:
self.prefixes.append(self.prefix)
return self.has_message
class CheckRunner:
def __init__(self, args, extra_args):
self.resource_dir = args.resource_dir
self.assume_file_name = args.assume_filename
self.input_file_name = args.input_file_name
self.check_name = args.check_name
self.temp_file_name = args.temp_file_name
self.original_file_name = self.temp_file_name + ".orig"
self.expect_clang_tidy_error = args.expect_clang_tidy_error
self.std = args.std
self.check_suffix = args.check_suffix
self.input_text = ''
self.has_check_fixes = False
self.has_check_messages = False
self.has_check_notes = False
self.fixes = MessagePrefix('CHECK-FIXES')
self.messages = MessagePrefix('CHECK-MESSAGES')
self.notes = MessagePrefix('CHECK-NOTES')
def __init__(self, args, extra_args):
self.resource_dir = args.resource_dir
self.assume_file_name = args.assume_filename
self.input_file_name = args.input_file_name
self.check_name = args.check_name
self.temp_file_name = args.temp_file_name
self.original_file_name = self.temp_file_name + ".orig"
self.expect_clang_tidy_error = args.expect_clang_tidy_error
self.std = args.std
self.check_suffix = args.check_suffix
self.input_text = ""
self.has_check_fixes = False
self.has_check_messages = False
self.has_check_notes = False
self.fixes = MessagePrefix("CHECK-FIXES")
self.messages = MessagePrefix("CHECK-MESSAGES")
self.notes = MessagePrefix("CHECK-NOTES")
file_name_with_extension = self.assume_file_name or self.input_file_name
_, extension = os.path.splitext(file_name_with_extension)
if extension not in ['.c', '.hpp', '.m', '.mm']:
extension = '.cpp'
self.temp_file_name = self.temp_file_name + extension
file_name_with_extension = self.assume_file_name or self.input_file_name
_, extension = os.path.splitext(file_name_with_extension)
if extension not in [".c", ".hpp", ".m", ".mm"]:
extension = ".cpp"
self.temp_file_name = self.temp_file_name + extension
self.clang_extra_args = []
self.clang_tidy_extra_args = extra_args
if '--' in extra_args:
i = self.clang_tidy_extra_args.index('--')
self.clang_extra_args = self.clang_tidy_extra_args[i + 1:]
self.clang_tidy_extra_args = self.clang_tidy_extra_args[:i]
self.clang_extra_args = []
self.clang_tidy_extra_args = extra_args
if "--" in extra_args:
i = self.clang_tidy_extra_args.index("--")
self.clang_extra_args = self.clang_tidy_extra_args[i + 1 :]
self.clang_tidy_extra_args = self.clang_tidy_extra_args[:i]
# If the test does not specify a config style, force an empty one; otherwise
# auto-detection logic can discover a ".clang-tidy" file that is not related to
# the test.
if not any([
re.match('^-?-config(-file)?=', arg)
for arg in self.clang_tidy_extra_args]):
self.clang_tidy_extra_args.append('--config={}')
# If the test does not specify a config style, force an empty one; otherwise
# auto-detection logic can discover a ".clang-tidy" file that is not related to
# the test.
if not any(
[re.match("^-?-config(-file)?=", arg) for arg in self.clang_tidy_extra_args]
):
self.clang_tidy_extra_args.append("--config={}")
if extension in ['.m', '.mm']:
self.clang_extra_args = ['-fobjc-abi-version=2', '-fobjc-arc', '-fblocks'] + \
self.clang_extra_args
if extension in [".m", ".mm"]:
self.clang_extra_args = [
"-fobjc-abi-version=2",
"-fobjc-arc",
"-fblocks",
] + self.clang_extra_args
if extension in ['.cpp', '.hpp', '.mm']:
self.clang_extra_args.append('-std=' + self.std)
if extension in [".cpp", ".hpp", ".mm"]:
self.clang_extra_args.append("-std=" + self.std)
# Tests should not rely on STL being available, and instead provide mock
# implementations of relevant APIs.
self.clang_extra_args.append('-nostdinc++')
# Tests should not rely on STL being available, and instead provide mock
# implementations of relevant APIs.
self.clang_extra_args.append("-nostdinc++")
if self.resource_dir is not None:
self.clang_extra_args.append('-resource-dir=%s' % self.resource_dir)
if self.resource_dir is not None:
self.clang_extra_args.append("-resource-dir=%s" % self.resource_dir)
def read_input(self):
with open(self.input_file_name, 'r', encoding='utf-8') as input_file:
self.input_text = input_file.read()
def read_input(self):
with open(self.input_file_name, "r", encoding="utf-8") as input_file:
self.input_text = input_file.read()
def get_prefixes(self):
for suffix in self.check_suffix:
if suffix and not re.match('^[A-Z0-9\\-]+$', suffix):
sys.exit('Only A..Z, 0..9 and "-" are allowed in check suffixes list,'
+ ' but "%s" was given' % suffix)
def get_prefixes(self):
for suffix in self.check_suffix:
if suffix and not re.match("^[A-Z0-9\\-]+$", suffix):
sys.exit(
'Only A..Z, 0..9 and "-" are allowed in check suffixes list,'
+ ' but "%s" was given' % suffix
)
file_check_suffix = ('-' + suffix) if suffix else ''
file_check_suffix = ("-" + suffix) if suffix else ""
has_check_fix = self.fixes.check(file_check_suffix, self.input_text)
self.has_check_fixes = self.has_check_fixes or has_check_fix
has_check_fix = self.fixes.check(file_check_suffix, self.input_text)
self.has_check_fixes = self.has_check_fixes or has_check_fix
has_check_message = self.messages.check(file_check_suffix, self.input_text)
self.has_check_messages = self.has_check_messages or has_check_message
has_check_message = self.messages.check(file_check_suffix, self.input_text)
self.has_check_messages = self.has_check_messages or has_check_message
has_check_note = self.notes.check(file_check_suffix, self.input_text)
self.has_check_notes = self.has_check_notes or has_check_note
has_check_note = self.notes.check(file_check_suffix, self.input_text)
self.has_check_notes = self.has_check_notes or has_check_note
if has_check_note and has_check_message:
sys.exit('Please use either %s or %s but not both' %
(self.notes.prefix, self.messages.prefix))
if has_check_note and has_check_message:
sys.exit(
"Please use either %s or %s but not both"
% (self.notes.prefix, self.messages.prefix)
)
if not has_check_fix and not has_check_message and not has_check_note:
sys.exit('%s, %s or %s not found in the input' %
(self.fixes.prefix, self.messages.prefix, self.notes.prefix))
if not has_check_fix and not has_check_message and not has_check_note:
sys.exit(
"%s, %s or %s not found in the input"
% (self.fixes.prefix, self.messages.prefix, self.notes.prefix)
)
assert self.has_check_fixes or self.has_check_messages or self.has_check_notes
assert self.has_check_fixes or self.has_check_messages or self.has_check_notes
def prepare_test_inputs(self):
# Remove the contents of the CHECK lines to avoid CHECKs matching on
# themselves. We need to keep the comments to preserve line numbers while
# avoiding empty lines which could potentially trigger formatting-related
# checks.
cleaned_test = re.sub('// *CHECK-[A-Z0-9\\-]*:[^\r\n]*', '//', self.input_text)
write_file(self.temp_file_name, cleaned_test)
write_file(self.original_file_name, cleaned_test)
def prepare_test_inputs(self):
# Remove the contents of the CHECK lines to avoid CHECKs matching on
# themselves. We need to keep the comments to preserve line numbers while
# avoiding empty lines which could potentially trigger formatting-related
# checks.
cleaned_test = re.sub("// *CHECK-[A-Z0-9\\-]*:[^\r\n]*", "//", self.input_text)
write_file(self.temp_file_name, cleaned_test)
write_file(self.original_file_name, cleaned_test)
def run_clang_tidy(self):
args = ['clang-tidy', self.temp_file_name, '-fix', '--checks=-*,' + self.check_name] + \
self.clang_tidy_extra_args + ['--'] + self.clang_extra_args
if self.expect_clang_tidy_error:
args.insert(0, 'not')
print('Running ' + repr(args) + '...')
clang_tidy_output = try_run(args)
print('------------------------ clang-tidy output -----------------------')
print(clang_tidy_output.encode(sys.stdout.encoding, errors="replace").decode(sys.stdout.encoding))
print('------------------------------------------------------------------')
def run_clang_tidy(self):
args = (
[
"clang-tidy",
self.temp_file_name,
"-fix",
"--checks=-*," + self.check_name,
]
+ self.clang_tidy_extra_args
+ ["--"]
+ self.clang_extra_args
)
if self.expect_clang_tidy_error:
args.insert(0, "not")
print("Running " + repr(args) + "...")
clang_tidy_output = try_run(args)
print("------------------------ clang-tidy output -----------------------")
print(
clang_tidy_output.encode(sys.stdout.encoding, errors="replace").decode(
sys.stdout.encoding
)
)
print("------------------------------------------------------------------")
diff_output = try_run(['diff', '-u', self.original_file_name, self.temp_file_name], False)
print('------------------------------ Fixes -----------------------------')
print(diff_output)
print('------------------------------------------------------------------')
return clang_tidy_output
diff_output = try_run(
["diff", "-u", self.original_file_name, self.temp_file_name], False
)
print("------------------------------ Fixes -----------------------------")
print(diff_output)
print("------------------------------------------------------------------")
return clang_tidy_output
def check_fixes(self):
if self.has_check_fixes:
try_run(['FileCheck', '-input-file=' + self.temp_file_name, self.input_file_name,
'-check-prefixes=' + ','.join(self.fixes.prefixes),
'-strict-whitespace'])
def check_fixes(self):
if self.has_check_fixes:
try_run(
[
"FileCheck",
"-input-file=" + self.temp_file_name,
self.input_file_name,
"-check-prefixes=" + ",".join(self.fixes.prefixes),
"-strict-whitespace",
]
)
def check_messages(self, clang_tidy_output):
if self.has_check_messages:
messages_file = self.temp_file_name + '.msg'
write_file(messages_file, clang_tidy_output)
try_run(['FileCheck', '-input-file=' + messages_file, self.input_file_name,
'-check-prefixes=' + ','.join(self.messages.prefixes),
'-implicit-check-not={{warning|error}}:'])
def check_messages(self, clang_tidy_output):
if self.has_check_messages:
messages_file = self.temp_file_name + ".msg"
write_file(messages_file, clang_tidy_output)
try_run(
[
"FileCheck",
"-input-file=" + messages_file,
self.input_file_name,
"-check-prefixes=" + ",".join(self.messages.prefixes),
"-implicit-check-not={{warning|error}}:",
]
)
def check_notes(self, clang_tidy_output):
if self.has_check_notes:
notes_file = self.temp_file_name + '.notes'
filtered_output = [line for line in clang_tidy_output.splitlines()
if not ("note: FIX-IT applied" in line)]
write_file(notes_file, '\n'.join(filtered_output))
try_run(['FileCheck', '-input-file=' + notes_file, self.input_file_name,
'-check-prefixes=' + ','.join(self.notes.prefixes),
'-implicit-check-not={{note|warning|error}}:'])
def check_notes(self, clang_tidy_output):
if self.has_check_notes:
notes_file = self.temp_file_name + ".notes"
filtered_output = [
line
for line in clang_tidy_output.splitlines()
if not ("note: FIX-IT applied" in line)
]
write_file(notes_file, "\n".join(filtered_output))
try_run(
[
"FileCheck",
"-input-file=" + notes_file,
self.input_file_name,
"-check-prefixes=" + ",".join(self.notes.prefixes),
"-implicit-check-not={{note|warning|error}}:",
]
)
def run(self):
self.read_input()
self.get_prefixes()
self.prepare_test_inputs()
clang_tidy_output = self.run_clang_tidy()
self.check_fixes()
self.check_messages(clang_tidy_output)
self.check_notes(clang_tidy_output)
def run(self):
self.read_input()
self.get_prefixes()
self.prepare_test_inputs()
clang_tidy_output = self.run_clang_tidy()
self.check_fixes()
self.check_messages(clang_tidy_output)
self.check_notes(clang_tidy_output)
def expand_std(std):
if std == 'c++98-or-later':
return ['c++98', 'c++11', 'c++14', 'c++17', 'c++20']
if std == 'c++11-or-later':
return ['c++11', 'c++14', 'c++17', 'c++20']
if std == 'c++14-or-later':
return ['c++14', 'c++17', 'c++20']
if std == 'c++17-or-later':
return ['c++17', 'c++20']
if std == 'c++20-or-later':
return ['c++20']
return [std]
if std == "c++98-or-later":
return ["c++98", "c++11", "c++14", "c++17", "c++20"]
if std == "c++11-or-later":
return ["c++11", "c++14", "c++17", "c++20"]
if std == "c++14-or-later":
return ["c++14", "c++17", "c++20"]
if std == "c++17-or-later":
return ["c++17", "c++20"]
if std == "c++20-or-later":
return ["c++20"]
return [std]
def csv(string):
return string.split(',')
return string.split(",")
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument('-expect-clang-tidy-error', action='store_true')
parser.add_argument('-resource-dir')
parser.add_argument('-assume-filename')
parser.add_argument('input_file_name')
parser.add_argument('check_name')
parser.add_argument('temp_file_name')
parser.add_argument(
'-check-suffix',
'-check-suffixes',
default=[''],
type=csv,
help='comma-separated list of FileCheck suffixes')
parser.add_argument('-std', type=csv, default=['c++11-or-later'])
return parser.parse_known_args()
parser = argparse.ArgumentParser()
parser.add_argument("-expect-clang-tidy-error", action="store_true")
parser.add_argument("-resource-dir")
parser.add_argument("-assume-filename")
parser.add_argument("input_file_name")
parser.add_argument("check_name")
parser.add_argument("temp_file_name")
parser.add_argument(
"-check-suffix",
"-check-suffixes",
default=[""],
type=csv,
help="comma-separated list of FileCheck suffixes",
)
parser.add_argument("-std", type=csv, default=["c++11-or-later"])
return parser.parse_known_args()
def main():
args, extra_args = parse_arguments()
args, extra_args = parse_arguments()
abbreviated_stds = args.std
for abbreviated_std in abbreviated_stds:
for std in expand_std(abbreviated_std):
args.std = std
CheckRunner(args, extra_args).run()
abbreviated_stds = args.std
for abbreviated_std in abbreviated_stds:
for std in expand_std(abbreviated_std):
args.std = std
CheckRunner(args, extra_args).run()
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

55
clang-tools-extra/test/lit.cfg.py
View File

@ -10,55 +10,70 @@ from lit.llvm import llvm_config
# Configuration file for the 'lit' test runner.
# name: The name of this test suite.
config.name = 'Clang Tools'
config.name = "Clang Tools"
# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.c', '.cpp', '.hpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s',
'.modularize', '.module-map-checker', '.test']
config.suffixes = [
".c",
".cpp",
".hpp",
".m",
".mm",
".cu",
".ll",
".cl",
".s",
".modularize",
".module-map-checker",
".test",
]
# Test-time dependencies located in directories called 'Inputs' are excluded
# from test suites; there won't be any lit tests within them.
config.excludes = ['Inputs']
config.excludes = ["Inputs"]
# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
# test_exec_root: The root path where tests should be run.
config.test_exec_root = os.path.join(config.clang_tools_binary_dir, 'test')
config.test_exec_root = os.path.join(config.clang_tools_binary_dir, "test")
# Tools need the same environment setup as clang (we don't need clang itself).
llvm_config.use_clang(required = False)
llvm_config.use_clang(required=False)
if config.clang_tidy_staticanalyzer:
config.available_features.add('static-analyzer')
config.available_features.add("static-analyzer")
python_exec = shlex.quote(config.python_executable)
check_clang_tidy = os.path.join(
config.test_source_root, "clang-tidy", "check_clang_tidy.py")
config.test_source_root, "clang-tidy", "check_clang_tidy.py"
)
config.substitutions.append(
('%check_clang_tidy',
'%s %s' % (python_exec, check_clang_tidy)) )
("%check_clang_tidy", "%s %s" % (python_exec, check_clang_tidy))
)
clang_tidy_diff = os.path.join(
config.test_source_root, "..", "clang-tidy", "tool", "clang-tidy-diff.py")
config.test_source_root, "..", "clang-tidy", "tool", "clang-tidy-diff.py"
)
config.substitutions.append(
('%clang_tidy_diff',
'%s %s' % (python_exec, clang_tidy_diff)) )
("%clang_tidy_diff", "%s %s" % (python_exec, clang_tidy_diff))
)
run_clang_tidy = os.path.join(
config.test_source_root, "..", "clang-tidy", "tool", "run-clang-tidy.py")
config.test_source_root, "..", "clang-tidy", "tool", "run-clang-tidy.py"
)
config.substitutions.append(
('%run_clang_tidy',
'%s %s' % (python_exec, run_clang_tidy)) )
("%run_clang_tidy", "%s %s" % (python_exec, run_clang_tidy))
)
clang_tidy_headers = os.path.join(
config.test_source_root, "clang-tidy", "checkers", "Inputs", "Headers")
config.substitutions.append(
("%clang_tidy_headers", clang_tidy_headers) )
config.test_source_root, "clang-tidy", "checkers", "Inputs", "Headers"
)
config.substitutions.append(("%clang_tidy_headers", clang_tidy_headers))
# Plugins (loadable modules)
if config.has_plugins and config.llvm_plugin_ext:
config.available_features.add('plugins')
config.available_features.add("plugins")
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

7
clang/bindings/python/clang/__init__.py
View File

@ -1,10 +1,10 @@
#===- __init__.py - Clang Python Bindings --------------------*- python -*--===#
# ===- __init__.py - Clang Python Bindings --------------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
r"""
Clang Library Bindings
@ -19,5 +19,4 @@ The available modules are:
Bindings for the Clang indexing library.
"""
__all__ = ['cindex']
__all__ = ["cindex"]

1709
clang/bindings/python/clang/cindex.py
View File

File diff suppressed because it is too large Load Diff

16
clang/bindings/python/clang/enumerations.py
View File

@ -1,10 +1,10 @@
#===- enumerations.py - Python Enumerations ------------------*- python -*--===#
# ===- enumerations.py - Python Enumerations ------------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
"""
Clang Enumerations
@ -23,11 +23,11 @@ automatically generated by scanning the libclang headers!
# Maps to CXTokenKind. Note that libclang maintains a separate set of token
# enumerations from the C++ API.
TokenKinds = [
('PUNCTUATION', 0),
('KEYWORD', 1),
('IDENTIFIER', 2),
('LITERAL', 3),
('COMMENT', 4),
("PUNCTUATION", 0),
("KEYWORD", 1),
("IDENTIFIER", 2),
("LITERAL", 3),
("COMMENT", 4),
]
__all__ = ['TokenKinds']
__all__ = ["TokenKinds"]

84
clang/bindings/python/examples/cindex/cindex-dump.py
View File

@ -1,26 +1,30 @@
#!/usr/bin/env python
#===- cindex-dump.py - cindex/Python Source Dump -------------*- python -*--===#
# ===- cindex-dump.py - cindex/Python Source Dump -------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
"""
A simple command line tool for dumping a source file using the Clang Index
Library.
"""
def get_diag_info(diag):
return { 'severity' : diag.severity,
'location' : diag.location,
'spelling' : diag.spelling,
'ranges' : diag.ranges,
'fixits' : diag.fixits }
def get_cursor_id(cursor, cursor_list = []):
def get_diag_info(diag):
return {
"severity": diag.severity,
"location": diag.location,
"spelling": diag.spelling,
"ranges": diag.ranges,
"fixits": diag.fixits,
}
def get_cursor_id(cursor, cursor_list=[]):
if not opts.showIDs:
return None
@ -29,28 +33,31 @@ def get_cursor_id(cursor, cursor_list = []):
# FIXME: This is really slow. It would be nice if the index API exposed
# something that let us hash cursors.
for i,c in enumerate(cursor_list):
for i, c in enumerate(cursor_list):
if cursor == c:
return i
cursor_list.append(cursor)
return len(cursor_list) - 1
def get_info(node, depth=0):
if opts.maxDepth is not None and depth >= opts.maxDepth:
children = None
else:
children = [get_info(c, depth+1)
for c in node.get_children()]
return { 'id' : get_cursor_id(node),
'kind' : node.kind,
'usr' : node.get_usr(),
'spelling' : node.spelling,
'location' : node.location,
'extent.start' : node.extent.start,
'extent.end' : node.extent.end,
'is_definition' : node.is_definition(),
'definition id' : get_cursor_id(node.get_definition()),
'children' : children }
children = [get_info(c, depth + 1) for c in node.get_children()]
return {
"id": get_cursor_id(node),
"kind": node.kind,
"usr": node.get_usr(),
"spelling": node.spelling,
"location": node.location,
"extent.start": node.extent.start,
"extent.end": node.extent.end,
"is_definition": node.is_definition(),
"definition id": get_cursor_id(node.get_definition()),
"children": children,
}
def main():
from clang.cindex import Index
@ -61,26 +68,37 @@ def main():
global opts
parser = OptionParser("usage: %prog [options] {filename} [clang-args*]")
parser.add_option("", "--show-ids", dest="showIDs",
help="Compute cursor IDs (very slow)",
action="store_true", default=False)
parser.add_option("", "--max-depth", dest="maxDepth",
help="Limit cursor expansion to depth N",
metavar="N", type=int, default=None)
parser.add_option(
"",
"--show-ids",
dest="showIDs",
help="Compute cursor IDs (very slow)",
action="store_true",
default=False,
)
parser.add_option(
"",
"--max-depth",
dest="maxDepth",
help="Limit cursor expansion to depth N",
metavar="N",
type=int,
default=None,
)
parser.disable_interspersed_args()
(opts, args) = parser.parse_args()
if len(args) == 0:
parser.error('invalid number arguments')
parser.error("invalid number arguments")
index = Index.create()
tu = index.parse(None, args)
if not tu:
parser.error("unable to load input")
pprint(('diags', [get_diag_info(d) for d in tu.diagnostics]))
pprint(('nodes', get_info(tu.cursor)))
pprint(("diags", [get_diag_info(d) for d in tu.diagnostics]))
pprint(("nodes", get_info(tu.cursor)))
if __name__ == '__main__':
if __name__ == "__main__":
main()

19
clang/bindings/python/examples/cindex/cindex-includes.py
View File

@ -1,18 +1,19 @@
#!/usr/bin/env python
#===- cindex-includes.py - cindex/Python Inclusion Graph -----*- python -*--===#
# ===- cindex-includes.py - cindex/Python Inclusion Graph -----*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
"""
A simple command line tool for dumping a Graphviz description (dot) that
describes include dependencies.
"""
def main():
import sys
from clang.cindex import Index
@ -23,7 +24,7 @@ def main():
parser.disable_interspersed_args()
(opts, args) = parser.parse_args()
if len(args) == 0:
parser.error('invalid number arguments')
parser.error("invalid number arguments")
# FIXME: Add an output file option
out = sys.stdout
@ -36,22 +37,22 @@ def main():
# A helper function for generating the node name.
def name(f):
if f:
return "\"" + f.name + "\""
return '"' + f.name + '"'
# Generate the include graph
out.write("digraph G {\n")
for i in tu.get_includes():
line = " ";
line = " "
if i.is_input_file:
# Always write the input file as a node just in case it doesn't
# actually include anything. This would generate a 1 node graph.
line += name(i.include)
else:
line += '%s->%s' % (name(i.source), name(i.include))
line += "\n";
line += "%s->%s" % (name(i.source), name(i.include))
line += "\n"
out.write(line)
out.write("}\n")
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

12
clang/bindings/python/tests/cindex/test_access_specifiers.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import AccessSpecifier
from clang.cindex import Cursor
@ -17,7 +18,8 @@ class TestAccessSpecifiers(unittest.TestCase):
def test_access_specifiers(self):
"""Ensure that C++ access specifiers are available on cursors"""
tu = get_tu("""
tu = get_tu(
"""
class test_class {
public:
void public_member_function();
@ -26,7 +28,9 @@ protected:
private:
void private_member_function();
};
""", lang = 'cpp')
""",
lang="cpp",
)
test_class = get_cursor(tu, "test_class")
self.assertEqual(test_class.access_specifier, AccessSpecifier.INVALID)

135
clang/bindings/python/tests/cindex/test_cdb.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import CompilationDatabase
from clang.cindex import CompilationDatabaseError
@ -15,10 +16,10 @@ from .util import skip_if_no_fspath
from .util import str_to_path
kInputsDir = os.path.join(os.path.dirname(__file__), 'INPUTS')
kInputsDir = os.path.join(os.path.dirname(__file__), "INPUTS")
@unittest.skipIf(sys.platform == 'win32', "TODO: Fix these tests on Windows")
@unittest.skipIf(sys.platform == "win32", "TODO: Fix these tests on Windows")
class TestCDB(unittest.TestCase):
def test_create_fail(self):
"""Check we fail loading a database with an assertion"""
@ -27,7 +28,7 @@ class TestCDB(unittest.TestCase):
# clang_CompilationDatabase_fromDirectory calls fprintf(stderr, ...)
# Suppress its output.
stderr = os.dup(2)
with open(os.devnull, 'wb') as null:
with open(os.devnull, "wb") as null:
os.dup2(null.fileno(), 2)
with self.assertRaises(CompilationDatabaseError) as cm:
cdb = CompilationDatabase.fromDirectory(path)
@ -35,8 +36,7 @@ class TestCDB(unittest.TestCase):
os.close(stderr)
e = cm.exception
self.assertEqual(e.cdb_error,
CompilationDatabaseError.ERROR_CANNOTLOADDATABASE)
self.assertEqual(e.cdb_error, CompilationDatabaseError.ERROR_CANNOTLOADDATABASE)
def test_create(self):
"""Check we can load a compilation database"""
@ -45,14 +45,16 @@ class TestCDB(unittest.TestCase):
def test_lookup_succeed(self):
"""Check we get some results if the file exists in the db"""
cdb = CompilationDatabase.fromDirectory(kInputsDir)
cmds = cdb.getCompileCommands('/home/john.doe/MyProject/project.cpp')
cmds = cdb.getCompileCommands("/home/john.doe/MyProject/project.cpp")
self.assertNotEqual(len(cmds), 0)
@skip_if_no_fspath
def test_lookup_succeed_pathlike(self):
"""Same as test_lookup_succeed, but with PathLikes"""
cdb = CompilationDatabase.fromDirectory(str_to_path(kInputsDir))
cmds = cdb.getCompileCommands(str_to_path('/home/john.doe/MyProject/project.cpp'))
cmds = cdb.getCompileCommands(
str_to_path("/home/john.doe/MyProject/project.cpp")
)
self.assertNotEqual(len(cmds), 0)
def test_all_compilecommand(self):
@ -61,71 +63,116 @@ class TestCDB(unittest.TestCase):
cmds = cdb.getAllCompileCommands()
self.assertEqual(len(cmds), 3)
expected = [
{ 'wd': '/home/john.doe/MyProject',
'file': '/home/john.doe/MyProject/project.cpp',
'line': ['clang++', '--driver-mode=g++', '-o', 'project.o', '-c',
'/home/john.doe/MyProject/project.cpp']},
{ 'wd': '/home/john.doe/MyProjectA',
'file': '/home/john.doe/MyProject/project2.cpp',
'line': ['clang++', '--driver-mode=g++', '-o', 'project2.o', '-c',
'/home/john.doe/MyProject/project2.cpp']},
{ 'wd': '/home/john.doe/MyProjectB',
'file': '/home/john.doe/MyProject/project2.cpp',
'line': ['clang++', '--driver-mode=g++', '-DFEATURE=1', '-o',
'project2-feature.o', '-c',
'/home/john.doe/MyProject/project2.cpp']},
]
{
"wd": "/home/john.doe/MyProject",
"file": "/home/john.doe/MyProject/project.cpp",
"line": [
"clang++",
"--driver-mode=g++",
"-o",
"project.o",
"-c",
"/home/john.doe/MyProject/project.cpp",
],
},
{
"wd": "/home/john.doe/MyProjectA",
"file": "/home/john.doe/MyProject/project2.cpp",
"line": [
"clang++",
"--driver-mode=g++",
"-o",
"project2.o",
"-c",
"/home/john.doe/MyProject/project2.cpp",
],
},
{
"wd": "/home/john.doe/MyProjectB",
"file": "/home/john.doe/MyProject/project2.cpp",
"line": [
"clang++",
"--driver-mode=g++",
"-DFEATURE=1",
"-o",
"project2-feature.o",
"-c",
"/home/john.doe/MyProject/project2.cpp",
],
},
]
for i in range(len(cmds)):
self.assertEqual(cmds[i].directory, expected[i]['wd'])
self.assertEqual(cmds[i].filename, expected[i]['file'])
for arg, exp in zip(cmds[i].arguments, expected[i]['line']):
self.assertEqual(cmds[i].directory, expected[i]["wd"])
self.assertEqual(cmds[i].filename, expected[i]["file"])
for arg, exp in zip(cmds[i].arguments, expected[i]["line"]):
self.assertEqual(arg, exp)
def test_1_compilecommand(self):
"""Check file with single compile command"""
cdb = CompilationDatabase.fromDirectory(kInputsDir)
file = '/home/john.doe/MyProject/project.cpp'
file = "/home/john.doe/MyProject/project.cpp"
cmds = cdb.getCompileCommands(file)
self.assertEqual(len(cmds), 1)
self.assertEqual(cmds[0].directory, os.path.dirname(file))
self.assertEqual(cmds[0].filename, file)
expected = [ 'clang++', '--driver-mode=g++', '-o', 'project.o', '-c',
'/home/john.doe/MyProject/project.cpp']
expected = [
"clang++",
"--driver-mode=g++",
"-o",
"project.o",
"-c",
"/home/john.doe/MyProject/project.cpp",
]
for arg, exp in zip(cmds[0].arguments, expected):
self.assertEqual(arg, exp)
def test_2_compilecommand(self):
"""Check file with 2 compile commands"""
cdb = CompilationDatabase.fromDirectory(kInputsDir)
cmds = cdb.getCompileCommands('/home/john.doe/MyProject/project2.cpp')
cmds = cdb.getCompileCommands("/home/john.doe/MyProject/project2.cpp")
self.assertEqual(len(cmds), 2)
expected = [
{ 'wd': '/home/john.doe/MyProjectA',
'line': ['clang++', '--driver-mode=g++', '-o', 'project2.o', '-c',
'/home/john.doe/MyProject/project2.cpp']},
{ 'wd': '/home/john.doe/MyProjectB',
'line': ['clang++', '--driver-mode=g++', '-DFEATURE=1', '-o',
'project2-feature.o', '-c',
'/home/john.doe/MyProject/project2.cpp']}
]
{
"wd": "/home/john.doe/MyProjectA",
"line": [
"clang++",
"--driver-mode=g++",
"-o",
"project2.o",
"-c",
"/home/john.doe/MyProject/project2.cpp",
],
},
{
"wd": "/home/john.doe/MyProjectB",
"line": [
"clang++",
"--driver-mode=g++",
"-DFEATURE=1",
"-o",
"project2-feature.o",
"-c",
"/home/john.doe/MyProject/project2.cpp",
],
},
]
for i in range(len(cmds)):
self.assertEqual(cmds[i].directory, expected[i]['wd'])
for arg, exp in zip(cmds[i].arguments, expected[i]['line']):
self.assertEqual(cmds[i].directory, expected[i]["wd"])
for arg, exp in zip(cmds[i].arguments, expected[i]["line"]):
self.assertEqual(arg, exp)
def test_compilecommand_iterator_stops(self):
"""Check that iterator stops after the correct number of elements"""
cdb = CompilationDatabase.fromDirectory(kInputsDir)
count = 0
for cmd in cdb.getCompileCommands('/home/john.doe/MyProject/project2.cpp'):
for cmd in cdb.getCompileCommands("/home/john.doe/MyProject/project2.cpp"):
count += 1
self.assertLessEqual(count, 2)
def test_compilationDB_references(self):
"""Ensure CompilationsCommands are independent of the database"""
cdb = CompilationDatabase.fromDirectory(kInputsDir)
cmds = cdb.getCompileCommands('/home/john.doe/MyProject/project.cpp')
cmds = cdb.getCompileCommands("/home/john.doe/MyProject/project.cpp")
del cdb
gc.collect()
workingdir = cmds[0].directory
@ -133,7 +180,7 @@ class TestCDB(unittest.TestCase):
def test_compilationCommands_references(self):
"""Ensure CompilationsCommand keeps a reference to CompilationCommands"""
cdb = CompilationDatabase.fromDirectory(kInputsDir)
cmds = cdb.getCompileCommands('/home/john.doe/MyProject/project.cpp')
cmds = cdb.getCompileCommands("/home/john.doe/MyProject/project.cpp")
del cdb
cmd0 = cmds[0]
del cmds

92
clang/bindings/python/tests/cindex/test_code_completion.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import TranslationUnit
@ -21,7 +22,10 @@ class TestCodeCompletion(unittest.TestCase):
self.assertIn(c, completions)
def test_code_complete(self):
files = [('fake.c', """
files = [
(
"fake.c",
"""
/// Aaa.
int test1;
@ -31,23 +35,34 @@ void test2(void);
void f() {
}
""")]
""",
)
]
tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
tu = TranslationUnit.from_source(
"fake.c",
["-std=c99"],
unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION,
)
cr = tu.codeComplete('fake.c', 9, 1, unsaved_files=files, include_brief_comments=True)
cr = tu.codeComplete(
"fake.c", 9, 1, unsaved_files=files, include_brief_comments=True
)
expected = [
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} | {';', SemiColon} || Priority: 40 || Availability: Available || Brief comment: None"
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} | {';', SemiColon} || Priority: 40 || Availability: Available || Brief comment: None",
]
self.check_completion_results(cr, expected)
@skip_if_no_fspath
def test_code_complete_pathlike(self):
files = [(str_to_path('fake.c'), """
files = [
(
str_to_path("fake.c"),
"""
/// Aaa.
int test1;
@ -57,22 +72,37 @@ void test2(void);
void f() {
}
""")]
""",
)
]
tu = TranslationUnit.from_source(str_to_path('fake.c'), ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
tu = TranslationUnit.from_source(
str_to_path("fake.c"),
["-std=c99"],
unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION,
)
cr = tu.codeComplete(str_to_path('fake.c'), 9, 1, unsaved_files=files, include_brief_comments=True)
cr = tu.codeComplete(
str_to_path("fake.c"),
9,
1,
unsaved_files=files,
include_brief_comments=True,
)
expected = [
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} | {';', SemiColon} || Priority: 40 || Availability: Available || Brief comment: None"
"{'int', ResultType} | {'test1', TypedText} || Priority: 50 || Availability: Available || Brief comment: Aaa.",
"{'void', ResultType} | {'test2', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 50 || Availability: Available || Brief comment: Bbb.",
"{'return', TypedText} | {';', SemiColon} || Priority: 40 || Availability: Available || Brief comment: None",
]
self.check_completion_results(cr, expected)
def test_code_complete_availability(self):
files = [('fake.cpp', """
files = [
(
"fake.cpp",
"""
class P {
protected:
int member;
@ -87,26 +117,30 @@ void f(P x, Q y) {
x.; // member is inaccessible
y.; // member is accessible
}
""")]
""",
)
]
tu = TranslationUnit.from_source('fake.cpp', ['-std=c++98'], unsaved_files=files)
tu = TranslationUnit.from_source(
"fake.cpp", ["-std=c++98"], unsaved_files=files
)
cr = tu.codeComplete('fake.cpp', 12, 5, unsaved_files=files)
cr = tu.codeComplete("fake.cpp", 12, 5, unsaved_files=files)
expected = [
"{'const', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
"{'volatile', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
"{'operator', TypedText} || Priority: 40 || Availability: Available || Brief comment: None",
"{'P', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
"{'Q', TypedText} || Priority: 50 || Availability: Available || Brief comment: None"
"{'const', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
"{'volatile', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
"{'operator', TypedText} || Priority: 40 || Availability: Available || Brief comment: None",
"{'P', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
"{'Q', TypedText} || Priority: 50 || Availability: Available || Brief comment: None",
]
self.check_completion_results(cr, expected)
cr = tu.codeComplete('fake.cpp', 13, 5, unsaved_files=files)
cr = tu.codeComplete("fake.cpp", 13, 5, unsaved_files=files)
expected = [
"{'P', TypedText} | {'::', Text} || Priority: 75 || Availability: Available || Brief comment: None",
"{'P &', ResultType} | {'operator=', TypedText} | {'(', LeftParen} | {'const P &', Placeholder} | {')', RightParen} || Priority: 79 || Availability: Available || Brief comment: None",
"{'int', ResultType} | {'member', TypedText} || Priority: 35 || Availability: NotAccessible || Brief comment: None",
"{'void', ResultType} | {'~P', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 79 || Availability: Available || Brief comment: None"
"{'void', ResultType} | {'~P', TypedText} | {'(', LeftParen} | {')', RightParen} || Priority: 79 || Availability: Available || Brief comment: None",
]
self.check_completion_results(cr, expected)

28
clang/bindings/python/tests/cindex/test_comment.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import TranslationUnit
from tests.cindex.util import get_cursor
@ -11,7 +12,10 @@ import unittest
class TestComment(unittest.TestCase):
def test_comment(self):
files = [('fake.c', """
files = [
(
"fake.c",
"""
/// Aaa.
int test1;
@ -22,11 +26,17 @@ void test2(void);
void f() {
}
""")]
""",
)
]
# make a comment-aware TU
tu = TranslationUnit.from_source('fake.c', ['-std=c99'], unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION)
test1 = get_cursor(tu, 'test1')
tu = TranslationUnit.from_source(
"fake.c",
["-std=c99"],
unsaved_files=files,
options=TranslationUnit.PARSE_INCLUDE_BRIEF_COMMENTS_IN_CODE_COMPLETION,
)
test1 = get_cursor(tu, "test1")
self.assertIsNotNone(test1, "Could not find test1.")
self.assertTrue(test1.type.is_pod())
raw = test1.raw_comment
@ -34,13 +44,13 @@ void f() {
self.assertEqual(raw, """/// Aaa.""")
self.assertEqual(brief, """Aaa.""")
test2 = get_cursor(tu, 'test2')
test2 = get_cursor(tu, "test2")
raw = test2.raw_comment
brief = test2.brief_comment
self.assertEqual(raw, """/// Bbb.\n/// x""")
self.assertEqual(brief, """Bbb. x""")
f = get_cursor(tu, 'f')
f = get_cursor(tu, "f")
raw = f.raw_comment
brief = f.brief_comment
self.assertIsNone(raw)

319
clang/bindings/python/tests/cindex/test_cursor.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
import ctypes
import gc
@ -53,6 +54,7 @@ kTemplateArgTest = """\
void foo<-7, float, true>();
"""
class TestCursor(unittest.TestCase):
def test_get_children(self):
tu = get_tu(kInput)
@ -66,9 +68,9 @@ class TestCursor(unittest.TestCase):
self.assertNotEqual(tu_nodes[0], tu_nodes[1])
self.assertEqual(tu_nodes[0].kind, CursorKind.STRUCT_DECL)
self.assertEqual(tu_nodes[0].spelling, 's0')
self.assertEqual(tu_nodes[0].spelling, "s0")
self.assertEqual(tu_nodes[0].is_definition(), True)
self.assertEqual(tu_nodes[0].location.file.name, 't.c')
self.assertEqual(tu_nodes[0].location.file.name, "t.c")
self.assertEqual(tu_nodes[0].location.line, 1)
self.assertEqual(tu_nodes[0].location.column, 8)
self.assertGreater(tu_nodes[0].hash, 0)
@ -77,25 +79,25 @@ class TestCursor(unittest.TestCase):
s0_nodes = list(tu_nodes[0].get_children())
self.assertEqual(len(s0_nodes), 2)
self.assertEqual(s0_nodes[0].kind, CursorKind.FIELD_DECL)
self.assertEqual(s0_nodes[0].spelling, 'a')
self.assertEqual(s0_nodes[0].spelling, "a")
self.assertEqual(s0_nodes[0].type.kind, TypeKind.INT)
self.assertEqual(s0_nodes[1].kind, CursorKind.FIELD_DECL)
self.assertEqual(s0_nodes[1].spelling, 'b')
self.assertEqual(s0_nodes[1].spelling, "b")
self.assertEqual(s0_nodes[1].type.kind, TypeKind.INT)
self.assertEqual(tu_nodes[1].kind, CursorKind.STRUCT_DECL)
self.assertEqual(tu_nodes[1].spelling, 's1')
self.assertEqual(tu_nodes[1].displayname, 's1')
self.assertEqual(tu_nodes[1].spelling, "s1")
self.assertEqual(tu_nodes[1].displayname, "s1")
self.assertEqual(tu_nodes[1].is_definition(), False)
self.assertEqual(tu_nodes[2].kind, CursorKind.FUNCTION_DECL)
self.assertEqual(tu_nodes[2].spelling, 'f0')
self.assertEqual(tu_nodes[2].displayname, 'f0(int, int)')
self.assertEqual(tu_nodes[2].spelling, "f0")
self.assertEqual(tu_nodes[2].displayname, "f0(int, int)")
self.assertEqual(tu_nodes[2].is_definition(), True)
def test_references(self):
"""Ensure that references to TranslationUnit are kept."""
tu = get_tu('int x;')
tu = get_tu("int x;")
cursors = list(tu.cursor.get_children())
self.assertGreater(len(cursors), 0)
@ -111,12 +113,12 @@ class TestCursor(unittest.TestCase):
parent = cursor.semantic_parent
def test_canonical(self):
source = 'struct X; struct X; struct X { int member; };'
source = "struct X; struct X; struct X { int member; };"
tu = get_tu(source)
cursors = []
for cursor in tu.cursor.get_children():
if cursor.spelling == 'X':
if cursor.spelling == "X":
cursors.append(cursor)
self.assertEqual(len(cursors), 3)
@ -124,12 +126,12 @@ class TestCursor(unittest.TestCase):
def test_is_const_method(self):
"""Ensure Cursor.is_const_method works."""
source = 'class X { void foo() const; void bar(); };'
tu = get_tu(source, lang='cpp')
source = "class X { void foo() const; void bar(); };"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
foo = get_cursor(tu, 'foo')
bar = get_cursor(tu, 'bar')
cls = get_cursor(tu, "X")
foo = get_cursor(tu, "foo")
bar = get_cursor(tu, "bar")
self.assertIsNotNone(cls)
self.assertIsNotNone(foo)
self.assertIsNotNone(bar)
@ -139,10 +141,10 @@ class TestCursor(unittest.TestCase):
def test_is_converting_constructor(self):
"""Ensure Cursor.is_converting_constructor works."""
source = 'class X { explicit X(int); X(double); X(); };'
tu = get_tu(source, lang='cpp')
source = "class X { explicit X(int); X(double); X(); };"
tu = get_tu(source, lang="cpp")
xs = get_cursors(tu, 'X')
xs = get_cursors(tu, "X")
self.assertEqual(len(xs), 4)
self.assertEqual(xs[0].kind, CursorKind.CLASS_DECL)
@ -155,13 +157,12 @@ class TestCursor(unittest.TestCase):
self.assertTrue(cs[1].is_converting_constructor())
self.assertFalse(cs[2].is_converting_constructor())
def test_is_copy_constructor(self):
"""Ensure Cursor.is_copy_constructor works."""
source = 'class X { X(); X(const X&); X(X&&); };'
tu = get_tu(source, lang='cpp')
source = "class X { X(); X(const X&); X(X&&); };"
tu = get_tu(source, lang="cpp")
xs = get_cursors(tu, 'X')
xs = get_cursors(tu, "X")
self.assertEqual(xs[0].kind, CursorKind.CLASS_DECL)
cs = xs[1:]
self.assertEqual(cs[0].kind, CursorKind.CONSTRUCTOR)
@ -174,10 +175,10 @@ class TestCursor(unittest.TestCase):
def test_is_default_constructor(self):
"""Ensure Cursor.is_default_constructor works."""
source = 'class X { X(); X(int); };'
tu = get_tu(source, lang='cpp')
source = "class X { X(); X(int); };"
tu = get_tu(source, lang="cpp")
xs = get_cursors(tu, 'X')
xs = get_cursors(tu, "X")
self.assertEqual(xs[0].kind, CursorKind.CLASS_DECL)
cs = xs[1:]
self.assertEqual(cs[0].kind, CursorKind.CONSTRUCTOR)
@ -188,10 +189,10 @@ class TestCursor(unittest.TestCase):
def test_is_move_constructor(self):
"""Ensure Cursor.is_move_constructor works."""
source = 'class X { X(); X(const X&); X(X&&); };'
tu = get_tu(source, lang='cpp')
source = "class X { X(); X(const X&); X(X&&); };"
tu = get_tu(source, lang="cpp")
xs = get_cursors(tu, 'X')
xs = get_cursors(tu, "X")
self.assertEqual(xs[0].kind, CursorKind.CLASS_DECL)
cs = xs[1:]
self.assertEqual(cs[0].kind, CursorKind.CONSTRUCTOR)
@ -204,11 +205,11 @@ class TestCursor(unittest.TestCase):
def test_is_default_method(self):
"""Ensure Cursor.is_default_method works."""
source = 'class X { X() = default; }; class Y { Y(); };'
tu = get_tu(source, lang='cpp')
source = "class X { X() = default; }; class Y { Y(); };"
tu = get_tu(source, lang="cpp")
xs = get_cursors(tu, 'X')
ys = get_cursors(tu, 'Y')
xs = get_cursors(tu, "X")
ys = get_cursors(tu, "Y")
self.assertEqual(len(xs), 2)
self.assertEqual(len(ys), 2)
@ -268,14 +269,22 @@ class TestCursor(unittest.TestCase):
self.assertEqual(len(move_assignment_operators_cursors), 8)
self.assertTrue(len(non_move_assignment_operators_cursors), 7)
self.assertTrue(all([
cursor.is_move_assignment_operator_method()
for cursor in move_assignment_operators_cursors
]))
self.assertFalse(any([
cursor.is_move_assignment_operator_method()
for cursor in non_move_assignment_operators_cursors
]))
self.assertTrue(
all(
[
cursor.is_move_assignment_operator_method()
for cursor in move_assignment_operators_cursors
]
)
)
self.assertFalse(
any(
[
cursor.is_move_assignment_operator_method()
for cursor in non_move_assignment_operators_cursors
]
)
)
def test_is_explicit_method(self):
"""Ensure Cursor.is_explicit_method works."""
@ -297,9 +306,7 @@ class TestCursor(unittest.TestCase):
explicit(false) operator float();
};
"""
tu_with_explicit_methods = get_tu(
source_with_explicit_methods, lang="cpp"
)
tu_with_explicit_methods = get_tu(source_with_explicit_methods, lang="cpp")
tu_without_explicit_methods = get_tu(
source_without_explicit_methods, lang="cpp"
)
@ -319,23 +326,23 @@ class TestCursor(unittest.TestCase):
self.assertEqual(len(explicit_methods_cursors), 4)
self.assertTrue(len(non_explicit_methods_cursors), 4)
self.assertTrue(all([
cursor.is_explicit_method()
for cursor in explicit_methods_cursors
]))
self.assertFalse(any([
cursor.is_explicit_method()
for cursor in non_explicit_methods_cursors
]))
self.assertTrue(
all([cursor.is_explicit_method() for cursor in explicit_methods_cursors])
)
self.assertFalse(
any(
[cursor.is_explicit_method() for cursor in non_explicit_methods_cursors]
)
)
def test_is_mutable_field(self):
"""Ensure Cursor.is_mutable_field works."""
source = 'class X { int x_; mutable int y_; };'
tu = get_tu(source, lang='cpp')
source = "class X { int x_; mutable int y_; };"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
x_ = get_cursor(tu, 'x_')
y_ = get_cursor(tu, 'y_')
cls = get_cursor(tu, "X")
x_ = get_cursor(tu, "x_")
y_ = get_cursor(tu, "y_")
self.assertIsNotNone(cls)
self.assertIsNotNone(x_)
self.assertIsNotNone(y_)
@ -346,12 +353,12 @@ class TestCursor(unittest.TestCase):
def test_is_static_method(self):
"""Ensure Cursor.is_static_method works."""
source = 'class X { static void foo(); void bar(); };'
tu = get_tu(source, lang='cpp')
source = "class X { static void foo(); void bar(); };"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
foo = get_cursor(tu, 'foo')
bar = get_cursor(tu, 'bar')
cls = get_cursor(tu, "X")
foo = get_cursor(tu, "foo")
bar = get_cursor(tu, "bar")
self.assertIsNotNone(cls)
self.assertIsNotNone(foo)
self.assertIsNotNone(bar)
@ -361,12 +368,12 @@ class TestCursor(unittest.TestCase):
def test_is_pure_virtual_method(self):
"""Ensure Cursor.is_pure_virtual_method works."""
source = 'class X { virtual void foo() = 0; virtual void bar(); };'
tu = get_tu(source, lang='cpp')
source = "class X { virtual void foo() = 0; virtual void bar(); };"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
foo = get_cursor(tu, 'foo')
bar = get_cursor(tu, 'bar')
cls = get_cursor(tu, "X")
foo = get_cursor(tu, "foo")
bar = get_cursor(tu, "bar")
self.assertIsNotNone(cls)
self.assertIsNotNone(foo)
self.assertIsNotNone(bar)
@ -376,12 +383,12 @@ class TestCursor(unittest.TestCase):
def test_is_virtual_method(self):
"""Ensure Cursor.is_virtual_method works."""
source = 'class X { virtual void foo(); void bar(); };'
tu = get_tu(source, lang='cpp')
source = "class X { virtual void foo(); void bar(); };"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
foo = get_cursor(tu, 'foo')
bar = get_cursor(tu, 'bar')
cls = get_cursor(tu, "X")
foo = get_cursor(tu, "foo")
bar = get_cursor(tu, "bar")
self.assertIsNotNone(cls)
self.assertIsNotNone(foo)
self.assertIsNotNone(bar)
@ -391,23 +398,23 @@ class TestCursor(unittest.TestCase):
def test_is_abstract_record(self):
"""Ensure Cursor.is_abstract_record works."""
source = 'struct X { virtual void x() = 0; }; struct Y : X { void x(); };'
tu = get_tu(source, lang='cpp')
source = "struct X { virtual void x() = 0; }; struct Y : X { void x(); };"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
cls = get_cursor(tu, "X")
self.assertTrue(cls.is_abstract_record())
cls = get_cursor(tu, 'Y')
cls = get_cursor(tu, "Y")
self.assertFalse(cls.is_abstract_record())
def test_is_scoped_enum(self):
"""Ensure Cursor.is_scoped_enum works."""
source = 'class X {}; enum RegularEnum {}; enum class ScopedEnum {};'
tu = get_tu(source, lang='cpp')
source = "class X {}; enum RegularEnum {}; enum class ScopedEnum {};"
tu = get_tu(source, lang="cpp")
cls = get_cursor(tu, 'X')
regular_enum = get_cursor(tu, 'RegularEnum')
scoped_enum = get_cursor(tu, 'ScopedEnum')
cls = get_cursor(tu, "X")
regular_enum = get_cursor(tu, "RegularEnum")
scoped_enum = get_cursor(tu, "ScopedEnum")
self.assertIsNotNone(cls)
self.assertIsNotNone(regular_enum)
self.assertIsNotNone(scoped_enum)
@ -417,8 +424,8 @@ class TestCursor(unittest.TestCase):
self.assertTrue(scoped_enum.is_scoped_enum())
def test_underlying_type(self):
tu = get_tu('typedef int foo;')
typedef = get_cursor(tu, 'foo')
tu = get_tu("typedef int foo;")
typedef = get_cursor(tu, "foo")
self.assertIsNotNone(typedef)
self.assertTrue(typedef.kind.is_declaration())
@ -426,25 +433,25 @@ class TestCursor(unittest.TestCase):
self.assertEqual(underlying.kind, TypeKind.INT)
def test_semantic_parent(self):
tu = get_tu(kParentTest, 'cpp')
curs = get_cursors(tu, 'f')
decl = get_cursor(tu, 'C')
tu = get_tu(kParentTest, "cpp")
curs = get_cursors(tu, "f")
decl = get_cursor(tu, "C")
self.assertEqual(len(curs), 2)
self.assertEqual(curs[0].semantic_parent, curs[1].semantic_parent)
self.assertEqual(curs[0].semantic_parent, decl)
def test_lexical_parent(self):
tu = get_tu(kParentTest, 'cpp')
curs = get_cursors(tu, 'f')
decl = get_cursor(tu, 'C')
tu = get_tu(kParentTest, "cpp")
curs = get_cursors(tu, "f")
decl = get_cursor(tu, "C")
self.assertEqual(len(curs), 2)
self.assertNotEqual(curs[0].lexical_parent, curs[1].lexical_parent)
self.assertEqual(curs[0].lexical_parent, decl)
self.assertEqual(curs[1].lexical_parent, tu.cursor)
def test_enum_type(self):
tu = get_tu('enum TEST { FOO=1, BAR=2 };')
enum = get_cursor(tu, 'TEST')
tu = get_tu("enum TEST { FOO=1, BAR=2 };")
enum = get_cursor(tu, "TEST")
self.assertIsNotNone(enum)
self.assertEqual(enum.kind, CursorKind.ENUM_DECL)
@ -452,23 +459,23 @@ class TestCursor(unittest.TestCase):
self.assertIn(enum_type.kind, (TypeKind.UINT, TypeKind.INT))
def test_enum_type_cpp(self):
tu = get_tu('enum TEST : long long { FOO=1, BAR=2 };', lang="cpp")
enum = get_cursor(tu, 'TEST')
tu = get_tu("enum TEST : long long { FOO=1, BAR=2 };", lang="cpp")
enum = get_cursor(tu, "TEST")
self.assertIsNotNone(enum)
self.assertEqual(enum.kind, CursorKind.ENUM_DECL)
self.assertEqual(enum.enum_type.kind, TypeKind.LONGLONG)
def test_objc_type_encoding(self):
tu = get_tu('int i;', lang='objc')
i = get_cursor(tu, 'i')
tu = get_tu("int i;", lang="objc")
i = get_cursor(tu, "i")
self.assertIsNotNone(i)
self.assertEqual(i.objc_type_encoding, 'i')
self.assertEqual(i.objc_type_encoding, "i")
def test_enum_values(self):
tu = get_tu('enum TEST { SPAM=1, EGG, HAM = EGG * 20};')
enum = get_cursor(tu, 'TEST')
tu = get_tu("enum TEST { SPAM=1, EGG, HAM = EGG * 20};")
enum = get_cursor(tu, "TEST")
self.assertIsNotNone(enum)
self.assertEqual(enum.kind, CursorKind.ENUM_DECL)
@ -486,8 +493,10 @@ class TestCursor(unittest.TestCase):
self.assertEqual(ham.enum_value, 40)
def test_enum_values_cpp(self):
tu = get_tu('enum TEST : long long { SPAM = -1, HAM = 0x10000000000};', lang="cpp")
enum = get_cursor(tu, 'TEST')
tu = get_tu(
"enum TEST : long long { SPAM = -1, HAM = 0x10000000000};", lang="cpp"
)
enum = get_cursor(tu, "TEST")
self.assertIsNotNone(enum)
self.assertEqual(enum.kind, CursorKind.ENUM_DECL)
@ -503,9 +512,11 @@ class TestCursor(unittest.TestCase):
self.assertEqual(ham.enum_value, 0x10000000000)
def test_annotation_attribute(self):
tu = get_tu('int foo (void) __attribute__ ((annotate("here be annotation attribute")));')
tu = get_tu(
'int foo (void) __attribute__ ((annotate("here be annotation attribute")));'
)
foo = get_cursor(tu, 'foo')
foo = get_cursor(tu, "foo")
self.assertIsNotNone(foo)
for c in foo.get_children():
@ -518,13 +529,13 @@ class TestCursor(unittest.TestCase):
def test_annotation_template(self):
annotation = '__attribute__ ((annotate("annotation")))'
for source, kind in [
('int foo (T value) %s;', CursorKind.FUNCTION_TEMPLATE),
('class %s foo {};', CursorKind.CLASS_TEMPLATE),
("int foo (T value) %s;", CursorKind.FUNCTION_TEMPLATE),
("class %s foo {};", CursorKind.CLASS_TEMPLATE),
]:
source = 'template<typename T> ' + (source % annotation)
source = "template<typename T> " + (source % annotation)
tu = get_tu(source, lang="cpp")
foo = get_cursor(tu, 'foo')
foo = get_cursor(tu, "foo")
self.assertIsNotNone(foo)
self.assertEqual(foo.kind, kind)
@ -536,8 +547,8 @@ class TestCursor(unittest.TestCase):
self.fail("Couldn't find annotation for {}".format(kind))
def test_result_type(self):
tu = get_tu('int foo();')
foo = get_cursor(tu, 'foo')
tu = get_tu("int foo();")
foo = get_cursor(tu, "foo")
self.assertIsNotNone(foo)
t = foo.result_type
@ -549,22 +560,22 @@ class TestCursor(unittest.TestCase):
-(void)voidMethod;
@end
"""
tu = get_tu(code, lang='objc')
cursor = get_cursor(tu, 'voidMethod')
tu = get_tu(code, lang="objc")
cursor = get_cursor(tu, "voidMethod")
result_type = cursor.result_type
self.assertEqual(cursor.kind, CursorKind.OBJC_INSTANCE_METHOD_DECL)
self.assertEqual(result_type.kind, TypeKind.VOID)
def test_availability(self):
tu = get_tu('class A { A(A const&) = delete; };', lang='cpp')
tu = get_tu("class A { A(A const&) = delete; };", lang="cpp")
# AvailabilityKind.AVAILABLE
cursor = get_cursor(tu, 'A')
cursor = get_cursor(tu, "A")
self.assertEqual(cursor.kind, CursorKind.CLASS_DECL)
self.assertEqual(cursor.availability, AvailabilityKind.AVAILABLE)
# AvailabilityKind.NOT_AVAILABLE
cursors = get_cursors(tu, 'A')
cursors = get_cursors(tu, "A")
for c in cursors:
if c.kind == CursorKind.CONSTRUCTOR:
self.assertEqual(c.availability, AvailabilityKind.NOT_AVAILABLE)
@ -573,26 +584,26 @@ class TestCursor(unittest.TestCase):
self.fail("Could not find cursor for deleted constructor")
# AvailabilityKind.DEPRECATED
tu = get_tu('void test() __attribute__((deprecated));', lang='cpp')
cursor = get_cursor(tu, 'test')
tu = get_tu("void test() __attribute__((deprecated));", lang="cpp")
cursor = get_cursor(tu, "test")
self.assertEqual(cursor.availability, AvailabilityKind.DEPRECATED)
# AvailabilityKind.NOT_ACCESSIBLE is only used in the code completion results
def test_get_tokens(self):
"""Ensure we can map cursors back to tokens."""
tu = get_tu('int foo(int i);')
foo = get_cursor(tu, 'foo')
tu = get_tu("int foo(int i);")
foo = get_cursor(tu, "foo")
tokens = list(foo.get_tokens())
self.assertEqual(len(tokens), 6)
self.assertEqual(tokens[0].spelling, 'int')
self.assertEqual(tokens[1].spelling, 'foo')
self.assertEqual(tokens[0].spelling, "int")
self.assertEqual(tokens[1].spelling, "foo")
def test_get_token_cursor(self):
"""Ensure we can map tokens to cursors."""
tu = get_tu('class A {}; int foo(A var = A());', lang='cpp')
foo = get_cursor(tu, 'foo')
tu = get_tu("class A {}; int foo(A var = A());", lang="cpp")
foo = get_cursor(tu, "foo")
for cursor in foo.walk_preorder():
if cursor.kind.is_expression() and not cursor.kind.is_statement():
@ -602,18 +613,18 @@ class TestCursor(unittest.TestCase):
tokens = list(cursor.get_tokens())
self.assertEqual(len(tokens), 4, [t.spelling for t in tokens])
self.assertEqual(tokens[0].spelling, '=')
self.assertEqual(tokens[1].spelling, 'A')
self.assertEqual(tokens[2].spelling, '(')
self.assertEqual(tokens[3].spelling, ')')
self.assertEqual(tokens[0].spelling, "=")
self.assertEqual(tokens[1].spelling, "A")
self.assertEqual(tokens[2].spelling, "(")
self.assertEqual(tokens[3].spelling, ")")
t_cursor = tokens[1].cursor
self.assertEqual(t_cursor.kind, CursorKind.TYPE_REF)
r_cursor = t_cursor.referenced # should not raise an exception
r_cursor = t_cursor.referenced # should not raise an exception
self.assertEqual(r_cursor.kind, CursorKind.CLASS_DECL)
def test_get_arguments(self):
tu = get_tu('void foo(int i, int j);')
foo = get_cursor(tu, 'foo')
tu = get_tu("void foo(int i, int j);")
foo = get_cursor(tu, "foo")
arguments = list(foo.get_arguments())
self.assertEqual(len(arguments), 2)
@ -621,43 +632,49 @@ class TestCursor(unittest.TestCase):
self.assertEqual(arguments[1].spelling, "j")
def test_get_num_template_arguments(self):
tu = get_tu(kTemplateArgTest, lang='cpp')
foos = get_cursors(tu, 'foo')
tu = get_tu(kTemplateArgTest, lang="cpp")
foos = get_cursors(tu, "foo")
self.assertEqual(foos[1].get_num_template_arguments(), 3)
def test_get_template_argument_kind(self):
tu = get_tu(kTemplateArgTest, lang='cpp')
foos = get_cursors(tu, 'foo')
tu = get_tu(kTemplateArgTest, lang="cpp")
foos = get_cursors(tu, "foo")
self.assertEqual(foos[1].get_template_argument_kind(0), TemplateArgumentKind.INTEGRAL)
self.assertEqual(foos[1].get_template_argument_kind(1), TemplateArgumentKind.TYPE)
self.assertEqual(foos[1].get_template_argument_kind(2), TemplateArgumentKind.INTEGRAL)
self.assertEqual(
foos[1].get_template_argument_kind(0), TemplateArgumentKind.INTEGRAL
)
self.assertEqual(
foos[1].get_template_argument_kind(1), TemplateArgumentKind.TYPE
)
self.assertEqual(
foos[1].get_template_argument_kind(2), TemplateArgumentKind.INTEGRAL
)
def test_get_template_argument_type(self):
tu = get_tu(kTemplateArgTest, lang='cpp')
foos = get_cursors(tu, 'foo')
tu = get_tu(kTemplateArgTest, lang="cpp")
foos = get_cursors(tu, "foo")
self.assertEqual(foos[1].get_template_argument_type(1).kind, TypeKind.FLOAT)
def test_get_template_argument_value(self):
tu = get_tu(kTemplateArgTest, lang='cpp')
foos = get_cursors(tu, 'foo')
tu = get_tu(kTemplateArgTest, lang="cpp")
foos = get_cursors(tu, "foo")
self.assertEqual(foos[1].get_template_argument_value(0), -7)
self.assertEqual(foos[1].get_template_argument_value(2), True)
def test_get_template_argument_unsigned_value(self):
tu = get_tu(kTemplateArgTest, lang='cpp')
foos = get_cursors(tu, 'foo')
tu = get_tu(kTemplateArgTest, lang="cpp")
foos = get_cursors(tu, "foo")
self.assertEqual(foos[1].get_template_argument_unsigned_value(0), 2 ** 32 - 7)
self.assertEqual(foos[1].get_template_argument_unsigned_value(0), 2**32 - 7)
self.assertEqual(foos[1].get_template_argument_unsigned_value(2), True)
def test_referenced(self):
tu = get_tu('void foo(); void bar() { foo(); }')
foo = get_cursor(tu, 'foo')
bar = get_cursor(tu, 'bar')
tu = get_tu("void foo(); void bar() { foo(); }")
foo = get_cursor(tu, "foo")
bar = get_cursor(tu, "bar")
for c in bar.get_children():
if c.kind == CursorKind.CALL_EXPR:
self.assertEqual(c.referenced.spelling, foo.spelling)
@ -667,12 +684,14 @@ class TestCursor(unittest.TestCase):
kInputForMangling = """\
int foo(int, int);
"""
tu = get_tu(kInputForMangling, lang='cpp')
foo = get_cursor(tu, 'foo')
tu = get_tu(kInputForMangling, lang="cpp")
foo = get_cursor(tu, "foo")
# Since libclang does not link in targets, we cannot pass a triple to it
# and force the target. To enable this test to pass on all platforms, accept
# all valid manglings.
# [c-index-test handles this by running the source through clang, emitting
# an AST file and running libclang on that AST file]
self.assertIn(foo.mangled_name, ('_Z3fooii', '__Z3fooii', '?foo@@YAHHH', '?foo@@YAHHH@Z'))
self.assertIn(
foo.mangled_name, ("_Z3fooii", "__Z3fooii", "?foo@@YAHHH", "?foo@@YAHHH@Z")
)

36
clang/bindings/python/tests/cindex/test_cursor_kind.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import CursorKind
@ -10,7 +11,7 @@ import unittest
class TestCursorKind(unittest.TestCase):
def test_name(self):
self.assertEqual(CursorKind.UNEXPOSED_DECL.name, 'UNEXPOSED_DECL')
self.assertEqual(CursorKind.UNEXPOSED_DECL.name, "UNEXPOSED_DECL")
def test_get_all_kinds(self):
kinds = CursorKind.get_all_kinds()
@ -43,16 +44,27 @@ class TestCursorKind(unittest.TestCase):
self.assertFalse(CursorKind.TYPE_REF.is_unexposed())
for k in CursorKind.get_all_kinds():
group = [n for n in ('is_declaration', 'is_reference', 'is_expression',
'is_statement', 'is_invalid', 'is_attribute')
if getattr(k, n)()]
group = [
n
for n in (
"is_declaration",
"is_reference",
"is_expression",
"is_statement",
"is_invalid",
"is_attribute",
)
if getattr(k, n)()
]
if k in ( CursorKind.TRANSLATION_UNIT,
CursorKind.MACRO_DEFINITION,
CursorKind.MACRO_INSTANTIATION,
CursorKind.INCLUSION_DIRECTIVE,
CursorKind.PREPROCESSING_DIRECTIVE,
CursorKind.OVERLOAD_CANDIDATE):
if k in (
CursorKind.TRANSLATION_UNIT,
CursorKind.MACRO_DEFINITION,
CursorKind.MACRO_INSTANTIATION,
CursorKind.INCLUSION_DIRECTIVE,
CursorKind.PREPROCESSING_DIRECTIVE,
CursorKind.OVERLOAD_CANDIDATE,
):
self.assertEqual(len(group), 0)
else:
self.assertEqual(len(group), 1)

57
clang/bindings/python/tests/cindex/test_diagnostics.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import *
from .util import get_tu
@ -14,41 +15,42 @@ import unittest
class TestDiagnostics(unittest.TestCase):
def test_diagnostic_warning(self):
tu = get_tu('int f0() {}\n')
tu = get_tu("int f0() {}\n")
self.assertEqual(len(tu.diagnostics), 1)
self.assertEqual(tu.diagnostics[0].severity, Diagnostic.Warning)
self.assertEqual(tu.diagnostics[0].location.line, 1)
self.assertEqual(tu.diagnostics[0].location.column, 11)
self.assertEqual(tu.diagnostics[0].spelling,
'non-void function does not return a value')
self.assertEqual(
tu.diagnostics[0].spelling, "non-void function does not return a value"
)
def test_diagnostic_note(self):
# FIXME: We aren't getting notes here for some reason.
tu = get_tu('#define A x\nvoid *A = 1;\n')
tu = get_tu("#define A x\nvoid *A = 1;\n")
self.assertEqual(len(tu.diagnostics), 1)
self.assertEqual(tu.diagnostics[0].severity, Diagnostic.Error)
self.assertEqual(tu.diagnostics[0].location.line, 2)
self.assertEqual(tu.diagnostics[0].location.column, 7)
self.assertIn('incompatible', tu.diagnostics[0].spelling)
# self.assertEqual(tu.diagnostics[1].severity, Diagnostic.Note)
# self.assertEqual(tu.diagnostics[1].location.line, 1)
# self.assertEqual(tu.diagnostics[1].location.column, 11)
# self.assertEqual(tu.diagnostics[1].spelling, 'instantiated from')
self.assertIn("incompatible", tu.diagnostics[0].spelling)
# self.assertEqual(tu.diagnostics[1].severity, Diagnostic.Note)
# self.assertEqual(tu.diagnostics[1].location.line, 1)
# self.assertEqual(tu.diagnostics[1].location.column, 11)
# self.assertEqual(tu.diagnostics[1].spelling, 'instantiated from')
def test_diagnostic_fixit(self):
tu = get_tu('struct { int f0; } x = { f0 : 1 };')
tu = get_tu("struct { int f0; } x = { f0 : 1 };")
self.assertEqual(len(tu.diagnostics), 1)
self.assertEqual(tu.diagnostics[0].severity, Diagnostic.Warning)
self.assertEqual(tu.diagnostics[0].location.line, 1)
self.assertEqual(tu.diagnostics[0].location.column, 26)
self.assertRegex(tu.diagnostics[0].spelling,
'use of GNU old-style.*')
self.assertRegex(tu.diagnostics[0].spelling, "use of GNU old-style.*")
self.assertEqual(len(tu.diagnostics[0].fixits), 1)
self.assertEqual(tu.diagnostics[0].fixits[0].range.start.line, 1)
self.assertEqual(tu.diagnostics[0].fixits[0].range.start.column, 26)
self.assertEqual(tu.diagnostics[0].fixits[0].range.end.line, 1)
self.assertEqual(tu.diagnostics[0].fixits[0].range.end.column, 30)
self.assertEqual(tu.diagnostics[0].fixits[0].value, '.f0 = ')
self.assertEqual(tu.diagnostics[0].fixits[0].value, ".f0 = ")
def test_diagnostic_range(self):
tu = get_tu('void f() { int i = "a"; }')
@ -56,8 +58,7 @@ class TestDiagnostics(unittest.TestCase):
self.assertEqual(tu.diagnostics[0].severity, Diagnostic.Error)
self.assertEqual(tu.diagnostics[0].location.line, 1)
self.assertEqual(tu.diagnostics[0].location.column, 16)
self.assertRegex(tu.diagnostics[0].spelling,
'incompatible pointer to.*')
self.assertRegex(tu.diagnostics[0].spelling, "incompatible pointer to.*")
self.assertEqual(len(tu.diagnostics[0].fixits), 0)
self.assertEqual(len(tu.diagnostics[0].ranges), 1)
self.assertEqual(tu.diagnostics[0].ranges[0].start.line, 1)
@ -69,7 +70,7 @@ class TestDiagnostics(unittest.TestCase):
def test_diagnostic_category(self):
"""Ensure that category properties work."""
tu = get_tu('int f(int i) { return 7; }', all_warnings=True)
tu = get_tu("int f(int i) { return 7; }", all_warnings=True)
self.assertEqual(len(tu.diagnostics), 1)
d = tu.diagnostics[0]
@ -78,33 +79,35 @@ class TestDiagnostics(unittest.TestCase):
self.assertEqual(d.location.column, 11)
self.assertEqual(d.category_number, 2)
self.assertEqual(d.category_name, 'Semantic Issue')
self.assertEqual(d.category_name, "Semantic Issue")
def test_diagnostic_option(self):
"""Ensure that category option properties work."""
tu = get_tu('int f(int i) { return 7; }', all_warnings=True)
tu = get_tu("int f(int i) { return 7; }", all_warnings=True)
self.assertEqual(len(tu.diagnostics), 1)
d = tu.diagnostics[0]
self.assertEqual(d.option, '-Wunused-parameter')
self.assertEqual(d.disable_option, '-Wno-unused-parameter')
self.assertEqual(d.option, "-Wunused-parameter")
self.assertEqual(d.disable_option, "-Wno-unused-parameter")
def test_diagnostic_children(self):
tu = get_tu('void f(int x) {} void g() { f(); }')
tu = get_tu("void f(int x) {} void g() { f(); }")
self.assertEqual(len(tu.diagnostics), 1)
d = tu.diagnostics[0]
children = d.children
self.assertEqual(len(children), 1)
self.assertEqual(children[0].severity, Diagnostic.Note)
self.assertRegex(children[0].spelling,
'.*declared here')
self.assertRegex(children[0].spelling, ".*declared here")
self.assertEqual(children[0].location.line, 1)
self.assertEqual(children[0].location.column, 6)
def test_diagnostic_string_repr(self):
tu = get_tu('struct MissingSemicolon{}')
tu = get_tu("struct MissingSemicolon{}")
self.assertEqual(len(tu.diagnostics), 1)
d = tu.diagnostics[0]
self.assertEqual(repr(d), '<Diagnostic severity 3, location <SourceLocation file \'t.c\', line 1, column 26>, spelling "expected \';\' after struct">')
self.assertEqual(
repr(d),
"<Diagnostic severity 3, location <SourceLocation file 't.c', line 1, column 26>, spelling \"expected ';' after struct\">",
)

13
clang/bindings/python/tests/cindex/test_exception_specification_kind.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
import clang.cindex
from clang.cindex import ExceptionSpecificationKind
@ -24,12 +25,12 @@ class TestExceptionSpecificationKind(unittest.TestCase):
int square2(int x) noexcept;
int square3(int x) noexcept(noexcept(x * x));"""
tu = get_tu(source, lang='cpp', flags=['-std=c++14'])
tu = get_tu(source, lang="cpp", flags=["-std=c++14"])
declarations = find_function_declarations(tu.cursor)
expected = [
('square1', ExceptionSpecificationKind.NONE),
('square2', ExceptionSpecificationKind.BASIC_NOEXCEPT),
('square3', ExceptionSpecificationKind.COMPUTED_NOEXCEPT)
("square1", ExceptionSpecificationKind.NONE),
("square2", ExceptionSpecificationKind.BASIC_NOEXCEPT),
("square3", ExceptionSpecificationKind.COMPUTED_NOEXCEPT),
]
self.assertListEqual(declarations, expected)

7
clang/bindings/python/tests/cindex/test_file.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import Index, File
@ -11,7 +12,7 @@ import unittest
class TestFile(unittest.TestCase):
def test_file(self):
index = Index.create()
tu = index.parse('t.c', unsaved_files = [('t.c', "")])
tu = index.parse("t.c", unsaved_files=[("t.c", "")])
file = File.from_name(tu, "t.c")
self.assertEqual(str(file), "t.c")
self.assertEqual(file.name, "t.c")

11
clang/bindings/python/tests/cindex/test_index.py
View File

@ -1,14 +1,15 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import *
import os
import unittest
kInputsDir = os.path.join(os.path.dirname(__file__), 'INPUTS')
kInputsDir = os.path.join(os.path.dirname(__file__), "INPUTS")
class TestIndex(unittest.TestCase):
@ -20,7 +21,7 @@ class TestIndex(unittest.TestCase):
def test_parse(self):
index = Index.create()
self.assertIsInstance(index, Index)
tu = index.parse(os.path.join(kInputsDir, 'hello.cpp'))
tu = index.parse(os.path.join(kInputsDir, "hello.cpp"))
self.assertIsInstance(tu, TranslationUnit)
tu = index.parse(None, ['-c', os.path.join(kInputsDir, 'hello.cpp')])
tu = index.parse(None, ["-c", os.path.join(kInputsDir, "hello.cpp")])
self.assertIsInstance(tu, TranslationUnit)

20
clang/bindings/python/tests/cindex/test_linkage.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import LinkageKind
from clang.cindex import Cursor
@ -17,22 +18,25 @@ class TestLinkage(unittest.TestCase):
def test_linkage(self):
"""Ensure that linkage specifers are available on cursors"""
tu = get_tu("""
tu = get_tu(
"""
void foo() { int no_linkage; }
static int internal;
namespace { struct unique_external_type {} }
unique_external_type unique_external;
extern int external;
""", lang = 'cpp')
""",
lang="cpp",
)
no_linkage = get_cursor(tu.cursor, 'no_linkage')
no_linkage = get_cursor(tu.cursor, "no_linkage")
self.assertEqual(no_linkage.linkage, LinkageKind.NO_LINKAGE)
internal = get_cursor(tu.cursor, 'internal')
internal = get_cursor(tu.cursor, "internal")
self.assertEqual(internal.linkage, LinkageKind.INTERNAL)
unique_external = get_cursor(tu.cursor, 'unique_external')
unique_external = get_cursor(tu.cursor, "unique_external")
self.assertEqual(unique_external.linkage, LinkageKind.UNIQUE_EXTERNAL)
external = get_cursor(tu.cursor, 'external')
external = get_cursor(tu.cursor, "external")
self.assertEqual(external.linkage, LinkageKind.EXTERNAL)

86
clang/bindings/python/tests/cindex/test_location.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import Cursor
from clang.cindex import File
@ -14,7 +15,7 @@ from .util import get_tu
import unittest
baseInput="int one;\nint two;\n"
baseInput = "int one;\nint two;\n"
class TestLocation(unittest.TestCase):
@ -25,43 +26,43 @@ class TestLocation(unittest.TestCase):
def test_location(self):
tu = get_tu(baseInput)
one = get_cursor(tu, 'one')
two = get_cursor(tu, 'two')
one = get_cursor(tu, "one")
two = get_cursor(tu, "two")
self.assertIsNotNone(one)
self.assertIsNotNone(two)
self.assert_location(one.location,line=1,column=5,offset=4)
self.assert_location(two.location,line=2,column=5,offset=13)
self.assert_location(one.location, line=1, column=5, offset=4)
self.assert_location(two.location, line=2, column=5, offset=13)
# adding a linebreak at top should keep columns same
tu = get_tu('\n' + baseInput)
one = get_cursor(tu, 'one')
two = get_cursor(tu, 'two')
tu = get_tu("\n" + baseInput)
one = get_cursor(tu, "one")
two = get_cursor(tu, "two")
self.assertIsNotNone(one)
self.assertIsNotNone(two)
self.assert_location(one.location,line=2,column=5,offset=5)
self.assert_location(two.location,line=3,column=5,offset=14)
self.assert_location(one.location, line=2, column=5, offset=5)
self.assert_location(two.location, line=3, column=5, offset=14)
# adding a space should affect column on first line only
tu = get_tu(' ' + baseInput)
one = get_cursor(tu, 'one')
two = get_cursor(tu, 'two')
tu = get_tu(" " + baseInput)
one = get_cursor(tu, "one")
two = get_cursor(tu, "two")
self.assert_location(one.location,line=1,column=6,offset=5)
self.assert_location(two.location,line=2,column=5,offset=14)
self.assert_location(one.location, line=1, column=6, offset=5)
self.assert_location(two.location, line=2, column=5, offset=14)
# define the expected location ourselves and see if it matches
# the returned location
tu = get_tu(baseInput)
file = File.from_name(tu, 't.c')
file = File.from_name(tu, "t.c")
location = SourceLocation.from_position(tu, file, 1, 5)
cursor = Cursor.from_location(tu, location)
one = get_cursor(tu, 'one')
one = get_cursor(tu, "one")
self.assertIsNotNone(one)
self.assertEqual(one, cursor)
@ -74,7 +75,7 @@ class TestLocation(unittest.TestCase):
offset_location = SourceLocation.from_offset(tu, file, 5)
cursor = Cursor.from_location(tu, offset_location)
verified = False
for n in [n for n in tu.cursor.get_children() if n.spelling == 'one']:
for n in [n for n in tu.cursor.get_children() if n.spelling == "one"]:
self.assertEqual(n, cursor)
verified = True
@ -82,18 +83,22 @@ class TestLocation(unittest.TestCase):
def test_extent(self):
tu = get_tu(baseInput)
one = get_cursor(tu, 'one')
two = get_cursor(tu, 'two')
one = get_cursor(tu, "one")
two = get_cursor(tu, "two")
self.assert_location(one.extent.start,line=1,column=1,offset=0)
self.assert_location(one.extent.end,line=1,column=8,offset=7)
self.assertEqual(baseInput[one.extent.start.offset:one.extent.end.offset], "int one")
self.assert_location(one.extent.start, line=1, column=1, offset=0)
self.assert_location(one.extent.end, line=1, column=8, offset=7)
self.assertEqual(
baseInput[one.extent.start.offset : one.extent.end.offset], "int one"
)
self.assert_location(two.extent.start,line=2,column=1,offset=9)
self.assert_location(two.extent.end,line=2,column=8,offset=16)
self.assertEqual(baseInput[two.extent.start.offset:two.extent.end.offset], "int two")
self.assert_location(two.extent.start, line=2, column=1, offset=9)
self.assert_location(two.extent.end, line=2, column=8, offset=16)
self.assertEqual(
baseInput[two.extent.start.offset : two.extent.end.offset], "int two"
)
file = File.from_name(tu, 't.c')
file = File.from_name(tu, "t.c")
location1 = SourceLocation.from_position(tu, file, 1, 1)
location2 = SourceLocation.from_position(tu, file, 1, 8)
@ -106,15 +111,22 @@ class TestLocation(unittest.TestCase):
self.assertNotEqual(range1, range3)
def test_is_system_location(self):
header = os.path.normpath('./fake/fake.h')
tu = TranslationUnit.from_source('fake.c', [f'-isystem{os.path.dirname(header)}'], unsaved_files = [
('fake.c', """
header = os.path.normpath("./fake/fake.h")
tu = TranslationUnit.from_source(
"fake.c",
[f"-isystem{os.path.dirname(header)}"],
unsaved_files=[
(
"fake.c",
"""
#include <fake.h>
int one;
"""),
(header, "int two();")
])
one = get_cursor(tu, 'one')
two = get_cursor(tu, 'two')
""",
),
(header, "int two();"),
],
)
one = get_cursor(tu, "one")
two = get_cursor(tu, "two")
self.assertFalse(one.location.is_in_system_header)
self.assertTrue(two.location.is_in_system_header)

54
clang/bindings/python/tests/cindex/test_tls_kind.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import TLSKind
from clang.cindex import Cursor
@ -17,38 +18,57 @@ class TestTLSKind(unittest.TestCase):
def test_tls_kind(self):
"""Ensure that thread-local storage kinds are available on cursors."""
tu = get_tu("""
tu = get_tu(
"""
int tls_none;
thread_local int tls_dynamic;
_Thread_local int tls_static;
""", lang = 'cpp')
""",
lang="cpp",
)
tls_none = get_cursor(tu.cursor, 'tls_none')
tls_none = get_cursor(tu.cursor, "tls_none")
self.assertEqual(tls_none.tls_kind, TLSKind.NONE)
tls_dynamic = get_cursor(tu.cursor, 'tls_dynamic')
tls_dynamic = get_cursor(tu.cursor, "tls_dynamic")
self.assertEqual(tls_dynamic.tls_kind, TLSKind.DYNAMIC)
tls_static = get_cursor(tu.cursor, 'tls_static')
tls_static = get_cursor(tu.cursor, "tls_static")
self.assertEqual(tls_static.tls_kind, TLSKind.STATIC)
# The following case tests '__declspec(thread)'. Since it is a Microsoft
# specific extension, specific flags are required for the parser to pick
# these up.
flags = ['-fms-extensions', '-target', 'x86_64-unknown-windows-win32',
'-fms-compatibility-version=18']
tu = get_tu("""
flags = [
"-fms-extensions",
"-target",
"x86_64-unknown-windows-win32",
"-fms-compatibility-version=18",
]
tu = get_tu(
"""
__declspec(thread) int tls_declspec_msvc18;
""", lang = 'cpp', flags=flags)
""",
lang="cpp",
flags=flags,
)
tls_declspec_msvc18 = get_cursor(tu.cursor, 'tls_declspec_msvc18')
tls_declspec_msvc18 = get_cursor(tu.cursor, "tls_declspec_msvc18")
self.assertEqual(tls_declspec_msvc18.tls_kind, TLSKind.STATIC)
flags = ['-fms-extensions', '-target', 'x86_64-unknown-windows-win32',
'-fms-compatibility-version=19']
tu = get_tu("""
flags = [
"-fms-extensions",
"-target",
"x86_64-unknown-windows-win32",
"-fms-compatibility-version=19",
]
tu = get_tu(
"""
__declspec(thread) int tls_declspec_msvc19;
""", lang = 'cpp', flags=flags)
""",
lang="cpp",
flags=flags,
)
tls_declspec_msvc19 = get_cursor(tu.cursor, 'tls_declspec_msvc19')
tls_declspec_msvc19 = get_cursor(tu.cursor, "tls_declspec_msvc19")
self.assertEqual(tls_declspec_msvc19.tls_kind, TLSKind.DYNAMIC)

15
clang/bindings/python/tests/cindex/test_token_kind.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import TokenKind
@ -12,16 +13,16 @@ class TestTokenKind(unittest.TestCase):
def test_constructor(self):
"""Ensure TokenKind constructor works as expected."""
t = TokenKind(5, 'foo')
t = TokenKind(5, "foo")
self.assertEqual(t.value, 5)
self.assertEqual(t.name, 'foo')
self.assertEqual(t.name, "foo")
def test_bad_register(self):
"""Ensure a duplicate value is rejected for registration."""
with self.assertRaises(ValueError):
TokenKind.register(2, 'foo')
TokenKind.register(2, "foo")
def test_unknown_value(self):
"""Ensure trying to fetch an unknown value raises."""
@ -31,7 +32,7 @@ class TestTokenKind(unittest.TestCase):
def test_registration(self):
"""Ensure that items registered appear as class attributes."""
self.assertTrue(hasattr(TokenKind, 'LITERAL'))
self.assertTrue(hasattr(TokenKind, "LITERAL"))
literal = TokenKind.LITERAL
self.assertIsInstance(literal, TokenKind)
@ -46,4 +47,4 @@ class TestTokenKind(unittest.TestCase):
"""Ensure repr() works."""
r = repr(TokenKind.LITERAL)
self.assertEqual(r, 'TokenKind.LITERAL')
self.assertEqual(r, "TokenKind.LITERAL")

19
clang/bindings/python/tests/cindex/test_tokens.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from clang.cindex import CursorKind
from clang.cindex import Index
@ -17,12 +18,12 @@ import unittest
class TestTokens(unittest.TestCase):
def test_token_to_cursor(self):
"""Ensure we can obtain a Cursor from a Token instance."""
tu = get_tu('int i = 5;')
r = tu.get_extent('t.c', (0, 9))
tu = get_tu("int i = 5;")
r = tu.get_extent("t.c", (0, 9))
tokens = list(tu.get_tokens(extent=r))
self.assertEqual(len(tokens), 4)
self.assertEqual(tokens[1].spelling, 'i')
self.assertEqual(tokens[1].spelling, "i")
self.assertEqual(tokens[1].kind, TokenKind.IDENTIFIER)
cursor = tokens[1].cursor
@ -32,8 +33,8 @@ class TestTokens(unittest.TestCase):
def test_token_location(self):
"""Ensure Token.location works."""
tu = get_tu('int foo = 10;')
r = tu.get_extent('t.c', (0, 11))
tu = get_tu("int foo = 10;")
r = tu.get_extent("t.c", (0, 11))
tokens = list(tu.get_tokens(extent=r))
self.assertEqual(len(tokens), 4)
@ -46,8 +47,8 @@ class TestTokens(unittest.TestCase):
def test_token_extent(self):
"""Ensure Token.extent works."""
tu = get_tu('int foo = 10;')
r = tu.get_extent('t.c', (0, 11))
tu = get_tu("int foo = 10;")
r = tu.get_extent("t.c", (0, 11))
tokens = list(tu.get_tokens(extent=r))
self.assertEqual(len(tokens), 4)

191
clang/bindings/python/tests/cindex/test_translation_unit.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
from contextlib import contextmanager
import gc
@ -25,7 +26,7 @@ from .util import skip_if_no_fspath
from .util import str_to_path
kInputsDir = os.path.join(os.path.dirname(__file__), 'INPUTS')
kInputsDir = os.path.join(os.path.dirname(__file__), "INPUTS")
@contextmanager
@ -52,79 +53,99 @@ def save_tu_pathlike(tu):
class TestTranslationUnit(unittest.TestCase):
def test_spelling(self):
path = os.path.join(kInputsDir, 'hello.cpp')
path = os.path.join(kInputsDir, "hello.cpp")
tu = TranslationUnit.from_source(path)
self.assertEqual(tu.spelling, path)
def test_cursor(self):
path = os.path.join(kInputsDir, 'hello.cpp')
path = os.path.join(kInputsDir, "hello.cpp")
tu = get_tu(path)
c = tu.cursor
self.assertIsInstance(c, Cursor)
self.assertIs(c.kind, CursorKind.TRANSLATION_UNIT)
def test_parse_arguments(self):
path = os.path.join(kInputsDir, 'parse_arguments.c')
tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi'])
path = os.path.join(kInputsDir, "parse_arguments.c")
tu = TranslationUnit.from_source(path, ["-DDECL_ONE=hello", "-DDECL_TWO=hi"])
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'hello')
self.assertEqual(spellings[-1], 'hi')
self.assertEqual(spellings[-2], "hello")
self.assertEqual(spellings[-1], "hi")
def test_reparse_arguments(self):
path = os.path.join(kInputsDir, 'parse_arguments.c')
tu = TranslationUnit.from_source(path, ['-DDECL_ONE=hello', '-DDECL_TWO=hi'])
path = os.path.join(kInputsDir, "parse_arguments.c")
tu = TranslationUnit.from_source(path, ["-DDECL_ONE=hello", "-DDECL_TWO=hi"])
tu.reparse()
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'hello')
self.assertEqual(spellings[-1], 'hi')
self.assertEqual(spellings[-2], "hello")
self.assertEqual(spellings[-1], "hi")
def test_unsaved_files(self):
tu = TranslationUnit.from_source('fake.c', ['-I./'], unsaved_files = [
('fake.c', """
tu = TranslationUnit.from_source(
"fake.c",
["-I./"],
unsaved_files=[
(
"fake.c",
"""
#include "fake.h"
int x;
int SOME_DEFINE;
"""),
('./fake.h', """
""",
),
(
"./fake.h",
"""
#define SOME_DEFINE y
""")
])
""",
),
],
)
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'x')
self.assertEqual(spellings[-1], 'y')
self.assertEqual(spellings[-2], "x")
self.assertEqual(spellings[-1], "y")
def test_unsaved_files_2(self):
if sys.version_info.major >= 3:
from io import StringIO
else:
from io import BytesIO as StringIO
tu = TranslationUnit.from_source('fake.c', unsaved_files = [
('fake.c', StringIO('int x;'))])
tu = TranslationUnit.from_source(
"fake.c", unsaved_files=[("fake.c", StringIO("int x;"))]
)
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-1], 'x')
self.assertEqual(spellings[-1], "x")
@skip_if_no_fspath
def test_from_source_accepts_pathlike(self):
tu = TranslationUnit.from_source(str_to_path('fake.c'), ['-Iincludes'], unsaved_files = [
(str_to_path('fake.c'), """
tu = TranslationUnit.from_source(
str_to_path("fake.c"),
["-Iincludes"],
unsaved_files=[
(
str_to_path("fake.c"),
"""
#include "fake.h"
int x;
int SOME_DEFINE;
"""),
(str_to_path('includes/fake.h'), """
""",
),
(
str_to_path("includes/fake.h"),
"""
#define SOME_DEFINE y
""")
])
""",
),
],
)
spellings = [c.spelling for c in tu.cursor.get_children()]
self.assertEqual(spellings[-2], 'x')
self.assertEqual(spellings[-1], 'y')
self.assertEqual(spellings[-2], "x")
self.assertEqual(spellings[-1], "y")
def assert_normpaths_equal(self, path1, path2):
""" Compares two paths for equality after normalizing them with
os.path.normpath
"""Compares two paths for equality after normalizing them with
os.path.normpath
"""
self.assertEqual(os.path.normpath(path1),
os.path.normpath(path2))
self.assertEqual(os.path.normpath(path1), os.path.normpath(path2))
def test_includes(self):
def eq(expected, actual):
@ -134,7 +155,7 @@ int SOME_DEFINE;
else:
self.assert_normpaths_equal(expected[1], actual.include.name)
src = os.path.join(kInputsDir, 'include.cpp')
src = os.path.join(kInputsDir, "include.cpp")
h1 = os.path.join(kInputsDir, "header1.h")
h2 = os.path.join(kInputsDir, "header2.h")
h3 = os.path.join(kInputsDir, "header3.h")
@ -145,21 +166,27 @@ int SOME_DEFINE;
eq(i[0], i[1])
def test_inclusion_directive(self):
src = os.path.join(kInputsDir, 'include.cpp')
src = os.path.join(kInputsDir, "include.cpp")
h1 = os.path.join(kInputsDir, "header1.h")
h2 = os.path.join(kInputsDir, "header2.h")
h3 = os.path.join(kInputsDir, "header3.h")
inc = [h1, h3, h2, h3, h1]
tu = TranslationUnit.from_source(src, options=TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD)
inclusion_directive_files = [c.get_included_file().name for c in tu.cursor.get_children() if c.kind == CursorKind.INCLUSION_DIRECTIVE]
tu = TranslationUnit.from_source(
src, options=TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
)
inclusion_directive_files = [
c.get_included_file().name
for c in tu.cursor.get_children()
if c.kind == CursorKind.INCLUSION_DIRECTIVE
]
for i in zip(inc, inclusion_directive_files):
self.assert_normpaths_equal(i[0], i[1])
def test_save(self):
"""Ensure TranslationUnit.save() works."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
with save_tu(tu) as path:
self.assertTrue(os.path.exists(path))
@ -169,7 +196,7 @@ int SOME_DEFINE;
def test_save_pathlike(self):
"""Ensure TranslationUnit.save() works with PathLike filename."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
with save_tu_pathlike(tu) as path:
self.assertTrue(os.path.exists(path))
@ -178,9 +205,9 @@ int SOME_DEFINE;
def test_save_translation_errors(self):
"""Ensure that saving to an invalid directory raises."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
path = '/does/not/exist/llvm-test.ast'
path = "/does/not/exist/llvm-test.ast"
self.assertFalse(os.path.exists(os.path.dirname(path)))
with self.assertRaises(TranslationUnitSaveError) as cm:
@ -192,7 +219,7 @@ int SOME_DEFINE;
def test_load(self):
"""Ensure TranslationUnits can be constructed from saved files."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
self.assertEqual(len(tu.diagnostics), 0)
with save_tu(tu) as path:
self.assertTrue(os.path.exists(path))
@ -201,7 +228,7 @@ int SOME_DEFINE;
tu2 = TranslationUnit.from_ast_file(filename=path)
self.assertEqual(len(tu2.diagnostics), 0)
foo = get_cursor(tu2, 'foo')
foo = get_cursor(tu2, "foo")
self.assertIsNotNone(foo)
# Just in case there is an open file descriptor somewhere.
@ -211,20 +238,20 @@ int SOME_DEFINE;
def test_load_pathlike(self):
"""Ensure TranslationUnits can be constructed from saved files -
PathLike variant."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
self.assertEqual(len(tu.diagnostics), 0)
with save_tu(tu) as path:
tu2 = TranslationUnit.from_ast_file(filename=str_to_path(path))
self.assertEqual(len(tu2.diagnostics), 0)
foo = get_cursor(tu2, 'foo')
foo = get_cursor(tu2, "foo")
self.assertIsNotNone(foo)
# Just in case there is an open file descriptor somewhere.
del tu2
def test_index_parse(self):
path = os.path.join(kInputsDir, 'hello.cpp')
path = os.path.join(kInputsDir, "hello.cpp")
index = Index.create()
tu = index.parse(path)
self.assertIsInstance(tu, TranslationUnit)
@ -232,97 +259,97 @@ int SOME_DEFINE;
def test_get_file(self):
"""Ensure tu.get_file() works appropriately."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
f = tu.get_file('t.c')
f = tu.get_file("t.c")
self.assertIsInstance(f, File)
self.assertEqual(f.name, 't.c')
self.assertEqual(f.name, "t.c")
with self.assertRaises(Exception):
f = tu.get_file('foobar.cpp')
f = tu.get_file("foobar.cpp")
@skip_if_no_fspath
def test_get_file_pathlike(self):
"""Ensure tu.get_file() works appropriately with PathLike filenames."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
f = tu.get_file(str_to_path('t.c'))
f = tu.get_file(str_to_path("t.c"))
self.assertIsInstance(f, File)
self.assertEqual(f.name, 't.c')
self.assertEqual(f.name, "t.c")
with self.assertRaises(Exception):
f = tu.get_file(str_to_path('foobar.cpp'))
f = tu.get_file(str_to_path("foobar.cpp"))
def test_get_source_location(self):
"""Ensure tu.get_source_location() works."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
location = tu.get_location('t.c', 2)
location = tu.get_location("t.c", 2)
self.assertIsInstance(location, SourceLocation)
self.assertEqual(location.offset, 2)
self.assertEqual(location.file.name, 't.c')
self.assertEqual(location.file.name, "t.c")
location = tu.get_location('t.c', (1, 3))
location = tu.get_location("t.c", (1, 3))
self.assertIsInstance(location, SourceLocation)
self.assertEqual(location.line, 1)
self.assertEqual(location.column, 3)
self.assertEqual(location.file.name, 't.c')
self.assertEqual(location.file.name, "t.c")
def test_get_source_range(self):
"""Ensure tu.get_source_range() works."""
tu = get_tu('int foo();')
tu = get_tu("int foo();")
r = tu.get_extent('t.c', (1,4))
r = tu.get_extent("t.c", (1, 4))
self.assertIsInstance(r, SourceRange)
self.assertEqual(r.start.offset, 1)
self.assertEqual(r.end.offset, 4)
self.assertEqual(r.start.file.name, 't.c')
self.assertEqual(r.end.file.name, 't.c')
self.assertEqual(r.start.file.name, "t.c")
self.assertEqual(r.end.file.name, "t.c")
r = tu.get_extent('t.c', ((1,2), (1,3)))
r = tu.get_extent("t.c", ((1, 2), (1, 3)))
self.assertIsInstance(r, SourceRange)
self.assertEqual(r.start.line, 1)
self.assertEqual(r.start.column, 2)
self.assertEqual(r.end.line, 1)
self.assertEqual(r.end.column, 3)
self.assertEqual(r.start.file.name, 't.c')
self.assertEqual(r.end.file.name, 't.c')
self.assertEqual(r.start.file.name, "t.c")
self.assertEqual(r.end.file.name, "t.c")
start = tu.get_location('t.c', 0)
end = tu.get_location('t.c', 5)
start = tu.get_location("t.c", 0)
end = tu.get_location("t.c", 5)
r = tu.get_extent('t.c', (start, end))
r = tu.get_extent("t.c", (start, end))
self.assertIsInstance(r, SourceRange)
self.assertEqual(r.start.offset, 0)
self.assertEqual(r.end.offset, 5)
self.assertEqual(r.start.file.name, 't.c')
self.assertEqual(r.end.file.name, 't.c')
self.assertEqual(r.start.file.name, "t.c")
self.assertEqual(r.end.file.name, "t.c")
def test_get_tokens_gc(self):
"""Ensures get_tokens() works properly with garbage collection."""
tu = get_tu('int foo();')
r = tu.get_extent('t.c', (0, 10))
tu = get_tu("int foo();")
r = tu.get_extent("t.c", (0, 10))
tokens = list(tu.get_tokens(extent=r))
self.assertEqual(tokens[0].spelling, 'int')
self.assertEqual(tokens[0].spelling, "int")
gc.collect()
self.assertEqual(tokens[0].spelling, 'int')
self.assertEqual(tokens[0].spelling, "int")
del tokens[1]
gc.collect()
self.assertEqual(tokens[0].spelling, 'int')
self.assertEqual(tokens[0].spelling, "int")
# May trigger segfault if we don't do our job properly.
del tokens
gc.collect()
gc.collect() # Just in case.
gc.collect() # Just in case.
def test_fail_from_source(self):
path = os.path.join(kInputsDir, 'non-existent.cpp')
path = os.path.join(kInputsDir, "non-existent.cpp")
try:
tu = TranslationUnit.from_source(path)
except TranslationUnitLoadError:
@ -330,7 +357,7 @@ int SOME_DEFINE;
self.assertEqual(tu, None)
def test_fail_from_ast_file(self):
path = os.path.join(kInputsDir, 'non-existent.ast')
path = os.path.join(kInputsDir, "non-existent.ast")
try:
tu = TranslationUnit.from_ast_file(path)
except TranslationUnitLoadError:

195
clang/bindings/python/tests/cindex/test_type.py
View File

@ -1,7 +1,8 @@
import os
from clang.cindex import Config
if 'CLANG_LIBRARY_PATH' in os.environ:
Config.set_library_path(os.environ['CLANG_LIBRARY_PATH'])
if "CLANG_LIBRARY_PATH" in os.environ:
Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
import gc
import unittest
@ -31,7 +32,7 @@ struct teststruct {
"""
constarrayInput="""
constarrayInput = """
struct teststruct {
void *A[2];
};
@ -42,81 +43,85 @@ class TestType(unittest.TestCase):
def test_a_struct(self):
tu = get_tu(kInput)
teststruct = get_cursor(tu, 'teststruct')
teststruct = get_cursor(tu, "teststruct")
self.assertIsNotNone(teststruct, "Could not find teststruct.")
fields = list(teststruct.get_children())
self.assertEqual(fields[0].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[0].translation_unit)
self.assertEqual(fields[0].spelling, 'a')
self.assertEqual(fields[0].spelling, "a")
self.assertFalse(fields[0].type.is_const_qualified())
self.assertEqual(fields[0].type.kind, TypeKind.INT)
self.assertEqual(fields[0].type.get_canonical().kind, TypeKind.INT)
self.assertEqual(fields[0].type.get_typedef_name(), '')
self.assertEqual(fields[0].type.get_typedef_name(), "")
self.assertEqual(fields[1].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[1].translation_unit)
self.assertEqual(fields[1].spelling, 'b')
self.assertEqual(fields[1].spelling, "b")
self.assertFalse(fields[1].type.is_const_qualified())
self.assertEqual(fields[1].type.kind, TypeKind.ELABORATED)
self.assertEqual(fields[1].type.get_canonical().kind, TypeKind.INT)
self.assertEqual(fields[1].type.get_declaration().spelling, 'I')
self.assertEqual(fields[1].type.get_typedef_name(), 'I')
self.assertEqual(fields[1].type.get_declaration().spelling, "I")
self.assertEqual(fields[1].type.get_typedef_name(), "I")
self.assertEqual(fields[2].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[2].translation_unit)
self.assertEqual(fields[2].spelling, 'c')
self.assertEqual(fields[2].spelling, "c")
self.assertFalse(fields[2].type.is_const_qualified())
self.assertEqual(fields[2].type.kind, TypeKind.LONG)
self.assertEqual(fields[2].type.get_canonical().kind, TypeKind.LONG)
self.assertEqual(fields[2].type.get_typedef_name(), '')
self.assertEqual(fields[2].type.get_typedef_name(), "")
self.assertEqual(fields[3].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[3].translation_unit)
self.assertEqual(fields[3].spelling, 'd')
self.assertEqual(fields[3].spelling, "d")
self.assertFalse(fields[3].type.is_const_qualified())
self.assertEqual(fields[3].type.kind, TypeKind.ULONG)
self.assertEqual(fields[3].type.get_canonical().kind, TypeKind.ULONG)
self.assertEqual(fields[3].type.get_typedef_name(), '')
self.assertEqual(fields[3].type.get_typedef_name(), "")
self.assertEqual(fields[4].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[4].translation_unit)
self.assertEqual(fields[4].spelling, 'e')
self.assertEqual(fields[4].spelling, "e")
self.assertFalse(fields[4].type.is_const_qualified())
self.assertEqual(fields[4].type.kind, TypeKind.LONG)
self.assertEqual(fields[4].type.get_canonical().kind, TypeKind.LONG)
self.assertEqual(fields[4].type.get_typedef_name(), '')
self.assertEqual(fields[4].type.get_typedef_name(), "")
self.assertEqual(fields[5].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[5].translation_unit)
self.assertEqual(fields[5].spelling, 'f')
self.assertEqual(fields[5].spelling, "f")
self.assertTrue(fields[5].type.is_const_qualified())
self.assertEqual(fields[5].type.kind, TypeKind.INT)
self.assertEqual(fields[5].type.get_canonical().kind, TypeKind.INT)
self.assertEqual(fields[5].type.get_typedef_name(), '')
self.assertEqual(fields[5].type.get_typedef_name(), "")
self.assertEqual(fields[6].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[6].translation_unit)
self.assertEqual(fields[6].spelling, 'g')
self.assertEqual(fields[6].spelling, "g")
self.assertFalse(fields[6].type.is_const_qualified())
self.assertEqual(fields[6].type.kind, TypeKind.POINTER)
self.assertEqual(fields[6].type.get_pointee().kind, TypeKind.INT)
self.assertEqual(fields[6].type.get_typedef_name(), '')
self.assertEqual(fields[6].type.get_typedef_name(), "")
self.assertEqual(fields[7].kind, CursorKind.FIELD_DECL)
self.assertIsNotNone(fields[7].translation_unit)
self.assertEqual(fields[7].spelling, 'h')
self.assertEqual(fields[7].spelling, "h")
self.assertFalse(fields[7].type.is_const_qualified())
self.assertEqual(fields[7].type.kind, TypeKind.POINTER)
self.assertEqual(fields[7].type.get_pointee().kind, TypeKind.POINTER)
self.assertEqual(fields[7].type.get_pointee().get_pointee().kind, TypeKind.POINTER)
self.assertEqual(fields[7].type.get_pointee().get_pointee().get_pointee().kind, TypeKind.INT)
self.assertEqual(fields[7].type.get_typedef_name(), '')
self.assertEqual(
fields[7].type.get_pointee().get_pointee().kind, TypeKind.POINTER
)
self.assertEqual(
fields[7].type.get_pointee().get_pointee().get_pointee().kind, TypeKind.INT
)
self.assertEqual(fields[7].type.get_typedef_name(), "")
def test_references(self):
"""Ensure that a Type maintains a reference to a TranslationUnit."""
tu = get_tu('int x;')
tu = get_tu("int x;")
children = list(tu.cursor.get_children())
self.assertGreater(len(children), 0)
@ -136,10 +141,10 @@ class TestType(unittest.TestCase):
def testConstantArray(self):
tu = get_tu(constarrayInput)
teststruct = get_cursor(tu, 'teststruct')
teststruct = get_cursor(tu, "teststruct")
self.assertIsNotNone(teststruct, "Didn't find teststruct??")
fields = list(teststruct.get_children())
self.assertEqual(fields[0].spelling, 'A')
self.assertEqual(fields[0].spelling, "A")
self.assertEqual(fields[0].type.kind, TypeKind.CONSTANTARRAY)
self.assertIsNotNone(fields[0].type.get_array_element_type())
self.assertEqual(fields[0].type.get_array_element_type().kind, TypeKind.POINTER)
@ -147,12 +152,12 @@ class TestType(unittest.TestCase):
def test_equal(self):
"""Ensure equivalence operators work on Type."""
source = 'int a; int b; void *v;'
source = "int a; int b; void *v;"
tu = get_tu(source)
a = get_cursor(tu, 'a')
b = get_cursor(tu, 'b')
v = get_cursor(tu, 'v')
a = get_cursor(tu, "a")
b = get_cursor(tu, "b")
v = get_cursor(tu, "v")
self.assertIsNotNone(a)
self.assertIsNotNone(b)
@ -162,15 +167,15 @@ class TestType(unittest.TestCase):
self.assertNotEqual(a.type, v.type)
self.assertNotEqual(a.type, None)
self.assertNotEqual(a.type, 'foo')
self.assertNotEqual(a.type, "foo")
def test_type_spelling(self):
"""Ensure Type.spelling works."""
tu = get_tu('int c[5]; void f(int i[]); int x; int v[x];')
c = get_cursor(tu, 'c')
i = get_cursor(tu, 'i')
x = get_cursor(tu, 'x')
v = get_cursor(tu, 'v')
tu = get_tu("int c[5]; void f(int i[]); int x; int v[x];")
c = get_cursor(tu, "c")
i = get_cursor(tu, "i")
x = get_cursor(tu, "x")
v = get_cursor(tu, "v")
self.assertIsNotNone(c)
self.assertIsNotNone(i)
self.assertIsNotNone(x)
@ -182,16 +187,16 @@ class TestType(unittest.TestCase):
def test_typekind_spelling(self):
"""Ensure TypeKind.spelling works."""
tu = get_tu('int a;')
a = get_cursor(tu, 'a')
tu = get_tu("int a;")
a = get_cursor(tu, "a")
self.assertIsNotNone(a)
self.assertEqual(a.type.kind.spelling, 'Int')
self.assertEqual(a.type.kind.spelling, "Int")
def test_function_argument_types(self):
"""Ensure that Type.argument_types() works as expected."""
tu = get_tu('void f(int, int);')
f = get_cursor(tu, 'f')
tu = get_tu("void f(int, int);")
f = get_cursor(tu, "f")
self.assertIsNotNone(f)
args = f.type.argument_types()
@ -213,20 +218,20 @@ class TestType(unittest.TestCase):
def test_argument_types_string_key(self):
"""Ensure that non-int keys raise a TypeError."""
tu = get_tu('void f(int, int);')
f = get_cursor(tu, 'f')
tu = get_tu("void f(int, int);")
f = get_cursor(tu, "f")
self.assertIsNotNone(f)
args = f.type.argument_types()
self.assertEqual(len(args), 2)
with self.assertRaises(TypeError):
args['foo']
args["foo"]
def test_argument_types_negative_index(self):
"""Ensure that negative indexes on argument_types Raises an IndexError."""
tu = get_tu('void f(int, int);')
f = get_cursor(tu, 'f')
tu = get_tu("void f(int, int);")
f = get_cursor(tu, "f")
args = f.type.argument_types()
with self.assertRaises(IndexError):
@ -234,8 +239,8 @@ class TestType(unittest.TestCase):
def test_argument_types_overflow_index(self):
"""Ensure that indexes beyond the length of Type.argument_types() raise."""
tu = get_tu('void f(int, int);')
f = get_cursor(tu, 'f')
tu = get_tu("void f(int, int);")
f = get_cursor(tu, "f")
args = f.type.argument_types()
with self.assertRaises(IndexError):
@ -243,8 +248,8 @@ class TestType(unittest.TestCase):
def test_argument_types_invalid_type(self):
"""Ensure that obtaining argument_types on a Type without them raises."""
tu = get_tu('int i;')
i = get_cursor(tu, 'i')
tu = get_tu("int i;")
i = get_cursor(tu, "i")
self.assertIsNotNone(i)
with self.assertRaises(Exception):
@ -252,9 +257,9 @@ class TestType(unittest.TestCase):
def test_is_pod(self):
"""Ensure Type.is_pod() works."""
tu = get_tu('int i; void f();')
i = get_cursor(tu, 'i')
f = get_cursor(tu, 'f')
tu = get_tu("int i; void f();")
i = get_cursor(tu, "i")
f = get_cursor(tu, "f")
self.assertIsNotNone(i)
self.assertIsNotNone(f)
@ -265,7 +270,7 @@ class TestType(unittest.TestCase):
def test_function_variadic(self):
"""Ensure Type.is_function_variadic works."""
source ="""
source = """
#include <stdarg.h>
void foo(int a, ...);
@ -273,8 +278,8 @@ class TestType(unittest.TestCase):
"""
tu = get_tu(source)
foo = get_cursor(tu, 'foo')
bar = get_cursor(tu, 'bar')
foo = get_cursor(tu, "foo")
bar = get_cursor(tu, "bar")
self.assertIsNotNone(foo)
self.assertIsNotNone(bar)
@ -285,10 +290,10 @@ class TestType(unittest.TestCase):
def test_element_type(self):
"""Ensure Type.element_type works."""
tu = get_tu('int c[5]; void f(int i[]); int x; int v[x];')
c = get_cursor(tu, 'c')
i = get_cursor(tu, 'i')
v = get_cursor(tu, 'v')
tu = get_tu("int c[5]; void f(int i[]); int x; int v[x];")
c = get_cursor(tu, "c")
i = get_cursor(tu, "i")
v = get_cursor(tu, "v")
self.assertIsNotNone(c)
self.assertIsNotNone(i)
self.assertIsNotNone(v)
@ -302,17 +307,17 @@ class TestType(unittest.TestCase):
def test_invalid_element_type(self):
"""Ensure Type.element_type raises if type doesn't have elements."""
tu = get_tu('int i;')
i = get_cursor(tu, 'i')
tu = get_tu("int i;")
i = get_cursor(tu, "i")
self.assertIsNotNone(i)
with self.assertRaises(Exception):
i.element_type
def test_element_count(self):
"""Ensure Type.element_count works."""
tu = get_tu('int i[5]; int j;')
i = get_cursor(tu, 'i')
j = get_cursor(tu, 'j')
tu = get_tu("int i[5]; int j;")
i = get_cursor(tu, "i")
j = get_cursor(tu, "j")
self.assertIsNotNone(i)
self.assertIsNotNone(j)
@ -325,10 +330,10 @@ class TestType(unittest.TestCase):
def test_is_volatile_qualified(self):
"""Ensure Type.is_volatile_qualified works."""
tu = get_tu('volatile int i = 4; int j = 2;')
tu = get_tu("volatile int i = 4; int j = 2;")
i = get_cursor(tu, 'i')
j = get_cursor(tu, 'j')
i = get_cursor(tu, "i")
j = get_cursor(tu, "j")
self.assertIsNotNone(i)
self.assertIsNotNone(j)
@ -340,10 +345,10 @@ class TestType(unittest.TestCase):
def test_is_restrict_qualified(self):
"""Ensure Type.is_restrict_qualified works."""
tu = get_tu('struct s { void * restrict i; void * j; };')
tu = get_tu("struct s { void * restrict i; void * j; };")
i = get_cursor(tu, 'i')
j = get_cursor(tu, 'j')
i = get_cursor(tu, "i")
j = get_cursor(tu, "j")
self.assertIsNotNone(i)
self.assertIsNotNone(j)
@ -356,7 +361,7 @@ class TestType(unittest.TestCase):
"""Ensure Cursor.type.get_size, Cursor.type.get_align and
Cursor.type.get_offset works."""
source ="""
source = """
struct a {
long a1;
long a2:3;
@ -364,15 +369,17 @@ class TestType(unittest.TestCase):
long long a4;
};
"""
tries=[(['-target','i386-linux-gnu'],(4,16,0,32,35,64)),
(['-target','nvptx64-unknown-unknown'],(8,24,0,64,67,128)),
(['-target','i386-pc-win32'],(8,16,0,32,35,64)),
(['-target','msp430-none-none'],(2,14,0,32,35,48))]
tries = [
(["-target", "i386-linux-gnu"], (4, 16, 0, 32, 35, 64)),
(["-target", "nvptx64-unknown-unknown"], (8, 24, 0, 64, 67, 128)),
(["-target", "i386-pc-win32"], (8, 16, 0, 32, 35, 64)),
(["-target", "msp430-none-none"], (2, 14, 0, 32, 35, 48)),
]
for flags, values in tries:
align,total,a1,a2,a3,a4 = values
align, total, a1, a2, a3, a4 = values
tu = get_tu(source, flags=flags)
teststruct = get_cursor(tu, 'a')
teststruct = get_cursor(tu, "a")
fields = list(teststruct.get_children())
self.assertEqual(teststruct.type.get_align(), align)
@ -390,7 +397,7 @@ class TestType(unittest.TestCase):
def test_offset(self):
"""Ensure Cursor.get_record_field_offset works in anonymous records"""
source="""
source = """
struct Test {
struct {int a;} typeanon;
struct {
@ -401,14 +408,16 @@ class TestType(unittest.TestCase):
};
int bar;
};"""
tries=[(['-target','i386-linux-gnu'],(4,16,0,32,64,96)),
(['-target','nvptx64-unknown-unknown'],(8,24,0,32,64,96)),
(['-target','i386-pc-win32'],(8,16,0,32,64,96)),
(['-target','msp430-none-none'],(2,14,0,32,64,96))]
tries = [
(["-target", "i386-linux-gnu"], (4, 16, 0, 32, 64, 96)),
(["-target", "nvptx64-unknown-unknown"], (8, 24, 0, 32, 64, 96)),
(["-target", "i386-pc-win32"], (8, 16, 0, 32, 64, 96)),
(["-target", "msp430-none-none"], (2, 14, 0, 32, 64, 96)),
]
for flags, values in tries:
align,total,f1,bariton,foo,bar = values
align, total, f1, bariton, foo, bar = values
tu = get_tu(source)
teststruct = get_cursor(tu, 'Test')
teststruct = get_cursor(tu, "Test")
children = list(teststruct.get_children())
fields = list(teststruct.type.get_fields())
self.assertEqual(children[0].kind, CursorKind.STRUCT_DECL)
@ -426,7 +435,7 @@ class TestType(unittest.TestCase):
"""Ensure decayed types are handled as the original type"""
tu = get_tu("void foo(int a[]);")
foo = get_cursor(tu, 'foo')
foo = get_cursor(tu, "foo")
a = foo.type.argument_types()[0]
self.assertEqual(a.kind, TypeKind.INCOMPLETEARRAY)
@ -435,9 +444,9 @@ class TestType(unittest.TestCase):
def test_addrspace(self):
"""Ensure the address space can be queried"""
tu = get_tu('__attribute__((address_space(2))) int testInteger = 3;', 'c')
tu = get_tu("__attribute__((address_space(2))) int testInteger = 3;", "c")
testInteger = get_cursor(tu, 'testInteger')
testInteger = get_cursor(tu, "testInteger")
self.assertIsNotNone(testInteger, "Could not find testInteger.")
self.assertEqual(testInteger.type.get_address_space(), 2)
@ -452,17 +461,17 @@ class TestType(unittest.TestCase):
Template<Foo> instance;
int bar;
"""
tu = get_tu(source, lang='cpp')
tu = get_tu(source, lang="cpp")
# Varible with a template argument.
cursor = get_cursor(tu, 'instance')
cursor = get_cursor(tu, "instance")
cursor_type = cursor.type
self.assertEqual(cursor.kind, CursorKind.VAR_DECL)
self.assertEqual(cursor_type.spelling, 'Template<Foo>')
self.assertEqual(cursor_type.spelling, "Template<Foo>")
self.assertEqual(cursor_type.get_num_template_arguments(), 1)
template_type = cursor_type.get_template_argument_type(0)
self.assertEqual(template_type.spelling, 'Foo')
self.assertEqual(template_type.spelling, "Foo")
# Variable without a template argument.
cursor = get_cursor(tu, 'bar')
cursor = get_cursor(tu, "bar")
self.assertEqual(cursor.get_num_template_arguments(), -1)

44
clang/bindings/python/tests/cindex/util.py
View File

@ -1,7 +1,8 @@
# This file provides common utility functions for the test suite.
import os
HAS_FSPATH = hasattr(os, 'fspath')
HAS_FSPATH = hasattr(os, "fspath")
if HAS_FSPATH:
from pathlib import Path as str_to_path
@ -13,7 +14,8 @@ import unittest
from clang.cindex import Cursor
from clang.cindex import TranslationUnit
def get_tu(source, lang='c', all_warnings=False, flags=[]):
def get_tu(source, lang="c", all_warnings=False, flags=[]):
"""Obtain a translation unit from source and language.
By default, the translation unit is created from source file "t.<ext>"
@ -25,20 +27,20 @@ def get_tu(source, lang='c', all_warnings=False, flags=[]):
all_warnings is a convenience argument to enable all compiler warnings.
"""
args = list(flags)
name = 't.c'
if lang == 'cpp':
name = 't.cpp'
args.append('-std=c++11')
elif lang == 'objc':
name = 't.m'
elif lang != 'c':
raise Exception('Unknown language: %s' % lang)
name = "t.c"
if lang == "cpp":
name = "t.cpp"
args.append("-std=c++11")
elif lang == "objc":
name = "t.m"
elif lang != "c":
raise Exception("Unknown language: %s" % lang)
if all_warnings:
args += ['-Wall', '-Wextra']
args += ["-Wall", "-Wextra"]
return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)])
return TranslationUnit.from_source(name, args, unsaved_files=[(name,
source)])
def get_cursor(source, spelling):
"""Obtain a cursor from a source object.
@ -58,6 +60,7 @@ def get_cursor(source, spelling):
return None
def get_cursors(source, spelling):
"""Obtain all cursors from a source object with a specific spelling.
@ -78,13 +81,14 @@ def get_cursors(source, spelling):
return cursors
skip_if_no_fspath = unittest.skipUnless(HAS_FSPATH,
"Requires file system path protocol / Python 3.6+")
skip_if_no_fspath = unittest.skipUnless(
HAS_FSPATH, "Requires file system path protocol / Python 3.6+"
)
__all__ = [
'get_cursor',
'get_cursors',
'get_tu',
'skip_if_no_fspath',
'str_to_path',
"get_cursor",
"get_cursors",
"get_tu",
"skip_if_no_fspath",
"str_to_path",
]

146
clang/docs/analyzer/conf.py
View File

@ -17,103 +17,103 @@ from datetime import date
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo', 'sphinx.ext.mathjax']
extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = u'Clang Static Analyzer'
copyright = u'2013-%d, Analyzer Team' % date.today().year
project = "Clang Static Analyzer"
copyright = "2013-%d, Analyzer Team" % date.today().year
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short version.
version = '17'
version = "17"
# The full version, including alpha/beta/rc tags.
release = '17'
release = "17"
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'haiku'
html_theme = "haiku"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
@ -122,88 +122,91 @@ html_static_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'ClangStaticAnalyzerdoc'
htmlhelp_basename = "ClangStaticAnalyzerdoc"
# -- Options for LaTeX output --------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'ClangStaticAnalyzer.tex', u'Clang Static Analyzer Documentation',
u'Analyzer Team', 'manual'),
(
"index",
"ClangStaticAnalyzer.tex",
"Clang Static Analyzer Documentation",
"Analyzer Team",
"manual",
),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
@ -211,12 +214,17 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'clangstaticanalyzer', u'Clang Static Analyzer Documentation',
[u'Analyzer Team'], 1)
(
"index",
"clangstaticanalyzer",
"Clang Static Analyzer Documentation",
["Analyzer Team"],
1,
)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# man_show_urls = False
# -- Options for Texinfo output ------------------------------------------------
@ -225,20 +233,26 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'ClangStaticAnalyzer', u'Clang Static Analyzer Documentation',
u'Analyzer Team', 'ClangStaticAnalyzer', 'One line description of project.',
'Miscellaneous'),
(
"index",
"ClangStaticAnalyzer",
"Clang Static Analyzer Documentation",
"Analyzer Team",
"ClangStaticAnalyzer",
"One line description of project.",
"Miscellaneous",
),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# texinfo_show_urls = 'footnote'
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}
intersphinx_mapping = {"http://docs.python.org/": None}

191
clang/docs/conf.py
View File

@ -18,80 +18,81 @@ from datetime import date
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo', 'sphinx.ext.mathjax']
extensions = ["sphinx.ext.todo", "sphinx.ext.mathjax"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = {
'.rst': 'restructuredtext',
".rst": "restructuredtext",
}
try:
import recommonmark
import recommonmark
except ImportError:
# manpages do not use any .md sources
if not tags.has('builder-man'):
raise
# manpages do not use any .md sources
if not tags.has("builder-man"):
raise
else:
import sphinx
if sphinx.version_info >= (3, 0):
# This requires 0.5 or later.
extensions.append('recommonmark')
else:
source_parsers = {'.md': 'recommonmark.parser.CommonMarkParser'}
source_suffix['.md'] = 'markdown'
import sphinx
if sphinx.version_info >= (3, 0):
# This requires 0.5 or later.
extensions.append("recommonmark")
else:
source_parsers = {".md": "recommonmark.parser.CommonMarkParser"}
source_suffix[".md"] = "markdown"
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = u'Clang'
copyright = u'2007-%d, The Clang Team' % date.today().year
project = "Clang"
copyright = "2007-%d, The Clang Team" % date.today().year
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'friendly'
pygments_style = "friendly"
in_progress_title = "(In-Progress) " if tags.has("PreRelease") else ""
@ -103,31 +104,31 @@ rst_epilog = f"""
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'haiku'
html_theme = "haiku"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
@ -136,88 +137,85 @@ html_static_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'Clangdoc'
htmlhelp_basename = "Clangdoc"
# -- Options for LaTeX output --------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'Clang.tex', u'Clang Documentation',
u'The Clang Team', 'manual'),
("index", "Clang.tex", "Clang Documentation", "The Clang Team", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
@ -229,38 +227,49 @@ man_pages = []
# Automatically derive the list of man pages from the contents of the command
# guide subdirectory. This was copied from llvm/docs/conf.py.
basedir = os.path.dirname(__file__)
man_page_authors = u'Maintained by the Clang / LLVM Team (<http://clang.llvm.org>)'
command_guide_subpath = 'CommandGuide'
man_page_authors = "Maintained by the Clang / LLVM Team (<http://clang.llvm.org>)"
command_guide_subpath = "CommandGuide"
command_guide_path = os.path.join(basedir, command_guide_subpath)
for name in os.listdir(command_guide_path):
# Ignore non-ReST files and the index page.
if not name.endswith('.rst') or name in ('index.rst',):
if not name.endswith(".rst") or name in ("index.rst",):
continue
# Otherwise, automatically extract the description.
file_subpath = os.path.join(command_guide_subpath, name)
with open(os.path.join(command_guide_path, name)) as f:
title = f.readline().rstrip('\n')
header = f.readline().rstrip('\n')
title = f.readline().rstrip("\n")
header = f.readline().rstrip("\n")
if len(header) != len(title):
print((
"error: invalid header in %r (does not match title)" % (
file_subpath,)), file=sys.stderr)
if ' - ' not in title:
print((
("error: invalid title in %r "
"(expected '<name> - <description>')") % (
file_subpath,)), file=sys.stderr)
print(
(
"error: invalid header in %r (does not match title)"
% (file_subpath,)
),
file=sys.stderr,
)
if " - " not in title:
print(
(
(
"error: invalid title in %r "
"(expected '<name> - <description>')"
)
% (file_subpath,)
),
file=sys.stderr,
)
# Split the name out of the title.
name,description = title.split(' - ', 1)
man_pages.append((file_subpath.replace('.rst',''), name,
description, man_page_authors, 1))
name, description = title.split(" - ", 1)
man_pages.append(
(file_subpath.replace(".rst", ""), name, description, man_page_authors, 1)
)
# If true, show URL addresses after external links.
#man_show_urls = False
# man_show_urls = False
# -- Options for Texinfo output ------------------------------------------------
@ -269,16 +278,22 @@ for name in os.listdir(command_guide_path):
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'Clang', u'Clang Documentation',
u'The Clang Team', 'Clang', 'One line description of project.',
'Miscellaneous'),
(
"index",
"Clang",
"Clang Documentation",
"The Clang Team",
"Clang",
"One line description of project.",
"Miscellaneous",
),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# texinfo_show_urls = 'footnote'

821
clang/docs/tools/dump_ast_matchers.py
View File

@ -5,24 +5,25 @@
import collections
import re
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
CLASS_INDEX_PAGE_URL = 'https://clang.llvm.org/doxygen/classes.html'
CLASS_INDEX_PAGE_URL = "https://clang.llvm.org/doxygen/classes.html"
try:
CLASS_INDEX_PAGE = urlopen(CLASS_INDEX_PAGE_URL).read().decode('utf-8')
CLASS_INDEX_PAGE = urlopen(CLASS_INDEX_PAGE_URL).read().decode("utf-8")
except Exception as e:
raise Exception('Unable to get %s: %s' % (CLASS_INDEX_PAGE_URL, e))
raise Exception("Unable to get %s: %s" % (CLASS_INDEX_PAGE_URL, e))
MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h'
MATCHERS_FILE = "../../include/clang/ASTMatchers/ASTMatchers.h"
# Each matcher is documented in one row of the form:
# result | name | argA
# The subsequent row contains the documentation and is hidden by default,
# becoming visible via javascript when the user clicks the matcher name.
TD_TEMPLATE="""
TD_TEMPLATE = """
<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>
<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>
"""
@ -40,189 +41,221 @@ ids = collections.defaultdict(int)
# Cache for doxygen urls we have already verified.
doxygen_probes = {}
def esc(text):
"""Escape any html in the given text."""
text = re.sub(r'&', '&amp;', text)
text = re.sub(r'<', '&lt;', text)
text = re.sub(r'>', '&gt;', text)
def link_if_exists(m):
"""Wrap a likely AST node name in a link to its clang docs.
We want to do this only if the page exists, in which case it will be
referenced from the class index page.
"""
name = m.group(1)
url = 'https://clang.llvm.org/doxygen/classclang_1_1%s.html' % name
if url not in doxygen_probes:
search_str = 'href="classclang_1_1%s.html"' % name
doxygen_probes[url] = search_str in CLASS_INDEX_PAGE
if not doxygen_probes[url]:
print('Did not find %s in class index page' % name)
if doxygen_probes[url]:
return r'Matcher&lt;<a href="%s">%s</a>&gt;' % (url, name)
else:
return m.group(0)
text = re.sub(
r'Matcher&lt;([^\*&]+)&gt;', link_if_exists, text)
return text
def esc(text):
"""Escape any html in the given text."""
text = re.sub(r"&", "&amp;", text)
text = re.sub(r"<", "&lt;", text)
text = re.sub(r">", "&gt;", text)
def link_if_exists(m):
"""Wrap a likely AST node name in a link to its clang docs.
We want to do this only if the page exists, in which case it will be
referenced from the class index page.
"""
name = m.group(1)
url = "https://clang.llvm.org/doxygen/classclang_1_1%s.html" % name
if url not in doxygen_probes:
search_str = 'href="classclang_1_1%s.html"' % name
doxygen_probes[url] = search_str in CLASS_INDEX_PAGE
if not doxygen_probes[url]:
print("Did not find %s in class index page" % name)
if doxygen_probes[url]:
return r'Matcher&lt;<a href="%s">%s</a>&gt;' % (url, name)
else:
return m.group(0)
text = re.sub(r"Matcher&lt;([^\*&]+)&gt;", link_if_exists, text)
return text
def extract_result_types(comment):
"""Extracts a list of result types from the given comment.
"""Extracts a list of result types from the given comment.
We allow annotations in the comment of the matcher to specify what
nodes a matcher can match on. Those comments have the form:
Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
We allow annotations in the comment of the matcher to specify what
nodes a matcher can match on. Those comments have the form:
Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
Returns the empty list if no 'Usable as' specification could be
parsed.
"""
result_types = []
m = re.search(r"Usable as: Any Matcher[\s\n]*$", comment, re.S)
if m:
return ["*"]
while True:
m = re.match(r"^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$", comment, re.S)
if not m:
if re.search(r"Usable as:\s*$", comment):
return result_types
else:
return None
result_types += [m.group(2)]
comment = m.group(1)
Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
Returns the empty list if no 'Usable as' specification could be
parsed.
"""
result_types = []
m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S)
if m:
return ['*']
while True:
m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S)
if not m:
if re.search(r'Usable as:\s*$', comment):
return result_types
else:
return None
result_types += [m.group(2)]
comment = m.group(1)
def strip_doxygen(comment):
"""Returns the given comment without \-escaped words."""
# If there is only a doxygen keyword in the line, delete the whole line.
comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M)
# If there is a doxygen \see command, change the \see prefix into "See also:".
# FIXME: it would be better to turn this into a link to the target instead.
comment = re.sub(r'\\see', r'See also:', comment)
# Delete the doxygen command and the following whitespace.
comment = re.sub(r'\\[^\s]+\s+', r'', comment)
return comment
"""Returns the given comment without \-escaped words."""
# If there is only a doxygen keyword in the line, delete the whole line.
comment = re.sub(r"^\\[^\s]+\n", r"", comment, flags=re.M)
# If there is a doxygen \see command, change the \see prefix into "See also:".
# FIXME: it would be better to turn this into a link to the target instead.
comment = re.sub(r"\\see", r"See also:", comment)
# Delete the doxygen command and the following whitespace.
comment = re.sub(r"\\[^\s]+\s+", r"", comment)
return comment
def unify_arguments(args):
"""Gets rid of anything the user doesn't care about in the argument list."""
args = re.sub(r'internal::', r'', args)
args = re.sub(r'extern const\s+(.*)&', r'\1 ', args)
args = re.sub(r'&', r' ', args)
args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args)
args = re.sub(r'BindableMatcher', r'Matcher', args)
args = re.sub(r'const Matcher', r'Matcher', args)
return args
"""Gets rid of anything the user doesn't care about in the argument list."""
args = re.sub(r"internal::", r"", args)
args = re.sub(r"extern const\s+(.*)&", r"\1 ", args)
args = re.sub(r"&", r" ", args)
args = re.sub(r"(^|\s)M\d?(\s)", r"\1Matcher<*>\2", args)
args = re.sub(r"BindableMatcher", r"Matcher", args)
args = re.sub(r"const Matcher", r"Matcher", args)
return args
def unify_type(result_type):
"""Gets rid of anything the user doesn't care about in the type name."""
result_type = re.sub(r'^internal::(Bindable)?Matcher<([a-zA-Z_][a-zA-Z0-9_]*)>$', r'\2', result_type)
return result_type
"""Gets rid of anything the user doesn't care about in the type name."""
result_type = re.sub(
r"^internal::(Bindable)?Matcher<([a-zA-Z_][a-zA-Z0-9_]*)>$", r"\2", result_type
)
return result_type
def add_matcher(result_type, name, args, comment, is_dyncast=False):
"""Adds a matcher to one of our categories."""
if name == 'id':
# FIXME: Figure out whether we want to support the 'id' matcher.
return
matcher_id = '%s%d' % (name, ids[name])
ids[name] += 1
args = unify_arguments(args)
result_type = unify_type(result_type)
"""Adds a matcher to one of our categories."""
if name == "id":
# FIXME: Figure out whether we want to support the 'id' matcher.
return
matcher_id = "%s%d" % (name, ids[name])
ids[name] += 1
args = unify_arguments(args)
result_type = unify_type(result_type)
docs_result_type = esc('Matcher<%s>' % result_type);
docs_result_type = esc("Matcher<%s>" % result_type)
if name == 'mapAnyOf':
args = "nodeMatcherFunction..."
docs_result_type = "<em>unspecified</em>"
if name == "mapAnyOf":
args = "nodeMatcherFunction..."
docs_result_type = "<em>unspecified</em>"
matcher_html = TD_TEMPLATE % {
"result": docs_result_type,
"name": name,
"args": esc(args),
"comment": esc(strip_doxygen(comment)),
"id": matcher_id,
}
if is_dyncast:
dict = node_matchers
lookup = result_type + name
# Use a heuristic to figure out whether a matcher is a narrowing or
# traversal matcher. By default, matchers that take other matchers as
# arguments (and are not node matchers) do traversal. We specifically
# exclude known narrowing matchers that also take other matchers as
# arguments.
elif "Matcher<" not in args or name in [
"allOf",
"anyOf",
"anything",
"unless",
"mapAnyOf",
]:
dict = narrowing_matchers
lookup = result_type + name + esc(args)
else:
dict = traversal_matchers
lookup = result_type + name + esc(args)
if dict.get(lookup) is None or len(dict.get(lookup)) < len(matcher_html):
dict[lookup] = matcher_html
matcher_html = TD_TEMPLATE % {
'result': docs_result_type,
'name': name,
'args': esc(args),
'comment': esc(strip_doxygen(comment)),
'id': matcher_id,
}
if is_dyncast:
dict = node_matchers
lookup = result_type + name
# Use a heuristic to figure out whether a matcher is a narrowing or
# traversal matcher. By default, matchers that take other matchers as
# arguments (and are not node matchers) do traversal. We specifically
# exclude known narrowing matchers that also take other matchers as
# arguments.
elif ('Matcher<' not in args or
name in ['allOf', 'anyOf', 'anything', 'unless', 'mapAnyOf']):
dict = narrowing_matchers
lookup = result_type + name + esc(args)
else:
dict = traversal_matchers
lookup = result_type + name + esc(args)
if dict.get(lookup) is None or len(dict.get(lookup)) < len(matcher_html):
dict[lookup] = matcher_html
def act_on_decl(declaration, comment, allowed_types):
"""Parse the matcher out of the given declaration and comment.
"""Parse the matcher out of the given declaration and comment.
If 'allowed_types' is set, it contains a list of node types the matcher
can match on, as extracted from the static type asserts in the matcher
definition.
"""
if declaration.strip():
If 'allowed_types' is set, it contains a list of node types the matcher
can match on, as extracted from the static type asserts in the matcher
definition.
"""
if declaration.strip():
if re.match(r'^\s?(#|namespace|using)', declaration): return
if re.match(r"^\s?(#|namespace|using)", declaration):
return
# Node matchers are defined by writing:
# VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
m = re.match(r""".*Variadic(?:DynCast)?AllOfMatcher\s*<
# Node matchers are defined by writing:
# VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
m = re.match(
r""".*Variadic(?:DynCast)?AllOfMatcher\s*<
\s*([^\s,]+)\s*(?:,
\s*([^\s>]+)\s*)?>
\s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X)
if m:
result, inner, name = m.groups()
if not inner:
inner = result
add_matcher(result, name, 'Matcher<%s>...' % inner,
comment, is_dyncast=True)
return
\s*([^\s;]+)\s*;\s*$""",
declaration,
flags=re.X,
)
if m:
result, inner, name = m.groups()
if not inner:
inner = result
add_matcher(
result, name, "Matcher<%s>..." % inner, comment, is_dyncast=True
)
return
# Special case of type matchers:
# AstTypeMatcher<ArgumentType> name
m = re.match(r""".*AstTypeMatcher\s*<
# Special case of type matchers:
# AstTypeMatcher<ArgumentType> name
m = re.match(
r""".*AstTypeMatcher\s*<
\s*([^\s>]+)\s*>
\s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X)
if m:
inner, name = m.groups()
add_matcher('Type', name, 'Matcher<%s>...' % inner,
comment, is_dyncast=True)
# FIXME: re-enable once we have implemented casting on the TypeLoc
# hierarchy.
# add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner,
# comment, is_dyncast=True)
return
\s*([^\s;]+)\s*;\s*$""",
declaration,
flags=re.X,
)
if m:
inner, name = m.groups()
add_matcher(
"Type", name, "Matcher<%s>..." % inner, comment, is_dyncast=True
)
# FIXME: re-enable once we have implemented casting on the TypeLoc
# hierarchy.
# add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner,
# comment, is_dyncast=True)
return
# Parse the various matcher definition macros.
m = re.match(""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER(?:_DECL)?\(
# Parse the various matcher definition macros.
m = re.match(
""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER(?:_DECL)?\(
\s*([^\s,]+\s*),
\s*(?:[^\s,]+\s*),
\s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
\)\s*;\s*$""", declaration, flags=re.X)
if m:
loc, name, results = m.groups()[0:3]
result_types = [r.strip() for r in results.split(',')]
\)\s*;\s*$""",
declaration,
flags=re.X,
)
if m:
loc, name, results = m.groups()[0:3]
result_types = [r.strip() for r in results.split(",")]
comment_result_types = extract_result_types(comment)
if (comment_result_types and
sorted(result_types) != sorted(comment_result_types)):
raise Exception('Inconsistent documentation for: %s' % name)
for result_type in result_types:
add_matcher(result_type, name, 'Matcher<Type>', comment)
# if loc:
# add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>',
# comment)
return
comment_result_types = extract_result_types(comment)
if comment_result_types and sorted(result_types) != sorted(
comment_result_types
):
raise Exception("Inconsistent documentation for: %s" % name)
for result_type in result_types:
add_matcher(result_type, name, "Matcher<Type>", comment)
# if loc:
# add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>',
# comment)
return
m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
m = re.match(
r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
\s*([^\s,]+)\s*,
\s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
(?:,\s*([^\s,]+)\s*
@ -230,45 +263,56 @@ def act_on_decl(declaration, comment, allowed_types):
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*\d+\s*)?
\)\s*{\s*$""", declaration, flags=re.X)
\)\s*{\s*$""",
declaration,
flags=re.X,
)
if m:
p, n, name, results = m.groups()[0:4]
args = m.groups()[4:]
result_types = [r.strip() for r in results.split(',')]
if allowed_types and allowed_types != result_types:
raise Exception('Inconsistent documentation for: %s' % name)
if n not in ['', '2']:
raise Exception('Cannot parse "%s"' % declaration)
args = ', '.join('%s %s' % (args[i], args[i+1])
for i in range(0, len(args), 2) if args[i])
for result_type in result_types:
add_matcher(result_type, name, args, comment)
return
if m:
p, n, name, results = m.groups()[0:4]
args = m.groups()[4:]
result_types = [r.strip() for r in results.split(",")]
if allowed_types and allowed_types != result_types:
raise Exception("Inconsistent documentation for: %s" % name)
if n not in ["", "2"]:
raise Exception('Cannot parse "%s"' % declaration)
args = ", ".join(
"%s %s" % (args[i], args[i + 1])
for i in range(0, len(args), 2)
if args[i]
)
for result_type in result_types:
add_matcher(result_type, name, args, comment)
return
m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER_REGEX(?:_OVERLOAD)?\(
m = re.match(
r"""^\s*AST_POLYMORPHIC_MATCHER_REGEX(?:_OVERLOAD)?\(
\s*([^\s,]+)\s*,
\s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),
\s*([^\s,]+)\s*
(?:,\s*\d+\s*)?
\)\s*{\s*$""", declaration, flags=re.X)
\)\s*{\s*$""",
declaration,
flags=re.X,
)
if m:
name, results, arg_name = m.groups()[0:3]
result_types = [r.strip() for r in results.split(',')]
if allowed_types and allowed_types != result_types:
raise Exception('Inconsistent documentation for: %s' % name)
arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
comment += """
if m:
name, results, arg_name = m.groups()[0:3]
result_types = [r.strip() for r in results.split(",")]
if allowed_types and allowed_types != result_types:
raise Exception("Inconsistent documentation for: %s" % name)
arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
comment += """
If the matcher is used in clang-query, RegexFlags parameter
should be passed as a quoted string. e.g: "NoFlags".
Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
"""
for result_type in result_types:
add_matcher(result_type, name, arg, comment)
return
for result_type in result_types:
add_matcher(result_type, name, arg, comment)
return
m = re.match(r"""^\s*AST_MATCHER_FUNCTION(_P)?(.?)(?:_OVERLOAD)?\(
m = re.match(
r"""^\s*AST_MATCHER_FUNCTION(_P)?(.?)(?:_OVERLOAD)?\(
(?:\s*([^\s,]+)\s*,)?
\s*([^\s,]+)\s*
(?:,\s*([^\s,]+)\s*
@ -276,18 +320,25 @@ Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*\d+\s*)?
\)\s*{\s*$""", declaration, flags=re.X)
if m:
p, n, result, name = m.groups()[0:4]
args = m.groups()[4:]
if n not in ['', '2']:
raise Exception('Cannot parse "%s"' % declaration)
args = ', '.join('%s %s' % (args[i], args[i+1])
for i in range(0, len(args), 2) if args[i])
add_matcher(result, name, args, comment)
return
\)\s*{\s*$""",
declaration,
flags=re.X,
)
if m:
p, n, result, name = m.groups()[0:4]
args = m.groups()[4:]
if n not in ["", "2"]:
raise Exception('Cannot parse "%s"' % declaration)
args = ", ".join(
"%s %s" % (args[i], args[i + 1])
for i in range(0, len(args), 2)
if args[i]
)
add_matcher(result, name, args, comment)
return
m = re.match(r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
m = re.match(
r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
(?:\s*([^\s,]+)\s*,)?
\s*([^\s,]+)\s*
(?:,\s*([^,]+)\s*
@ -295,160 +346,197 @@ Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*\d+\s*)?
\)\s*{""", declaration, flags=re.X)
if m:
p, n, result, name = m.groups()[0:4]
args = m.groups()[4:]
if not result:
if not allowed_types:
raise Exception('Did not find allowed result types for: %s' % name)
result_types = allowed_types
else:
result_types = [result]
if n not in ['', '2']:
raise Exception('Cannot parse "%s"' % declaration)
args = ', '.join('%s %s' % (args[i], args[i+1])
for i in range(0, len(args), 2) if args[i])
for result_type in result_types:
add_matcher(result_type, name, args, comment)
return
\)\s*{""",
declaration,
flags=re.X,
)
if m:
p, n, result, name = m.groups()[0:4]
args = m.groups()[4:]
if not result:
if not allowed_types:
raise Exception("Did not find allowed result types for: %s" % name)
result_types = allowed_types
else:
result_types = [result]
if n not in ["", "2"]:
raise Exception('Cannot parse "%s"' % declaration)
args = ", ".join(
"%s %s" % (args[i], args[i + 1])
for i in range(0, len(args), 2)
if args[i]
)
for result_type in result_types:
add_matcher(result_type, name, args, comment)
return
m = re.match(r"""^\s*AST_MATCHER_REGEX(?:_OVERLOAD)?\(
m = re.match(
r"""^\s*AST_MATCHER_REGEX(?:_OVERLOAD)?\(
\s*([^\s,]+)\s*,
\s*([^\s,]+)\s*,
\s*([^\s,]+)\s*
(?:,\s*\d+\s*)?
\)\s*{""", declaration, flags=re.X)
if m:
result, name, arg_name = m.groups()[0:3]
if not result:
if not allowed_types:
raise Exception('Did not find allowed result types for: %s' % name)
result_types = allowed_types
else:
result_types = [result]
arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
comment += """
\)\s*{""",
declaration,
flags=re.X,
)
if m:
result, name, arg_name = m.groups()[0:3]
if not result:
if not allowed_types:
raise Exception("Did not find allowed result types for: %s" % name)
result_types = allowed_types
else:
result_types = [result]
arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
comment += """
If the matcher is used in clang-query, RegexFlags parameter
should be passed as a quoted string. e.g: "NoFlags".
Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
"""
for result_type in result_types:
add_matcher(result_type, name, arg, comment)
return
for result_type in result_types:
add_matcher(result_type, name, arg, comment)
return
# Parse ArgumentAdapting matchers.
m = re.match(
r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*
# Parse ArgumentAdapting matchers.
m = re.match(
r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*
([a-zA-Z]*);$""",
declaration, flags=re.X)
if m:
name = m.groups()[0]
add_matcher('*', name, 'Matcher<*>', comment)
return
declaration,
flags=re.X,
)
if m:
name = m.groups()[0]
add_matcher("*", name, "Matcher<*>", comment)
return
# Parse Variadic functions.
m = re.match(
r"""^.*internal::VariadicFunction\s*<\s*([^,]+),\s*([^,]+),\s*[^>]+>\s*
# Parse Variadic functions.
m = re.match(
r"""^.*internal::VariadicFunction\s*<\s*([^,]+),\s*([^,]+),\s*[^>]+>\s*
([a-zA-Z]*);$""",
declaration, flags=re.X)
if m:
result, arg, name = m.groups()[:3]
add_matcher(result, name, '%s, ..., %s' % (arg, arg), comment)
return
declaration,
flags=re.X,
)
if m:
result, arg, name = m.groups()[:3]
add_matcher(result, name, "%s, ..., %s" % (arg, arg), comment)
return
m = re.match(
r"""^.*internal::VariadicFunction\s*<\s*
m = re.match(
r"""^.*internal::VariadicFunction\s*<\s*
internal::PolymorphicMatcher<[\S\s]+
AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),\s*(.*);$""",
declaration, flags=re.X)
declaration,
flags=re.X,
)
if m:
results, trailing = m.groups()
trailing, name = trailing.rsplit(">", 1)
name = name.strip()
trailing, _ = trailing.rsplit(",", 1)
_, arg = trailing.rsplit(",", 1)
arg = arg.strip()
if m:
results, trailing = m.groups()
trailing, name = trailing.rsplit(">", 1)
name = name.strip()
trailing, _ = trailing.rsplit(",", 1)
_, arg = trailing.rsplit(",", 1)
arg = arg.strip()
result_types = [r.strip() for r in results.split(',')]
for result_type in result_types:
add_matcher(result_type, name, '%s, ..., %s' % (arg, arg), comment)
return
result_types = [r.strip() for r in results.split(",")]
for result_type in result_types:
add_matcher(result_type, name, "%s, ..., %s" % (arg, arg), comment)
return
# Parse Variadic operator matchers.
m = re.match(
r"""^.*VariadicOperatorMatcherFunc\s*<\s*([^,]+),\s*([^\s]+)\s*>\s*
# Parse Variadic operator matchers.
m = re.match(
r"""^.*VariadicOperatorMatcherFunc\s*<\s*([^,]+),\s*([^\s]+)\s*>\s*
([a-zA-Z]*);$""",
declaration, flags=re.X)
if m:
min_args, max_args, name = m.groups()[:3]
if max_args == '1':
add_matcher('*', name, 'Matcher<*>', comment)
return
elif max_args == 'std::numeric_limits<unsigned>::max()':
add_matcher('*', name, 'Matcher<*>, ..., Matcher<*>', comment)
return
declaration,
flags=re.X,
)
if m:
min_args, max_args, name = m.groups()[:3]
if max_args == "1":
add_matcher("*", name, "Matcher<*>", comment)
return
elif max_args == "std::numeric_limits<unsigned>::max()":
add_matcher("*", name, "Matcher<*>, ..., Matcher<*>", comment)
return
m = re.match(
r"""^.*MapAnyOfMatcher<.*>\s*
m = re.match(
r"""^.*MapAnyOfMatcher<.*>\s*
([a-zA-Z]*);$""",
declaration, flags=re.X)
if m:
name = m.groups()[0]
add_matcher('*', name, 'Matcher<*>...Matcher<*>', comment)
return
declaration,
flags=re.X,
)
if m:
name = m.groups()[0]
add_matcher("*", name, "Matcher<*>...Matcher<*>", comment)
return
# Parse free standing matcher functions, like:
# Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
m = re.match(r"""^\s*(?:template\s+<\s*(?:class|typename)\s+(.+)\s*>\s+)?
# Parse free standing matcher functions, like:
# Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
m = re.match(
r"""^\s*(?:template\s+<\s*(?:class|typename)\s+(.+)\s*>\s+)?
(.*)\s+
([^\s\(]+)\s*\(
(.*)
\)\s*{""", declaration, re.X)
if m:
template_name, result, name, args = m.groups()
if template_name:
matcherTemplateArgs = re.findall(r'Matcher<\s*(%s)\s*>' % template_name, args)
templateArgs = re.findall(r'(?:^|[\s,<])(%s)(?:$|[\s,>])' % template_name, args)
if len(matcherTemplateArgs) < len(templateArgs):
# The template name is used naked, so don't replace with `*`` later on
template_name = None
else :
args = re.sub(r'(^|[\s,<])%s($|[\s,>])' % template_name, r'\1*\2', args)
args = ', '.join(p.strip() for p in args.split(','))
m = re.match(r'(?:^|.*\s+)internal::(?:Bindable)?Matcher<([^>]+)>$', result)
if m:
result_types = [m.group(1)]
if template_name and len(result_types) == 1 and result_types[0] == template_name:
result_types = ['*']
else:
result_types = extract_result_types(comment)
if not result_types:
if not comment:
# Only overloads don't have their own doxygen comments; ignore those.
print('Ignoring "%s"' % name)
\)\s*{""",
declaration,
re.X,
)
if m:
template_name, result, name, args = m.groups()
if template_name:
matcherTemplateArgs = re.findall(
r"Matcher<\s*(%s)\s*>" % template_name, args
)
templateArgs = re.findall(
r"(?:^|[\s,<])(%s)(?:$|[\s,>])" % template_name, args
)
if len(matcherTemplateArgs) < len(templateArgs):
# The template name is used naked, so don't replace with `*`` later on
template_name = None
else:
args = re.sub(
r"(^|[\s,<])%s($|[\s,>])" % template_name, r"\1*\2", args
)
args = ", ".join(p.strip() for p in args.split(","))
m = re.match(r"(?:^|.*\s+)internal::(?:Bindable)?Matcher<([^>]+)>$", result)
if m:
result_types = [m.group(1)]
if (
template_name
and len(result_types) == 1
and result_types[0] == template_name
):
result_types = ["*"]
else:
result_types = extract_result_types(comment)
if not result_types:
if not comment:
# Only overloads don't have their own doxygen comments; ignore those.
print('Ignoring "%s"' % name)
else:
print('Cannot determine result type for "%s"' % name)
else:
for result_type in result_types:
add_matcher(result_type, name, args, comment)
else:
print('Cannot determine result type for "%s"' % name)
else:
for result_type in result_types:
add_matcher(result_type, name, args, comment)
else:
print('*** Unparsable: "' + declaration + '" ***')
print('*** Unparsable: "' + declaration + '" ***')
def sort_table(matcher_type, matcher_map):
"""Returns the sorted html table for the given row map."""
table = ''
for key in sorted(matcher_map.keys()):
table += matcher_map[key] + '\n'
return ('<!-- START_%(type)s_MATCHERS -->\n' +
'%(table)s' +
'<!--END_%(type)s_MATCHERS -->') % {
'type': matcher_type,
'table': table,
}
"""Returns the sorted html table for the given row map."""
table = ""
for key in sorted(matcher_map.keys()):
table += matcher_map[key] + "\n"
return (
"<!-- START_%(type)s_MATCHERS -->\n"
+ "%(table)s"
+ "<!--END_%(type)s_MATCHERS -->"
) % {
"type": matcher_type,
"table": table,
}
# Parse the ast matchers.
# We alternate between two modes:
@ -457,51 +545,64 @@ def sort_table(matcher_type, matcher_map):
# definition might contain static asserts that specify the result
# type.
# body = False: We parse the comments and declaration of the matcher.
comment = ''
declaration = ''
comment = ""
declaration = ""
allowed_types = []
body = False
for line in open(MATCHERS_FILE).read().splitlines():
if body:
if line.strip() and line[0] == '}':
if declaration:
act_on_decl(declaration, comment, allowed_types)
comment = ''
declaration = ''
allowed_types = []
body = False
if body:
if line.strip() and line[0] == "}":
if declaration:
act_on_decl(declaration, comment, allowed_types)
comment = ""
declaration = ""
allowed_types = []
body = False
else:
m = re.search(r"is_base_of<([^,]+), NodeType>", line)
if m and m.group(1):
allowed_types += [m.group(1)]
continue
if line.strip() and line.lstrip()[0] == "/":
comment += re.sub(r"^/+\s?", "", line) + "\n"
else:
m = re.search(r'is_base_of<([^,]+), NodeType>', line)
if m and m.group(1):
allowed_types += [m.group(1)]
continue
if line.strip() and line.lstrip()[0] == '/':
comment += re.sub(r'^/+\s?', '', line) + '\n'
else:
declaration += ' ' + line
if ((not line.strip()) or
line.rstrip()[-1] == ';' or
(line.rstrip()[-1] == '{' and line.rstrip()[-3:] != '= {')):
if line.strip() and line.rstrip()[-1] == '{':
body = True
else:
act_on_decl(declaration, comment, allowed_types)
comment = ''
declaration = ''
allowed_types = []
declaration += " " + line
if (
(not line.strip())
or line.rstrip()[-1] == ";"
or (line.rstrip()[-1] == "{" and line.rstrip()[-3:] != "= {")
):
if line.strip() and line.rstrip()[-1] == "{":
body = True
else:
act_on_decl(declaration, comment, allowed_types)
comment = ""
declaration = ""
allowed_types = []
node_matcher_table = sort_table('DECL', node_matchers)
narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers)
traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers)
node_matcher_table = sort_table("DECL", node_matchers)
narrowing_matcher_table = sort_table("NARROWING", narrowing_matchers)
traversal_matcher_table = sort_table("TRAVERSAL", traversal_matchers)
reference = open('../LibASTMatchersReference.html').read()
reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->',
node_matcher_table, reference, flags=re.S)
reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->',
narrowing_matcher_table, reference, flags=re.S)
reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->',
traversal_matcher_table, reference, flags=re.S)
with open('../LibASTMatchersReference.html', 'w', newline='\n') as output:
output.write(reference)
reference = open("../LibASTMatchersReference.html").read()
reference = re.sub(
r"<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->",
node_matcher_table,
reference,
flags=re.S,
)
reference = re.sub(
r"<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->",
narrowing_matcher_table,
reference,
flags=re.S,
)
reference = re.sub(
r"<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->",
traversal_matcher_table,
reference,
flags=re.S,
)
with open("../LibASTMatchersReference.html", "w", newline="\n") as output:
output.write(reference)

34
clang/docs/tools/dump_format_help.py
View File

@ -7,28 +7,27 @@ import re
import subprocess
import sys
CLANG_DIR = os.path.join(os.path.dirname(__file__), '../..')
DOC_FILE = os.path.join(CLANG_DIR, 'docs/ClangFormat.rst')
CLANG_DIR = os.path.join(os.path.dirname(__file__), "../..")
DOC_FILE = os.path.join(CLANG_DIR, "docs/ClangFormat.rst")
def substitute(text, tag, contents):
replacement = '\n.. START_%s\n\n%s\n\n.. END_%s\n' % (tag, contents, tag)
pattern = r'\n\.\. START_%s\n.*\n\.\. END_%s\n' % (tag, tag)
return re.sub(pattern, '%s', text, flags=re.S) % replacement
replacement = "\n.. START_%s\n\n%s\n\n.. END_%s\n" % (tag, contents, tag)
pattern = r"\n\.\. START_%s\n.*\n\.\. END_%s\n" % (tag, tag)
return re.sub(pattern, "%s", text, flags=re.S) % replacement
def indent(text, columns, indent_first_line=True):
indent_str = ' ' * columns
s = re.sub(r'\n([^\n])', '\n' + indent_str + '\\1', text, flags=re.S)
if not indent_first_line or s.startswith('\n'):
indent_str = " " * columns
s = re.sub(r"\n([^\n])", "\n" + indent_str + "\\1", text, flags=re.S)
if not indent_first_line or s.startswith("\n"):
return s
return indent_str + s
def get_help_output():
args = ["clang-format", "--help"]
cmd = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
cmd = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, _ = cmd.communicate()
out = out.decode(sys.stdout.encoding)
return out
@ -36,12 +35,15 @@ def get_help_output():
def get_help_text():
out = get_help_output()
out = re.sub(r' clang-format\.exe ', ' clang-format ', out)
out = re.sub(r" clang-format\.exe ", " clang-format ", out)
out = '''.. code-block:: console
out = (
""".. code-block:: console
$ clang-format -help
''' + out
"""
+ out
)
out = indent(out, 2, indent_first_line=False)
return out
@ -49,7 +51,7 @@ $ clang-format -help
def validate(text, columns):
for line in text.splitlines():
if len(line) > columns:
print('warning: line too long:\n', line, file=sys.stderr)
print("warning: line too long:\n", line, file=sys.stderr)
help_text = get_help_text()
@ -58,7 +60,7 @@ validate(help_text, 95)
with open(DOC_FILE) as f:
contents = f.read()
contents = substitute(contents, 'FORMAT_HELP', help_text)
contents = substitute(contents, "FORMAT_HELP", help_text)
with open(DOC_FILE, 'wb') as output:
with open(DOC_FILE, "wb") as output:
output.write(contents.encode())

634
clang/docs/tools/dump_format_style.py
View File

@ -10,354 +10,412 @@ import sys
from io import TextIOWrapper
from typing import Set
CLANG_DIR = os.path.join(os.path.dirname(__file__), '../..')
FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, 'include/clang/Format/Format.h')
INCLUDE_STYLE_FILE = os.path.join(CLANG_DIR, 'include/clang/Tooling/Inclusions/IncludeStyle.h')
DOC_FILE = os.path.join(CLANG_DIR, 'docs/ClangFormatStyleOptions.rst')
CLANG_DIR = os.path.join(os.path.dirname(__file__), "../..")
FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, "include/clang/Format/Format.h")
INCLUDE_STYLE_FILE = os.path.join(
CLANG_DIR, "include/clang/Tooling/Inclusions/IncludeStyle.h"
)
DOC_FILE = os.path.join(CLANG_DIR, "docs/ClangFormatStyleOptions.rst")
PLURALS_FILE = os.path.join(os.path.dirname(__file__), 'plurals.txt')
PLURALS_FILE = os.path.join(os.path.dirname(__file__), "plurals.txt")
plurals: Set[str] = set()
with open(PLURALS_FILE, 'a+') as f:
f.seek(0)
plurals = set(f.read().splitlines())
with open(PLURALS_FILE, "a+") as f:
f.seek(0)
plurals = set(f.read().splitlines())
def substitute(text, tag, contents):
replacement = '\n.. START_%s\n\n%s\n\n.. END_%s\n' % (tag, contents, tag)
pattern = r'\n\.\. START_%s\n.*\n\.\. END_%s\n' % (tag, tag)
return re.sub(pattern, '%s', text, flags=re.S) % replacement
replacement = "\n.. START_%s\n\n%s\n\n.. END_%s\n" % (tag, contents, tag)
pattern = r"\n\.\. START_%s\n.*\n\.\. END_%s\n" % (tag, tag)
return re.sub(pattern, "%s", text, flags=re.S) % replacement
def register_plural(singular: str, plural: str):
if plural not in plurals:
if not hasattr(register_plural, "generated_new_plural"):
print('Plural generation: you can use '
f'`git checkout -- {os.path.relpath(PLURALS_FILE)}` '
'to reemit warnings or `git add` to include new plurals\n')
register_plural.generated_new_plural = True
if plural not in plurals:
if not hasattr(register_plural, "generated_new_plural"):
print(
"Plural generation: you can use "
f"`git checkout -- {os.path.relpath(PLURALS_FILE)}` "
"to reemit warnings or `git add` to include new plurals\n"
)
register_plural.generated_new_plural = True
plurals.add(plural)
with open(PLURALS_FILE, "a") as f:
f.write(plural + "\n")
cf = inspect.currentframe()
lineno = ""
if cf and cf.f_back:
lineno = ":" + str(cf.f_back.f_lineno)
print(
f"{__file__}{lineno} check if plural of {singular} is {plural}",
file=sys.stderr,
)
return plural
plurals.add(plural)
with open(PLURALS_FILE, 'a') as f:
f.write(plural + '\n')
cf = inspect.currentframe()
lineno = ''
if cf and cf.f_back:
lineno = ':' + str(cf.f_back.f_lineno)
print(f'{__file__}{lineno} check if plural of {singular} is {plural}', file=sys.stderr)
return plural
def pluralize(word: str):
lword = word.lower()
if len(lword) >= 2 and lword[-1] == 'y' and lword[-2] not in 'aeiou':
return register_plural(word, word[:-1] + 'ies')
elif lword.endswith(('s', 'sh', 'ch', 'x', 'z')):
return register_plural(word, word[:-1] + 'es')
elif lword.endswith('fe'):
return register_plural(word, word[:-2] + 'ves')
elif lword.endswith('f') and not lword.endswith('ff'):
return register_plural(word, word[:-1] + 'ves')
else:
return register_plural(word, word + 's')
lword = word.lower()
if len(lword) >= 2 and lword[-1] == "y" and lword[-2] not in "aeiou":
return register_plural(word, word[:-1] + "ies")
elif lword.endswith(("s", "sh", "ch", "x", "z")):
return register_plural(word, word[:-1] + "es")
elif lword.endswith("fe"):
return register_plural(word, word[:-2] + "ves")
elif lword.endswith("f") and not lword.endswith("ff"):
return register_plural(word, word[:-1] + "ves")
else:
return register_plural(word, word + "s")
def to_yaml_type(typestr: str):
if typestr == 'bool':
return 'Boolean'
elif typestr == 'int':
return 'Integer'
elif typestr == 'unsigned':
return 'Unsigned'
elif typestr == 'std::string':
return 'String'
if typestr == "bool":
return "Boolean"
elif typestr == "int":
return "Integer"
elif typestr == "unsigned":
return "Unsigned"
elif typestr == "std::string":
return "String"
match = re.match(r'std::vector<(.*)>$', typestr)
if match:
return 'List of ' + pluralize(to_yaml_type(match.group(1)))
match = re.match(r"std::vector<(.*)>$", typestr)
if match:
return "List of " + pluralize(to_yaml_type(match.group(1)))
match = re.match(r'std::optional<(.*)>$', typestr)
if match:
return to_yaml_type(match.group(1))
match = re.match(r"std::optional<(.*)>$", typestr)
if match:
return to_yaml_type(match.group(1))
return typestr
return typestr
def doxygen2rst(text):
text = re.sub(r'<tt>\s*(.*?)\s*<\/tt>', r'``\1``', text)
text = re.sub(r'\\c ([^ ,;\.]+)', r'``\1``', text)
text = re.sub(r'\\\w+ ', '', text)
return text
text = re.sub(r"<tt>\s*(.*?)\s*<\/tt>", r"``\1``", text)
text = re.sub(r"\\c ([^ ,;\.]+)", r"``\1``", text)
text = re.sub(r"\\\w+ ", "", text)
return text
def indent(text, columns, indent_first_line=True):
indent_str = ' ' * columns
s = re.sub(r'\n([^\n])', '\n' + indent_str + '\\1', text, flags=re.S)
if not indent_first_line or s.startswith('\n'):
return s
return indent_str + s
indent_str = " " * columns
s = re.sub(r"\n([^\n])", "\n" + indent_str + "\\1", text, flags=re.S)
if not indent_first_line or s.startswith("\n"):
return s
return indent_str + s
class Option(object):
def __init__(self, name, opt_type, comment, version):
self.name = name
self.type = opt_type
self.comment = comment.strip()
self.enum = None
self.nested_struct = None
self.version = version
def __init__(self, name, opt_type, comment, version):
self.name = name
self.type = opt_type
self.comment = comment.strip()
self.enum = None
self.nested_struct = None
self.version = version
def __str__(self):
s = ".. _%s:\n\n**%s** (``%s``) " % (
self.name,
self.name,
to_yaml_type(self.type),
)
if self.version:
s += ":versionbadge:`clang-format %s` " % self.version
s += ":ref:`¶ <%s>`\n%s" % (self.name, doxygen2rst(indent(self.comment, 2)))
if self.enum and self.enum.values:
s += indent("\n\nPossible values:\n\n%s\n" % self.enum, 2)
if self.nested_struct:
s += indent(
"\n\nNested configuration flags:\n\n%s\n" % self.nested_struct, 2
)
return s
def __str__(self):
s = ".. _%s:\n\n**%s** (``%s``) " % (self.name, self.name, to_yaml_type(self.type))
if self.version:
s += ':versionbadge:`clang-format %s` ' % self.version
s += ':ref:`¶ <%s>`\n%s' % (self.name, doxygen2rst(indent(self.comment, 2)))
if self.enum and self.enum.values:
s += indent('\n\nPossible values:\n\n%s\n' % self.enum, 2)
if self.nested_struct:
s += indent('\n\nNested configuration flags:\n\n%s\n' %self.nested_struct,
2)
return s
class NestedStruct(object):
def __init__(self, name, comment):
self.name = name
self.comment = comment.strip()
self.values = []
def __init__(self, name, comment):
self.name = name
self.comment = comment.strip()
self.values = []
def __str__(self):
return self.comment + "\n" + "\n".join(map(str, self.values))
def __str__(self):
return self.comment + '\n' + '\n'.join(map(str, self.values))
class NestedField(object):
def __init__(self, name, comment):
self.name = name
self.comment = comment.strip()
def __init__(self, name, comment):
self.name = name
self.comment = comment.strip()
def __str__(self):
return "\n* ``%s`` %s" % (
self.name,
doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
)
def __str__(self):
return '\n* ``%s`` %s' % (
self.name,
doxygen2rst(indent(self.comment, 2, indent_first_line=False)))
class Enum(object):
def __init__(self, name, comment):
self.name = name
self.comment = comment.strip()
self.values = []
def __init__(self, name, comment):
self.name = name
self.comment = comment.strip()
self.values = []
def __str__(self):
return "\n".join(map(str, self.values))
def __str__(self):
return '\n'.join(map(str, self.values))
class NestedEnum(object):
def __init__(self, name, enumtype, comment, values):
self.name = name
self.comment = comment
self.values = values
self.type = enumtype
def __init__(self, name, enumtype, comment, values):
self.name = name
self.comment = comment
self.values = values
self.type = enumtype
def __str__(self):
s = "\n* ``%s %s``\n%s" % (
to_yaml_type(self.type),
self.name,
doxygen2rst(indent(self.comment, 2)),
)
s += indent("\nPossible values:\n\n", 2)
s += indent("\n".join(map(str, self.values)), 2)
return s
def __str__(self):
s = '\n* ``%s %s``\n%s' % (to_yaml_type(self.type), self.name,
doxygen2rst(indent(self.comment, 2)))
s += indent('\nPossible values:\n\n', 2)
s += indent('\n'.join(map(str, self.values)), 2)
return s
class EnumValue(object):
def __init__(self, name, comment, config):
self.name = name
self.comment = comment
self.config = config
def __init__(self, name, comment, config):
self.name = name
self.comment = comment
self.config = config
def __str__(self):
return '* ``%s`` (in configuration: ``%s``)\n%s' % (
self.name,
re.sub('.*_', '', self.config),
doxygen2rst(indent(self.comment, 2)))
def __str__(self):
return "* ``%s`` (in configuration: ``%s``)\n%s" % (
self.name,
re.sub(".*_", "", self.config),
doxygen2rst(indent(self.comment, 2)),
)
class OptionsReader:
def __init__(self, header: TextIOWrapper):
self.header = header
self.in_code_block = False
self.code_indent = 0
self.lineno = 0
self.last_err_lineno = -1
def __init__(self, header: TextIOWrapper):
self.header = header
self.in_code_block = False
self.code_indent = 0
self.lineno = 0
self.last_err_lineno = -1
def __file_path(self):
return os.path.relpath(self.header.name)
def __file_path(self):
return os.path.relpath(self.header.name)
def __print_line(self, line: str):
print(f'{self.lineno:>6} | {line}', file=sys.stderr)
def __print_line(self, line: str):
print(f"{self.lineno:>6} | {line}", file=sys.stderr)
def __warning(self, msg: str, line: str):
print(f'{self.__file_path()}:{self.lineno}: warning: {msg}:', file=sys.stderr)
self.__print_line(line)
def __clean_comment_line(self, line: str):
match = re.match(r'^/// (?P<indent> +)?\\code(\{.(?P<lang>\w+)\})?$', line)
if match:
if self.in_code_block:
self.__warning('`\\code` in another `\\code`', line)
self.in_code_block = True
indent_str = match.group('indent')
if not indent_str:
indent_str = ''
self.code_indent = len(indent_str)
lang = match.group('lang')
if not lang:
lang = 'c++'
return f'\n{indent_str}.. code-block:: {lang}\n\n'
endcode_match = re.match(r'^/// +\\endcode$', line)
if endcode_match:
if not self.in_code_block:
self.__warning('no correct `\\code` found before this `\\endcode`', line)
self.in_code_block = False
return ''
# check code block indentation
if (self.in_code_block and not line == '///' and not
line.startswith('/// ' + ' ' * self.code_indent)):
if self.last_err_lineno == self.lineno - 1:
def __warning(self, msg: str, line: str):
print(f"{self.__file_path()}:{self.lineno}: warning: {msg}:", file=sys.stderr)
self.__print_line(line)
else:
self.__warning('code block should be indented', line)
self.last_err_lineno = self.lineno
match = re.match(r'^/// \\warning$', line)
if match:
return '\n.. warning:: \n\n'
def __clean_comment_line(self, line: str):
match = re.match(r"^/// (?P<indent> +)?\\code(\{.(?P<lang>\w+)\})?$", line)
if match:
if self.in_code_block:
self.__warning("`\\code` in another `\\code`", line)
self.in_code_block = True
indent_str = match.group("indent")
if not indent_str:
indent_str = ""
self.code_indent = len(indent_str)
lang = match.group("lang")
if not lang:
lang = "c++"
return f"\n{indent_str}.. code-block:: {lang}\n\n"
endwarning_match = re.match(r'^/// +\\endwarning$', line)
if endwarning_match:
return ''
return line[4:] + '\n'
endcode_match = re.match(r"^/// +\\endcode$", line)
if endcode_match:
if not self.in_code_block:
self.__warning(
"no correct `\\code` found before this `\\endcode`", line
)
self.in_code_block = False
return ""
def read_options(self):
class State:
BeforeStruct, Finished, InStruct, InNestedStruct, InNestedFieldComment, \
InFieldComment, InEnum, InEnumMemberComment = range(8)
state = State.BeforeStruct
# check code block indentation
if (
self.in_code_block
and not line == "///"
and not line.startswith("/// " + " " * self.code_indent)
):
if self.last_err_lineno == self.lineno - 1:
self.__print_line(line)
else:
self.__warning("code block should be indented", line)
self.last_err_lineno = self.lineno
options = []
enums = {}
nested_structs = {}
comment = ''
enum = None
nested_struct = None
version = None
match = re.match(r"^/// \\warning$", line)
if match:
return "\n.. warning:: \n\n"
for line in self.header:
self.lineno += 1
line = line.strip()
if state == State.BeforeStruct:
if line in ('struct FormatStyle {', 'struct IncludeStyle {'):
state = State.InStruct
elif state == State.InStruct:
if line.startswith('///'):
state = State.InFieldComment
comment = self.__clean_comment_line(line)
elif line == '};':
state = State.Finished
break
elif state == State.InFieldComment:
if line.startswith(r'/// \version'):
match = re.match(r'/// \\version\s*(?P<version>[0-9.]+)*', line)
if match:
version = match.group('version')
elif line.startswith('///'):
comment += self.__clean_comment_line(line)
elif line.startswith('enum'):
state = State.InEnum
name = re.sub(r'enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{', '\\1', line)
enum = Enum(name, comment)
elif line.startswith('struct'):
state = State.InNestedStruct
name = re.sub(r'struct\s+(\w+)\s*\{', '\\1', line)
nested_struct = NestedStruct(name, comment)
elif line.endswith(';'):
prefix = '// '
if line.startswith(prefix):
line = line[len(prefix):]
state = State.InStruct
field_type, field_name = re.match(r'([<>:\w(,\s)]+)\s+(\w+);',
line).groups()
endwarning_match = re.match(r"^/// +\\endwarning$", line)
if endwarning_match:
return ""
return line[4:] + "\n"
if not version:
self.__warning(f'missing version for {field_name}', line)
option = Option(str(field_name), str(field_type), comment, version)
options.append(option)
version = None
else:
raise Exception('Invalid format, expected comment, field or enum\n' + line)
elif state == State.InNestedStruct:
if line.startswith('///'):
state = State.InNestedFieldComment
comment = self.__clean_comment_line(line)
elif line == '};':
state = State.InStruct
nested_structs[nested_struct.name] = nested_struct
elif state == State.InNestedFieldComment:
if line.startswith('///'):
comment += self.__clean_comment_line(line)
else:
state = State.InNestedStruct
field_type, field_name = re.match(r'([<>:\w(,\s)]+)\s+(\w+);', line).groups()
if field_type in enums:
nested_struct.values.append(NestedEnum(field_name,
field_type,
comment,
enums[field_type].values))
else:
nested_struct.values.append(NestedField(field_type + " " + field_name, comment))
def read_options(self):
class State:
(
BeforeStruct,
Finished,
InStruct,
InNestedStruct,
InNestedFieldComment,
InFieldComment,
InEnum,
InEnumMemberComment,
) = range(8)
elif state == State.InEnum:
if line.startswith('///'):
state = State.InEnumMemberComment
comment = self.__clean_comment_line(line)
elif line == '};':
state = State.InStruct
enums[enum.name] = enum
else:
# Enum member without documentation. Must be documented where the enum
# is used.
pass
elif state == State.InEnumMemberComment:
if line.startswith('///'):
comment += self.__clean_comment_line(line)
else:
state = State.InEnum
val = line.replace(',', '')
pos = val.find(" // ")
if pos != -1:
config = val[pos + 4:]
val = val[:pos]
else:
config = val
enum.values.append(EnumValue(val, comment, config))
if state != State.Finished:
raise Exception('Not finished by the end of file')
state = State.BeforeStruct
for option in options:
if option.type not in ['bool', 'unsigned', 'int', 'std::string',
'std::vector<std::string>',
'std::vector<IncludeCategory>',
'std::vector<RawStringFormat>',
'std::optional<unsigned>']:
if option.type in enums:
option.enum = enums[option.type]
elif option.type in nested_structs:
option.nested_struct = nested_structs[option.type]
else:
raise Exception('Unknown type: %s' % option.type)
return options
options = []
enums = {}
nested_structs = {}
comment = ""
enum = None
nested_struct = None
version = None
for line in self.header:
self.lineno += 1
line = line.strip()
if state == State.BeforeStruct:
if line in ("struct FormatStyle {", "struct IncludeStyle {"):
state = State.InStruct
elif state == State.InStruct:
if line.startswith("///"):
state = State.InFieldComment
comment = self.__clean_comment_line(line)
elif line == "};":
state = State.Finished
break
elif state == State.InFieldComment:
if line.startswith(r"/// \version"):
match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)
if match:
version = match.group("version")
elif line.startswith("///"):
comment += self.__clean_comment_line(line)
elif line.startswith("enum"):
state = State.InEnum
name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)
enum = Enum(name, comment)
elif line.startswith("struct"):
state = State.InNestedStruct
name = re.sub(r"struct\s+(\w+)\s*\{", "\\1", line)
nested_struct = NestedStruct(name, comment)
elif line.endswith(";"):
prefix = "// "
if line.startswith(prefix):
line = line[len(prefix) :]
state = State.InStruct
field_type, field_name = re.match(
r"([<>:\w(,\s)]+)\s+(\w+);", line
).groups()
if not version:
self.__warning(f"missing version for {field_name}", line)
option = Option(str(field_name), str(field_type), comment, version)
options.append(option)
version = None
else:
raise Exception(
"Invalid format, expected comment, field or enum\n" + line
)
elif state == State.InNestedStruct:
if line.startswith("///"):
state = State.InNestedFieldComment
comment = self.__clean_comment_line(line)
elif line == "};":
state = State.InStruct
nested_structs[nested_struct.name] = nested_struct
elif state == State.InNestedFieldComment:
if line.startswith("///"):
comment += self.__clean_comment_line(line)
else:
state = State.InNestedStruct
field_type, field_name = re.match(
r"([<>:\w(,\s)]+)\s+(\w+);", line
).groups()
if field_type in enums:
nested_struct.values.append(
NestedEnum(
field_name,
field_type,
comment,
enums[field_type].values,
)
)
else:
nested_struct.values.append(
NestedField(field_type + " " + field_name, comment)
)
elif state == State.InEnum:
if line.startswith("///"):
state = State.InEnumMemberComment
comment = self.__clean_comment_line(line)
elif line == "};":
state = State.InStruct
enums[enum.name] = enum
else:
# Enum member without documentation. Must be documented where the enum
# is used.
pass
elif state == State.InEnumMemberComment:
if line.startswith("///"):
comment += self.__clean_comment_line(line)
else:
state = State.InEnum
val = line.replace(",", "")
pos = val.find(" // ")
if pos != -1:
config = val[pos + 4 :]
val = val[:pos]
else:
config = val
enum.values.append(EnumValue(val, comment, config))
if state != State.Finished:
raise Exception("Not finished by the end of file")
for option in options:
if option.type not in [
"bool",
"unsigned",
"int",
"std::string",
"std::vector<std::string>",
"std::vector<IncludeCategory>",
"std::vector<RawStringFormat>",
"std::optional<unsigned>",
]:
if option.type in enums:
option.enum = enums[option.type]
elif option.type in nested_structs:
option.nested_struct = nested_structs[option.type]
else:
raise Exception("Unknown type: %s" % option.type)
return options
with open(FORMAT_STYLE_FILE) as f:
opts = OptionsReader(f).read_options()
opts = OptionsReader(f).read_options()
with open(INCLUDE_STYLE_FILE) as f:
opts += OptionsReader(f).read_options()
opts += OptionsReader(f).read_options()
opts = sorted(opts, key=lambda x: x.name)
options_text = '\n\n'.join(map(str, opts))
options_text = "\n\n".join(map(str, opts))
with open(DOC_FILE) as f:
contents = f.read()
contents = f.read()
contents = substitute(contents, 'FORMAT_STYLE_OPTIONS', options_text)
contents = substitute(contents, "FORMAT_STYLE_OPTIONS", options_text)
with open(DOC_FILE, 'wb') as output:
output.write(contents.encode())
with open(DOC_FILE, "wb") as output:
output.write(contents.encode())

79
clang/docs/tools/generate_formatted_state.py
View File

@ -9,14 +9,17 @@ from datetime import datetime
def get_git_revision_short_hash():
""" Get the get SHA in short hash form. """
return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']
).decode(sys.stdout.encoding).strip()
"""Get the get SHA in short hash form."""
return (
subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])
.decode(sys.stdout.encoding)
.strip()
)
def get_style(count, passed):
""" Determine if this directory is good based on the number of clean
files vs the number of files in total. """
"""Determine if this directory is good based on the number of clean
files vs the number of files in total."""
if passed == count:
return ":good:"
if passed != 0:
@ -24,10 +27,10 @@ def get_style(count, passed):
return ":none:"
TOP_DIR = os.path.join(os.path.dirname(__file__), '../../..')
CLANG_DIR = os.path.join(os.path.dirname(__file__), '../..')
DOC_FILE = os.path.join(CLANG_DIR, 'docs/ClangFormattedStatus.rst')
CLEAN_FILE = os.path.join(CLANG_DIR, 'docs/tools/clang-formatted-files.txt')
TOP_DIR = os.path.join(os.path.dirname(__file__), "../../..")
CLANG_DIR = os.path.join(os.path.dirname(__file__), "../..")
DOC_FILE = os.path.join(CLANG_DIR, "docs/ClangFormattedStatus.rst")
CLEAN_FILE = os.path.join(CLANG_DIR, "docs/tools/clang-formatted-files.txt")
rootdir = TOP_DIR
@ -75,15 +78,14 @@ TABLE_ROW = """\
- {style2}`{percent}%`
"""
FNULL = open(os.devnull, 'w')
FNULL = open(os.devnull, "w")
with open(DOC_FILE, 'wb') as output:
with open(DOC_FILE, "wb") as output:
cleanfiles = open(CLEAN_FILE, "wb")
sha = get_git_revision_short_hash()
today = datetime.now().strftime("%B %d, %Y %H:%M:%S")
output.write(bytes(RST_PREFIX.format(today=today,
sha=sha).encode("utf-8")))
output.write(bytes(RST_PREFIX.format(today=today, sha=sha).encode("utf-8")))
total_files_count = 0
total_files_pass = 0
@ -100,13 +102,14 @@ with open(DOC_FILE, 'wb') as output:
git_check = subprocess.Popen(
["git", "ls-files", "--error-unmatch", act_sub_dir],
stdout=FNULL,
stderr=FNULL)
stderr=FNULL,
)
if git_check.wait() != 0:
print("Skipping directory: ", act_sub_dir)
subdirs.remove(subdir)
path = os.path.relpath(root, TOP_DIR)
path = path.replace('\\', '/')
path = path.replace("\\", "/")
file_count = 0
file_pass = 0
@ -124,8 +127,8 @@ with open(DOC_FILE, 'wb') as output:
stdout, err = cmd.communicate()
relpath = os.path.relpath(file_path, TOP_DIR)
relpath = relpath.replace('\\', '/')
if err.decode(sys.stdout.encoding).find(': warning:') > 0:
relpath = relpath.replace("\\", "/")
if err.decode(sys.stdout.encoding).find(": warning:") > 0:
print(relpath, ":", "FAIL")
file_fail += 1
else:
@ -139,25 +142,39 @@ with open(DOC_FILE, 'wb') as output:
total_files_fail += file_fail
if file_count > 0:
percent = (int(100.0 * (float(file_pass)/float(file_count))))
percent = int(100.0 * (float(file_pass) / float(file_count)))
style = get_style(file_count, file_pass)
output.write(bytes(TABLE_ROW.format(path=path,
count=file_count,
passes=file_pass,
fails=file_fail,
percent=str(percent), style="",
style2=style).encode("utf-8")))
output.write(
bytes(
TABLE_ROW.format(
path=path,
count=file_count,
passes=file_pass,
fails=file_fail,
percent=str(percent),
style="",
style2=style,
).encode("utf-8")
)
)
output.flush()
print("----\n")
print(path, file_count, file_pass, file_fail, percent)
print("----\n")
total_percent = (float(total_files_pass)/float(total_files_count))
total_percent = float(total_files_pass) / float(total_files_count)
percent_str = str(int(100.0 * total_percent))
output.write(bytes(TABLE_ROW.format(path="Total",
count=total_files_count,
passes=total_files_pass,
fails=total_files_fail,
percent=percent_str, style=":total:",
style2=":total:").encode("utf-8")))
output.write(
bytes(
TABLE_ROW.format(
path="Total",
count=total_files_count,
passes=total_files_pass,
fails=total_files_fail,
percent=percent_str,
style=":total:",
style2=":total:",
).encode("utf-8")
)
)

343
clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
View File

@ -8,22 +8,24 @@ import filecmp
import shutil
import argparse
class Generator(object):
implementationContent = ''
implementationContent = ""
RefClades = {"DeclarationNameInfo",
RefClades = {
"DeclarationNameInfo",
"NestedNameSpecifierLoc",
"TemplateArgumentLoc",
"TypeLoc"}
"TypeLoc",
}
def __init__(self, templateClasses):
self.templateClasses = templateClasses
def GeneratePrologue(self):
self.implementationContent += \
"""
self.implementationContent += """
/*===- Generated file -------------------------------------------*- C++ -*-===*\
|* *|
|* Introspection of available AST node SourceLocations *|
@ -63,20 +65,23 @@ std::vector<clang::TypeLoc> &TLRG;
if CladeName in self.RefClades:
InstanceDecoration = "&"
self.implementationContent += \
"""
self.implementationContent += """
void GetLocationsImpl(SharedLocationCall const& Prefix,
clang::{0} const {1}Object, SourceLocationMap &Locs,
SourceRangeMap &Rngs,
std::vector<clang::TypeLoc> &TypeLocRecursionGuard);
""".format(CladeName, InstanceDecoration)
""".format(
CladeName, InstanceDecoration
)
def GenerateSrcLocMethod(self,
ClassName, ClassData, CreateLocalRecursionGuard):
def GenerateSrcLocMethod(self, ClassName, ClassData, CreateLocalRecursionGuard):
NormalClassName = ClassName
RecursionGuardParam = ('' if CreateLocalRecursionGuard else \
', std::vector<clang::TypeLoc>& TypeLocRecursionGuard')
RecursionGuardParam = (
""
if CreateLocalRecursionGuard
else ", std::vector<clang::TypeLoc>& TypeLocRecursionGuard"
)
if "templateParms" in ClassData:
TemplatePreamble = "template <typename "
@ -92,147 +97,163 @@ void GetLocationsImpl(SharedLocationCall const& Prefix,
TemplatePreamble += TA
ClassName += ">"
TemplatePreamble += ">\n";
TemplatePreamble += ">\n"
self.implementationContent += TemplatePreamble
self.implementationContent += \
"""
self.implementationContent += """
static void GetLocations{0}(SharedLocationCall const& Prefix,
clang::{1} const &Object,
SourceLocationMap &Locs, SourceRangeMap &Rngs {2})
{{
""".format(NormalClassName, ClassName, RecursionGuardParam)
""".format(
NormalClassName, ClassName, RecursionGuardParam
)
if 'sourceLocations' in ClassData:
for locName in ClassData['sourceLocations']:
self.implementationContent += \
"""
if "sourceLocations" in ClassData:
for locName in ClassData["sourceLocations"]:
self.implementationContent += """
Locs.insert(LocationAndString(Object.{0}(),
llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}")));
""".format(locName)
""".format(
locName
)
self.implementationContent += '\n'
self.implementationContent += "\n"
if 'sourceRanges' in ClassData:
for rngName in ClassData['sourceRanges']:
self.implementationContent += \
"""
if "sourceRanges" in ClassData:
for rngName in ClassData["sourceRanges"]:
self.implementationContent += """
Rngs.insert(RangeAndString(Object.{0}(),
llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}")));
""".format(rngName)
""".format(
rngName
)
self.implementationContent += '\n'
self.implementationContent += "\n"
if 'typeLocs' in ClassData or 'typeSourceInfos' in ClassData \
or 'nestedNameLocs' in ClassData \
or 'declNameInfos' in ClassData:
if (
"typeLocs" in ClassData
or "typeSourceInfos" in ClassData
or "nestedNameLocs" in ClassData
or "declNameInfos" in ClassData
):
if CreateLocalRecursionGuard:
self.implementationContent += \
'std::vector<clang::TypeLoc> TypeLocRecursionGuard;\n'
self.implementationContent += (
"std::vector<clang::TypeLoc> TypeLocRecursionGuard;\n"
)
self.implementationContent += '\n'
self.implementationContent += "\n"
if 'typeLocs' in ClassData:
for typeLoc in ClassData['typeLocs']:
if "typeLocs" in ClassData:
for typeLoc in ClassData["typeLocs"]:
self.implementationContent += \
"""
self.implementationContent += """
if (Object.{0}()) {{
GetLocationsImpl(
llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}"),
Object.{0}(), Locs, Rngs, TypeLocRecursionGuard);
}}
""".format(typeLoc)
""".format(
typeLoc
)
self.implementationContent += '\n'
if 'typeSourceInfos' in ClassData:
for tsi in ClassData['typeSourceInfos']:
self.implementationContent += \
"""
self.implementationContent += "\n"
if "typeSourceInfos" in ClassData:
for tsi in ClassData["typeSourceInfos"]:
self.implementationContent += """
if (Object.{0}()) {{
GetLocationsImpl(llvm::makeIntrusiveRefCnt<LocationCall>(
llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}",
LocationCall::ReturnsPointer), "getTypeLoc"),
Object.{0}()->getTypeLoc(), Locs, Rngs, TypeLocRecursionGuard);
}}
""".format(tsi)
""".format(
tsi
)
self.implementationContent += '\n'
self.implementationContent += "\n"
if 'nestedNameLocs' in ClassData:
for NN in ClassData['nestedNameLocs']:
self.implementationContent += \
"""
if "nestedNameLocs" in ClassData:
for NN in ClassData["nestedNameLocs"]:
self.implementationContent += """
if (Object.{0}())
GetLocationsImpl(
llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}"),
Object.{0}(), Locs, Rngs, TypeLocRecursionGuard);
""".format(NN)
""".format(
NN
)
if 'declNameInfos' in ClassData:
for declName in ClassData['declNameInfos']:
if "declNameInfos" in ClassData:
for declName in ClassData["declNameInfos"]:
self.implementationContent += \
"""
self.implementationContent += """
GetLocationsImpl(
llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}"),
Object.{0}(), Locs, Rngs, TypeLocRecursionGuard);
""".format(declName)
""".format(
declName
)
self.implementationContent += '}\n'
self.implementationContent += "}\n"
def GenerateFiles(self, OutputFile):
with open(os.path.join(os.getcwd(),
OutputFile), 'w') as f:
with open(os.path.join(os.getcwd(), OutputFile), "w") as f:
f.write(self.implementationContent)
def GenerateBaseGetLocationsFunction(self, ASTClassNames,
ClassEntries, CladeName, InheritanceMap,
CreateLocalRecursionGuard):
def GenerateBaseGetLocationsFunction(
self,
ASTClassNames,
ClassEntries,
CladeName,
InheritanceMap,
CreateLocalRecursionGuard,
):
MethodReturnType = 'NodeLocationAccessors'
MethodReturnType = "NodeLocationAccessors"
InstanceDecoration = "*"
if CladeName in self.RefClades:
InstanceDecoration = "&"
Signature = \
'GetLocations(clang::{0} const {1}Object)'.format(
CladeName, InstanceDecoration)
ImplSignature = \
"""
Signature = "GetLocations(clang::{0} const {1}Object)".format(
CladeName, InstanceDecoration
)
ImplSignature = """
GetLocationsImpl(SharedLocationCall const& Prefix,
clang::{0} const {1}Object, SourceLocationMap &Locs,
SourceRangeMap &Rngs,
std::vector<clang::TypeLoc> &TypeLocRecursionGuard)
""".format(CladeName, InstanceDecoration)
""".format(
CladeName, InstanceDecoration
)
self.implementationContent += 'void {0} {{ '.format(ImplSignature)
self.implementationContent += "void {0} {{ ".format(ImplSignature)
if CladeName == "TypeLoc":
self.implementationContent += 'if (Object.isNull()) return;'
self.implementationContent += "if (Object.isNull()) return;"
self.implementationContent += \
"""
self.implementationContent += """
if (llvm::find(TypeLocRecursionGuard, Object) != TypeLocRecursionGuard.end())
return;
TypeLocRecursionGuard.push_back(Object);
RecursionPopper RAII(TypeLocRecursionGuard);
"""
RecursionGuardParam = ''
RecursionGuardParam = ""
if not CreateLocalRecursionGuard:
RecursionGuardParam = ', TypeLocRecursionGuard'
RecursionGuardParam = ", TypeLocRecursionGuard"
ArgPrefix = '*'
ArgPrefix = "*"
if CladeName in self.RefClades:
ArgPrefix = ''
self.implementationContent += \
'GetLocations{0}(Prefix, {1}Object, Locs, Rngs {2});'.format(
CladeName, ArgPrefix, RecursionGuardParam)
ArgPrefix = ""
self.implementationContent += (
"GetLocations{0}(Prefix, {1}Object, Locs, Rngs {2});".format(
CladeName, ArgPrefix, RecursionGuardParam
)
)
if CladeName == "TypeLoc":
self.implementationContent += \
'''
self.implementationContent += """
if (auto QTL = Object.getAs<clang::QualifiedTypeLoc>()) {
auto Dequalified = QTL.getNextTypeLoc();
return GetLocationsImpl(llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "getNextTypeLoc"),
@ -240,7 +261,7 @@ static void GetLocations{0}(SharedLocationCall const& Prefix,
Locs,
Rngs,
TypeLocRecursionGuard);
}'''
}"""
for ASTClassName in ASTClassNames:
if ASTClassName in self.templateClasses:
@ -248,21 +269,22 @@ static void GetLocations{0}(SharedLocationCall const& Prefix,
if ASTClassName == CladeName:
continue
if CladeName != "TypeLoc":
self.implementationContent += \
"""
self.implementationContent += """
if (auto Derived = llvm::dyn_cast<clang::{0}>(Object)) {{
GetLocations{0}(Prefix, *Derived, Locs, Rngs {1});
}}
""".format(ASTClassName, RecursionGuardParam)
""".format(
ASTClassName, RecursionGuardParam
)
continue
self.GenerateBaseTypeLocVisit(ASTClassName, ClassEntries,
RecursionGuardParam, InheritanceMap)
self.GenerateBaseTypeLocVisit(
ASTClassName, ClassEntries, RecursionGuardParam, InheritanceMap
)
self.implementationContent += '}'
self.implementationContent += "}"
self.implementationContent += \
"""
self.implementationContent += """
{0} NodeIntrospection::{1} {{
NodeLocationAccessors Result;
SharedLocationCall Prefix;
@ -270,108 +292,125 @@ if (auto Derived = llvm::dyn_cast<clang::{0}>(Object)) {{
GetLocationsImpl(Prefix, Object, Result.LocationAccessors,
Result.RangeAccessors, TypeLocRecursionGuard);
""".format(MethodReturnType, Signature)
""".format(
MethodReturnType, Signature
)
self.implementationContent += 'return Result; }'
self.implementationContent += "return Result; }"
def GenerateBaseTypeLocVisit(self, ASTClassName, ClassEntries,
RecursionGuardParam, InheritanceMap):
CallPrefix = 'Prefix'
if ASTClassName != 'TypeLoc':
CallPrefix = \
'''llvm::makeIntrusiveRefCnt<LocationCall>(Prefix,
def GenerateBaseTypeLocVisit(
self, ASTClassName, ClassEntries, RecursionGuardParam, InheritanceMap
):
CallPrefix = "Prefix"
if ASTClassName != "TypeLoc":
CallPrefix = """llvm::makeIntrusiveRefCnt<LocationCall>(Prefix,
"getAs<clang::{0}>", LocationCall::IsCast)
'''.format(ASTClassName)
""".format(
ASTClassName
)
if ASTClassName in ClassEntries:
self.implementationContent += \
"""
self.implementationContent += """
if (auto ConcreteTL = Object.getAs<clang::{0}>())
GetLocations{1}({2}, ConcreteTL, Locs, Rngs {3});
""".format(ASTClassName, ASTClassName,
CallPrefix, RecursionGuardParam)
""".format(
ASTClassName, ASTClassName, CallPrefix, RecursionGuardParam
)
if ASTClassName in InheritanceMap:
for baseTemplate in self.templateClasses:
if baseTemplate in InheritanceMap[ASTClassName]:
self.implementationContent += \
"""
self.implementationContent += """
if (auto ConcreteTL = Object.getAs<clang::{0}>())
GetLocations{1}({2}, ConcreteTL, Locs, Rngs {3});
""".format(InheritanceMap[ASTClassName], baseTemplate,
CallPrefix, RecursionGuardParam)
""".format(
InheritanceMap[ASTClassName],
baseTemplate,
CallPrefix,
RecursionGuardParam,
)
def GenerateDynNodeVisitor(self, CladeNames):
MethodReturnType = 'NodeLocationAccessors'
MethodReturnType = "NodeLocationAccessors"
Signature = \
'GetLocations(clang::DynTypedNode const &Node)'
Signature = "GetLocations(clang::DynTypedNode const &Node)"
self.implementationContent += MethodReturnType \
+ ' NodeIntrospection::' + Signature + '{'
self.implementationContent += (
MethodReturnType + " NodeIntrospection::" + Signature + "{"
)
for CladeName in CladeNames:
if CladeName == "DeclarationNameInfo":
continue
self.implementationContent += \
"""
self.implementationContent += """
if (const auto *N = Node.get<{0}>())
""".format(CladeName)
""".format(
CladeName
)
ArgPrefix = ""
if CladeName in self.RefClades:
ArgPrefix = "*"
self.implementationContent += \
"""
return GetLocations({0}const_cast<{1} *>(N));""".format(ArgPrefix, CladeName)
self.implementationContent += """
return GetLocations({0}const_cast<{1} *>(N));""".format(
ArgPrefix, CladeName
)
self.implementationContent += '\nreturn {}; }'
self.implementationContent += "\nreturn {}; }"
def GenerateEpilogue(self):
self.implementationContent += '''
self.implementationContent += """
}
}
'''
"""
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--json-input-path',
help='Read API description from FILE', metavar='FILE')
parser.add_argument('--output-file', help='Generate output in FILEPATH',
metavar='FILEPATH')
parser.add_argument('--use-empty-implementation',
help='Generate empty implementation',
action="store", type=int)
parser.add_argument('--empty-implementation',
help='Copy empty implementation from FILEPATH',
action="store", metavar='FILEPATH')
parser.add_argument(
"--json-input-path", help="Read API description from FILE", metavar="FILE"
)
parser.add_argument(
"--output-file", help="Generate output in FILEPATH", metavar="FILEPATH"
)
parser.add_argument(
"--use-empty-implementation",
help="Generate empty implementation",
action="store",
type=int,
)
parser.add_argument(
"--empty-implementation",
help="Copy empty implementation from FILEPATH",
action="store",
metavar="FILEPATH",
)
options = parser.parse_args()
use_empty_implementation = options.use_empty_implementation
if (not use_empty_implementation
and not os.path.exists(options.json_input_path)):
if not use_empty_implementation and not os.path.exists(options.json_input_path):
use_empty_implementation = True
if not use_empty_implementation:
with open(options.json_input_path) as f:
jsonData = json.load(f)
if not 'classesInClade' in jsonData or not jsonData["classesInClade"]:
if not "classesInClade" in jsonData or not jsonData["classesInClade"]:
use_empty_implementation = True
if use_empty_implementation:
if not os.path.exists(options.output_file) or \
not filecmp.cmp(options.empty_implementation, options.output_file):
if not os.path.exists(options.output_file) or not filecmp.cmp(
options.empty_implementation, options.output_file
):
shutil.copyfile(options.empty_implementation, options.output_file)
sys.exit(0)
templateClasses = []
for (ClassName, ClassAccessors) in jsonData['classEntries'].items():
for (ClassName, ClassAccessors) in jsonData["classEntries"].items():
if "templateParms" in ClassAccessors:
templateClasses.append(ClassName)
@ -379,33 +418,35 @@ def main():
g.GeneratePrologue()
for (CladeName, ClassNameData) in jsonData['classesInClade'].items():
for (CladeName, ClassNameData) in jsonData["classesInClade"].items():
g.GenerateBaseGetLocationsDeclaration(CladeName)
def getCladeName(ClassName):
for (CladeName, ClassNameData) in jsonData['classesInClade'].items():
if ClassName in ClassNameData:
return CladeName
for (CladeName, ClassNameData) in jsonData["classesInClade"].items():
if ClassName in ClassNameData:
return CladeName
for (ClassName, ClassAccessors) in jsonData['classEntries'].items():
for (ClassName, ClassAccessors) in jsonData["classEntries"].items():
cladeName = getCladeName(ClassName)
g.GenerateSrcLocMethod(
ClassName, ClassAccessors,
cladeName not in Generator.RefClades)
ClassName, ClassAccessors, cladeName not in Generator.RefClades
)
for (CladeName, ClassNameData) in jsonData['classesInClade'].items():
for (CladeName, ClassNameData) in jsonData["classesInClade"].items():
g.GenerateBaseGetLocationsFunction(
ClassNameData,
jsonData['classEntries'],
jsonData["classEntries"],
CladeName,
jsonData["classInheritance"],
CladeName not in Generator.RefClades)
CladeName not in Generator.RefClades,
)
g.GenerateDynNodeVisitor(jsonData['classesInClade'].keys())
g.GenerateDynNodeVisitor(jsonData["classesInClade"].keys())
g.GenerateEpilogue()
g.GenerateFiles(options.output_file)
if __name__ == '__main__':
if __name__ == "__main__":
main()

116
clang/test/AST/gen_ast_dump_json_test.py
View File

@ -22,23 +22,22 @@ def normalize(dict_var):
normalize(e)
elif type(v) is str:
if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):
dict_var[k] = '0x{{.*}}'
dict_var[k] = "0x{{.*}}"
elif os.path.isfile(v):
dict_var[k] = '{{.*}}'
dict_var[k] = "{{.*}}"
else:
splits = (v.split(' '))
splits = v.split(" ")
out_splits = []
for split in splits:
inner_splits = split.rsplit(':',2)
inner_splits = split.rsplit(":", 2)
if os.path.isfile(inner_splits[0]):
out_splits.append(
'{{.*}}:%s:%s'
%(inner_splits[1],
inner_splits[2]))
"{{.*}}:%s:%s" % (inner_splits[1], inner_splits[2])
)
continue
out_splits.append(split)
dict_var[k] = ' '.join(out_splits)
dict_var[k] = " ".join(out_splits)
def filter_json(dict_var, filters, out):
@ -64,19 +63,39 @@ def default_clang_path():
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--clang", help="The clang binary (could be a relative or absolute path)",
action="store", default=default_clang_path())
parser.add_argument("--source", help="the source file(s). Without --update, the command used to generate the JSON "
"will be of the format <clang> -cc1 -ast-dump=json <opts> <source>",
action="store", nargs=argparse.ONE_OR_MORE, required=True)
parser.add_argument("--filters", help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType",
action="store", default='')
parser.add_argument(
"--clang",
help="The clang binary (could be a relative or absolute path)",
action="store",
default=default_clang_path(),
)
parser.add_argument(
"--source",
help="the source file(s). Without --update, the command used to generate the JSON "
"will be of the format <clang> -cc1 -ast-dump=json <opts> <source>",
action="store",
nargs=argparse.ONE_OR_MORE,
required=True,
)
parser.add_argument(
"--filters",
help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType",
action="store",
default="",
)
update_or_generate_group = parser.add_mutually_exclusive_group()
update_or_generate_group.add_argument("--update", help="Update the file in-place", action="store_true")
update_or_generate_group.add_argument("--opts", help="other options",
action="store", default='', type=str)
parser.add_argument("--update-manual", help="When using --update, also update files that do not have the "
"autogenerated disclaimer", action="store_true")
update_or_generate_group.add_argument(
"--update", help="Update the file in-place", action="store_true"
)
update_or_generate_group.add_argument(
"--opts", help="other options", action="store", default="", type=str
)
parser.add_argument(
"--update-manual",
help="When using --update, also update files that do not have the "
"autogenerated disclaimer",
action="store_true",
)
args = parser.parse_args()
if not args.source:
@ -87,15 +106,22 @@ def main():
sys.exit("clang binary specified not present.")
for src in args.source:
process_file(src, clang_binary, cmdline_filters=args.filters,
cmdline_opts=args.opts, do_update=args.update,
force_update=args.update_manual)
process_file(
src,
clang_binary,
cmdline_filters=args.filters,
cmdline_opts=args.opts,
do_update=args.update,
force_update=args.update_manual,
)
def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
do_update, force_update):
note_firstline = "// NOTE: CHECK lines have been autogenerated by " \
"gen_ast_dump_json_test.py"
def process_file(
source_file, clang_binary, cmdline_filters, cmdline_opts, do_update, force_update
):
note_firstline = (
"// NOTE: CHECK lines have been autogenerated by " "gen_ast_dump_json_test.py"
)
filters_line_prefix = "// using --filters="
note = note_firstline
@ -110,21 +136,27 @@ def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
if found_autogenerated_line:
# print("Filters line: '", line.rstrip(), "'", sep="")
if line.startswith(filters_line_prefix):
filters_line = line[len(filters_line_prefix):].rstrip()
filters_line = line[len(filters_line_prefix) :].rstrip()
break
if line.startswith(note_firstline):
found_autogenerated_line = True
# print("Found autogenerated disclaimer at line", i + 1)
if not found_autogenerated_line and not force_update:
print("Not updating", source_file, "since it is not autogenerated.",
file=sys.stderr)
print(
"Not updating",
source_file,
"since it is not autogenerated.",
file=sys.stderr,
)
return
if not cmdline_filters and filters_line:
cmdline_filters = filters_line
print("Inferred filters as '" + cmdline_filters + "'")
if "RUN: %clang_cc1 " not in first_line:
sys.exit("When using --update the first line of the input file must contain RUN: %clang_cc1")
sys.exit(
"When using --update the first line of the input file must contain RUN: %clang_cc1"
)
clang_start = first_line.find("%clang_cc1") + len("%clang_cc1")
file_check_idx = first_line.rfind("| FileCheck")
if file_check_idx:
@ -142,13 +174,13 @@ def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
options = cmdline_opts.split()
options.append("-ast-dump=json")
cmd.extend(options)
using_ast_dump_filter = any('ast-dump-filter' in arg for arg in cmd)
using_ast_dump_filter = any("ast-dump-filter" in arg for arg in cmd)
cmd.append(source_file)
print("Will run", cmd)
filters = set()
if cmdline_filters:
note += "\n" + filters_line_prefix + cmdline_filters
filters = set(cmdline_filters.split(','))
filters = set(cmdline_filters.split(","))
print("Will use the following filters:", filters)
try:
@ -156,7 +188,7 @@ def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
except Exception as ex:
print("The clang command failed with %s" % ex)
return -1
out_asts = []
if using_ast_dump_filter:
# If we're using a filter, then we might have multiple JSON objects
@ -183,7 +215,7 @@ def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
out_asts.append(j)
else:
filter_json(j, filters, out_asts)
with tempfile.NamedTemporaryFile("w", delete=False) as f:
with open(source_file, "r") as srcf:
for line in srcf.readlines():
@ -194,16 +226,16 @@ def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
f.write(note + "\n")
for out_ast in out_asts:
append_str = json.dumps(out_ast, indent=1, ensure_ascii=False)
out_str = '\n\n'
out_str = "\n\n"
out_str += "// CHECK-NOT: {{^}}Dumping\n"
index = 0
for append_line in append_str.splitlines()[2:]:
if index == 0:
out_str += '// CHECK: %s\n' %(append_line.rstrip())
out_str += "// CHECK: %s\n" % (append_line.rstrip())
index += 1
else:
out_str += '// CHECK-NEXT: %s\n' %(append_line.rstrip())
out_str += "// CHECK-NEXT: %s\n" % (append_line.rstrip())
f.write(out_str)
f.flush()
f.close()
@ -211,13 +243,13 @@ def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
print("Updating json appended source file to %s." % source_file)
copyfile(f.name, source_file)
else:
partition = source_file.rpartition('.')
dest_path = '%s-json%s%s' % (partition[0], partition[1], partition[2])
partition = source_file.rpartition(".")
dest_path = "%s-json%s%s" % (partition[0], partition[1], partition[2])
print("Writing json appended source file to %s." % dest_path)
copyfile(f.name, dest_path)
os.remove(f.name)
return 0
if __name__ == '__main__':
if __name__ == "__main__":
main()

31
clang/test/Analysis/analyzer_test.py
View File

@ -3,7 +3,6 @@ import lit.TestRunner
# Custom format class for static analyzer tests
class AnalyzerTest(lit.formats.ShTest):
def __init__(self, execute_external, use_z3_solver=False):
super(AnalyzerTest, self).__init__(execute_external)
self.use_z3_solver = use_z3_solver
@ -15,18 +14,24 @@ class AnalyzerTest(lit.formats.ShTest):
saved_test = test
lit.TestRunner.parseIntegratedTestScript(test)
if 'z3' not in test.requires:
results.append(self.executeWithAnalyzeSubstitution(
saved_test, litConfig, '-analyzer-constraints=range'))
if "z3" not in test.requires:
results.append(
self.executeWithAnalyzeSubstitution(
saved_test, litConfig, "-analyzer-constraints=range"
)
)
if results[-1].code == lit.Test.FAIL:
return results[-1]
# If z3 backend available, add an additional run line for it
if self.use_z3_solver == '1':
assert(test.config.clang_staticanalyzer_z3 == '1')
results.append(self.executeWithAnalyzeSubstitution(
saved_test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3'))
if self.use_z3_solver == "1":
assert test.config.clang_staticanalyzer_z3 == "1"
results.append(
self.executeWithAnalyzeSubstitution(
saved_test, litConfig, "-analyzer-constraints=z3 -DANALYZER_CM_Z3"
)
)
# Combine all result outputs into the last element
for x in results:
@ -35,14 +40,14 @@ class AnalyzerTest(lit.formats.ShTest):
if results:
return results[-1]
return lit.Test.Result(lit.Test.UNSUPPORTED,
"Test requires the following unavailable features: z3")
return lit.Test.Result(
lit.Test.UNSUPPORTED, "Test requires the following unavailable features: z3"
)
def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
saved_substitutions = list(test.config.substitutions)
test.config.substitutions.append(('%analyze', substitution))
result = lit.TestRunner.executeShTest(test, litConfig,
self.execute_external)
test.config.substitutions.append(("%analyze", substitution))
result = lit.TestRunner.executeShTest(test, litConfig, self.execute_external)
test.config.substitutions = saved_substitutions
return result

89
clang/test/Analysis/check-analyzer-fixit.py
View File

@ -1,16 +1,16 @@
#!/usr/bin/env python
#
#===- check-analyzer-fixit.py - Static Analyzer test helper ---*- python -*-===#
# ===- check-analyzer-fixit.py - Static Analyzer test helper ---*- python -*-===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
#
# This file copy-pasted mostly from the Clang-Tidy's 'check_clang_tidy.py'.
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
r"""
Clang Static Analyzer test helper
@ -33,7 +33,7 @@ import sys
def write_file(file_name, text):
with open(file_name, 'w') as f:
with open(file_name, "w") as f:
f.write(text)
@ -44,18 +44,18 @@ def run_test_once(args, extra_args):
file_name_with_extension = input_file_name
_, extension = os.path.splitext(file_name_with_extension)
if extension not in ['.c', '.hpp', '.m', '.mm']:
extension = '.cpp'
if extension not in [".c", ".hpp", ".m", ".mm"]:
extension = ".cpp"
temp_file_name = temp_file_name + extension
with open(input_file_name, 'r') as input_file:
with open(input_file_name, "r") as input_file:
input_text = input_file.read()
# Remove the contents of the CHECK lines to avoid CHECKs matching on
# themselves. We need to keep the comments to preserve line numbers while
# avoiding empty lines which could potentially trigger formatting-related
# checks.
cleaned_test = re.sub('// *CHECK-[A-Z0-9\-]*:[^\r\n]*', '//', input_text)
cleaned_test = re.sub("// *CHECK-[A-Z0-9\-]*:[^\r\n]*", "//", input_text)
write_file(temp_file_name, cleaned_test)
original_file_name = temp_file_name + ".orig"
@ -63,59 +63,82 @@ def run_test_once(args, extra_args):
try:
builtin_include_dir = subprocess.check_output(
['clang', '-print-file-name=include'], stderr=subprocess.STDOUT).decode()
["clang", "-print-file-name=include"], stderr=subprocess.STDOUT
).decode()
except subprocess.CalledProcessError as e:
print('Cannot print Clang include directory: ' + e.output.decode())
print("Cannot print Clang include directory: " + e.output.decode())
builtin_include_dir = os.path.normpath(builtin_include_dir)
args = (['clang', '-cc1', '-internal-isystem', builtin_include_dir,
'-nostdsysteminc', '-analyze', '-analyzer-constraints=range',
'-analyzer-config', 'apply-fixits=true']
+ clang_analyzer_extra_args + ['-verify', temp_file_name])
args = (
[
"clang",
"-cc1",
"-internal-isystem",
builtin_include_dir,
"-nostdsysteminc",
"-analyze",
"-analyzer-constraints=range",
"-analyzer-config",
"apply-fixits=true",
]
+ clang_analyzer_extra_args
+ ["-verify", temp_file_name]
)
print('Running ' + str(args) + '...')
print("Running " + str(args) + "...")
try:
clang_analyzer_output = \
subprocess.check_output(args, stderr=subprocess.STDOUT).decode()
clang_analyzer_output = subprocess.check_output(
args, stderr=subprocess.STDOUT
).decode()
except subprocess.CalledProcessError as e:
print('Clang Static Analyzer test failed:\n' + e.output.decode())
print("Clang Static Analyzer test failed:\n" + e.output.decode())
raise
print('----------------- Clang Static Analyzer output -----------------\n' +
clang_analyzer_output +
'\n--------------------------------------------------------------')
print(
"----------------- Clang Static Analyzer output -----------------\n"
+ clang_analyzer_output
+ "\n--------------------------------------------------------------"
)
try:
diff_output = subprocess.check_output(
['diff', '-u', original_file_name, temp_file_name],
stderr=subprocess.STDOUT)
["diff", "-u", original_file_name, temp_file_name], stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError as e:
diff_output = e.output
print('----------------------------- Fixes ----------------------------\n' +
diff_output.decode() +
'\n--------------------------------------------------------------')
print(
"----------------------------- Fixes ----------------------------\n"
+ diff_output.decode()
+ "\n--------------------------------------------------------------"
)
try:
subprocess.check_output(
['FileCheck', '-input-file=' + temp_file_name, input_file_name,
'-check-prefixes=CHECK-FIXES', '-strict-whitespace'],
stderr=subprocess.STDOUT)
[
"FileCheck",
"-input-file=" + temp_file_name,
input_file_name,
"-check-prefixes=CHECK-FIXES",
"-strict-whitespace",
],
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
print('FileCheck failed:\n' + e.output.decode())
print("FileCheck failed:\n" + e.output.decode())
raise
def main():
parser = argparse.ArgumentParser()
parser.add_argument('input_file_name')
parser.add_argument('temp_file_name')
parser.add_argument("input_file_name")
parser.add_argument("temp_file_name")
args, extra_args = parser.parse_known_args()
run_test_once(args, extra_args)
if __name__ == '__main__':
if __name__ == "__main__":
main()

20
clang/test/Analysis/exploded-graph-rewriter/lit.local.cfg
View File

@ -7,11 +7,17 @@ import os
use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
config.test_format = lit.formats.ShTest(use_lit_shell == "0")
config.substitutions.append(('%exploded_graph_rewriter',
'\'%s\' %s --dump-dot-only' % (
config.python_executable,
os.path.join(
config.clang_src_dir,
'utils', 'analyzer', 'exploded-graph-rewriter.py'))))
config.substitutions.append(
(
"%exploded_graph_rewriter",
"'%s' %s --dump-dot-only"
% (
config.python_executable,
os.path.join(
config.clang_src_dir, "utils", "analyzer", "exploded-graph-rewriter.py"
),
),
)
)
config.suffixes.add('.dot')
config.suffixes.add(".dot")

36
clang/test/Analysis/lit.local.cfg
View File

@ -6,23 +6,37 @@ import site
# is available.
site.addsitedir(os.path.dirname(__file__))
import analyzer_test
config.test_format = analyzer_test.AnalyzerTest(
config.test_format.execute_external, config.use_z3_solver)
config.test_format.execute_external, config.use_z3_solver
)
# Filtering command used by Clang Analyzer tests (when comparing .plist files
# with reference output)
config.substitutions.append(('%normalize_plist',
"grep -Ev '%s|%s|%s'" %
('^[[:space:]]*<string>.* version .*</string>[[:space:]]*$',
'^[[:space:]]*<string>/.*</string>[[:space:]]*$',
'^[[:space:]]*<string>.:.*</string>[[:space:]]*$')))
config.substitutions.append(
(
"%normalize_plist",
"grep -Ev '%s|%s|%s'"
% (
"^[[:space:]]*<string>.* version .*</string>[[:space:]]*$",
"^[[:space:]]*<string>/.*</string>[[:space:]]*$",
"^[[:space:]]*<string>.:.*</string>[[:space:]]*$",
),
)
)
# Filtering command for testing SARIF output against reference output.
config.substitutions.append(('%normalize_sarif',
"grep -Ev '^[[:space:]]*(%s|%s|%s)[[:space:]]*$'" %
('"uri": "file:.*%basename_t"',
'"version": ".* version .*"',
'"version": "2.1.0"')))
config.substitutions.append(
(
"%normalize_sarif",
"grep -Ev '^[[:space:]]*(%s|%s|%s)[[:space:]]*$'"
% (
'"uri": "file:.*%basename_t"',
'"version": ".* version .*"',
'"version": "2.1.0"',
),
)
)
if not config.root.clang_staticanalyzer:
config.unsupported = True

22
clang/test/Analysis/scan-build/lit.local.cfg
View File

@ -9,12 +9,16 @@ config.test_format = lit.formats.ShTest(use_lit_shell == "0")
clang_path = config.clang if config.have_llvm_driver else os.path.realpath(config.clang)
config.substitutions.append(('%scan-build',
'\'%s\' --use-analyzer=%s ' % (
lit.util.which('scan-build',
os.path.join(
config.clang_src_dir,
'tools',
'scan-build',
'bin')),
clang_path)))
config.substitutions.append(
(
"%scan-build",
"'%s' --use-analyzer=%s "
% (
lit.util.which(
"scan-build",
os.path.join(config.clang_src_dir, "tools", "scan-build", "bin"),
),
clang_path,
),
)
)

637
clang/test/CodeGen/builtins-nvptx-mma.py
View File

@ -12,352 +12,395 @@ from collections import defaultdict
from itertools import product
from string import Template
class MMAFrag:
def __init__(self, geom, frag, ptx_elt_type):
self.geom = geom
self.frag = frag
self.ptx_type = ptx_elt_type;
def __repr__(self):
return "%s:%s:%s" % (self.geom, self.frag, self.ptx_type)
class MMAFrag:
def __init__(self, geom, frag, ptx_elt_type):
self.geom = geom
self.frag = frag
self.ptx_type = ptx_elt_type
def __repr__(self):
return "%s:%s:%s" % (self.geom, self.frag, self.ptx_type)
class MMAOp:
def __init__(self, a, b, c, d, b1op=""):
self.a = a
self.b = b
self.c = c
self.d = d
self.b1op = b1op
def __init__(self, a, b, c, d, b1op=""):
self.a = a
self.b = b
self.c = c
self.d = d
self.b1op = b1op
def __repr__(self):
return "{A:%s, B:%s, C:%s, D:%s}" % (self.a, self.b, self.c, self.d)
def __repr__(self):
return ("{A:%s, B:%s, C:%s, D:%s}" % (self.a, self.b, self.c, self.d ))
def make_mma_ops(geoms, types_a, types_b, types_c, types_d, b1ops=None):
ops = []
if b1ops is None:
b1ops = [""]
for geom, type_a, type_c in product( geoms, types_a, types_c):
for type_b, type_d in product(types_b if types_b else [type_a],
types_d if types_d else [type_c]):
ops += [
MMAOp(MMAFrag(geom, "a", type_a),
MMAFrag(geom, "b", type_b),
MMAFrag(geom, "c", type_c),
MMAFrag(geom, "d", type_d), b1op)
for b1op in b1ops]
return ops
ops = []
if b1ops is None:
b1ops = [""]
for geom, type_a, type_c in product(geoms, types_a, types_c):
for type_b, type_d in product(
types_b if types_b else [type_a], types_d if types_d else [type_c]
):
ops += [
MMAOp(
MMAFrag(geom, "a", type_a),
MMAFrag(geom, "b", type_b),
MMAFrag(geom, "c", type_c),
MMAFrag(geom, "d", type_d),
b1op,
)
for b1op in b1ops
]
return ops
def make_ldst_ops(geoms, frags, types):
return [MMAFrag(geom, frag, ptx_type) for (geom, frag, ptx_type)
in product(geoms, frags, types)]
return [
MMAFrag(geom, frag, ptx_type)
for (geom, frag, ptx_type) in product(geoms, frags, types)
]
def get_mma_ops():
return (make_mma_ops(["m16n16k8"],
["tf32"], [], ["f32"], []) +
make_mma_ops(["m16n16k16", "m32n8k16", "m8n32k16"],
["bf16"], [], ["f32"], []) +
make_mma_ops(["m8n8k4"],
["f64"], [], ["f64"], []) +
make_mma_ops(["m16n16k16", "m32n8k16", "m8n32k16"],
["f16"], [], ["f16", "f32"], ["f16", "f32"]) +
make_mma_ops(["m16n16k16", "m32n8k16", "m8n32k16"],
["s8", "u8"], [], ["s32"], []) +
make_mma_ops(["m8n8k32"],
["s4", "u4"], [], ["s32"], []) +
make_mma_ops(["m8n8k128"],
["b1"], [], ["s32"], [],
[".xor.popc", ".and.popc"]))
return (
make_mma_ops(["m16n16k8"], ["tf32"], [], ["f32"], [])
+ make_mma_ops(["m16n16k16", "m32n8k16", "m8n32k16"], ["bf16"], [], ["f32"], [])
+ make_mma_ops(["m8n8k4"], ["f64"], [], ["f64"], [])
+ make_mma_ops(
["m16n16k16", "m32n8k16", "m8n32k16"],
["f16"],
[],
["f16", "f32"],
["f16", "f32"],
)
+ make_mma_ops(
["m16n16k16", "m32n8k16", "m8n32k16"], ["s8", "u8"], [], ["s32"], []
)
+ make_mma_ops(["m8n8k32"], ["s4", "u4"], [], ["s32"], [])
+ make_mma_ops(
["m8n8k128"], ["b1"], [], ["s32"], [], [".xor.popc", ".and.popc"]
)
)
def get_ldst_ops():
# NOTE: fragemts are from the point of view of PTX.
# fragment `d` is only for store ops, others for both loads and stores.
return (make_ldst_ops(["m16n16k16", "m32n8k16", "m8n32k16"],
["a", "b"], ["f16", "u8", "s8", "bf16"]) +
make_ldst_ops(["m16n16k16", "m32n8k16", "m8n32k16"],
["c", "d"], ["f16", "f32", "s32"]) +
make_ldst_ops(["m8n8k32"], ["a", "b"], ["s4","u4"]) +
make_ldst_ops(["m8n8k128"], ["a", "b"], ["b1"]) +
make_ldst_ops(["m8n8k32", "m8n8k128"], ["c", "d"], ["s32"]) +
make_ldst_ops(["m8n8k4"], ["a", "b", "c", "d"], ["f64"]) +
# TF32 m16n16k8 is odd.
# For fragment 'C' it uses __mma_*tf32*_m16n16k8_ld_c
# but 'D' calls __mma_m16n16k8_st_c_*f32*.
make_ldst_ops(["m16n16k8"], ["a", "b", "c"], ["tf32"]) +
make_ldst_ops(["m16n16k8"], ["d"], ["f32"]))
# NOTE: fragemts are from the point of view of PTX.
# fragment `d` is only for store ops, others for both loads and stores.
return (
make_ldst_ops(
["m16n16k16", "m32n8k16", "m8n32k16"],
["a", "b"],
["f16", "u8", "s8", "bf16"],
)
+ make_ldst_ops(
["m16n16k16", "m32n8k16", "m8n32k16"], ["c", "d"], ["f16", "f32", "s32"]
)
+ make_ldst_ops(["m8n8k32"], ["a", "b"], ["s4", "u4"])
+ make_ldst_ops(["m8n8k128"], ["a", "b"], ["b1"])
+ make_ldst_ops(["m8n8k32", "m8n8k128"], ["c", "d"], ["s32"])
+ make_ldst_ops(["m8n8k4"], ["a", "b", "c", "d"], ["f64"])
+
# TF32 m16n16k8 is odd.
# For fragment 'C' it uses __mma_*tf32*_m16n16k8_ld_c
# but 'D' calls __mma_m16n16k8_st_c_*f32*.
make_ldst_ops(["m16n16k8"], ["a", "b", "c"], ["tf32"])
+ make_ldst_ops(["m16n16k8"], ["d"], ["f32"])
)
def is_geom_supported(geom):
# geometries for FP and ints.
if geom in ["m8n32k16", "m32n8k16"]:
return ptx_version >= 61
# geometries for sub-ints.
if geom in ["m8n8k32", "m8n8k128"]:
return ptx_version >= 63 and gpu_arch >= 75
if geom == "m16n16k16":
return ptx_version >= 60
if geom in ["m16n16k8", "m8n8k4"]:
return ptx_version >= 70 and gpu_arch >= 80
assert(False) # Unexpected geometry.
# geometries for FP and ints.
if geom in ["m8n32k16", "m32n8k16"]:
return ptx_version >= 61
# geometries for sub-ints.
if geom in ["m8n8k32", "m8n8k128"]:
return ptx_version >= 63 and gpu_arch >= 75
if geom == "m16n16k16":
return ptx_version >= 60
if geom in ["m16n16k8", "m8n8k4"]:
return ptx_version >= 70 and gpu_arch >= 80
assert False # Unexpected geometry.
def is_type_supported(ptx_type):
if ptx_type in ["s8", "u8", "s32"]:
return ptx_version >= 63 and gpu_arch >= 72
if ptx_type in ["s4", "u4", "b1"]:
return ptx_version >= 63 and gpu_arch >= 75
if ptx_type in ["bf16", "tf32", "f64"]:
return ptx_version >= 70 and gpu_arch >= 80
return ptx_version >= 60 and gpu_arch >= 70
if ptx_type in ["s8", "u8", "s32"]:
return ptx_version >= 63 and gpu_arch >= 72
if ptx_type in ["s4", "u4", "b1"]:
return ptx_version >= 63 and gpu_arch >= 75
if ptx_type in ["bf16", "tf32", "f64"]:
return ptx_version >= 70 and gpu_arch >= 80
return ptx_version >= 60 and gpu_arch >= 70
def is_rnd_supported(op):
# rnd is only supported for FP64 WMMA
return op.a.ptx_type == "f64"
# rnd is only supported for FP64 WMMA
return op.a.ptx_type == "f64"
def is_mma_variant_supported(op, layout_a, layout_b, satf):
if not (is_type_supported(op.a.ptx_type)
and is_geom_supported(op.a.geom)):
return False
if not (is_type_supported(op.a.ptx_type) and is_geom_supported(op.a.geom)):
return False
if satf and not op.a.ptx_type in ["f16", "s8", "u8", "s4", "u4"]:
return False
if satf and not op.a.ptx_type in ["f16", "s8", "u8", "s4", "u4"]:
return False
# sub-integer types require row/col layout.
if op.a.ptx_type in ["s4", "u4", "b1"]:
return layout_a == "row" and layout_b == "col"
return True
# sub-integer types require row/col layout.
if op.a.ptx_type in ["s4", "u4", "b1"]:
return layout_a == "row" and layout_b == "col"
return True
def is_ldst_variant_supported(frag, layout):
if not (is_type_supported(frag.ptx_type)
and is_geom_supported(frag.geom)):
return False
if frag.ptx_type in ["s4", "u4", "b1"]:
# sub-integer types require sm_75 and ptx63, row/col layout for a/b.
return ((frag.frag == "a" and layout == "row")
if not (is_type_supported(frag.ptx_type) and is_geom_supported(frag.geom)):
return False
if frag.ptx_type in ["s4", "u4", "b1"]:
# sub-integer types require sm_75 and ptx63, row/col layout for a/b.
return (
(frag.frag == "a" and layout == "row")
or (frag.frag == "b" and layout == "col")
or frag.frag in ["c", "d"])
return True
or frag.frag in ["c", "d"]
)
return True
def get_builtin_prefix(frag):
prefix = None
if frag.geom in ["m16n16k16", "m32n8k16", "m8n32k16"]:
if frag.ptx_type in ["f16", "f32"]:
prefix = "__hmma"
elif frag.ptx_type == "bf16":
prefix = "__mma_bf16"
else:
prefix = "__imma"
elif frag.geom == "m8n8k32":
prefix = "__imma" # sub-integers
elif frag.geom == "m8n8k128":
prefix = "__bmma"
elif frag.geom == "m8n8k4":
prefix = "__dmma"
elif frag.geom == "m16n16k8":
if frag.ptx_type == "f32":
prefix = "__mma"
else:
prefix = "__mma_tf32"
assert prefix
return prefix
prefix = None
if frag.geom in ["m16n16k16", "m32n8k16", "m8n32k16"]:
if frag.ptx_type in ["f16", "f32"]:
prefix = "__hmma"
elif frag.ptx_type == "bf16":
prefix = "__mma_bf16"
else:
prefix = "__imma"
elif frag.geom == "m8n8k32":
prefix = "__imma" # sub-integers
elif frag.geom == "m8n8k128":
prefix = "__bmma"
elif frag.geom == "m8n8k4":
prefix = "__dmma"
elif frag.geom == "m16n16k8":
if frag.ptx_type == "f32":
prefix = "__mma"
else:
prefix = "__mma_tf32"
assert prefix
return prefix
def get_ldst_builtin_name(frag):
prefix = get_builtin_prefix(frag)
prefix = get_builtin_prefix(frag)
if prefix == "__hmma":
suffix = "" if frag.frag in ["a","b"] else frag.ptx_type
elif prefix in ["__dmma", "__mma_bf16", "__mma_tf32"]:
suffix = "" if frag.frag in ["a","b","c"] else frag.ptx_type
else:
suffix = "" if frag.frag == "c" else frag.ptx_type
if suffix == "s32":
suffix = "i32"
if prefix == "__hmma":
suffix = "" if frag.frag in ["a", "b"] else frag.ptx_type
elif prefix in ["__dmma", "__mma_bf16", "__mma_tf32"]:
suffix = "" if frag.frag in ["a", "b", "c"] else frag.ptx_type
else:
suffix = "" if frag.frag == "c" else frag.ptx_type
if suffix == "s32":
suffix = "i32"
if frag.frag == "d":
ifrag = "c"
op = "st"
else:
ifrag = frag.frag
op = "ld"
if frag.frag == "d":
ifrag = "c"
op = "st"
else:
ifrag = frag.frag
op = "ld"
name = "%s_%s_%s_%s%s" % (
prefix,
frag.geom,
op,
ifrag,
"_" + suffix if suffix else "",
)
return name
name = "%s_%s_%s_%s%s" % (prefix, frag.geom, op, ifrag,
"_" + suffix if suffix else "")
return name
def get_mma_builtin_name(op):
prefix = get_builtin_prefix(op.a)
prefix = get_builtin_prefix(op.a)
if prefix == "__hmma":
suffix = op.d.ptx_type + op.c.ptx_type
elif prefix in ["__mma_bf16", "__mma_tf32"]:
suffix = op.d.ptx_type
else:
suffix = op.a.ptx_type
if prefix == "__hmma":
suffix = op.d.ptx_type + op.c.ptx_type
elif prefix in ["__mma_bf16", "__mma_tf32"]:
suffix = op.d.ptx_type
else:
suffix = op.a.ptx_type
name = "{prefix}_{geom}_mma{b1op}_{suffix}".format(
prefix=prefix, geom=op.a.geom, b1op=op.b1op.replace(".", "_"), suffix=suffix
)
return name
name = "{prefix}_{geom}_mma{b1op}_{suffix}".format(
prefix = prefix,
geom = op.a.geom,
b1op = op.b1op.replace(".","_"),
suffix = suffix)
return name
def get_required_sm(frag, b1op=""):
if frag.ptx_type in ["f64", "bf16", "tf32"]:
return 80
if frag.ptx_type in ["u4", "s4", "b1"]:
if b1op == ".and.popc":
return 80
return 75
if frag.ptx_type in ["s8", "u8"]:
return 72
if frag.ptx_type == "s32":
if frag.geom in ["m8n8k32", "m8n8k128"]: # s4/u4/b1
return 75
else: # s8/u8
return 72
if frag.ptx_type in ["f16", "f32"]:
if frag.geom == "m16n16k8":
return 80
else:
return 70
assert(False)
if frag.ptx_type in ["f64", "bf16", "tf32"]:
return 80
if frag.ptx_type in ["u4", "s4", "b1"]:
if b1op == ".and.popc":
return 80
return 75
if frag.ptx_type in ["s8", "u8"]:
return 72
if frag.ptx_type == "s32":
if frag.geom in ["m8n8k32", "m8n8k128"]: # s4/u4/b1
return 75
else: # s8/u8
return 72
if frag.ptx_type in ["f16", "f32"]:
if frag.geom == "m16n16k8":
return 80
else:
return 70
assert False
def get_required_ptx(frag, b1op=""):
if frag.ptx_type == "b1" and b1op == ".and.popc":
return 71
if frag.ptx_type in ["f64", "bf16", "tf32"]:
return 70
if frag.ptx_type in ["f16", "f32"]:
if frag.geom == "m16n16k16":
return 60
if frag.geom == "m16n16k8":
return 70
return 61
return 63
if frag.ptx_type == "b1" and b1op == ".and.popc":
return 71
if frag.ptx_type in ["f64", "bf16", "tf32"]:
return 70
if frag.ptx_type in ["f16", "f32"]:
if frag.geom == "m16n16k16":
return 60
if frag.geom == "m16n16k8":
return 70
return 61
return 63
def get_src_dst_prefix(frag):
if frag.ptx_type == "f32":
return "f"
if frag.ptx_type == "f64":
return "d"
if frag.ptx_type == "tf32" and frag.frag in ["c", "d"]:
return "f"
return ""
if frag.ptx_type == "f32":
return "f"
if frag.ptx_type == "f64":
return "d"
if frag.ptx_type == "tf32" and frag.frag in ["c", "d"]:
return "f"
return ""
def gen_wmma_ldst_tests(results):
load_template = """
load_template = """
// CHECK${check_suffix}: call {{.*}} @${intrinsic}
// expected-error-re@+1 {{'${builtin}' needs target feature (sm_${min_sm}{{.*}},(ptx${min_ptx}{{.*}}}}
${builtin}(${dst}, ${src}, ldm, ${blayout});
""".rstrip()
intrinsic_template = "llvm.nvvm.wmma.${geom}.${op}.${frag}.${ilayout}.stride.${itype}"
intrinsic_template = (
"llvm.nvvm.wmma.${geom}.${op}.${frag}.${ilayout}.stride.${itype}"
)
for frag, layout in sorted(product(get_ldst_ops(), ["row","col"]), key=str):
for frag, layout in sorted(product(get_ldst_ops(), ["row", "col"]), key=str):
if not is_ldst_variant_supported(frag, layout):
continue
if not is_ldst_variant_supported(frag, layout):
continue
src_dst_prefix = get_src_dst_prefix(frag)
src_dst_prefix = get_src_dst_prefix(frag)
min_sm = get_required_sm(frag)
min_ptx = get_required_ptx(frag)
# TF32 uses f32 for accumulator loads.
if frag.geom == "m16n16k8" and frag.frag =="c":
assert frag.ptx_type == "tf32"
itype = "f32"
else:
itype = frag.ptx_type
min_sm = get_required_sm(frag)
min_ptx = get_required_ptx(frag)
# TF32 uses f32 for accumulator loads.
if frag.geom == "m16n16k8" and frag.frag == "c":
assert frag.ptx_type == "tf32"
itype = "f32"
else:
itype = frag.ptx_type
params = {
"check_suffix" : "_PTX%d_SM%d" % (min_ptx, min_sm),
"builtin" : get_ldst_builtin_name(frag),
"min_ptx" : min_ptx,
"min_sm" : min_sm,
"dst": src_dst_prefix + "dst",
"src": src_dst_prefix + "src",
"blayout" : 0 if layout == "row" else 1,
"intrinsic" : Template(intrinsic_template).substitute({
"frag" : frag.frag,
"geom" : frag.geom,
"ilayout" : layout,
"itype" : itype,
"op" : "store" if frag.frag == "d" else "load",
})
}
results[(min_ptx,min_sm)] += Template(load_template).substitute(params)
params = {
"check_suffix": "_PTX%d_SM%d" % (min_ptx, min_sm),
"builtin": get_ldst_builtin_name(frag),
"min_ptx": min_ptx,
"min_sm": min_sm,
"dst": src_dst_prefix + "dst",
"src": src_dst_prefix + "src",
"blayout": 0 if layout == "row" else 1,
"intrinsic": Template(intrinsic_template).substitute(
{
"frag": frag.frag,
"geom": frag.geom,
"ilayout": layout,
"itype": itype,
"op": "store" if frag.frag == "d" else "load",
}
),
}
results[(min_ptx, min_sm)] += Template(load_template).substitute(params)
return results
return results
def mma_signature(op):
if op.a.ptx_type == "f16":
# FP16 ops identified by accumulator & result type.
return "%s.%s" % (op.d.ptx_type, op.c.ptx_type)
else:
# other ops are identified by input type.
return op.a.ptx_type
if op.a.ptx_type == "f16":
# FP16 ops identified by accumulator & result type.
return "%s.%s" % (op.d.ptx_type, op.c.ptx_type)
else:
# other ops are identified by input type.
return op.a.ptx_type
# Get numeric value for rowcol parameter of the builtin
# AFAICT it uses the encoding accepted by NVVM intrinsics:
# https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#nvvm-intrin-warp-level-matrix-mma
def get_ilayout(a, b):
return {
"row.row" : 0,
"row.col" : 1,
"col.row" : 2,
"col.col" : 3
}[a + "." + b]
return {"row.row": 0, "row.col": 1, "col.row": 2, "col.col": 3}[a + "." + b]
def gen_wmma_mma_tests(results):
mma_template = """
mma_template = """
// CHECK${check_suffix}: call {{.*}} @${intrinsic}
// expected-error-re@+1 {{'${builtin}' needs target feature (sm_${min_sm}{{.*}},(ptx${min_ptx}{{.*}}}}
${builtin}(${dst}, ${asrc}, ${asrc}, ${csrc}, ${ilayout}${maybe_satf});
""".rstrip()
intrinsic_template = "llvm.nvvm.wmma.${geom}.mma${b1op}.${alayout}.${blayout}.${intrinsic_signature}${satf}"
intrinsic_template = "llvm.nvvm.wmma.${geom}.mma${b1op}.${alayout}.${blayout}.${intrinsic_signature}${satf}"
for op, alayout, blayout, satf in sorted(product( get_mma_ops(),
["row","col"],
["row","col"],
[".satfinite", ""]),
key=str):
for op, alayout, blayout, satf in sorted(
product(get_mma_ops(), ["row", "col"], ["row", "col"], [".satfinite", ""]),
key=str,
):
if not is_mma_variant_supported(op, alayout, blayout, satf):
continue
if not is_mma_variant_supported(op, alayout, blayout, satf):
continue
asrc_prefix = get_src_dst_prefix(op.a)
csrc_prefix = get_src_dst_prefix(op.c)
ddst_prefix = get_src_dst_prefix(op.d)
if op.a.ptx_type == "b1": # .b1 MMA has no satf argument.
isatf_arg = ""
else:
isatf_arg = ", 1" if satf else ", 0"
min_sm = get_required_sm(op.a, op.b1op)
min_ptx = get_required_ptx(op.a, op.b1op)
params = {
"check_suffix" : "_PTX%d_SM%d" % (min_ptx, min_sm),
"builtin" : get_mma_builtin_name(op),
"min_ptx" : min_ptx,
"min_sm" : min_sm,
"dst": ddst_prefix + "dst",
"asrc": asrc_prefix + "src",
"csrc": csrc_prefix + "src",
"ilayout" : get_ilayout(alayout, blayout),
"maybe_satf" : isatf_arg,
"intrinsic" : Template(intrinsic_template).substitute({
"geom" : op.a.geom,
"alayout" : alayout,
"blayout" : blayout,
"intrinsic_signature" : mma_signature(op),
"satf" : satf,
"b1op" : op.b1op
})
}
results[(min_ptx, min_sm)] += Template(mma_template).substitute(params)
asrc_prefix = get_src_dst_prefix(op.a)
csrc_prefix = get_src_dst_prefix(op.c)
ddst_prefix = get_src_dst_prefix(op.d)
if op.a.ptx_type == "b1": # .b1 MMA has no satf argument.
isatf_arg = ""
else:
isatf_arg = ", 1" if satf else ", 0"
min_sm = get_required_sm(op.a, op.b1op)
min_ptx = get_required_ptx(op.a, op.b1op)
params = {
"check_suffix": "_PTX%d_SM%d" % (min_ptx, min_sm),
"builtin": get_mma_builtin_name(op),
"min_ptx": min_ptx,
"min_sm": min_sm,
"dst": ddst_prefix + "dst",
"asrc": asrc_prefix + "src",
"csrc": csrc_prefix + "src",
"ilayout": get_ilayout(alayout, blayout),
"maybe_satf": isatf_arg,
"intrinsic": Template(intrinsic_template).substitute(
{
"geom": op.a.geom,
"alayout": alayout,
"blayout": blayout,
"intrinsic_signature": mma_signature(op),
"satf": satf,
"b1op": op.b1op,
}
),
}
results[(min_ptx, min_sm)] += Template(mma_template).substitute(params)
return results
return results
def gen_tests():
results = gen_wmma_ldst_tests(defaultdict(str))
results = gen_wmma_mma_tests(results)
results = gen_wmma_ldst_tests(defaultdict(str))
results = gen_wmma_mma_tests(results)
run_template = r"""
run_template = r"""
//
// *** DO NOT EDIT ***
//
@ -376,20 +419,30 @@ def gen_tests():
// ${run}: -DPTX=${ptx} -DSM=${sm} -fcuda-is-device -S -o /dev/null -x cuda \
// ${run}: -verify %s
"""
def supported_variants(ptx, sm, results):
return [(ptx_, sm_) for ptx_, sm_ in results if ptx_ <= ptx and sm_ <= sm]
print(Template(run_template).substitute({
"run" : "RUN", # To avoid lit misinterpreting the template
"ptx" : ptx_version,
"sm" : gpu_arch,
"check_labels" : ",".join(["CHECK_PTX%d_SM%d" % (ptx_, sm_)
for ptx_, sm_
in supported_variants(ptx_version, gpu_arch,
results)])
}))
def supported_variants(ptx, sm, results):
return [(ptx_, sm_) for ptx_, sm_ in results if ptx_ <= ptx and sm_ <= sm]
print("""
print(
Template(run_template).substitute(
{
"run": "RUN", # To avoid lit misinterpreting the template
"ptx": ptx_version,
"sm": gpu_arch,
"check_labels": ",".join(
[
"CHECK_PTX%d_SM%d" % (ptx_, sm_)
for ptx_, sm_ in supported_variants(
ptx_version, gpu_arch, results
)
]
),
}
)
)
print(
"""
#if !defined(CUDA_VERSION)
#define __device__ __attribute__((device))
#define __global__ __attribute__((global))
@ -403,15 +456,17 @@ typedef unsigned long long uint64_t;
__device__ void test_wmma_buitins(int *src, int *dst,
float *fsrc, float *fdst,
double *dsrc, double *ddst, int ldm) {
""");
"""
)
for (ptx, sm), tests in sorted(results.items()):
print()
print("#if (PTX >= %d) && (SM >= %d)" % (ptx, sm))
print(tests)
print("#endif // (PTX >= %d) && (SM >= %d)"% (ptx, sm))
for (ptx, sm), tests in sorted(results.items()):
print()
print("#if (PTX >= %d) && (SM >= %d)" % (ptx, sm))
print(tests)
print("#endif // (PTX >= %d) && (SM >= %d)" % (ptx, sm))
print("}")
print("}")
parser = argparse.ArgumentParser()
parser.add_argument("--ptx", type=int, default=60)

2
clang/test/CodeGenHIP/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.cpp', '.hip']
config.suffixes = [".cpp", ".hip"]

2
clang/test/CodeGenHLSL/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.c', '.hlsl']
config.suffixes = [".c", ".hlsl"]

4
clang/test/CodeGenObjC/lit.local.cfg
View File

@ -1,2 +1,2 @@
if any(target in config.target_triple for target in ('aix', 'zos')):
config.unsupported = True
if any(target in config.target_triple for target in ("aix", "zos")):
config.unsupported = True

4
clang/test/CodeGenObjCXX/lit.local.cfg
View File

@ -1,2 +1,2 @@
if any(target in config.target_triple for target in ('aix', 'zos')):
config.unsupported = True
if any(target in config.target_triple for target in ("aix", "zos")):
config.unsupported = True

27
clang/test/Driver/XRay/lit.local.cfg
View File

@ -2,22 +2,29 @@ import platform
# Only run the tests in platforms where XRay instrumentation is supported.
supported_targets = [
'amd64', 'x86_64', 'x86_64h', 'arm', 'aarch64', 'arm64', 'powerpc64le', 'mips',
'mipsel', 'mips64', 'mips64el'
"amd64",
"x86_64",
"x86_64h",
"arm",
"aarch64",
"arm64",
"powerpc64le",
"mips",
"mipsel",
"mips64",
"mips64el",
]
# Only on platforms we support.
supported_oses = [
'Linux', 'FreeBSD', 'Darwin'
]
supported_oses = ["Linux", "FreeBSD", "Darwin"]
triple_set = set(config.target_triple.split('-'))
triple_set = set(config.target_triple.split("-"))
if len(triple_set.intersection(supported_targets)) == 0:
config.unsupported = True
config.unsupported = True
# Do not run for 'android' despite being linux.
if platform.system() not in supported_oses or 'android' in triple_set:
config.unsupported = True
if platform.system() not in supported_oses or "android" in triple_set:
config.unsupported = True
if config.enable_shared:
config.available_features.update(['enable_shared'])
config.available_features.update(["enable_shared"])

34
clang/test/Driver/ftime-trace-sections.py
View File

@ -2,15 +2,21 @@
import json, sys, time
def is_inside(range1, range2):
a = range1["ts"]; b = a + range1["dur"]
c = range2["ts"]; d = c + range2["dur"]
a = range1["ts"]
b = a + range1["dur"]
c = range2["ts"]
d = c + range2["dur"]
return (a >= c and a <= d) and (b >= c and b <= d)
def is_before(range1, range2):
b = range1["ts"] + range1["dur"]; c = range2["ts"]
b = range1["ts"] + range1["dur"]
c = range2["ts"]
return b <= c
log_contents = json.loads(sys.stdin.read())
events = log_contents["traceEvents"]
codegens = [event for event in events if event["name"] == "CodeGen Function"]
@ -22,13 +28,23 @@ seconds_since_epoch = time.time()
# Make sure that the 'beginningOfTime' is not later than now.
if beginning_of_time > seconds_since_epoch:
sys.exit("'beginningOfTime' should represent the absolute time when the "
"process has started")
sys.exit(
"'beginningOfTime' should represent the absolute time when the "
"process has started"
)
if not all([any([is_inside(codegen, frontend) for frontend in frontends])
for codegen in codegens]):
if not all(
[
any([is_inside(codegen, frontend) for frontend in frontends])
for codegen in codegens
]
):
sys.exit("Not all CodeGen sections are inside any Frontend section!")
if not all([all([is_before(frontend, backend) for frontend in frontends])
for backend in backends]):
if not all(
[
all([is_before(frontend, backend) for frontend in frontends])
for backend in backends
]
):
sys.exit("Not all Frontend section are before all Backend sections!")

51
clang/test/Driver/lit.local.cfg
View File

@ -1,26 +1,49 @@
from lit.llvm import llvm_config
config.suffixes = ['.c', '.cpp', '.cppm', '.h', '.m', '.mm', '.S', '.s', '.f90', '.F90', '.f95',
'.cu', '.rs', '.cl', '.clcpp', '.hip', '.hipi', '.hlsl']
config.suffixes = [
".c",
".cpp",
".cppm",
".h",
".m",
".mm",
".S",
".s",
".f90",
".F90",
".f95",
".cu",
".rs",
".cl",
".clcpp",
".hip",
".hipi",
".hlsl",
]
config.substitutions = list(config.substitutions)
config.substitutions.insert(0,
('%clang_cc1',
"""*** Do not use 'clang -cc1' in Driver tests. ***""") )
config.substitutions.insert(
0, ("%clang_cc1", """*** Do not use 'clang -cc1' in Driver tests. ***""")
)
# Remove harmful environmental variables for clang Driver tests.
# Some might be useful for other tests so they are only removed here.
driver_overwrite_env_vars = ['MACOSX_DEPLOYMENT_TARGET',
'IPHONEOS_DEPLOYMENT_TARGET',
'SDKROOT', 'CCC_OVERRIDE_OPTIONS',
'CC_PRINT_OPTIONS', 'CC_PRINT_HEADERS',
'CC_LOG_DIAGNOSTICS', 'CC_PRINT_PROC_STAT']
driver_overwrite_env_vars = [
"MACOSX_DEPLOYMENT_TARGET",
"IPHONEOS_DEPLOYMENT_TARGET",
"SDKROOT",
"CCC_OVERRIDE_OPTIONS",
"CC_PRINT_OPTIONS",
"CC_PRINT_HEADERS",
"CC_LOG_DIAGNOSTICS",
"CC_PRINT_PROC_STAT",
]
for name in driver_overwrite_env_vars:
if name in config.environment:
del config.environment[name]
if name in config.environment:
del config.environment[name]
if llvm_config.use_lld(required=False):
config.available_features.add('lld')
config.available_features.add("lld")
if config.ppc_linux_default_ieeelongdouble:
config.available_features.add('ppc_linux_default_ieeelongdouble')
config.available_features.add("ppc_linux_default_ieeelongdouble")

21
clang/test/Format/lit.local.cfg
View File

@ -1,4 +1,19 @@
# Suffixes supported by clang-format.
config.suffixes = ['.c', '.cc', '.cpp', '.h', '.m', '.mm', '.java', '.js',
'.ts', '.proto', '.protodevel', '.pb.txt', '.textproto',
'.textpb', '.asciipb', '.td']
config.suffixes = [
".c",
".cc",
".cpp",
".h",
".m",
".mm",
".java",
".js",
".ts",
".proto",
".protodevel",
".pb.txt",
".textproto",
".textpb",
".asciipb",
".td",
]

2
clang/test/Frontend/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.c', '.cpp', '.m', '.mm', '.ll', '.cl', '.test']
config.suffixes = [".c", ".cpp", ".m", ".mm", ".ll", ".cl", ".test"]

4
clang/test/Headers/lit.local.cfg
View File

@ -1,4 +1,6 @@
config.substitutions = list(config.substitutions)
# Enable substituting Clang Sema source directory for TableGen input.
config.substitutions.append(('%clang_src_sema_dir', os.path.join(config.clang_src_dir, 'lib', 'Sema')))
config.substitutions.append(
("%clang_src_sema_dir", os.path.join(config.clang_src_dir, "lib", "Sema"))
)

2
clang/test/Index/skip-parsed-bodies/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.json']
config.suffixes = [".json"]

4
clang/test/Interpreter/lit.local.cfg
View File

@ -1,2 +1,2 @@
if 'host-supports-jit' not in config.available_features:
config.unsupported = True
if "host-supports-jit" not in config.available_features:
config.unsupported = True

4
clang/test/LibClang/lit.local.cfg
View File

@ -1,2 +1,4 @@
config.substitutions.append(('%libclang', os.path.join(config.clang_lib_dir, 'libclang.so')))
config.substitutions.append(
("%libclang", os.path.join(config.clang_lib_dir, "libclang.so"))
)
config.pipefail = False

4
clang/test/OpenMP/lit.local.cfg
View File

@ -1,5 +1,5 @@
# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
from lit.llvm.subst import ToolSubst
fc = ToolSubst('FileCheck', unresolved='fatal')
config.substitutions.insert(0, (fc.regex, 'FileCheck --allow-unused-prefixes'))
fc = ToolSubst("FileCheck", unresolved="fatal")
config.substitutions.insert(0, (fc.regex, "FileCheck --allow-unused-prefixes"))

2
clang/test/ParserHLSL/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.hlsl']
config.suffixes = [".hlsl"]

6
clang/test/Sema/lit.local.cfg
View File

@ -1,4 +1,4 @@
config.substitutions = list(config.substitutions)
config.substitutions.insert(0,
(r'%clang\b',
"""*** Do not use the driver in Sema tests. ***""") )
config.substitutions.insert(
0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
)

6
clang/test/SemaCUDA/lit.local.cfg
View File

@ -1,4 +1,4 @@
config.substitutions = list(config.substitutions)
config.substitutions.insert(0,
(r'%clang\b',
"""*** Do not use the driver in Sema tests. ***""") )
config.substitutions.insert(
0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
)

2
clang/test/SemaHLSL/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.hlsl']
config.suffixes = [".hlsl"]

6
clang/test/SemaObjCXX/lit.local.cfg
View File

@ -1,4 +1,4 @@
config.substitutions = list(config.substitutions)
config.substitutions.insert(0,
(r'%clang\b',
"""*** Do not use the driver in Sema tests. ***""") )
config.substitutions.insert(
0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
)

6
clang/test/SemaOpenCL/lit.local.cfg
View File

@ -1,4 +1,4 @@
config.substitutions = list(config.substitutions)
config.substitutions.insert(0,
(r'%clang\b',
"""*** Do not use the driver in Sema tests. ***""") )
config.substitutions.insert(
0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
)

2
clang/test/TableGen/lit.local.cfg
View File

@ -1 +1 @@
config.suffixes = ['.td']
config.suffixes = [".td"]

70
clang/test/Unit/lit.cfg.py
View File

@ -10,67 +10,73 @@ import lit.formats
import lit.util
# name: The name of this test suite.
config.name = 'Clang-Unit'
config.name = "Clang-Unit"
# suffixes: A list of file extensions to treat as test files.
config.suffixes = []
# test_source_root: The root path where tests are located.
# test_exec_root: The root path where tests should be run.
config.test_exec_root = os.path.join(config.clang_obj_root, 'unittests')
config.test_exec_root = os.path.join(config.clang_obj_root, "unittests")
config.test_source_root = config.test_exec_root
# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, 'Tests')
config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")
# Propagate the temp directory. Windows requires this because it uses \Windows\
# if none of these are present.
if 'TMP' in os.environ:
config.environment['TMP'] = os.environ['TMP']
if 'TEMP' in os.environ:
config.environment['TEMP'] = os.environ['TEMP']
if "TMP" in os.environ:
config.environment["TMP"] = os.environ["TMP"]
if "TEMP" in os.environ:
config.environment["TEMP"] = os.environ["TEMP"]
if 'HOME' in os.environ:
config.environment['HOME'] = os.environ['HOME']
if "HOME" in os.environ:
config.environment["HOME"] = os.environ["HOME"]
# Propagate sanitizer options.
for var in [
'ASAN_SYMBOLIZER_PATH',
'HWASAN_SYMBOLIZER_PATH',
'MSAN_SYMBOLIZER_PATH',
'TSAN_SYMBOLIZER_PATH',
'UBSAN_SYMBOLIZER_PATH',
'ASAN_OPTIONS',
'HWASAN_OPTIONS',
'MSAN_OPTIONS',
'TSAN_OPTIONS',
'UBSAN_OPTIONS',
"ASAN_SYMBOLIZER_PATH",
"HWASAN_SYMBOLIZER_PATH",
"MSAN_SYMBOLIZER_PATH",
"TSAN_SYMBOLIZER_PATH",
"UBSAN_SYMBOLIZER_PATH",
"ASAN_OPTIONS",
"HWASAN_OPTIONS",
"MSAN_OPTIONS",
"TSAN_OPTIONS",
"UBSAN_OPTIONS",
]:
if var in os.environ:
config.environment[var] = os.environ[var]
def find_shlibpath_var():
if platform.system() in ['Linux', 'FreeBSD', 'NetBSD', 'OpenBSD', 'SunOS']:
yield 'LD_LIBRARY_PATH'
elif platform.system() == 'Darwin':
yield 'DYLD_LIBRARY_PATH'
elif platform.system() == 'Windows':
yield 'PATH'
elif platform.system() == 'AIX':
yield 'LIBPATH'
if platform.system() in ["Linux", "FreeBSD", "NetBSD", "OpenBSD", "SunOS"]:
yield "LD_LIBRARY_PATH"
elif platform.system() == "Darwin":
yield "DYLD_LIBRARY_PATH"
elif platform.system() == "Windows":
yield "PATH"
elif platform.system() == "AIX":
yield "LIBPATH"
for shlibpath_var in find_shlibpath_var():
# in stand-alone builds, shlibdir is clang's build tree
# while llvm_libs_dir is installed LLVM (and possibly older clang)
shlibpath = os.path.pathsep.join(
(config.shlibdir,
config.llvm_libs_dir,
config.environment.get(shlibpath_var, '')))
(
config.shlibdir,
config.llvm_libs_dir,
config.environment.get(shlibpath_var, ""),
)
)
config.environment[shlibpath_var] = shlibpath
break
else:
lit_config.warning("unable to inject shared library path on '{}'"
.format(platform.system()))
lit_config.warning(
"unable to inject shared library path on '{}'".format(platform.system())
)
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

284
clang/test/lit.cfg.py
View File

@ -16,7 +16,7 @@ from lit.llvm.subst import FindTool
# Configuration file for the 'lit' test runner.
# name: The name of this test suite.
config.name = 'Clang'
config.name = "Clang"
# testFormat: The test format to use to interpret tests.
#
@ -25,31 +25,54 @@ config.name = 'Clang'
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.c', '.cpp', '.i', '.cppm', '.m', '.mm', '.cu', '.hip', '.hlsl',
'.ll', '.cl', '.clcpp', '.s', '.S', '.modulemap', '.test', '.rs', '.ifs', '.rc']
config.suffixes = [
".c",
".cpp",
".i",
".cppm",
".m",
".mm",
".cu",
".hip",
".hlsl",
".ll",
".cl",
".clcpp",
".s",
".S",
".modulemap",
".test",
".rs",
".ifs",
".rc",
]
# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
# subdirectories contain auxiliary inputs for various tests in their parent
# directories.
config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt', 'debuginfo-tests']
config.excludes = [
"Inputs",
"CMakeLists.txt",
"README.txt",
"LICENSE.txt",
"debuginfo-tests",
]
# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
# test_exec_root: The root path where tests should be run.
config.test_exec_root = os.path.join(config.clang_obj_root, 'test')
config.test_exec_root = os.path.join(config.clang_obj_root, "test")
llvm_config.use_default_substitutions()
llvm_config.use_clang()
config.substitutions.append(
('%src_include_dir', config.clang_src_dir + '/include'))
config.substitutions.append(("%src_include_dir", config.clang_src_dir + "/include"))
config.substitutions.append(
('%target_triple', config.target_triple))
config.substitutions.append(("%target_triple", config.target_triple))
config.substitutions.append(('%PATH%', config.environment['PATH']))
config.substitutions.append(("%PATH%", config.environment["PATH"]))
# For each occurrence of a clang tool name, replace it with the full path to
@ -59,207 +82,262 @@ config.substitutions.append(('%PATH%', config.environment['PATH']))
tool_dirs = [config.clang_tools_dir, config.llvm_tools_dir]
tools = [
'apinotes-test', 'c-index-test', 'clang-diff', 'clang-format', 'clang-repl', 'clang-offload-packager',
'clang-tblgen', 'clang-scan-deps', 'opt', 'llvm-ifs', 'yaml2obj', 'clang-linker-wrapper',
'llvm-lto', 'llvm-lto2', 'llvm-profdata',
ToolSubst('%clang_extdef_map', command=FindTool(
'clang-extdef-mapping'), unresolved='ignore'),
"apinotes-test",
"c-index-test",
"clang-diff",
"clang-format",
"clang-repl",
"clang-offload-packager",
"clang-tblgen",
"clang-scan-deps",
"opt",
"llvm-ifs",
"yaml2obj",
"clang-linker-wrapper",
"llvm-lto",
"llvm-lto2",
"llvm-profdata",
ToolSubst(
"%clang_extdef_map",
command=FindTool("clang-extdef-mapping"),
unresolved="ignore",
),
]
if config.clang_examples:
config.available_features.add('examples')
config.available_features.add("examples")
def have_host_jit_feature_support(feature_name):
clang_repl_exe = lit.util.which('clang-repl', config.clang_tools_dir)
clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
if not clang_repl_exe:
return False
try:
clang_repl_cmd = subprocess.Popen(
[clang_repl_exe, '--host-supports-' + feature_name], stdout=subprocess.PIPE)
[clang_repl_exe, "--host-supports-" + feature_name], stdout=subprocess.PIPE
)
except OSError:
print('could not exec clang-repl')
print("could not exec clang-repl")
return False
clang_repl_out = clang_repl_cmd.stdout.read().decode('ascii')
clang_repl_out = clang_repl_cmd.stdout.read().decode("ascii")
clang_repl_cmd.wait()
return 'true' in clang_repl_out
return "true" in clang_repl_out
if have_host_jit_feature_support('jit'):
config.available_features.add('host-supports-jit')
if have_host_jit_feature_support("jit"):
config.available_features.add("host-supports-jit")
if config.clang_staticanalyzer:
config.available_features.add('staticanalyzer')
tools.append('clang-check')
config.available_features.add("staticanalyzer")
tools.append("clang-check")
if config.clang_staticanalyzer_z3:
config.available_features.add('z3')
config.available_features.add("z3")
else:
config.available_features.add('no-z3')
config.available_features.add("no-z3")
check_analyzer_fixit_path = os.path.join(
config.test_source_root, "Analysis", "check-analyzer-fixit.py")
config.test_source_root, "Analysis", "check-analyzer-fixit.py"
)
config.substitutions.append(
('%check_analyzer_fixit',
'"%s" %s' % (config.python_executable, check_analyzer_fixit_path)))
(
"%check_analyzer_fixit",
'"%s" %s' % (config.python_executable, check_analyzer_fixit_path),
)
)
llvm_config.add_tool_substitutions(tools, tool_dirs)
config.substitutions.append(
('%hmaptool', "'%s' %s" % (config.python_executable,
os.path.join(config.clang_src_dir, 'utils', 'hmaptool', 'hmaptool'))))
(
"%hmaptool",
"'%s' %s"
% (
config.python_executable,
os.path.join(config.clang_src_dir, "utils", "hmaptool", "hmaptool"),
),
)
)
config.substitutions.append(
('%deps-to-rsp',
'"%s" %s' % (config.python_executable, os.path.join(config.clang_src_dir, 'utils',
'module-deps-to-rsp.py'))))
(
"%deps-to-rsp",
'"%s" %s'
% (
config.python_executable,
os.path.join(config.clang_src_dir, "utils", "module-deps-to-rsp.py"),
),
)
)
config.substitutions.append(('%host_cc', config.host_cc))
config.substitutions.append(('%host_cxx', config.host_cxx))
config.substitutions.append(("%host_cc", config.host_cc))
config.substitutions.append(("%host_cxx", config.host_cxx))
# Plugins (loadable modules)
if config.has_plugins and config.llvm_plugin_ext:
config.available_features.add('plugins')
config.available_features.add("plugins")
if config.clang_default_pie_on_linux:
config.available_features.add('default-pie-on-linux')
config.available_features.add("default-pie-on-linux")
# Set available features we allow tests to conditionalize on.
#
if config.clang_default_cxx_stdlib != '':
config.available_features.add('default-cxx-stdlib={}'.format(config.clang_default_cxx_stdlib))
if config.clang_default_cxx_stdlib != "":
config.available_features.add(
"default-cxx-stdlib={}".format(config.clang_default_cxx_stdlib)
)
# As of 2011.08, crash-recovery tests still do not pass on FreeBSD.
if platform.system() not in ['FreeBSD']:
config.available_features.add('crash-recovery')
if platform.system() not in ["FreeBSD"]:
config.available_features.add("crash-recovery")
# ANSI escape sequences in non-dumb terminal
if platform.system() not in ['Windows']:
config.available_features.add('ansi-escape-sequences')
if platform.system() not in ["Windows"]:
config.available_features.add("ansi-escape-sequences")
# Capability to print utf8 to the terminal.
# Windows expects codepage, unless Wide API.
if platform.system() not in ['Windows']:
config.available_features.add('utf8-capable-terminal')
if platform.system() not in ["Windows"]:
config.available_features.add("utf8-capable-terminal")
# Support for libgcc runtime. Used to rule out tests that require
# clang to run with -rtlib=libgcc.
if platform.system() not in ['Darwin', 'Fuchsia']:
config.available_features.add('libgcc')
if platform.system() not in ["Darwin", "Fuchsia"]:
config.available_features.add("libgcc")
# Case-insensitive file system
def is_filesystem_case_insensitive():
handle, path = tempfile.mkstemp(
prefix='case-test', dir=config.test_exec_root)
handle, path = tempfile.mkstemp(prefix="case-test", dir=config.test_exec_root)
isInsensitive = os.path.exists(
os.path.join(
os.path.dirname(path),
os.path.basename(path).upper()
))
os.path.join(os.path.dirname(path), os.path.basename(path).upper())
)
os.close(handle)
os.remove(path)
return isInsensitive
if is_filesystem_case_insensitive():
config.available_features.add('case-insensitive-filesystem')
config.available_features.add("case-insensitive-filesystem")
# Tests that require the /dev/fd filesystem.
if os.path.exists('/dev/fd/0') and sys.platform not in ['cygwin']:
config.available_features.add('dev-fd-fs')
if os.path.exists("/dev/fd/0") and sys.platform not in ["cygwin"]:
config.available_features.add("dev-fd-fs")
# Set on native MS environment.
if re.match(r'.*-(windows-msvc)$', config.target_triple):
config.available_features.add('ms-sdk')
if re.match(r".*-(windows-msvc)$", config.target_triple):
config.available_features.add("ms-sdk")
# [PR8833] LLP64-incompatible tests
if not re.match(r'^(aarch64|x86_64).*-(windows-msvc|windows-gnu)$', config.target_triple):
config.available_features.add('LP64')
if not re.match(
r"^(aarch64|x86_64).*-(windows-msvc|windows-gnu)$", config.target_triple
):
config.available_features.add("LP64")
# Tests that are specific to the Apple Silicon macOS.
if re.match(r'^arm64(e)?-apple-(macos|darwin)', config.target_triple):
config.available_features.add('apple-silicon-mac')
if re.match(r"^arm64(e)?-apple-(macos|darwin)", config.target_triple):
config.available_features.add("apple-silicon-mac")
# [PR18856] Depends to remove opened file. On win32, a file could be removed
# only if all handles were closed.
if platform.system() not in ['Windows']:
config.available_features.add('can-remove-opened-file')
if platform.system() not in ["Windows"]:
config.available_features.add("can-remove-opened-file")
# Features
known_arches = ["x86_64", "mips64", "ppc64", "aarch64"]
if (any(config.target_triple.startswith(x) for x in known_arches)):
config.available_features.add("clang-target-64-bits")
if any(config.target_triple.startswith(x) for x in known_arches):
config.available_features.add("clang-target-64-bits")
def calculate_arch_features(arch_string):
features = []
for arch in arch_string.split():
features.append(arch.lower() + '-registered-target')
features.append(arch.lower() + "-registered-target")
return features
llvm_config.feature_config(
[('--assertion-mode', {'ON': 'asserts'}),
('--cxxflags', {r'-D_GLIBCXX_DEBUG\b': 'libstdcxx-safe-mode'}),
('--targets-built', calculate_arch_features),
])
[
("--assertion-mode", {"ON": "asserts"}),
("--cxxflags", {r"-D_GLIBCXX_DEBUG\b": "libstdcxx-safe-mode"}),
("--targets-built", calculate_arch_features),
]
)
if lit.util.which('xmllint'):
config.available_features.add('xmllint')
if lit.util.which("xmllint"):
config.available_features.add("xmllint")
if config.enable_backtrace:
config.available_features.add('backtrace')
config.available_features.add("backtrace")
if config.enable_threads:
config.available_features.add('thread_support')
config.available_features.add("thread_support")
# Check if we should allow outputs to console.
run_console_tests = int(lit_config.params.get('enable_console', '0'))
run_console_tests = int(lit_config.params.get("enable_console", "0"))
if run_console_tests != 0:
config.available_features.add('console')
config.available_features.add("console")
lit.util.usePlatformSdkOnDarwin(config, lit_config)
macOSSDKVersion = lit.util.findPlatformSdkVersionOnMacOS(config, lit_config)
if macOSSDKVersion is not None:
config.available_features.add('macos-sdk-' + str(macOSSDKVersion))
config.available_features.add("macos-sdk-" + str(macOSSDKVersion))
if os.path.exists('/etc/gentoo-release'):
config.available_features.add('gentoo')
if os.path.exists("/etc/gentoo-release"):
config.available_features.add("gentoo")
if config.enable_shared:
config.available_features.add("enable_shared")
# Add a vendor-specific feature.
if config.clang_vendor_uti:
config.available_features.add('clang-vendor=' + config.clang_vendor_uti)
config.available_features.add("clang-vendor=" + config.clang_vendor_uti)
if config.have_llvm_driver:
config.available_features.add('llvm-driver')
config.available_features.add("llvm-driver")
def exclude_unsupported_files_for_aix(dirname):
for filename in os.listdir(dirname):
source_path = os.path.join( dirname, filename)
source_path = os.path.join(dirname, filename)
if os.path.isdir(source_path):
continue
f = open(source_path, 'r', encoding='ISO-8859-1')
f = open(source_path, "r", encoding="ISO-8859-1")
try:
data = f.read()
# 64-bit object files are not supported on AIX, so exclude the tests.
if (any(option in data for option in ('-emit-obj', '-fmodule-format=obj', '-fintegrated-as')) and
'64' in config.target_triple):
config.excludes += [ filename ]
data = f.read()
# 64-bit object files are not supported on AIX, so exclude the tests.
if (
any(
option in data
for option in (
"-emit-obj",
"-fmodule-format=obj",
"-fintegrated-as",
)
)
and "64" in config.target_triple
):
config.excludes += [filename]
finally:
f.close()
f.close()
if 'aix' in config.target_triple:
for directory in ('/CodeGenCXX', '/Misc', '/Modules', '/PCH', '/Driver',
'/ASTMerge/anonymous-fields', '/ASTMerge/injected-class-name-decl'):
if "aix" in config.target_triple:
for directory in (
"/CodeGenCXX",
"/Misc",
"/Modules",
"/PCH",
"/Driver",
"/ASTMerge/anonymous-fields",
"/ASTMerge/injected-class-name-decl",
):
exclude_unsupported_files_for_aix(config.test_source_root + directory)
# Some tests perform deep recursion, which requires a larger pthread stack size
@ -268,10 +346,10 @@ if 'aix' in config.target_triple:
# a larger pthread stack size for the tests. Various applications and runtime
# libraries on AIX use a default pthread stack size of 4 MiB, so we will use
# that as a default value here.
if 'AIXTHREAD_STK' in os.environ:
config.environment['AIXTHREAD_STK'] = os.environ['AIXTHREAD_STK']
elif platform.system() == 'AIX':
config.environment['AIXTHREAD_STK'] = '4194304'
if "AIXTHREAD_STK" in os.environ:
config.environment["AIXTHREAD_STK"] = os.environ["AIXTHREAD_STK"]
elif platform.system() == "AIX":
config.environment["AIXTHREAD_STK"] = "4194304"
# The llvm-nm tool supports an environment variable "OBJECT_MODE" on AIX OS, which
# controls the kind of objects they will support. If there is no "OBJECT_MODE"
@ -280,9 +358,9 @@ elif platform.system() == 'AIX':
# 32-bit and 64-bit objects by default, set the environment variable
# "OBJECT_MODE" to 'any' for llvm-nm on AIX OS.
if 'system-aix' in config.available_features:
config.substitutions.append(('llvm-nm', 'env OBJECT_MODE=any llvm-nm'))
config.substitutions.append(('llvm-ar', 'env OBJECT_MODE=any llvm-ar'))
if "system-aix" in config.available_features:
config.substitutions.append(("llvm-nm", "env OBJECT_MODE=any llvm-nm"))
config.substitutions.append(("llvm-ar", "env OBJECT_MODE=any llvm-ar"))
# It is not realistically possible to account for all options that could
# possibly be present in system and user configuration files, so disable

39
clang/test/utils/update_cc_test_checks/lit.local.cfg
View File

@ -20,30 +20,41 @@ if config.standalone_build:
else:
config.test_format = lit.formats.ShTest(execute_external=False)
config.suffixes = ['.test']
config.suffixes = [".test"]
clang_path = os.path.join(config.clang_tools_dir, 'clang')
extra_args = '--clang ' + shell_quote(clang_path)
opt_path = os.path.join(config.llvm_tools_dir, 'opt')
extra_args += ' --opt ' + shell_quote(opt_path)
clang_path = os.path.join(config.clang_tools_dir, "clang")
extra_args = "--clang " + shell_quote(clang_path)
opt_path = os.path.join(config.llvm_tools_dir, "opt")
extra_args += " --opt " + shell_quote(opt_path)
# Specify an explicit default version in UTC tests, so that the --version
# embedded in UTC_ARGS does not change in all test expectations every time
# the default is bumped.
extra_args += ' --version=1'
script_path = os.path.join(config.llvm_src_root, 'utils',
'update_cc_test_checks.py')
extra_args += " --version=1"
script_path = os.path.join(
config.llvm_src_root, "utils", "update_cc_test_checks.py"
)
assert os.path.isfile(script_path)
# Windows: llvm-lit.py, Linux: llvm-lit
if config.llvm_external_lit:
lit = config.llvm_external_lit
else:
lit = shell_quote(glob.glob(os.path.join(config.llvm_tools_dir, 'llvm-lit*'))[0])
lit = shell_quote(
glob.glob(os.path.join(config.llvm_tools_dir, "llvm-lit*"))[0]
)
python = shell_quote(config.python_executable)
config.substitutions.append(
('%update_cc_test_checks', "%s %s %s" % (
python, shell_quote(script_path), extra_args)))
(
"%update_cc_test_checks",
"%s %s %s" % (python, shell_quote(script_path), extra_args),
)
)
config.substitutions.append(
('%clang_tools_dir', shell_quote(config.clang_tools_dir)))
("%clang_tools_dir", shell_quote(config.clang_tools_dir))
)
config.substitutions.append(
('%lit', "%s %s -Dclang_lit_site_cfg=%s -j1 -vv" % (
python, lit, shell_quote(config.clang_lit_site_cfg))))
(
"%lit",
"%s %s -Dclang_lit_site_cfg=%s -j1 -vv"
% (python, lit, shell_quote(config.clang_lit_site_cfg)),
)
)

254
clang/tools/clang-format/clang-format-diff.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python3
#
#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===#
# ===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
"""
This script reads input from a unified diff and reformats all the changed
@ -36,116 +36,156 @@ else:
def main():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=
argparse.RawDescriptionHelpFormatter)
parser.add_argument('-i', action='store_true', default=False,
help='apply edits to files instead of displaying a diff')
parser.add_argument('-p', metavar='NUM', default=0,
help='strip the smallest prefix containing P slashes')
parser.add_argument('-regex', metavar='PATTERN', default=None,
help='custom pattern selecting file paths to reformat '
'(case sensitive, overrides -iregex)')
parser.add_argument('-iregex', metavar='PATTERN', default=
r'.*\.(cpp|cc|c\+\+|cxx|cppm|ccm|cxxm|c\+\+m|c|cl|h|hh|hpp|hxx'
r'|m|mm|inc|js|ts|proto|protodevel|java|cs|json)',
help='custom pattern selecting file paths to reformat '
'(case insensitive, overridden by -regex)')
parser.add_argument('-sort-includes', action='store_true', default=False,
help='let clang-format sort include blocks')
parser.add_argument('-v', '--verbose', action='store_true',
help='be more verbose, ineffective without -i')
parser.add_argument('-style',
help='formatting style to apply (LLVM, GNU, Google, Chromium, '
'Microsoft, Mozilla, WebKit)')
parser.add_argument('-fallback-style',
help='The name of the predefined style used as a'
'fallback in case clang-format is invoked with'
'-style=file, but can not find the .clang-format'
'file to use.')
parser.add_argument('-binary', default='clang-format',
help='location of binary to use for clang-format')
args = parser.parse_args()
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument(
"-i",
action="store_true",
default=False,
help="apply edits to files instead of displaying a diff",
)
parser.add_argument(
"-p",
metavar="NUM",
default=0,
help="strip the smallest prefix containing P slashes",
)
parser.add_argument(
"-regex",
metavar="PATTERN",
default=None,
help="custom pattern selecting file paths to reformat "
"(case sensitive, overrides -iregex)",
)
parser.add_argument(
"-iregex",
metavar="PATTERN",
default=r".*\.(cpp|cc|c\+\+|cxx|cppm|ccm|cxxm|c\+\+m|c|cl|h|hh|hpp|hxx"
r"|m|mm|inc|js|ts|proto|protodevel|java|cs|json)",
help="custom pattern selecting file paths to reformat "
"(case insensitive, overridden by -regex)",
)
parser.add_argument(
"-sort-includes",
action="store_true",
default=False,
help="let clang-format sort include blocks",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="be more verbose, ineffective without -i",
)
parser.add_argument(
"-style",
help="formatting style to apply (LLVM, GNU, Google, Chromium, "
"Microsoft, Mozilla, WebKit)",
)
parser.add_argument(
"-fallback-style",
help="The name of the predefined style used as a"
"fallback in case clang-format is invoked with"
"-style=file, but can not find the .clang-format"
"file to use.",
)
parser.add_argument(
"-binary",
default="clang-format",
help="location of binary to use for clang-format",
)
args = parser.parse_args()
# Extract changed lines for each file.
filename = None
lines_by_file = {}
for line in sys.stdin:
match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
if match:
filename = match.group(2)
if filename is None:
continue
# Extract changed lines for each file.
filename = None
lines_by_file = {}
for line in sys.stdin:
match = re.search(r"^\+\+\+\ (.*?/){%s}(\S*)" % args.p, line)
if match:
filename = match.group(2)
if filename is None:
continue
if args.regex is not None:
if not re.match('^%s$' % args.regex, filename):
continue
else:
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
continue
if args.regex is not None:
if not re.match("^%s$" % args.regex, filename):
continue
else:
if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
continue
match = re.search(r'^@@.*\+(\d+)(?:,(\d+))?', line)
if match:
start_line = int(match.group(1))
line_count = 1
if match.group(2):
line_count = int(match.group(2))
# The input is something like
#
# @@ -1, +0,0 @@
#
# which means no lines were added.
if line_count == 0:
continue
# Also format lines range if line_count is 0 in case of deleting
# surrounding statements.
end_line = start_line
if line_count != 0:
end_line += line_count - 1
lines_by_file.setdefault(filename, []).extend(
['-lines', str(start_line) + ':' + str(end_line)])
match = re.search(r"^@@.*\+(\d+)(?:,(\d+))?", line)
if match:
start_line = int(match.group(1))
line_count = 1
if match.group(2):
line_count = int(match.group(2))
# The input is something like
#
# @@ -1, +0,0 @@
#
# which means no lines were added.
if line_count == 0:
continue
# Also format lines range if line_count is 0 in case of deleting
# surrounding statements.
end_line = start_line
if line_count != 0:
end_line += line_count - 1
lines_by_file.setdefault(filename, []).extend(
["-lines", str(start_line) + ":" + str(end_line)]
)
# Reformat files containing changes in place.
for filename, lines in lines_by_file.items():
if args.i and args.verbose:
print('Formatting {}'.format(filename))
command = [args.binary, filename]
if args.i:
command.append('-i')
if args.sort_includes:
command.append('-sort-includes')
command.extend(lines)
if args.style:
command.extend(['-style', args.style])
if args.fallback_style:
command.extend(['-fallback-style', args.fallback_style])
# Reformat files containing changes in place.
for filename, lines in lines_by_file.items():
if args.i and args.verbose:
print("Formatting {}".format(filename))
command = [args.binary, filename]
if args.i:
command.append("-i")
if args.sort_includes:
command.append("-sort-includes")
command.extend(lines)
if args.style:
command.extend(["-style", args.style])
if args.fallback_style:
command.extend(["-fallback-style", args.fallback_style])
try:
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=None,
stdin=subprocess.PIPE,
universal_newlines=True)
except OSError as e:
# Give the user more context when clang-format isn't
# found/isn't executable, etc.
raise RuntimeError(
'Failed to run "%s" - %s"' % (" ".join(command), e.strerror))
try:
p = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=None,
stdin=subprocess.PIPE,
universal_newlines=True,
)
except OSError as e:
# Give the user more context when clang-format isn't
# found/isn't executable, etc.
raise RuntimeError(
'Failed to run "%s" - %s"' % (" ".join(command), e.strerror)
)
stdout, stderr = p.communicate()
if p.returncode != 0:
sys.exit(p.returncode)
stdout, stderr = p.communicate()
if p.returncode != 0:
sys.exit(p.returncode)
if not args.i:
with open(filename) as f:
code = f.readlines()
formatted_code = StringIO(stdout).readlines()
diff = difflib.unified_diff(code, formatted_code,
filename, filename,
'(before formatting)', '(after formatting)')
diff_string = ''.join(diff)
if len(diff_string) > 0:
sys.stdout.write(diff_string)
if not args.i:
with open(filename) as f:
code = f.readlines()
formatted_code = StringIO(stdout).readlines()
diff = difflib.unified_diff(
code,
formatted_code,
filename,
filename,
"(before formatting)",
"(after formatting)",
)
diff_string = "".join(diff)
if len(diff_string) > 0:
sys.stdout.write(diff_string)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

77
clang/tools/clang-format/clang-format-sublime.py
View File

@ -18,7 +18,7 @@ import sublime_plugin
import subprocess
# Change this to the full path if clang-format is not on the path.
binary = 'clang-format'
binary = "clang-format"
# Change this to format according to other formatting styles. See the output of
# 'clang-format --help' for a list of supported styles. The default looks for
@ -26,35 +26,48 @@ binary = 'clang-format'
# used.
style = None
class ClangFormatCommand(sublime_plugin.TextCommand):
def run(self, edit):
encoding = self.view.encoding()
if encoding == 'Undefined':
encoding = 'utf-8'
regions = []
command = [binary]
if style:
command.extend(['-style', style])
for region in self.view.sel():
regions.append(region)
region_offset = min(region.a, region.b)
region_length = abs(region.b - region.a)
command.extend(['-offset', str(region_offset),
'-length', str(region_length),
'-assume-filename', str(self.view.file_name())])
old_viewport_position = self.view.viewport_position()
buf = self.view.substr(sublime.Region(0, self.view.size()))
p = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stdin=subprocess.PIPE)
output, error = p.communicate(buf.encode(encoding))
if error:
print(error)
self.view.replace(
edit, sublime.Region(0, self.view.size()),
output.decode(encoding))
self.view.sel().clear()
for region in regions:
self.view.sel().add(region)
# FIXME: Without the 10ms delay, the viewport sometimes jumps.
sublime.set_timeout(lambda: self.view.set_viewport_position(
old_viewport_position, False), 10)
def run(self, edit):
encoding = self.view.encoding()
if encoding == "Undefined":
encoding = "utf-8"
regions = []
command = [binary]
if style:
command.extend(["-style", style])
for region in self.view.sel():
regions.append(region)
region_offset = min(region.a, region.b)
region_length = abs(region.b - region.a)
command.extend(
[
"-offset",
str(region_offset),
"-length",
str(region_length),
"-assume-filename",
str(self.view.file_name()),
]
)
old_viewport_position = self.view.viewport_position()
buf = self.view.substr(sublime.Region(0, self.view.size()))
p = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
output, error = p.communicate(buf.encode(encoding))
if error:
print(error)
self.view.replace(
edit, sublime.Region(0, self.view.size()), output.decode(encoding)
)
self.view.sel().clear()
for region in regions:
self.view.sel().add(region)
# FIXME: Without the 10ms delay, the viewport sometimes jumps.
sublime.set_timeout(
lambda: self.view.set_viewport_position(old_viewport_position, False), 10
)

184
clang/tools/clang-format/clang-format.py
View File

@ -49,9 +49,9 @@ import vim
# set g:clang_format_path to the path to clang-format if it is not on the path
# Change this to the full path if clang-format is not on the path.
binary = 'clang-format'
binary = "clang-format"
if vim.eval('exists("g:clang_format_path")') == "1":
binary = vim.eval('g:clang_format_path')
binary = vim.eval("g:clang_format_path")
# Change this to format according to other formatting styles. See the output of
# 'clang-format --help' for a list of supported styles. The default looks for
@ -60,99 +60,109 @@ if vim.eval('exists("g:clang_format_path")') == "1":
style = None
fallback_style = None
if vim.eval('exists("g:clang_format_fallback_style")') == "1":
fallback_style = vim.eval('g:clang_format_fallback_style')
fallback_style = vim.eval("g:clang_format_fallback_style")
def get_buffer(encoding):
if platform.python_version_tuple()[0] == '3':
return vim.current.buffer
return [ line.decode(encoding) for line in vim.current.buffer ]
if platform.python_version_tuple()[0] == "3":
return vim.current.buffer
return [line.decode(encoding) for line in vim.current.buffer]
def main():
# Get the current text.
encoding = vim.eval("&encoding")
buf = get_buffer(encoding)
# Join the buffer into a single string with a terminating newline
text = ('\n'.join(buf) + '\n').encode(encoding)
# Get the current text.
encoding = vim.eval("&encoding")
buf = get_buffer(encoding)
# Join the buffer into a single string with a terminating newline
text = ("\n".join(buf) + "\n").encode(encoding)
# Determine range to format.
if vim.eval('exists("l:lines")') == '1':
lines = ['-lines', vim.eval('l:lines')]
elif vim.eval('exists("l:formatdiff")') == '1' and \
os.path.exists(vim.current.buffer.name):
with open(vim.current.buffer.name, 'r') as f:
ondisk = f.read().splitlines();
sequence = difflib.SequenceMatcher(None, ondisk, vim.current.buffer)
lines = []
for op in reversed(sequence.get_opcodes()):
if op[0] not in ['equal', 'delete']:
lines += ['-lines', '%s:%s' % (op[3] + 1, op[4])]
if lines == []:
return
else:
lines = ['-lines', '%s:%s' % (vim.current.range.start + 1,
vim.current.range.end + 1)]
# Determine range to format.
if vim.eval('exists("l:lines")') == "1":
lines = ["-lines", vim.eval("l:lines")]
elif vim.eval('exists("l:formatdiff")') == "1" and os.path.exists(
vim.current.buffer.name
):
with open(vim.current.buffer.name, "r") as f:
ondisk = f.read().splitlines()
sequence = difflib.SequenceMatcher(None, ondisk, vim.current.buffer)
lines = []
for op in reversed(sequence.get_opcodes()):
if op[0] not in ["equal", "delete"]:
lines += ["-lines", "%s:%s" % (op[3] + 1, op[4])]
if lines == []:
return
else:
lines = [
"-lines",
"%s:%s" % (vim.current.range.start + 1, vim.current.range.end + 1),
]
# Convert cursor (line, col) to bytes.
# Don't use line2byte: https://github.com/vim/vim/issues/5930
_, cursor_line, cursor_col, _ = vim.eval('getpos(".")') # 1-based
cursor_byte = 0
for line in text.split(b'\n')[:int(cursor_line) - 1]:
cursor_byte += len(line) + 1
cursor_byte += int(cursor_col) - 1
if cursor_byte < 0:
print('Couldn\'t determine cursor position. Is your file empty?')
return
# Convert cursor (line, col) to bytes.
# Don't use line2byte: https://github.com/vim/vim/issues/5930
_, cursor_line, cursor_col, _ = vim.eval('getpos(".")') # 1-based
cursor_byte = 0
for line in text.split(b"\n")[: int(cursor_line) - 1]:
cursor_byte += len(line) + 1
cursor_byte += int(cursor_col) - 1
if cursor_byte < 0:
print("Couldn't determine cursor position. Is your file empty?")
return
# Avoid flashing an ugly, ugly cmd prompt on Windows when invoking clang-format.
startupinfo = None
if sys.platform.startswith('win32'):
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
# Avoid flashing an ugly, ugly cmd prompt on Windows when invoking clang-format.
startupinfo = None
if sys.platform.startswith("win32"):
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
# Call formatter.
command = [binary, '-cursor', str(cursor_byte)]
if lines != ['-lines', 'all']:
command += lines
if style:
command.extend(['-style', style])
if fallback_style:
command.extend(['-fallback-style', fallback_style])
if vim.current.buffer.name:
command.extend(['-assume-filename', vim.current.buffer.name])
p = subprocess.Popen(command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE, startupinfo=startupinfo)
stdout, stderr = p.communicate(input=text)
# If successful, replace buffer contents.
if stderr:
print(stderr)
if not stdout:
print(
'No output from clang-format (crashed?).\n'
'Please report to bugs.llvm.org.'
# Call formatter.
command = [binary, "-cursor", str(cursor_byte)]
if lines != ["-lines", "all"]:
command += lines
if style:
command.extend(["-style", style])
if fallback_style:
command.extend(["-fallback-style", fallback_style])
if vim.current.buffer.name:
command.extend(["-assume-filename", vim.current.buffer.name])
p = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
startupinfo=startupinfo,
)
else:
header, content = stdout.split(b'\n', 1)
header = json.loads(header.decode('utf-8'))
# Strip off the trailing newline (added above).
# This maintains trailing empty lines present in the buffer if
# the -lines specification requests them to remain unchanged.
lines = content.decode(encoding).split('\n')[:-1]
sequence = difflib.SequenceMatcher(None, buf, lines)
for op in reversed(sequence.get_opcodes()):
if op[0] != 'equal':
vim.current.buffer[op[1]:op[2]] = lines[op[3]:op[4]]
if header.get('IncompleteFormat'):
print('clang-format: incomplete (syntax errors)')
# Convert cursor bytes to (line, col)
# Don't use goto: https://github.com/vim/vim/issues/5930
cursor_byte = int(header['Cursor'])
prefix = content[0:cursor_byte]
cursor_line = 1 + prefix.count(b'\n')
cursor_column = 1 + len(prefix.rsplit(b'\n', 1)[-1])
vim.command('call cursor(%d, %d)' % (cursor_line, cursor_column))
stdout, stderr = p.communicate(input=text)
# If successful, replace buffer contents.
if stderr:
print(stderr)
if not stdout:
print(
"No output from clang-format (crashed?).\n"
"Please report to bugs.llvm.org."
)
else:
header, content = stdout.split(b"\n", 1)
header = json.loads(header.decode("utf-8"))
# Strip off the trailing newline (added above).
# This maintains trailing empty lines present in the buffer if
# the -lines specification requests them to remain unchanged.
lines = content.decode(encoding).split("\n")[:-1]
sequence = difflib.SequenceMatcher(None, buf, lines)
for op in reversed(sequence.get_opcodes()):
if op[0] != "equal":
vim.current.buffer[op[1] : op[2]] = lines[op[3] : op[4]]
if header.get("IncompleteFormat"):
print("clang-format: incomplete (syntax errors)")
# Convert cursor bytes to (line, col)
# Don't use goto: https://github.com/vim/vim/issues/5930
cursor_byte = int(header["Cursor"])
prefix = content[0:cursor_byte]
cursor_line = 1 + prefix.count(b"\n")
cursor_column = 1 + len(prefix.rsplit(b"\n", 1)[-1])
vim.command("call cursor(%d, %d)" % (cursor_line, cursor_column))
main()

36
clang/tools/clang-rename/clang-rename.py
View File

@ -1,4 +1,4 @@
'''
"""
Minimal clang-rename integration with Vim.
Before installing make sure one of the following is satisfied:
@ -20,39 +20,43 @@ IMPORTANT NOTE: Before running the tool, make sure you saved the file.
All you have to do now is to place a cursor on a variable/function/class which
you would like to rename and press '<leader>cr'. You will be prompted for a new
name if the cursor points to a valid symbol.
'''
"""
from __future__ import absolute_import, division, print_function
import vim
import subprocess
import sys
def main():
binary = 'clang-rename'
binary = "clang-rename"
if vim.eval('exists("g:clang_rename_path")') == "1":
binary = vim.eval('g:clang_rename_path')
binary = vim.eval("g:clang_rename_path")
# Get arguments for clang-rename binary.
offset = int(vim.eval('line2byte(line("."))+col(".")')) - 2
if offset < 0:
print('Couldn\'t determine cursor position. Is your file empty?',
file=sys.stderr)
print(
"Couldn't determine cursor position. Is your file empty?", file=sys.stderr
)
return
filename = vim.current.buffer.name
new_name_request_message = 'type new name:'
new_name_request_message = "type new name:"
new_name = vim.eval("input('{}\n')".format(new_name_request_message))
# Call clang-rename.
command = [binary,
filename,
'-i',
'-offset', str(offset),
'-new-name', str(new_name)]
command = [
binary,
filename,
"-i",
"-offset",
str(offset),
"-new-name",
str(new_name),
]
# FIXME: make it possible to run the tool on unsaved file.
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if stderr:
@ -62,5 +66,5 @@ def main():
vim.command("checktime")
if __name__ == '__main__':
if __name__ == "__main__":
main()

303
clang/tools/include-mapping/cppreference_parser.py
View File

@ -1,11 +1,11 @@
#!/usr/bin/env python
#===- cppreference_parser.py - ------------------------------*- python -*--===#
# ===- cppreference_parser.py - ------------------------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
from bs4 import BeautifulSoup, NavigableString
@ -18,176 +18,189 @@ import sys
class Symbol:
def __init__(self, name, namespace, headers):
# unqualifed symbol name, e.g. "move"
self.name = name
# namespace of the symbol (with trailing "::"), e.g. "std::", "" (global scope)
# None for C symbols.
self.namespace = namespace
# a list of corresponding headers
self.headers = headers
def __init__(self, name, namespace, headers):
# unqualifed symbol name, e.g. "move"
self.name = name
# namespace of the symbol (with trailing "::"), e.g. "std::", "" (global scope)
# None for C symbols.
self.namespace = namespace
# a list of corresponding headers
self.headers = headers
def __lt__(self, other):
if self.namespace != other.namespace:
return str(self.namespace) < str(other.namespace)
return self.name < other.name
def __lt__(self, other):
if self.namespace != other.namespace:
return str(self.namespace) < str(other.namespace)
return self.name < other.name
def _HasClass(tag, *classes):
for c in tag.get('class', []):
if c in classes:
return True
return False
for c in tag.get("class", []):
if c in classes:
return True
return False
def _ParseSymbolPage(symbol_page_html, symbol_name):
"""Parse symbol page and retrieve the include header defined in this page.
The symbol page provides header for the symbol, specifically in
"Defined in header <header>" section. An example:
"""Parse symbol page and retrieve the include header defined in this page.
The symbol page provides header for the symbol, specifically in
"Defined in header <header>" section. An example:
<tr class="t-dsc-header">
<td colspan="2"> <div>Defined in header <code>&lt;ratio&gt;</code> </div>
</td></tr>
<tr class="t-dsc-header">
<td colspan="2"> <div>Defined in header <code>&lt;ratio&gt;</code> </div>
</td></tr>
Returns a list of headers.
"""
headers = set()
all_headers = set()
Returns a list of headers.
"""
headers = set()
all_headers = set()
soup = BeautifulSoup(symbol_page_html, "html.parser")
# Rows in table are like:
# Defined in header <foo> .t-dsc-header
# Defined in header <bar> .t-dsc-header
# decl1 .t-dcl
# Defined in header <baz> .t-dsc-header
# decl2 .t-dcl
for table in soup.select('table.t-dcl-begin, table.t-dsc-begin'):
current_headers = []
was_decl = False
for row in table.select('tr'):
if _HasClass(row, 't-dcl', 't-dsc'):
was_decl = True
# Symbols are in the first cell.
found_symbols = row.find('td').stripped_strings
if not symbol_name in found_symbols:
continue
headers.update(current_headers)
elif _HasClass(row, 't-dsc-header'):
# If we saw a decl since the last header, this is a new block of headers
# for a new block of decls.
if was_decl:
current_headers = []
soup = BeautifulSoup(symbol_page_html, "html.parser")
# Rows in table are like:
# Defined in header <foo> .t-dsc-header
# Defined in header <bar> .t-dsc-header
# decl1 .t-dcl
# Defined in header <baz> .t-dsc-header
# decl2 .t-dcl
for table in soup.select("table.t-dcl-begin, table.t-dsc-begin"):
current_headers = []
was_decl = False
# There are also .t-dsc-header for "defined in namespace".
if not "Defined in header " in row.text:
continue
# The interesting header content (e.g. <cstdlib>) is wrapped in <code>.
for header_code in row.find_all("code"):
current_headers.append(header_code.text)
all_headers.add(header_code.text)
# If the symbol was never named, consider all named headers.
return headers or all_headers
for row in table.select("tr"):
if _HasClass(row, "t-dcl", "t-dsc"):
was_decl = True
# Symbols are in the first cell.
found_symbols = row.find("td").stripped_strings
if not symbol_name in found_symbols:
continue
headers.update(current_headers)
elif _HasClass(row, "t-dsc-header"):
# If we saw a decl since the last header, this is a new block of headers
# for a new block of decls.
if was_decl:
current_headers = []
was_decl = False
# There are also .t-dsc-header for "defined in namespace".
if not "Defined in header " in row.text:
continue
# The interesting header content (e.g. <cstdlib>) is wrapped in <code>.
for header_code in row.find_all("code"):
current_headers.append(header_code.text)
all_headers.add(header_code.text)
# If the symbol was never named, consider all named headers.
return headers or all_headers
def _ParseIndexPage(index_page_html):
"""Parse index page.
The index page lists all std symbols and hrefs to their detailed pages
(which contain the defined header). An example:
"""Parse index page.
The index page lists all std symbols and hrefs to their detailed pages
(which contain the defined header). An example:
<a href="abs.html" title="abs"><tt>abs()</tt></a> (int) <br>
<a href="acos.html" title="acos"><tt>acos()</tt></a> <br>
<a href="abs.html" title="abs"><tt>abs()</tt></a> (int) <br>
<a href="acos.html" title="acos"><tt>acos()</tt></a> <br>
Returns a list of tuple (symbol_name, relative_path_to_symbol_page, variant).
"""
symbols = []
soup = BeautifulSoup(index_page_html, "html.parser")
for symbol_href in soup.select("a[title]"):
# Ignore annotated symbols like "acos<>() (std::complex)".
# These tend to be overloads, and we the primary is more useful.
# This accidentally accepts begin/end despite the (iterator) caption: the
# (since C++11) note is first. They are good symbols, so the bug is unfixed.
caption = symbol_href.next_sibling
variant = None
if isinstance(caption, NavigableString) and "(" in caption:
variant = caption.text.strip(" ()")
symbol_tt = symbol_href.find("tt")
if symbol_tt:
symbols.append((symbol_tt.text.rstrip("<>()"), # strip any trailing <>()
symbol_href["href"], variant))
return symbols
Returns a list of tuple (symbol_name, relative_path_to_symbol_page, variant).
"""
symbols = []
soup = BeautifulSoup(index_page_html, "html.parser")
for symbol_href in soup.select("a[title]"):
# Ignore annotated symbols like "acos<>() (std::complex)".
# These tend to be overloads, and we the primary is more useful.
# This accidentally accepts begin/end despite the (iterator) caption: the
# (since C++11) note is first. They are good symbols, so the bug is unfixed.
caption = symbol_href.next_sibling
variant = None
if isinstance(caption, NavigableString) and "(" in caption:
variant = caption.text.strip(" ()")
symbol_tt = symbol_href.find("tt")
if symbol_tt:
symbols.append(
(
symbol_tt.text.rstrip("<>()"), # strip any trailing <>()
symbol_href["href"],
variant,
)
)
return symbols
def _ReadSymbolPage(path, name):
with open(path) as f:
return _ParseSymbolPage(f.read(), name)
with open(path) as f:
return _ParseSymbolPage(f.read(), name)
def _GetSymbols(pool, root_dir, index_page_name, namespace, variants_to_accept):
"""Get all symbols listed in the index page. All symbols should be in the
given namespace.
"""Get all symbols listed in the index page. All symbols should be in the
given namespace.
Returns a list of Symbols.
"""
Returns a list of Symbols.
"""
# Workflow steps:
# 1. Parse index page which lists all symbols to get symbol
# name (unqualified name) and its href link to the symbol page which
# contains the defined header.
# 2. Parse the symbol page to get the defined header.
index_page_path = os.path.join(root_dir, index_page_name)
with open(index_page_path, "r") as f:
# Read each symbol page in parallel.
results = [] # (symbol_name, promise of [header...])
for symbol_name, symbol_page_path, variant in _ParseIndexPage(f.read()):
# Variant symbols (e.g. the std::locale version of isalpha) add ambiguity.
# FIXME: use these as a fallback rather than ignoring entirely.
variants_for_symbol = variants_to_accept.get(
(namespace or "") + symbol_name, ())
if variant and variant not in variants_for_symbol:
continue
path = os.path.join(root_dir, symbol_page_path)
if os.path.isfile(path):
results.append((symbol_name,
pool.apply_async(_ReadSymbolPage, (path, symbol_name))))
else:
sys.stderr.write("Discarding information for symbol: %s. Page %s does not exist.\n"
% (symbol_name, path))
# Workflow steps:
# 1. Parse index page which lists all symbols to get symbol
# name (unqualified name) and its href link to the symbol page which
# contains the defined header.
# 2. Parse the symbol page to get the defined header.
index_page_path = os.path.join(root_dir, index_page_name)
with open(index_page_path, "r") as f:
# Read each symbol page in parallel.
results = [] # (symbol_name, promise of [header...])
for symbol_name, symbol_page_path, variant in _ParseIndexPage(f.read()):
# Variant symbols (e.g. the std::locale version of isalpha) add ambiguity.
# FIXME: use these as a fallback rather than ignoring entirely.
variants_for_symbol = variants_to_accept.get(
(namespace or "") + symbol_name, ()
)
if variant and variant not in variants_for_symbol:
continue
path = os.path.join(root_dir, symbol_page_path)
if os.path.isfile(path):
results.append(
(
symbol_name,
pool.apply_async(_ReadSymbolPage, (path, symbol_name)),
)
)
else:
sys.stderr.write(
"Discarding information for symbol: %s. Page %s does not exist.\n"
% (symbol_name, path)
)
# Build map from symbol name to a set of headers.
symbol_headers = collections.defaultdict(set)
for symbol_name, lazy_headers in results:
symbol_headers[symbol_name].update(lazy_headers.get())
# Build map from symbol name to a set of headers.
symbol_headers = collections.defaultdict(set)
for symbol_name, lazy_headers in results:
symbol_headers[symbol_name].update(lazy_headers.get())
symbols = []
for name, headers in sorted(symbol_headers.items(), key=lambda t : t[0]):
symbols.append(Symbol(name, namespace, list(headers)))
return symbols
symbols = []
for name, headers in sorted(symbol_headers.items(), key=lambda t: t[0]):
symbols.append(Symbol(name, namespace, list(headers)))
return symbols
def GetSymbols(parse_pages):
"""Get all symbols by parsing the given pages.
"""Get all symbols by parsing the given pages.
Args:
parse_pages: a list of tuples (page_root_dir, index_page_name, namespace)
"""
# By default we prefer the non-variant versions, as they're more common. But
# there are some symbols, whose variant is more common. This list describes
# those symbols.
variants_to_accept = {
# std::remove<> has variant algorithm.
"std::remove": ("algorithm"),
}
symbols = []
# Run many workers to process individual symbol pages under the symbol index.
# Don't allow workers to capture Ctrl-C.
pool = multiprocessing.Pool(
initializer=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
try:
for root_dir, page_name, namespace in parse_pages:
symbols.extend(_GetSymbols(pool, root_dir, page_name, namespace,
variants_to_accept))
finally:
pool.terminate()
pool.join()
return sorted(symbols)
Args:
parse_pages: a list of tuples (page_root_dir, index_page_name, namespace)
"""
# By default we prefer the non-variant versions, as they're more common. But
# there are some symbols, whose variant is more common. This list describes
# those symbols.
variants_to_accept = {
# std::remove<> has variant algorithm.
"std::remove": ("algorithm"),
}
symbols = []
# Run many workers to process individual symbol pages under the symbol index.
# Don't allow workers to capture Ctrl-C.
pool = multiprocessing.Pool(
initializer=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)
)
try:
for root_dir, page_name, namespace in parse_pages:
symbols.extend(
_GetSymbols(pool, root_dir, page_name, namespace, variants_to_accept)
)
finally:
pool.terminate()
pool.join()
return sorted(symbols)

445
clang/tools/include-mapping/gen_std.py
View File

@ -1,11 +1,11 @@
#!/usr/bin/env python
#===- gen_std.py - ------------------------------------------*- python -*--===#
# ===- gen_std.py - ------------------------------------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
"""gen_std.py is a tool to generate a lookup table (from qualified names to
include headers) for C/C++ Standard Library symbols by parsing archived HTML
@ -55,244 +55,235 @@ CODE_PREFIX = """\
//===----------------------------------------------------------------------===//
"""
def ParseArg():
parser = argparse.ArgumentParser(description='Generate StdGen file')
parser.add_argument('-cppreference', metavar='PATH',
default='',
help='path to the cppreference offline HTML directory',
required=True
)
parser.add_argument('-symbols',
default='cpp',
help='Generate c or cpp (removed) symbols. One of {cpp, c, cpp_removed}.',
required=True)
return parser.parse_args()
parser = argparse.ArgumentParser(description="Generate StdGen file")
parser.add_argument(
"-cppreference",
metavar="PATH",
default="",
help="path to the cppreference offline HTML directory",
required=True,
)
parser.add_argument(
"-symbols",
default="cpp",
help="Generate c or cpp (removed) symbols. One of {cpp, c, cpp_removed}.",
required=True,
)
return parser.parse_args()
def AdditionalHeadersForIOSymbols(symbol):
# IO-related symbols declared in the <iosfwd> header, per C++
# [iosfwd.syn 31.3.1]:
iosfwd_symbols = [
'basic_ios',
'basic_streambuf',
'basic_istream',
'basic_ostream',
'basic_iostream',
# IO-related symbols declared in the <iosfwd> header, per C++
# [iosfwd.syn 31.3.1]:
iosfwd_symbols = [
"basic_ios",
"basic_streambuf",
"basic_istream",
"basic_ostream",
"basic_iostream",
"basic_stringbuf",
"basic_istringstream",
"basic_ostringstream",
"basic_stringstream",
"basic_spanbuf",
"basic_ispanstream",
"basic_ospanstream",
"basic_spanstream",
"basic_filebuf",
"basic_ifstream",
"basic_ofstream",
"basic_fstream",
"basic_syncbuf",
"basic_osyncstream",
"istreambuf_iterator",
"ostreambuf_iterator",
"ios",
"wios",
"streambuf",
"istream",
"ostream",
"iostream",
"stringbuf",
"istringstream",
"ostringstream",
"stringstream",
"spanbuf",
"ispanstream",
"ospanstream",
"spanstream",
"filebuf",
"ifstream",
"ofstream",
"fstream",
"syncbuf",
"osyncstream",
"wstreambuf",
"wistream",
"wostream",
"wiostream",
"wstringbuf",
"wistringstream",
"wostringstream",
"wstringstream",
"wspanbuf",
"wispanstream",
"wospanstream",
"wspanstream",
"wfilebuf",
"wifstream",
"wofstream",
"wfstream",
"wsyncbuf",
"wosyncstream",
"fpos",
"streampos",
"wstreampos",
"u8streampos",
"u16streampos",
"u32streampos",
]
assert len(symbol.headers) == 1
sym_header = symbol.headers[0]
headers = []
# <iostream> is preferred than <iosfwd>
'basic_stringbuf',
'basic_istringstream',
'basic_ostringstream',
'basic_stringstream',
# <iostream> is an alternative of <streambuf>, <istream>, <ostream>, <ios>.
# per C++ [iostream.syn 31.4.1]
if sym_header in ["<ios>", "<istream>", "<ostream>", "<streambuf>"]:
headers.append("<iostream>")
'basic_spanbuf',
'basic_ispanstream',
'basic_ospanstream',
'basic_spanstream',
if symbol.name in iosfwd_symbols:
headers.append("<iosfwd>")
'basic_filebuf',
'basic_ifstream',
'basic_ofstream',
'basic_fstream',
'basic_syncbuf',
'basic_osyncstream',
'istreambuf_iterator',
'ostreambuf_iterator',
'ios',
'wios',
'streambuf',
'istream',
'ostream',
'iostream',
'stringbuf',
'istringstream',
'ostringstream',
'stringstream',
'spanbuf',
'ispanstream',
'ospanstream',
'spanstream',
'filebuf',
'ifstream',
'ofstream',
'fstream',
'syncbuf',
'osyncstream',
'wstreambuf',
'wistream',
'wostream',
'wiostream',
'wstringbuf',
'wistringstream',
'wostringstream',
'wstringstream',
'wspanbuf',
'wispanstream',
'wospanstream',
'wspanstream',
'wfilebuf',
'wifstream',
'wofstream',
'wfstream',
'wsyncbuf',
'wosyncstream',
'fpos',
'streampos',
'wstreampos',
'u8streampos',
'u16streampos',
'u32streampos',
]
assert(len(symbol.headers) == 1)
sym_header = symbol.headers[0]
headers = []
# <iostream> is preferred than <iosfwd>
# <iostream> is an alternative of <streambuf>, <istream>, <ostream>, <ios>.
# per C++ [iostream.syn 31.4.1]
if sym_header in ["<ios>", "<istream>", "<ostream>", "<streambuf>"]:
headers.append("<iostream>")
if symbol.name in iosfwd_symbols:
headers.append("<iosfwd>")
return headers
return headers
def GetCCompatibilitySymbols(symbol):
# C++ form of the C standard headers.
c_compat_headers = {
"<cassert>",
"<cctype>",
"<cerrno>",
"<cfenv>",
"<cfloat>",
"<cinttypes>",
"<climits>",
"<clocale>",
"<cmath>",
"<csetjmp>",
"<csignal>",
"<cstdarg>",
"<cstddef>",
"<cstdint>",
"<cstdio>",
"<cstdlib>",
"<cstring>",
"<ctime>",
"<cuchar>",
"<cwchar>",
"<cwctype>",
}
# C++ [support.c.headers.other] 17.14.7
# ..., behaves as if each name placed in the standard library namespace by
# the corresponding <cname> header is placed within the global namespace
# scope, except for the functions described in [sf.cmath], the
# std​::​lerp function overloads ([c.math.lerp]), the declaration of
# std​::​byte ([cstddef.syn]), and the functions and function templates
# described in [support.types.byteops].
exception_symbols = {
"(assoc_)?laguerre[f|l]?",
"(assoc_|sph_)?legendre[f|l]?",
"beta[f|l]?",
"(comp_)?ellint_[1-3][f|l]?",
"(cyl_|sph_)?bessel_[i-k][f|l]?",
"(cyl_|sph_)?neumann[f|l]?",
"expint[f|l]?",
"hermite[f|l]?",
"riemann_zeta[f|l]?",
"lerp",
"byte",
}
assert(len(symbol.headers) == 1)
header = symbol.headers[0]
if header not in c_compat_headers:
return []
if any(re.fullmatch(x, symbol.name) for x in exception_symbols):
return []
# C++ form of the C standard headers.
c_compat_headers = {
"<cassert>",
"<cctype>",
"<cerrno>",
"<cfenv>",
"<cfloat>",
"<cinttypes>",
"<climits>",
"<clocale>",
"<cmath>",
"<csetjmp>",
"<csignal>",
"<cstdarg>",
"<cstddef>",
"<cstdint>",
"<cstdio>",
"<cstdlib>",
"<cstring>",
"<ctime>",
"<cuchar>",
"<cwchar>",
"<cwctype>",
}
# C++ [support.c.headers.other] 17.14.7
# ..., behaves as if each name placed in the standard library namespace by
# the corresponding <cname> header is placed within the global namespace
# scope, except for the functions described in [sf.cmath], the
# std​::​lerp function overloads ([c.math.lerp]), the declaration of
# std​::​byte ([cstddef.syn]), and the functions and function templates
# described in [support.types.byteops].
exception_symbols = {
"(assoc_)?laguerre[f|l]?",
"(assoc_|sph_)?legendre[f|l]?",
"beta[f|l]?",
"(comp_)?ellint_[1-3][f|l]?",
"(cyl_|sph_)?bessel_[i-k][f|l]?",
"(cyl_|sph_)?neumann[f|l]?",
"expint[f|l]?",
"hermite[f|l]?",
"riemann_zeta[f|l]?",
"lerp",
"byte",
}
assert len(symbol.headers) == 1
header = symbol.headers[0]
if header not in c_compat_headers:
return []
if any(re.fullmatch(x, symbol.name) for x in exception_symbols):
return []
# Introduce two more entries, both in the global namespace, one using the
# C++-compat header and another using the C header.
results = []
if symbol.namespace != None:
# avoid printing duplicated entries, for C macros!
results.append(cppreference_parser.Symbol(symbol.name, None, [header]))
c_header = "<" + header[2:-1] + ".h>" # <cstdio> => <stdio.h>
results.append(cppreference_parser.Symbol(symbol.name, None, [c_header]))
return results
# Introduce two more entries, both in the global namespace, one using the
# C++-compat header and another using the C header.
results = []
if symbol.namespace != None:
# avoid printing duplicated entries, for C macros!
results.append(cppreference_parser.Symbol(symbol.name, None, [header]))
c_header = "<" + header[2:-1] + ".h>" # <cstdio> => <stdio.h>
results.append(cppreference_parser.Symbol(symbol.name, None, [c_header]))
return results
def main():
args = ParseArg()
if args.symbols == 'cpp':
page_root = os.path.join(args.cppreference, "en", "cpp")
symbol_index_root = os.path.join(page_root, "symbol_index")
parse_pages = [
(page_root, "symbol_index.html", "std::"),
# std sub-namespace symbols have separated pages.
# We don't index std literal operators (e.g.
# std::literals::chrono_literals::operator""d), these symbols can't be
# accessed by std::<symbol_name>.
#
# std::placeholders symbols are handled manually in StdSpecialSymbolMap.inc
(symbol_index_root, "chrono.html", "std::chrono::"),
(symbol_index_root, "execution.html", "std::execution::"),
(symbol_index_root, "numbers.html", "std::numbers::"),
(symbol_index_root, "filesystem.html", "std::filesystem::"),
(symbol_index_root, "pmr.html", "std::pmr::"),
(symbol_index_root, "ranges.html", "std::ranges::"),
(symbol_index_root, "regex_constants.html", "std::regex_constants::"),
(symbol_index_root, "this_thread.html", "std::this_thread::"),
# Zombie symbols that were available from the Standard Library, but are
# removed in the following standards.
(symbol_index_root, "zombie_names.html", "std::"),
(symbol_index_root, "macro.html", None),
]
elif args.symbols == 'c':
page_root = os.path.join(args.cppreference, "en", "c")
symbol_index_root = page_root
parse_pages = [(page_root, "index.html", None)]
if not os.path.exists(symbol_index_root):
exit("Path %s doesn't exist!" % symbol_index_root)
args = ParseArg()
if args.symbols == "cpp":
page_root = os.path.join(args.cppreference, "en", "cpp")
symbol_index_root = os.path.join(page_root, "symbol_index")
parse_pages = [
(page_root, "symbol_index.html", "std::"),
# std sub-namespace symbols have separated pages.
# We don't index std literal operators (e.g.
# std::literals::chrono_literals::operator""d), these symbols can't be
# accessed by std::<symbol_name>.
#
# std::placeholders symbols are handled manually in StdSpecialSymbolMap.inc
(symbol_index_root, "chrono.html", "std::chrono::"),
(symbol_index_root, "execution.html", "std::execution::"),
(symbol_index_root, "numbers.html", "std::numbers::"),
(symbol_index_root, "filesystem.html", "std::filesystem::"),
(symbol_index_root, "pmr.html", "std::pmr::"),
(symbol_index_root, "ranges.html", "std::ranges::"),
(symbol_index_root, "regex_constants.html", "std::regex_constants::"),
(symbol_index_root, "this_thread.html", "std::this_thread::"),
# Zombie symbols that were available from the Standard Library, but are
# removed in the following standards.
(symbol_index_root, "zombie_names.html", "std::"),
(symbol_index_root, "macro.html", None),
]
elif args.symbols == "c":
page_root = os.path.join(args.cppreference, "en", "c")
symbol_index_root = page_root
parse_pages = [(page_root, "index.html", None)]
symbols = cppreference_parser.GetSymbols(parse_pages)
# We don't have version information from the unzipped offline HTML files.
# so we use the modified time of the symbol_index.html as the version.
index_page_path = os.path.join(page_root, "index.html")
cppreference_modified_date = datetime.datetime.fromtimestamp(
os.stat(index_page_path).st_mtime).strftime('%Y-%m-%d')
print(CODE_PREFIX % (args.symbols.upper(), cppreference_modified_date))
for symbol in symbols:
if len(symbol.headers) == 1:
augmented_symbols = [symbol]
augmented_symbols.extend(GetCCompatibilitySymbols(symbol))
for s in augmented_symbols:
s.headers.extend(AdditionalHeadersForIOSymbols(s))
for header in s.headers:
# SYMBOL(unqualified_name, namespace, header)
print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace,
header))
elif len(symbol.headers) == 0:
sys.stderr.write("No header found for symbol %s\n" % symbol.name)
else:
# FIXME: support symbols with multiple headers (e.g. std::move).
sys.stderr.write("Ambiguous header for symbol %s: %s\n" % (
symbol.name, ', '.join(symbol.headers)))
if not os.path.exists(symbol_index_root):
exit("Path %s doesn't exist!" % symbol_index_root)
symbols = cppreference_parser.GetSymbols(parse_pages)
# We don't have version information from the unzipped offline HTML files.
# so we use the modified time of the symbol_index.html as the version.
index_page_path = os.path.join(page_root, "index.html")
cppreference_modified_date = datetime.datetime.fromtimestamp(
os.stat(index_page_path).st_mtime
).strftime("%Y-%m-%d")
print(CODE_PREFIX % (args.symbols.upper(), cppreference_modified_date))
for symbol in symbols:
if len(symbol.headers) == 1:
augmented_symbols = [symbol]
augmented_symbols.extend(GetCCompatibilitySymbols(symbol))
for s in augmented_symbols:
s.headers.extend(AdditionalHeadersForIOSymbols(s))
for header in s.headers:
# SYMBOL(unqualified_name, namespace, header)
print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace, header))
elif len(symbol.headers) == 0:
sys.stderr.write("No header found for symbol %s\n" % symbol.name)
else:
# FIXME: support symbols with multiple headers (e.g. std::move).
sys.stderr.write(
"Ambiguous header for symbol %s: %s\n"
% (symbol.name, ", ".join(symbol.headers))
)
if __name__ == '__main__':
main()
if __name__ == "__main__":
main()

95
clang/tools/include-mapping/test.py
View File

@ -1,20 +1,20 @@
#!/usr/bin/env python
#===- test.py - ---------------------------------------------*- python -*--===#
# ===- test.py - ---------------------------------------------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# ===------------------------------------------------------------------------===#
from cppreference_parser import _ParseSymbolPage, _ParseIndexPage
import unittest
class TestStdGen(unittest.TestCase):
def testParseIndexPage(self):
html = """
class TestStdGen(unittest.TestCase):
def testParseIndexPage(self):
html = """
<a href="abs.html" title="abs"><tt>abs()</tt></a> (int) <br>
<a href="complex/abs.html" title="abs"><tt>abs&lt;&gt;()</tt></a> (std::complex) <br>
<a href="acos.html" title="acos"><tt>acos()</tt></a> <br>
@ -22,24 +22,23 @@ class TestStdGen(unittest.TestCase):
<a href="as_bytes.html" title="as bytes"><tt>as_bytes&lt;&gt;()</tt></a> <span class="t-mark-rev t-since-cxx20">(since C++20)</span> <br>
"""
actual = _ParseIndexPage(html)
expected = [
("abs", "abs.html", 'int'),
("abs", "complex/abs.html", 'std::complex'),
("acos", "acos.html", None),
("acosh", "acosh.html", None),
("as_bytes", "as_bytes.html", None),
]
self.assertEqual(len(actual), len(expected))
for i in range(0, len(actual)):
self.assertEqual(expected[i][0], actual[i][0])
self.assertTrue(actual[i][1].endswith(expected[i][1]))
self.assertEqual(expected[i][2], actual[i][2])
actual = _ParseIndexPage(html)
expected = [
("abs", "abs.html", "int"),
("abs", "complex/abs.html", "std::complex"),
("acos", "acos.html", None),
("acosh", "acosh.html", None),
("as_bytes", "as_bytes.html", None),
]
self.assertEqual(len(actual), len(expected))
for i in range(0, len(actual)):
self.assertEqual(expected[i][0], actual[i][0])
self.assertTrue(actual[i][1].endswith(expected[i][1]))
self.assertEqual(expected[i][2], actual[i][2])
def testParseSymbolPage_SingleHeader(self):
# Defined in header <cmath>
html = """
def testParseSymbolPage_SingleHeader(self):
# Defined in header <cmath>
html = """
<table class="t-dcl-begin"><tbody>
<tr class="t-dsc-header">
<td> <div>Defined in header <code><a href="cmath.html" title="cmath">&lt;cmath&gt;</a></code>
@ -53,14 +52,13 @@ class TestStdGen(unittest.TestCase):
</tr>
</tbody></table>
"""
self.assertEqual(_ParseSymbolPage(html, 'foo'), set(['<cmath>']))
self.assertEqual(_ParseSymbolPage(html, "foo"), set(["<cmath>"]))
def testParseSymbolPage_MulHeaders(self):
# Defined in header <cstddef>
# Defined in header <cstdio>
# Defined in header <cstdlib>
html = """
def testParseSymbolPage_MulHeaders(self):
# Defined in header <cstddef>
# Defined in header <cstdio>
# Defined in header <cstdlib>
html = """
<table class="t-dcl-begin"><tbody>
<tr class="t-dsc-header">
<td> <div>Defined in header <code><a href="cstddef.html" title="cstddef">&lt;cstddef&gt;</a></code>
@ -94,15 +92,13 @@ class TestStdGen(unittest.TestCase):
</tr>
</tbody></table>
"""
self.assertEqual(_ParseSymbolPage(html, "foo"),
set(['<cstdio>', '<cstdlib>']))
self.assertEqual(_ParseSymbolPage(html, "foo"), set(["<cstdio>", "<cstdlib>"]))
def testParseSymbolPage_MulHeadersInSameDiv(self):
# Multile <code> blocks in a Div.
# Defined in header <algorithm>
# Defined in header <utility>
html = """
def testParseSymbolPage_MulHeadersInSameDiv(self):
# Multile <code> blocks in a Div.
# Defined in header <algorithm>
# Defined in header <utility>
html = """
<table class="t-dcl-begin"><tbody>
<tr class="t-dsc-header">
<td><div>
@ -121,14 +117,15 @@ class TestStdGen(unittest.TestCase):
</tr>
</tbody></table>
"""
self.assertEqual(_ParseSymbolPage(html, "foo"),
set(['<algorithm>', '<utility>']))
self.assertEqual(
_ParseSymbolPage(html, "foo"), set(["<algorithm>", "<utility>"])
)
def testParseSymbolPage_MulSymbolsInSameTd(self):
# defined in header <cstdint>
# int8_t
# int16_t
html = """
def testParseSymbolPage_MulSymbolsInSameTd(self):
# defined in header <cstdint>
# int8_t
# int16_t
html = """
<table class="t-dcl-begin"><tbody>
<tr class="t-dsc-header">
<td><div>
@ -145,11 +142,9 @@ class TestStdGen(unittest.TestCase):
</tr>
</tbody></table>
"""
self.assertEqual(_ParseSymbolPage(html, "int8_t"),
set(['<cstdint>']))
self.assertEqual(_ParseSymbolPage(html, "int16_t"),
set(['<cstdint>']))
self.assertEqual(_ParseSymbolPage(html, "int8_t"), set(["<cstdint>"]))
self.assertEqual(_ParseSymbolPage(html, "int16_t"), set(["<cstdint>"]))
if __name__ == '__main__':
unittest.main()
if __name__ == "__main__":
unittest.main()

4
clang/tools/libclang/linker-script-to-export-list.py
View File

@ -3,9 +3,9 @@ import os
import sys
input_file = open(sys.argv[1])
output_file = open(sys.argv[2], 'w')
output_file = open(sys.argv[2], "w")
for line in input_file:
m = re.search('^\s+(clang_[^;]+)', line)
m = re.search("^\s+(clang_[^;]+)", line)
if m:
output_file.write(m.group(1) + "\n")

155
clang/tools/scan-build-py/lib/libear/__init__.py
View File

@ -13,42 +13,43 @@ import shutil
import contextlib
import logging
__all__ = ['build_libear']
__all__ = ["build_libear"]
def build_libear(compiler, dst_dir):
""" Returns the full path to the 'libear' library. """
"""Returns the full path to the 'libear' library."""
try:
src_dir = os.path.dirname(os.path.realpath(__file__))
toolset = make_toolset(src_dir)
toolset.set_compiler(compiler)
toolset.set_language_standard('c99')
toolset.add_definitions(['-D_GNU_SOURCE'])
toolset.set_language_standard("c99")
toolset.add_definitions(["-D_GNU_SOURCE"])
configure = do_configure(toolset)
configure.check_function_exists('execve', 'HAVE_EXECVE')
configure.check_function_exists('execv', 'HAVE_EXECV')
configure.check_function_exists('execvpe', 'HAVE_EXECVPE')
configure.check_function_exists('execvp', 'HAVE_EXECVP')
configure.check_function_exists('execvP', 'HAVE_EXECVP2')
configure.check_function_exists('exect', 'HAVE_EXECT')
configure.check_function_exists('execl', 'HAVE_EXECL')
configure.check_function_exists('execlp', 'HAVE_EXECLP')
configure.check_function_exists('execle', 'HAVE_EXECLE')
configure.check_function_exists('posix_spawn', 'HAVE_POSIX_SPAWN')
configure.check_function_exists('posix_spawnp', 'HAVE_POSIX_SPAWNP')
configure.check_symbol_exists('_NSGetEnviron', 'crt_externs.h',
'HAVE_NSGETENVIRON')
configure.check_function_exists("execve", "HAVE_EXECVE")
configure.check_function_exists("execv", "HAVE_EXECV")
configure.check_function_exists("execvpe", "HAVE_EXECVPE")
configure.check_function_exists("execvp", "HAVE_EXECVP")
configure.check_function_exists("execvP", "HAVE_EXECVP2")
configure.check_function_exists("exect", "HAVE_EXECT")
configure.check_function_exists("execl", "HAVE_EXECL")
configure.check_function_exists("execlp", "HAVE_EXECLP")
configure.check_function_exists("execle", "HAVE_EXECLE")
configure.check_function_exists("posix_spawn", "HAVE_POSIX_SPAWN")
configure.check_function_exists("posix_spawnp", "HAVE_POSIX_SPAWNP")
configure.check_symbol_exists(
"_NSGetEnviron", "crt_externs.h", "HAVE_NSGETENVIRON"
)
configure.write_by_template(
os.path.join(src_dir, 'config.h.in'),
os.path.join(dst_dir, 'config.h'))
os.path.join(src_dir, "config.h.in"), os.path.join(dst_dir, "config.h")
)
target = create_shared_library('ear', toolset)
target = create_shared_library("ear", toolset)
target.add_include(dst_dir)
target.add_sources('ear.c')
target.add_sources("ear.c")
target.link_against(toolset.dl_libraries())
target.link_against(['pthread'])
target.link_against(["pthread"])
target.build_release(dst_dir)
return os.path.join(dst_dir, target.name)
@ -59,10 +60,11 @@ def build_libear(compiler, dst_dir):
def execute(cmd, *args, **kwargs):
""" Make subprocess execution silent. """
"""Make subprocess execution silent."""
import subprocess
kwargs.update({'stdout': subprocess.PIPE, 'stderr': subprocess.STDOUT})
kwargs.update({"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT})
return subprocess.check_call(cmd, *args, **kwargs)
@ -76,7 +78,7 @@ def TemporaryDirectory(**kwargs):
class Toolset(object):
""" Abstract class to represent different toolset. """
"""Abstract class to represent different toolset."""
def __init__(self, src_dir):
self.src_dir = src_dir
@ -84,15 +86,15 @@ class Toolset(object):
self.c_flags = []
def set_compiler(self, compiler):
""" part of public interface """
"""part of public interface"""
self.compiler = compiler
def set_language_standard(self, standard):
""" part of public interface """
self.c_flags.append('-std=' + standard)
"""part of public interface"""
self.c_flags.append("-std=" + standard)
def add_definitions(self, defines):
""" part of public interface """
"""part of public interface"""
self.c_flags.extend(defines)
def dl_libraries(self):
@ -102,8 +104,8 @@ class Toolset(object):
raise NotImplementedError()
def shared_library_c_flags(self, release):
extra = ['-DNDEBUG', '-O3'] if release else []
return extra + ['-fPIC'] + self.c_flags
extra = ["-DNDEBUG", "-O3"] if release else []
return extra + ["-fPIC"] + self.c_flags
def shared_library_ld_flags(self, release, name):
raise NotImplementedError()
@ -117,11 +119,11 @@ class DarwinToolset(Toolset):
return []
def shared_library_name(self, name):
return 'lib' + name + '.dylib'
return "lib" + name + ".dylib"
def shared_library_ld_flags(self, release, name):
extra = ['-dead_strip'] if release else []
return extra + ['-dynamiclib', '-install_name', '@rpath/' + name]
extra = ["-dead_strip"] if release else []
return extra + ["-dynamiclib", "-install_name", "@rpath/" + name]
class UnixToolset(Toolset):
@ -132,11 +134,11 @@ class UnixToolset(Toolset):
return []
def shared_library_name(self, name):
return 'lib' + name + '.so'
return "lib" + name + ".so"
def shared_library_ld_flags(self, release, name):
extra = [] if release else []
return extra + ['-shared', '-Wl,-soname,' + name]
return extra + ["-shared", "-Wl,-soname," + name]
class LinuxToolset(UnixToolset):
@ -144,16 +146,16 @@ class LinuxToolset(UnixToolset):
UnixToolset.__init__(self, src_dir)
def dl_libraries(self):
return ['dl']
return ["dl"]
def make_toolset(src_dir):
platform = sys.platform
if platform in {'win32', 'cygwin'}:
raise RuntimeError('not implemented on this platform')
elif platform == 'darwin':
if platform in {"win32", "cygwin"}:
raise RuntimeError("not implemented on this platform")
elif platform == "darwin":
return DarwinToolset(src_dir)
elif platform in {'linux', 'linux2'}:
elif platform in {"linux", "linux2"}:
return LinuxToolset(src_dir)
else:
return UnixToolset(src_dir)
@ -162,17 +164,16 @@ def make_toolset(src_dir):
class Configure(object):
def __init__(self, toolset):
self.ctx = toolset
self.results = {'APPLE': sys.platform == 'darwin'}
self.results = {"APPLE": sys.platform == "darwin"}
def _try_to_compile_and_link(self, source):
try:
with TemporaryDirectory() as work_dir:
src_file = 'check.c'
with open(os.path.join(work_dir, src_file), 'w') as handle:
src_file = "check.c"
with open(os.path.join(work_dir, src_file), "w") as handle:
handle.write(source)
execute([self.ctx.compiler, src_file] + self.ctx.c_flags,
cwd=work_dir)
execute([self.ctx.compiler, src_file] + self.ctx.c_flags, cwd=work_dir)
return True
except Exception:
return False
@ -181,39 +182,41 @@ class Configure(object):
template = "int FUNCTION(); int main() { return FUNCTION(); }"
source = template.replace("FUNCTION", function)
logging.debug('Checking function %s', function)
logging.debug("Checking function %s", function)
found = self._try_to_compile_and_link(source)
logging.debug('Checking function %s -- %s', function,
'found' if found else 'not found')
logging.debug(
"Checking function %s -- %s", function, "found" if found else "not found"
)
self.results.update({name: found})
def check_symbol_exists(self, symbol, include, name):
template = """#include <INCLUDE>
int main() { return ((int*)(&SYMBOL))[0]; }"""
source = template.replace('INCLUDE', include).replace("SYMBOL", symbol)
source = template.replace("INCLUDE", include).replace("SYMBOL", symbol)
logging.debug('Checking symbol %s', symbol)
logging.debug("Checking symbol %s", symbol)
found = self._try_to_compile_and_link(source)
logging.debug('Checking symbol %s -- %s', symbol,
'found' if found else 'not found')
logging.debug(
"Checking symbol %s -- %s", symbol, "found" if found else "not found"
)
self.results.update({name: found})
def write_by_template(self, template, output):
def transform(line, definitions):
pattern = re.compile(r'^#cmakedefine\s+(\S+)')
pattern = re.compile(r"^#cmakedefine\s+(\S+)")
m = pattern.match(line)
if m:
key = m.group(1)
if key not in definitions or not definitions[key]:
return '/* #undef {0} */{1}'.format(key, os.linesep)
return "/* #undef {0} */{1}".format(key, os.linesep)
else:
return '#define {0}{1}'.format(key, os.linesep)
return "#define {0}{1}".format(key, os.linesep)
return line
with open(template, 'r') as src_handle:
logging.debug('Writing config to %s', output)
with open(output, 'w') as dst_handle:
with open(template, "r") as src_handle:
logging.debug("Writing config to %s", output)
with open(output, "w") as dst_handle:
for line in src_handle:
dst_handle.write(transform(line, self.results))
@ -231,28 +234,38 @@ class SharedLibrary(object):
self.lib = []
def add_include(self, directory):
self.inc.extend(['-I', directory])
self.inc.extend(["-I", directory])
def add_sources(self, source):
self.src.append(source)
def link_against(self, libraries):
self.lib.extend(['-l' + lib for lib in libraries])
self.lib.extend(["-l" + lib for lib in libraries])
def build_release(self, directory):
for src in self.src:
logging.debug('Compiling %s', src)
logging.debug("Compiling %s", src)
execute(
[self.ctx.compiler, '-c', os.path.join(self.ctx.src_dir, src),
'-o', src + '.o'] + self.inc +
self.ctx.shared_library_c_flags(True),
cwd=directory)
logging.debug('Linking %s', self.name)
[
self.ctx.compiler,
"-c",
os.path.join(self.ctx.src_dir, src),
"-o",
src + ".o",
]
+ self.inc
+ self.ctx.shared_library_c_flags(True),
cwd=directory,
)
logging.debug("Linking %s", self.name)
execute(
[self.ctx.compiler] + [src + '.o' for src in self.src] +
['-o', self.name] + self.lib +
self.ctx.shared_library_ld_flags(True, self.name),
cwd=directory)
[self.ctx.compiler]
+ [src + ".o" for src in self.src]
+ ["-o", self.name]
+ self.lib
+ self.ctx.shared_library_ld_flags(True, self.name),
cwd=directory,
)
def create_shared_library(name, toolset):

108
clang/tools/scan-build-py/lib/libscanbuild/__init__.py
View File

@ -14,22 +14,23 @@ import shlex
import subprocess
import sys
ENVIRONMENT_KEY = 'INTERCEPT_BUILD'
ENVIRONMENT_KEY = "INTERCEPT_BUILD"
Execution = collections.namedtuple('Execution', ['pid', 'cwd', 'cmd'])
Execution = collections.namedtuple("Execution", ["pid", "cwd", "cmd"])
CtuConfig = collections.namedtuple('CtuConfig', ['collect', 'analyze', 'dir',
'extdef_map_cmd'])
CtuConfig = collections.namedtuple(
"CtuConfig", ["collect", "analyze", "dir", "extdef_map_cmd"]
)
def duplicate_check(method):
""" Predicate to detect duplicated entries.
"""Predicate to detect duplicated entries.
Unique hash method can be use to detect duplicates. Entries are
represented as dictionaries, which has no default hash method.
This implementation uses a set datatype to store the unique hash values.
This method returns a method which can detect the duplicate values. """
This method returns a method which can detect the duplicate values."""
def predicate(entry):
entry_hash = predicate.unique(entry)
@ -44,35 +45,36 @@ def duplicate_check(method):
def run_build(command, *args, **kwargs):
""" Run and report build command execution
"""Run and report build command execution
:param command: array of tokens
:return: exit code of the process
"""
environment = kwargs.get('env', os.environ)
logging.debug('run build %s, in environment: %s', command, environment)
environment = kwargs.get("env", os.environ)
logging.debug("run build %s, in environment: %s", command, environment)
exit_code = subprocess.call(command, *args, **kwargs)
logging.debug('build finished with exit code: %d', exit_code)
logging.debug("build finished with exit code: %d", exit_code)
return exit_code
def run_command(command, cwd=None):
""" Run a given command and report the execution.
"""Run a given command and report the execution.
:param command: array of tokens
:param cwd: the working directory where the command will be executed
:return: output of the command
"""
def decode_when_needed(result):
""" check_output returns bytes or string depend on python version """
return result.decode('utf-8') if isinstance(result, bytes) else result
"""check_output returns bytes or string depend on python version"""
return result.decode("utf-8") if isinstance(result, bytes) else result
try:
directory = os.path.abspath(cwd) if cwd else os.getcwd()
logging.debug('exec command %s in %s', command, directory)
output = subprocess.check_output(command,
cwd=directory,
stderr=subprocess.STDOUT)
logging.debug("exec command %s in %s", command, directory)
output = subprocess.check_output(
command, cwd=directory, stderr=subprocess.STDOUT
)
return decode_when_needed(output).splitlines()
except subprocess.CalledProcessError as ex:
ex.output = decode_when_needed(ex.output).splitlines()
@ -80,7 +82,7 @@ def run_command(command, cwd=None):
def reconfigure_logging(verbose_level):
""" Reconfigure logging level and format based on the verbose flag.
"""Reconfigure logging level and format based on the verbose flag.
:param verbose_level: number of `-v` flags received by the command
:return: no return value
@ -95,45 +97,48 @@ def reconfigure_logging(verbose_level):
root.setLevel(level)
# Be verbose with messages.
if verbose_level <= 3:
fmt_string = '%(name)s: %(levelname)s: %(message)s'
fmt_string = "%(name)s: %(levelname)s: %(message)s"
else:
fmt_string = '%(name)s: %(levelname)s: %(funcName)s: %(message)s'
fmt_string = "%(name)s: %(levelname)s: %(funcName)s: %(message)s"
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(fmt=fmt_string))
root.handlers = [handler]
def command_entry_point(function):
""" Decorator for command entry methods.
"""Decorator for command entry methods.
The decorator initialize/shutdown logging and guard on programming
errors (catch exceptions).
The decorated method can have arbitrary parameters, the return value will
be the exit code of the process. """
be the exit code of the process."""
@functools.wraps(function)
def wrapper(*args, **kwargs):
""" Do housekeeping tasks and execute the wrapped method. """
"""Do housekeeping tasks and execute the wrapped method."""
try:
logging.basicConfig(format='%(name)s: %(message)s',
level=logging.WARNING,
stream=sys.stdout)
logging.basicConfig(
format="%(name)s: %(message)s", level=logging.WARNING, stream=sys.stdout
)
# This hack to get the executable name as %(name).
logging.getLogger().name = os.path.basename(sys.argv[0])
return function(*args, **kwargs)
except KeyboardInterrupt:
logging.warning('Keyboard interrupt')
logging.warning("Keyboard interrupt")
return 130 # Signal received exit code for bash.
except Exception:
logging.exception('Internal error.')
logging.exception("Internal error.")
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.error("Please report this bug and attach the output "
"to the bug report")
logging.error(
"Please report this bug and attach the output " "to the bug report"
)
else:
logging.error("Please run this command again and turn on "
"verbose mode (add '-vvvv' as argument).")
logging.error(
"Please run this command again and turn on "
"verbose mode (add '-vvvv' as argument)."
)
return 64 # Some non used exit code for internal errors.
finally:
logging.shutdown()
@ -142,7 +147,7 @@ def command_entry_point(function):
def compiler_wrapper(function):
""" Implements compiler wrapper base functionality.
"""Implements compiler wrapper base functionality.
A compiler wrapper executes the real compiler, then implement some
functionality, then returns with the real compiler exit code.
@ -155,53 +160,56 @@ def compiler_wrapper(function):
The :param function: will receive the following arguments:
:param result: the exit code of the compilation.
:param execution: the command executed by the wrapper. """
:param execution: the command executed by the wrapper."""
def is_cxx_compiler():
""" Find out was it a C++ compiler call. Compiler wrapper names
"""Find out was it a C++ compiler call. Compiler wrapper names
contain the compiler type. C++ compiler wrappers ends with `c++`,
but might have `.exe` extension on windows. """
but might have `.exe` extension on windows."""
wrapper_command = os.path.basename(sys.argv[0])
return re.match(r'(.+)c\+\+(.*)', wrapper_command)
return re.match(r"(.+)c\+\+(.*)", wrapper_command)
def run_compiler(executable):
""" Execute compilation with the real compiler. """
"""Execute compilation with the real compiler."""
command = executable + sys.argv[1:]
logging.debug('compilation: %s', command)
logging.debug("compilation: %s", command)
result = subprocess.call(command)
logging.debug('compilation exit code: %d', result)
logging.debug("compilation exit code: %d", result)
return result
# Get relevant parameters from environment.
parameters = json.loads(os.environ[ENVIRONMENT_KEY])
reconfigure_logging(parameters['verbose'])
reconfigure_logging(parameters["verbose"])
# Execute the requested compilation. Do crash if anything goes wrong.
cxx = is_cxx_compiler()
compiler = parameters['cxx'] if cxx else parameters['cc']
compiler = parameters["cxx"] if cxx else parameters["cc"]
result = run_compiler(compiler)
# Call the wrapped method and ignore it's return value.
try:
call = Execution(
pid=os.getpid(),
cwd=os.getcwd(),
cmd=['c++' if cxx else 'cc'] + sys.argv[1:])
cmd=["c++" if cxx else "cc"] + sys.argv[1:],
)
function(result, call)
except:
logging.exception('Compiler wrapper failed complete.')
logging.exception("Compiler wrapper failed complete.")
finally:
# Always return the real compiler exit code.
return result
def wrapper_environment(args):
""" Set up environment for interpose compiler wrapper."""
"""Set up environment for interpose compiler wrapper."""
return {
ENVIRONMENT_KEY: json.dumps({
'verbose': args.verbose,
'cc': shlex.split(args.cc),
'cxx': shlex.split(args.cxx)
})
ENVIRONMENT_KEY: json.dumps(
{
"verbose": args.verbose,
"cc": shlex.split(args.cc),
"cxx": shlex.split(args.cxx),
}
)
}

653
clang/tools/scan-build-py/lib/libscanbuild/analyze.py
View File

File diff suppressed because it is too large Load Diff

500
clang/tools/scan-build-py/lib/libscanbuild/arguments.py
View File

@ -21,65 +21,68 @@ import tempfile
from libscanbuild import reconfigure_logging, CtuConfig
from libscanbuild.clang import get_checkers, is_ctu_capable
__all__ = ['parse_args_for_intercept_build', 'parse_args_for_analyze_build',
'parse_args_for_scan_build']
__all__ = [
"parse_args_for_intercept_build",
"parse_args_for_analyze_build",
"parse_args_for_scan_build",
]
def parse_args_for_intercept_build():
""" Parse and validate command-line arguments for intercept-build. """
"""Parse and validate command-line arguments for intercept-build."""
parser = create_intercept_parser()
args = parser.parse_args()
reconfigure_logging(args.verbose)
logging.debug('Raw arguments %s', sys.argv)
logging.debug("Raw arguments %s", sys.argv)
# short validation logic
if not args.build:
parser.error(message='missing build command')
parser.error(message="missing build command")
logging.debug('Parsed arguments: %s', args)
logging.debug("Parsed arguments: %s", args)
return args
def parse_args_for_analyze_build():
""" Parse and validate command-line arguments for analyze-build. """
"""Parse and validate command-line arguments for analyze-build."""
from_build_command = False
parser = create_analyze_parser(from_build_command)
args = parser.parse_args()
reconfigure_logging(args.verbose)
logging.debug('Raw arguments %s', sys.argv)
logging.debug("Raw arguments %s", sys.argv)
normalize_args_for_analyze(args, from_build_command)
validate_args_for_analyze(parser, args, from_build_command)
logging.debug('Parsed arguments: %s', args)
logging.debug("Parsed arguments: %s", args)
return args
def parse_args_for_scan_build():
""" Parse and validate command-line arguments for scan-build. """
"""Parse and validate command-line arguments for scan-build."""
from_build_command = True
parser = create_analyze_parser(from_build_command)
args = parser.parse_args()
reconfigure_logging(args.verbose)
logging.debug('Raw arguments %s', sys.argv)
logging.debug("Raw arguments %s", sys.argv)
normalize_args_for_analyze(args, from_build_command)
validate_args_for_analyze(parser, args, from_build_command)
logging.debug('Parsed arguments: %s', args)
logging.debug("Parsed arguments: %s", args)
return args
def normalize_args_for_analyze(args, from_build_command):
""" Normalize parsed arguments for analyze-build and scan-build.
"""Normalize parsed arguments for analyze-build and scan-build.
:param args: Parsed argument object. (Will be mutated.)
:param from_build_command: Boolean value tells is the command suppose
to run the analyzer against a build command or a compilation db. """
to run the analyzer against a build command or a compilation db."""
# make plugins always a list. (it might be None when not specified.)
if args.plugins is None:
@ -96,16 +99,19 @@ def normalize_args_for_analyze(args, from_build_command):
# which have good default value.)
if from_build_command:
# add cdb parameter invisibly to make report module working.
args.cdb = 'compile_commands.json'
args.cdb = "compile_commands.json"
# Make ctu_dir an abspath as it is needed inside clang
if not from_build_command and hasattr(args, 'ctu_phases') \
and hasattr(args.ctu_phases, 'dir'):
if (
not from_build_command
and hasattr(args, "ctu_phases")
and hasattr(args.ctu_phases, "dir")
):
args.ctu_dir = os.path.abspath(args.ctu_dir)
def validate_args_for_analyze(parser, args, from_build_command):
""" Command line parsing is done by the argparse module, but semantic
"""Command line parsing is done by the argparse module, but semantic
validation still needs to be done. This method is doing it for
analyze-build and scan-build commands.
@ -114,7 +120,7 @@ def validate_args_for_analyze(parser, args, from_build_command):
:param from_build_command: Boolean value tells is the command suppose
to run the analyzer against a build command or a compilation db.
:return: No return value, but this call might throw when validation
fails. """
fails."""
if args.help_checkers_verbose:
print_checkers(get_checkers(args.clang, args.plugins))
@ -123,25 +129,33 @@ def validate_args_for_analyze(parser, args, from_build_command):
print_active_checkers(get_checkers(args.clang, args.plugins))
parser.exit(status=0)
elif from_build_command and not args.build:
parser.error(message='missing build command')
parser.error(message="missing build command")
elif not from_build_command and not os.path.exists(args.cdb):
parser.error(message='compilation database is missing')
parser.error(message="compilation database is missing")
# If the user wants CTU mode
if not from_build_command and hasattr(args, 'ctu_phases') \
and hasattr(args.ctu_phases, 'dir'):
if (
not from_build_command
and hasattr(args, "ctu_phases")
and hasattr(args.ctu_phases, "dir")
):
# If CTU analyze_only, the input directory should exist
if args.ctu_phases.analyze and not args.ctu_phases.collect \
and not os.path.exists(args.ctu_dir):
parser.error(message='missing CTU directory')
if (
args.ctu_phases.analyze
and not args.ctu_phases.collect
and not os.path.exists(args.ctu_dir)
):
parser.error(message="missing CTU directory")
# Check CTU capability via checking clang-extdef-mapping
if not is_ctu_capable(args.extdef_map_cmd):
parser.error(message="""This version of clang does not support CTU
functionality or clang-extdef-mapping command not found.""")
parser.error(
message="""This version of clang does not support CTU
functionality or clang-extdef-mapping command not found."""
)
def create_intercept_parser():
""" Creates a parser for command-line arguments to 'intercept'. """
"""Creates a parser for command-line arguments to 'intercept'."""
parser = create_default_parser()
parser_add_cdb(parser)
@ -149,23 +163,25 @@ def create_intercept_parser():
parser_add_prefer_wrapper(parser)
parser_add_compilers(parser)
advanced = parser.add_argument_group('advanced options')
advanced = parser.add_argument_group("advanced options")
group = advanced.add_mutually_exclusive_group()
group.add_argument(
'--append',
action='store_true',
"--append",
action="store_true",
help="""Extend existing compilation database with new entries.
Duplicate entries are detected and not present in the final output.
The output is not continuously updated, it's done when the build
command finished. """)
command finished. """,
)
parser.add_argument(
dest='build', nargs=argparse.REMAINDER, help="""Command to run.""")
dest="build", nargs=argparse.REMAINDER, help="""Command to run."""
)
return parser
def create_analyze_parser(from_build_command):
""" Creates a parser for command-line arguments to 'analyze'. """
"""Creates a parser for command-line arguments to 'analyze'."""
parser = create_default_parser()
@ -174,155 +190,174 @@ def create_analyze_parser(from_build_command):
parser_add_compilers(parser)
parser.add_argument(
'--intercept-first',
action='store_true',
"--intercept-first",
action="store_true",
help="""Run the build commands first, intercept compiler
calls and then run the static analyzer afterwards.
Generally speaking it has better coverage on build commands.
With '--override-compiler' it use compiler wrapper, but does
not run the analyzer till the build is finished.""")
not run the analyzer till the build is finished.""",
)
else:
parser_add_cdb(parser)
parser.add_argument(
'--status-bugs',
action='store_true',
"--status-bugs",
action="store_true",
help="""The exit status of '%(prog)s' is the same as the executed
build command. This option ignores the build exit status and sets to
be non zero if it found potential bugs or zero otherwise.""")
be non zero if it found potential bugs or zero otherwise.""",
)
parser.add_argument(
'--exclude',
metavar='<directory>',
dest='excludes',
action='append',
"--exclude",
metavar="<directory>",
dest="excludes",
action="append",
default=[],
help="""Do not run static analyzer against files found in this
directory. (You can specify this option multiple times.)
Could be useful when project contains 3rd party libraries.""")
Could be useful when project contains 3rd party libraries.""",
)
output = parser.add_argument_group('output control options')
output = parser.add_argument_group("output control options")
output.add_argument(
'--output',
'-o',
metavar='<path>',
"--output",
"-o",
metavar="<path>",
default=tempfile.gettempdir(),
help="""Specifies the output directory for analyzer reports.
Subdirectory will be created if default directory is targeted.""")
Subdirectory will be created if default directory is targeted.""",
)
output.add_argument(
'--keep-empty',
action='store_true',
"--keep-empty",
action="store_true",
help="""Don't remove the build results directory even if no issues
were reported.""")
were reported.""",
)
output.add_argument(
'--html-title',
metavar='<title>',
"--html-title",
metavar="<title>",
help="""Specify the title used on generated HTML pages.
If not specified, a default title will be used.""")
If not specified, a default title will be used.""",
)
format_group = output.add_mutually_exclusive_group()
format_group.add_argument(
'--plist',
'-plist',
dest='output_format',
const='plist',
default='html',
action='store_const',
help="""Cause the results as a set of .plist files.""")
"--plist",
"-plist",
dest="output_format",
const="plist",
default="html",
action="store_const",
help="""Cause the results as a set of .plist files.""",
)
format_group.add_argument(
'--plist-html',
'-plist-html',
dest='output_format',
const='plist-html',
default='html',
action='store_const',
help="""Cause the results as a set of .html and .plist files.""")
"--plist-html",
"-plist-html",
dest="output_format",
const="plist-html",
default="html",
action="store_const",
help="""Cause the results as a set of .html and .plist files.""",
)
format_group.add_argument(
'--plist-multi-file',
'-plist-multi-file',
dest='output_format',
const='plist-multi-file',
default='html',
action='store_const',
"--plist-multi-file",
"-plist-multi-file",
dest="output_format",
const="plist-multi-file",
default="html",
action="store_const",
help="""Cause the results as a set of .plist files with extra
information on related files.""")
information on related files.""",
)
format_group.add_argument(
'--sarif',
'-sarif',
dest='output_format',
const='sarif',
default='html',
action='store_const',
help="""Cause the results as a result.sarif file.""")
"--sarif",
"-sarif",
dest="output_format",
const="sarif",
default="html",
action="store_const",
help="""Cause the results as a result.sarif file.""",
)
format_group.add_argument(
'--sarif-html',
'-sarif-html',
dest='output_format',
const='sarif-html',
default='html',
action='store_const',
help="""Cause the results as a result.sarif file and .html files.""")
"--sarif-html",
"-sarif-html",
dest="output_format",
const="sarif-html",
default="html",
action="store_const",
help="""Cause the results as a result.sarif file and .html files.""",
)
advanced = parser.add_argument_group('advanced options')
advanced = parser.add_argument_group("advanced options")
advanced.add_argument(
'--use-analyzer',
metavar='<path>',
dest='clang',
default='clang',
"--use-analyzer",
metavar="<path>",
dest="clang",
default="clang",
help="""'%(prog)s' uses the 'clang' executable relative to itself for
static analysis. One can override this behavior with this option by
using the 'clang' packaged with Xcode (on OS X) or from the PATH.""")
using the 'clang' packaged with Xcode (on OS X) or from the PATH.""",
)
advanced.add_argument(
'--no-failure-reports',
'-no-failure-reports',
dest='output_failures',
action='store_false',
"--no-failure-reports",
"-no-failure-reports",
dest="output_failures",
action="store_false",
help="""Do not create a 'failures' subdirectory that includes analyzer
crash reports and preprocessed source files.""")
crash reports and preprocessed source files.""",
)
parser.add_argument(
'--analyze-headers',
action='store_true',
"--analyze-headers",
action="store_true",
help="""Also analyze functions in #included files. By default, such
functions are skipped unless they are called by functions within the
main source file.""")
main source file.""",
)
advanced.add_argument(
'--stats',
'-stats',
action='store_true',
help="""Generates visitation statistics for the project.""")
"--stats",
"-stats",
action="store_true",
help="""Generates visitation statistics for the project.""",
)
advanced.add_argument(
'--internal-stats',
action='store_true',
help="""Generate internal analyzer statistics.""")
"--internal-stats",
action="store_true",
help="""Generate internal analyzer statistics.""",
)
advanced.add_argument(
'--maxloop',
'-maxloop',
metavar='<loop count>',
"--maxloop",
"-maxloop",
metavar="<loop count>",
type=int,
help="""Specify the number of times a block can be visited before
giving up. Increase for more comprehensive coverage at a cost of
speed.""")
speed.""",
)
advanced.add_argument(
'--store',
'-store',
metavar='<model>',
dest='store_model',
choices=['region', 'basic'],
"--store",
"-store",
metavar="<model>",
dest="store_model",
choices=["region", "basic"],
help="""Specify the store model used by the analyzer. 'region'
specifies a field- sensitive store model. 'basic' which is far less
precise but can more quickly analyze code. 'basic' was the default
store model for checker-0.221 and earlier.""")
store model for checker-0.221 and earlier.""",
)
advanced.add_argument(
'--constraints',
'-constraints',
metavar='<model>',
dest='constraints_model',
choices=['range', 'basic'],
"--constraints",
"-constraints",
metavar="<model>",
dest="constraints_model",
choices=["range", "basic"],
help="""Specify the constraint engine used by the analyzer. Specifying
'basic' uses a simpler, less powerful constraint model used by
checker-0.160 and earlier.""")
checker-0.160 and earlier.""",
)
advanced.add_argument(
'--analyzer-config',
'-analyzer-config',
metavar='<options>',
"--analyzer-config",
"-analyzer-config",
metavar="<options>",
help="""Provide options to pass through to the analyzer's
-analyzer-config flag. Several options are separated with comma:
'key1=val1,key2=val2'
@ -332,134 +367,148 @@ def create_analyze_parser(from_build_command):
Switch the page naming to:
report-<filename>-<function/method name>-<id>.html
instead of report-XXXXXX.html""")
instead of report-XXXXXX.html""",
)
advanced.add_argument(
'--force-analyze-debug-code',
dest='force_debug',
action='store_true',
"--force-analyze-debug-code",
dest="force_debug",
action="store_true",
help="""Tells analyzer to enable assertions in code even if they were
disabled during compilation, enabling more precise results.""")
disabled during compilation, enabling more precise results.""",
)
plugins = parser.add_argument_group('checker options')
plugins = parser.add_argument_group("checker options")
plugins.add_argument(
'--load-plugin',
'-load-plugin',
metavar='<plugin library>',
dest='plugins',
action='append',
help="""Loading external checkers using the clang plugin interface.""")
"--load-plugin",
"-load-plugin",
metavar="<plugin library>",
dest="plugins",
action="append",
help="""Loading external checkers using the clang plugin interface.""",
)
plugins.add_argument(
'--enable-checker',
'-enable-checker',
metavar='<checker name>',
"--enable-checker",
"-enable-checker",
metavar="<checker name>",
action=AppendCommaSeparated,
help="""Enable specific checker.""")
help="""Enable specific checker.""",
)
plugins.add_argument(
'--disable-checker',
'-disable-checker',
metavar='<checker name>',
"--disable-checker",
"-disable-checker",
metavar="<checker name>",
action=AppendCommaSeparated,
help="""Disable specific checker.""")
help="""Disable specific checker.""",
)
plugins.add_argument(
'--help-checkers',
action='store_true',
"--help-checkers",
action="store_true",
help="""A default group of checkers is run unless explicitly disabled.
Exactly which checkers constitute the default group is a function of
the operating system in use. These can be printed with this flag.""")
the operating system in use. These can be printed with this flag.""",
)
plugins.add_argument(
'--help-checkers-verbose',
action='store_true',
help="""Print all available checkers and mark the enabled ones.""")
"--help-checkers-verbose",
action="store_true",
help="""Print all available checkers and mark the enabled ones.""",
)
if from_build_command:
parser.add_argument(
dest='build', nargs=argparse.REMAINDER, help="""Command to run.""")
dest="build", nargs=argparse.REMAINDER, help="""Command to run."""
)
else:
ctu = parser.add_argument_group('cross translation unit analysis')
ctu = parser.add_argument_group("cross translation unit analysis")
ctu_mutex_group = ctu.add_mutually_exclusive_group()
ctu_mutex_group.add_argument(
'--ctu',
action='store_const',
const=CtuConfig(collect=True, analyze=True,
dir='', extdef_map_cmd=''),
dest='ctu_phases',
"--ctu",
action="store_const",
const=CtuConfig(collect=True, analyze=True, dir="", extdef_map_cmd=""),
dest="ctu_phases",
help="""Perform cross translation unit (ctu) analysis (both collect
and analyze phases) using default <ctu-dir> for temporary output.
At the end of the analysis, the temporary directory is removed.""")
At the end of the analysis, the temporary directory is removed.""",
)
ctu.add_argument(
'--ctu-dir',
metavar='<ctu-dir>',
dest='ctu_dir',
default='ctu-dir',
"--ctu-dir",
metavar="<ctu-dir>",
dest="ctu_dir",
default="ctu-dir",
help="""Defines the temporary directory used between ctu
phases.""")
phases.""",
)
ctu_mutex_group.add_argument(
'--ctu-collect-only',
action='store_const',
const=CtuConfig(collect=True, analyze=False,
dir='', extdef_map_cmd=''),
dest='ctu_phases',
"--ctu-collect-only",
action="store_const",
const=CtuConfig(collect=True, analyze=False, dir="", extdef_map_cmd=""),
dest="ctu_phases",
help="""Perform only the collect phase of ctu.
Keep <ctu-dir> for further use.""")
Keep <ctu-dir> for further use.""",
)
ctu_mutex_group.add_argument(
'--ctu-analyze-only',
action='store_const',
const=CtuConfig(collect=False, analyze=True,
dir='', extdef_map_cmd=''),
dest='ctu_phases',
"--ctu-analyze-only",
action="store_const",
const=CtuConfig(collect=False, analyze=True, dir="", extdef_map_cmd=""),
dest="ctu_phases",
help="""Perform only the analyze phase of ctu. <ctu-dir> should be
present and will not be removed after analysis.""")
present and will not be removed after analysis.""",
)
ctu.add_argument(
'--use-extdef-map-cmd',
metavar='<path>',
dest='extdef_map_cmd',
default='clang-extdef-mapping',
"--use-extdef-map-cmd",
metavar="<path>",
dest="extdef_map_cmd",
default="clang-extdef-mapping",
help="""'%(prog)s' uses the 'clang-extdef-mapping' executable
relative to itself for generating external definition maps for
static analysis. One can override this behavior with this option
by using the 'clang-extdef-mapping' packaged with Xcode (on OS X)
or from the PATH.""")
or from the PATH.""",
)
return parser
def create_default_parser():
""" Creates command line parser for all build wrapper commands. """
"""Creates command line parser for all build wrapper commands."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
'--verbose',
'-v',
action='count',
"--verbose",
"-v",
action="count",
default=0,
help="""Enable verbose output from '%(prog)s'. A second, third and
fourth flags increases verbosity.""")
fourth flags increases verbosity.""",
)
return parser
def parser_add_cdb(parser):
parser.add_argument(
'--cdb',
metavar='<file>',
"--cdb",
metavar="<file>",
default="compile_commands.json",
help="""The JSON compilation database.""")
help="""The JSON compilation database.""",
)
def parser_add_prefer_wrapper(parser):
parser.add_argument(
'--override-compiler',
action='store_true',
"--override-compiler",
action="store_true",
help="""Always resort to the compiler wrapper even when better
intercept methods are available.""")
intercept methods are available.""",
)
def parser_add_compilers(parser):
parser.add_argument(
'--use-cc',
metavar='<path>',
dest='cc',
default=os.getenv('CC', 'cc'),
"--use-cc",
metavar="<path>",
dest="cc",
default=os.getenv("CC", "cc"),
help="""When '%(prog)s' analyzes a project by interposing a compiler
wrapper, which executes a real compiler for compilation and do other
tasks (record the compiler invocation). Because of this interposing,
@ -468,17 +517,19 @@ def parser_add_compilers(parser):
your default compiler.
If you need '%(prog)s' to use a specific compiler for *compilation*
then you can use this option to specify a path to that compiler.""")
then you can use this option to specify a path to that compiler.""",
)
parser.add_argument(
'--use-c++',
metavar='<path>',
dest='cxx',
default=os.getenv('CXX', 'c++'),
help="""This is the same as "--use-cc" but for C++ code.""")
"--use-c++",
metavar="<path>",
dest="cxx",
default=os.getenv("CXX", "c++"),
help="""This is the same as "--use-cc" but for C++ code.""",
)
class AppendCommaSeparated(argparse.Action):
""" argparse Action class to support multiple comma separated lists. """
"""argparse Action class to support multiple comma separated lists."""
def __call__(self, __parser, namespace, values, __option_string):
# getattr(obj, attr, default) does not really returns default but none
@ -486,32 +537,31 @@ class AppendCommaSeparated(argparse.Action):
setattr(namespace, self.dest, [])
# once it's fixed we can use as expected
actual = getattr(namespace, self.dest)
actual.extend(values.split(','))
actual.extend(values.split(","))
setattr(namespace, self.dest, actual)
def print_active_checkers(checkers):
""" Print active checkers to stdout. """
"""Print active checkers to stdout."""
for name in sorted(name for name, (_, active) in checkers.items()
if active):
for name in sorted(name for name, (_, active) in checkers.items() if active):
print(name)
def print_checkers(checkers):
""" Print verbose checker help to stdout. """
"""Print verbose checker help to stdout."""
print('')
print('available checkers:')
print('')
print("")
print("available checkers:")
print("")
for name in sorted(checkers.keys()):
description, active = checkers[name]
prefix = '+' if active else ' '
prefix = "+" if active else " "
if len(name) > 30:
print(' {0} {1}'.format(prefix, name))
print(' ' * 35 + description)
print(" {0} {1}".format(prefix, name))
print(" " * 35 + description)
else:
print(' {0} {1: <30} {2}'.format(prefix, name, description))
print('')
print(" {0} {1: <30} {2}".format(prefix, name, description))
print("")
print('NOTE: "+" indicates that an analysis is enabled by default.')
print('')
print("")

91
clang/tools/scan-build-py/lib/libscanbuild/clang.py
View File

@ -12,11 +12,16 @@ import re
from libscanbuild import run_command
from libscanbuild.shell import decode
__all__ = ['get_version', 'get_arguments', 'get_checkers', 'is_ctu_capable',
'get_triple_arch']
__all__ = [
"get_version",
"get_arguments",
"get_checkers",
"is_ctu_capable",
"get_triple_arch",
]
# regex for activated checker
ACTIVE_CHECKER_PATTERN = re.compile(r'^-analyzer-checker=(.*)$')
ACTIVE_CHECKER_PATTERN = re.compile(r"^-analyzer-checker=(.*)$")
class ClangErrorException(Exception):
@ -25,38 +30,38 @@ class ClangErrorException(Exception):
def get_version(clang):
""" Returns the compiler version as string.
"""Returns the compiler version as string.
:param clang: the compiler we are using
:return: the version string printed to stderr """
:return: the version string printed to stderr"""
output = run_command([clang, '-v'])
output = run_command([clang, "-v"])
# the relevant version info is in the first line
return output[0]
def get_arguments(command, cwd):
""" Capture Clang invocation.
"""Capture Clang invocation.
:param command: the compilation command
:param cwd: the current working directory
:return: the detailed front-end invocation command """
:return: the detailed front-end invocation command"""
cmd = command[:]
cmd.insert(1, '-###')
cmd.append('-fno-color-diagnostics')
cmd.insert(1, "-###")
cmd.append("-fno-color-diagnostics")
output = run_command(cmd, cwd=cwd)
# The relevant information is in the last line of the output.
# Don't check if finding last line fails, would throw exception anyway.
last_line = output[-1]
if re.search(r'clang(.*): error:', last_line):
if re.search(r"clang(.*): error:", last_line):
raise ClangErrorException(last_line)
return decode(last_line)
def get_active_checkers(clang, plugins):
""" Get the active checker list.
"""Get the active checker list.
:param clang: the compiler we are using
:param plugins: list of plugins which was requested by the user
@ -65,40 +70,42 @@ def get_active_checkers(clang, plugins):
To get the default checkers we execute Clang to print how this
compilation would be called. And take out the enabled checker from the
arguments. For input file we specify stdin and pass only language
information. """
information."""
def get_active_checkers_for(language):
""" Returns a list of active checkers for the given language. """
"""Returns a list of active checkers for the given language."""
load_args = [arg
for plugin in plugins
for arg in ['-Xclang', '-load', '-Xclang', plugin]]
cmd = [clang, '--analyze'] + load_args + ['-x', language, '-']
return [ACTIVE_CHECKER_PATTERN.match(arg).group(1)
for arg in get_arguments(cmd, '.')
if ACTIVE_CHECKER_PATTERN.match(arg)]
load_args = [
arg for plugin in plugins for arg in ["-Xclang", "-load", "-Xclang", plugin]
]
cmd = [clang, "--analyze"] + load_args + ["-x", language, "-"]
return [
ACTIVE_CHECKER_PATTERN.match(arg).group(1)
for arg in get_arguments(cmd, ".")
if ACTIVE_CHECKER_PATTERN.match(arg)
]
result = set()
for language in ['c', 'c++', 'objective-c', 'objective-c++']:
for language in ["c", "c++", "objective-c", "objective-c++"]:
result.update(get_active_checkers_for(language))
return frozenset(result)
def is_active(checkers):
""" Returns a method, which classifies the checker active or not,
based on the received checker name list. """
"""Returns a method, which classifies the checker active or not,
based on the received checker name list."""
def predicate(checker):
""" Returns True if the given checker is active. """
"""Returns True if the given checker is active."""
return any(pattern.match(checker) for pattern in predicate.patterns)
predicate.patterns = [re.compile(r'^' + a + r'(\.|$)') for a in checkers]
predicate.patterns = [re.compile(r"^" + a + r"(\.|$)") for a in checkers]
return predicate
def parse_checkers(stream):
""" Parse clang -analyzer-checker-help output.
"""Parse clang -analyzer-checker-help output.
Below the line 'CHECKERS:' are there the name description pairs.
Many of them are in one line, but some long named checker has the
@ -112,40 +119,40 @@ def parse_checkers(stream):
:param stream: list of lines to parse
:return: generator of tuples
(<checker name>, <checker description>) """
(<checker name>, <checker description>)"""
lines = iter(stream)
# find checkers header
for line in lines:
if re.match(r'^CHECKERS:', line):
if re.match(r"^CHECKERS:", line):
break
# find entries
state = None
for line in lines:
if state and not re.match(r'^\s\s\S', line):
if state and not re.match(r"^\s\s\S", line):
yield (state, line.strip())
state = None
elif re.match(r'^\s\s\S+$', line.rstrip()):
elif re.match(r"^\s\s\S+$", line.rstrip()):
state = line.strip()
else:
pattern = re.compile(r'^\s\s(?P<key>\S*)\s*(?P<value>.*)')
pattern = re.compile(r"^\s\s(?P<key>\S*)\s*(?P<value>.*)")
match = pattern.match(line.rstrip())
if match:
current = match.groupdict()
yield (current['key'], current['value'])
yield (current["key"], current["value"])
def get_checkers(clang, plugins):
""" Get all the available checkers from default and from the plugins.
"""Get all the available checkers from default and from the plugins.
:param clang: the compiler we are using
:param plugins: list of plugins which was requested by the user
:return: a dictionary of all available checkers and its status
{<checker name>: (<checker description>, <is active by default>)} """
{<checker name>: (<checker description>, <is active by default>)}"""
load = [elem for plugin in plugins for elem in ['-load', plugin]]
cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help']
load = [elem for plugin in plugins for elem in ["-load", plugin]]
cmd = [clang, "-cc1"] + load + ["-analyzer-checker-help"]
lines = run_command(cmd)
@ -156,17 +163,17 @@ def get_checkers(clang, plugins):
for name, description in parse_checkers(lines)
}
if not checkers:
raise Exception('Could not query Clang for available checkers.')
raise Exception("Could not query Clang for available checkers.")
return checkers
def is_ctu_capable(extdef_map_cmd):
""" Detects if the current (or given) clang and external definition mapping
executables are CTU compatible. """
"""Detects if the current (or given) clang and external definition mapping
executables are CTU compatible."""
try:
run_command([extdef_map_cmd, '-version'])
run_command([extdef_map_cmd, "-version"])
except (OSError, subprocess.CalledProcessError):
return False
return True
@ -174,7 +181,7 @@ def is_ctu_capable(extdef_map_cmd):
def get_triple_arch(command, cwd):
"""Returns the architecture part of the target triple for the given
compilation command. """
compilation command."""
cmd = get_arguments(command, cwd)
try:

109
clang/tools/scan-build-py/lib/libscanbuild/compilation.py
View File

@ -8,7 +8,7 @@ import re
import os
import collections
__all__ = ['split_command', 'classify_source', 'compiler_language']
__all__ = ["split_command", "classify_source", "compiler_language"]
# Ignored compiler options map for compilation database creation.
# The map is used in `split_command` method. (Which does ignore and classify
@ -19,55 +19,56 @@ __all__ = ['split_command', 'classify_source', 'compiler_language']
IGNORED_FLAGS = {
# compiling only flag, ignored because the creator of compilation
# database will explicitly set it.
'-c': 0,
"-c": 0,
# preprocessor macros, ignored because would cause duplicate entries in
# the output (the only difference would be these flags). this is actual
# finding from users, who suffered longer execution time caused by the
# duplicates.
'-MD': 0,
'-MMD': 0,
'-MG': 0,
'-MP': 0,
'-MF': 1,
'-MT': 1,
'-MQ': 1,
"-MD": 0,
"-MMD": 0,
"-MG": 0,
"-MP": 0,
"-MF": 1,
"-MT": 1,
"-MQ": 1,
# linker options, ignored because for compilation database will contain
# compilation commands only. so, the compiler would ignore these flags
# anyway. the benefit to get rid of them is to make the output more
# readable.
'-static': 0,
'-shared': 0,
'-s': 0,
'-rdynamic': 0,
'-l': 1,
'-L': 1,
'-u': 1,
'-z': 1,
'-T': 1,
'-Xlinker': 1
"-static": 0,
"-shared": 0,
"-s": 0,
"-rdynamic": 0,
"-l": 1,
"-L": 1,
"-u": 1,
"-z": 1,
"-T": 1,
"-Xlinker": 1,
}
# Known C/C++ compiler executable name patterns
COMPILER_PATTERNS = frozenset([
re.compile(r'^(intercept-|analyze-|)c(c|\+\+)$'),
re.compile(r'^([^-]*-)*[mg](cc|\+\+)(-\d+(\.\d+){0,2})?$'),
re.compile(r'^([^-]*-)*clang(\+\+)?(-\d+(\.\d+){0,2})?$'),
re.compile(r'^llvm-g(cc|\+\+)$'),
])
COMPILER_PATTERNS = frozenset(
[
re.compile(r"^(intercept-|analyze-|)c(c|\+\+)$"),
re.compile(r"^([^-]*-)*[mg](cc|\+\+)(-\d+(\.\d+){0,2})?$"),
re.compile(r"^([^-]*-)*clang(\+\+)?(-\d+(\.\d+){0,2})?$"),
re.compile(r"^llvm-g(cc|\+\+)$"),
]
)
def split_command(command):
""" Returns a value when the command is a compilation, None otherwise.
"""Returns a value when the command is a compilation, None otherwise.
The value on success is a named tuple with the following attributes:
files: list of source files
flags: list of compile options
compiler: string value of 'c' or 'c++' """
compiler: string value of 'c' or 'c++'"""
# the result of this method
result = collections.namedtuple('Compilation',
['compiler', 'flags', 'files'])
result = collections.namedtuple("Compilation", ["compiler", "flags", "files"])
result.compiler = compiler_language(command)
result.flags = []
result.files = []
@ -78,20 +79,20 @@ def split_command(command):
args = iter(command[1:])
for arg in args:
# quit when compilation pass is not involved
if arg in {'-E', '-S', '-cc1', '-M', '-MM', '-###'}:
if arg in {"-E", "-S", "-cc1", "-M", "-MM", "-###"}:
return None
# ignore some flags
elif arg in IGNORED_FLAGS:
count = IGNORED_FLAGS[arg]
for _ in range(count):
next(args)
elif re.match(r'^-(l|L|Wl,).+', arg):
elif re.match(r"^-(l|L|Wl,).+", arg):
pass
# some parameters could look like filename, take as compile option
elif arg in {'-D', '-I'}:
elif arg in {"-D", "-I"}:
result.flags.extend([arg, next(args)])
# parameter which looks source file is taken...
elif re.match(r'^[^-].+', arg) and classify_source(arg):
elif re.match(r"^[^-].+", arg) and classify_source(arg):
result.files.append(arg)
# and consider everything else as compile option.
else:
@ -101,25 +102,25 @@ def split_command(command):
def classify_source(filename, c_compiler=True):
""" Return the language from file name extension. """
"""Return the language from file name extension."""
mapping = {
'.c': 'c' if c_compiler else 'c++',
'.i': 'c-cpp-output' if c_compiler else 'c++-cpp-output',
'.ii': 'c++-cpp-output',
'.m': 'objective-c',
'.mi': 'objective-c-cpp-output',
'.mm': 'objective-c++',
'.mii': 'objective-c++-cpp-output',
'.C': 'c++',
'.cc': 'c++',
'.CC': 'c++',
'.cp': 'c++',
'.cpp': 'c++',
'.cxx': 'c++',
'.c++': 'c++',
'.C++': 'c++',
'.txx': 'c++'
".c": "c" if c_compiler else "c++",
".i": "c-cpp-output" if c_compiler else "c++-cpp-output",
".ii": "c++-cpp-output",
".m": "objective-c",
".mi": "objective-c-cpp-output",
".mm": "objective-c++",
".mii": "objective-c++-cpp-output",
".C": "c++",
".cc": "c++",
".CC": "c++",
".cp": "c++",
".cpp": "c++",
".cxx": "c++",
".c++": "c++",
".C++": "c++",
".txx": "c++",
}
__, extension = os.path.splitext(os.path.basename(filename))
@ -127,14 +128,14 @@ def classify_source(filename, c_compiler=True):
def compiler_language(command):
""" A predicate to decide the command is a compiler call or not.
"""A predicate to decide the command is a compiler call or not.
Returns 'c' or 'c++' when it match. None otherwise. """
Returns 'c' or 'c++' when it match. None otherwise."""
cplusplus = re.compile(r'^(.+)(\+\+)(-.+|)$')
cplusplus = re.compile(r"^(.+)(\+\+)(-.+|)$")
if command:
executable = os.path.basename(command[0])
if any(pattern.match(executable) for pattern in COMPILER_PATTERNS):
return 'c++' if cplusplus.match(executable) else 'c'
return "c++" if cplusplus.match(executable) else "c"
return None

177
clang/tools/scan-build-py/lib/libscanbuild/intercept.py
View File

@ -28,216 +28,225 @@ import json
import glob
import logging
from libear import build_libear, TemporaryDirectory
from libscanbuild import command_entry_point, compiler_wrapper, \
wrapper_environment, run_command, run_build
from libscanbuild import (
command_entry_point,
compiler_wrapper,
wrapper_environment,
run_command,
run_build,
)
from libscanbuild import duplicate_check
from libscanbuild.compilation import split_command
from libscanbuild.arguments import parse_args_for_intercept_build
from libscanbuild.shell import encode, decode
__all__ = ['capture', 'intercept_build', 'intercept_compiler_wrapper']
__all__ = ["capture", "intercept_build", "intercept_compiler_wrapper"]
GS = chr(0x1d)
RS = chr(0x1e)
US = chr(0x1f)
GS = chr(0x1D)
RS = chr(0x1E)
US = chr(0x1F)
COMPILER_WRAPPER_CC = 'intercept-cc'
COMPILER_WRAPPER_CXX = 'intercept-c++'
TRACE_FILE_EXTENSION = '.cmd' # same as in ear.c
WRAPPER_ONLY_PLATFORMS = frozenset({'win32', 'cygwin'})
COMPILER_WRAPPER_CC = "intercept-cc"
COMPILER_WRAPPER_CXX = "intercept-c++"
TRACE_FILE_EXTENSION = ".cmd" # same as in ear.c
WRAPPER_ONLY_PLATFORMS = frozenset({"win32", "cygwin"})
@command_entry_point
def intercept_build():
""" Entry point for 'intercept-build' command. """
"""Entry point for 'intercept-build' command."""
args = parse_args_for_intercept_build()
return capture(args)
def capture(args):
""" The entry point of build command interception. """
"""The entry point of build command interception."""
def post_processing(commands):
""" To make a compilation database, it needs to filter out commands
"""To make a compilation database, it needs to filter out commands
which are not compiler calls. Needs to find the source file name
from the arguments. And do shell escaping on the command.
To support incremental builds, it is desired to read elements from
an existing compilation database from a previous run. These elements
shall be merged with the new elements. """
shall be merged with the new elements."""
# create entries from the current run
current = itertools.chain.from_iterable(
# creates a sequence of entry generators from an exec,
format_entry(command) for command in commands)
format_entry(command)
for command in commands
)
# read entries from previous run
if 'append' in args and args.append and os.path.isfile(args.cdb):
if "append" in args and args.append and os.path.isfile(args.cdb):
with open(args.cdb) as handle:
previous = iter(json.load(handle))
else:
previous = iter([])
# filter out duplicate entries from both
duplicate = duplicate_check(entry_hash)
return (entry
for entry in itertools.chain(previous, current)
if os.path.exists(entry['file']) and not duplicate(entry))
return (
entry
for entry in itertools.chain(previous, current)
if os.path.exists(entry["file"]) and not duplicate(entry)
)
with TemporaryDirectory(prefix='intercept-') as tmp_dir:
with TemporaryDirectory(prefix="intercept-") as tmp_dir:
# run the build command
environment = setup_environment(args, tmp_dir)
exit_code = run_build(args.build, env=environment)
# read the intercepted exec calls
exec_traces = itertools.chain.from_iterable(
parse_exec_trace(os.path.join(tmp_dir, filename))
for filename in sorted(glob.iglob(os.path.join(tmp_dir, '*.cmd'))))
for filename in sorted(glob.iglob(os.path.join(tmp_dir, "*.cmd")))
)
# do post processing
entries = post_processing(exec_traces)
# dump the compilation database
with open(args.cdb, 'w+') as handle:
with open(args.cdb, "w+") as handle:
json.dump(list(entries), handle, sort_keys=True, indent=4)
return exit_code
def setup_environment(args, destination):
""" Sets up the environment for the build command.
"""Sets up the environment for the build command.
It sets the required environment variables and execute the given command.
The exec calls will be logged by the 'libear' preloaded library or by the
'wrapper' programs. """
'wrapper' programs."""
c_compiler = args.cc if 'cc' in args else 'cc'
cxx_compiler = args.cxx if 'cxx' in args else 'c++'
c_compiler = args.cc if "cc" in args else "cc"
cxx_compiler = args.cxx if "cxx" in args else "c++"
libear_path = None if args.override_compiler or is_preload_disabled(
sys.platform) else build_libear(c_compiler, destination)
libear_path = (
None
if args.override_compiler or is_preload_disabled(sys.platform)
else build_libear(c_compiler, destination)
)
environment = dict(os.environ)
environment.update({'INTERCEPT_BUILD_TARGET_DIR': destination})
environment.update({"INTERCEPT_BUILD_TARGET_DIR": destination})
if not libear_path:
logging.debug('intercept gonna use compiler wrappers')
logging.debug("intercept gonna use compiler wrappers")
environment.update(wrapper_environment(args))
environment.update({
'CC': COMPILER_WRAPPER_CC,
'CXX': COMPILER_WRAPPER_CXX
})
elif sys.platform == 'darwin':
logging.debug('intercept gonna preload libear on OSX')
environment.update({
'DYLD_INSERT_LIBRARIES': libear_path,
'DYLD_FORCE_FLAT_NAMESPACE': '1'
})
environment.update({"CC": COMPILER_WRAPPER_CC, "CXX": COMPILER_WRAPPER_CXX})
elif sys.platform == "darwin":
logging.debug("intercept gonna preload libear on OSX")
environment.update(
{"DYLD_INSERT_LIBRARIES": libear_path, "DYLD_FORCE_FLAT_NAMESPACE": "1"}
)
else:
logging.debug('intercept gonna preload libear on UNIX')
environment.update({'LD_PRELOAD': libear_path})
logging.debug("intercept gonna preload libear on UNIX")
environment.update({"LD_PRELOAD": libear_path})
return environment
@command_entry_point
def intercept_compiler_wrapper():
""" Entry point for `intercept-cc` and `intercept-c++`. """
"""Entry point for `intercept-cc` and `intercept-c++`."""
return compiler_wrapper(intercept_compiler_wrapper_impl)
def intercept_compiler_wrapper_impl(_, execution):
""" Implement intercept compiler wrapper functionality.
"""Implement intercept compiler wrapper functionality.
It does generate execution report into target directory.
The target directory name is from environment variables. """
The target directory name is from environment variables."""
message_prefix = 'execution report might be incomplete: %s'
message_prefix = "execution report might be incomplete: %s"
target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR')
target_dir = os.getenv("INTERCEPT_BUILD_TARGET_DIR")
if not target_dir:
logging.warning(message_prefix, 'missing target directory')
logging.warning(message_prefix, "missing target directory")
return
# write current execution info to the pid file
try:
target_file_name = str(os.getpid()) + TRACE_FILE_EXTENSION
target_file = os.path.join(target_dir, target_file_name)
logging.debug('writing execution report to: %s', target_file)
logging.debug("writing execution report to: %s", target_file)
write_exec_trace(target_file, execution)
except IOError:
logging.warning(message_prefix, 'io problem')
logging.warning(message_prefix, "io problem")
def write_exec_trace(filename, entry):
""" Write execution report file.
"""Write execution report file.
This method shall be sync with the execution report writer in interception
library. The entry in the file is a JSON objects.
:param filename: path to the output execution trace file,
:param entry: the Execution object to append to that file. """
:param entry: the Execution object to append to that file."""
with open(filename, 'ab') as handler:
with open(filename, "ab") as handler:
pid = str(entry.pid)
command = US.join(entry.cmd) + US
content = RS.join([pid, pid, 'wrapper', entry.cwd, command]) + GS
handler.write(content.encode('utf-8'))
content = RS.join([pid, pid, "wrapper", entry.cwd, command]) + GS
handler.write(content.encode("utf-8"))
def parse_exec_trace(filename):
""" Parse the file generated by the 'libear' preloaded library.
"""Parse the file generated by the 'libear' preloaded library.
Given filename points to a file which contains the basic report
generated by the interception library or wrapper command. A single
report file _might_ contain multiple process creation info. """
report file _might_ contain multiple process creation info."""
logging.debug('parse exec trace file: %s', filename)
with open(filename, 'r') as handler:
logging.debug("parse exec trace file: %s", filename)
with open(filename, "r") as handler:
content = handler.read()
for group in filter(bool, content.split(GS)):
records = group.split(RS)
yield {
'pid': records[0],
'ppid': records[1],
'function': records[2],
'directory': records[3],
'command': records[4].split(US)[:-1]
"pid": records[0],
"ppid": records[1],
"function": records[2],
"directory": records[3],
"command": records[4].split(US)[:-1],
}
def format_entry(exec_trace):
""" Generate the desired fields for compilation database entries. """
"""Generate the desired fields for compilation database entries."""
def abspath(cwd, name):
""" Create normalized absolute path from input filename. """
"""Create normalized absolute path from input filename."""
fullname = name if os.path.isabs(name) else os.path.join(cwd, name)
return os.path.normpath(fullname)
logging.debug('format this command: %s', exec_trace['command'])
compilation = split_command(exec_trace['command'])
logging.debug("format this command: %s", exec_trace["command"])
compilation = split_command(exec_trace["command"])
if compilation:
for source in compilation.files:
compiler = 'c++' if compilation.compiler == 'c++' else 'cc'
command = [compiler, '-c'] + compilation.flags + [source]
logging.debug('formated as: %s', command)
compiler = "c++" if compilation.compiler == "c++" else "cc"
command = [compiler, "-c"] + compilation.flags + [source]
logging.debug("formated as: %s", command)
yield {
'directory': exec_trace['directory'],
'command': encode(command),
'file': abspath(exec_trace['directory'], source)
"directory": exec_trace["directory"],
"command": encode(command),
"file": abspath(exec_trace["directory"], source),
}
def is_preload_disabled(platform):
""" Library-based interposition will fail silently if SIP is enabled,
"""Library-based interposition will fail silently if SIP is enabled,
so this should be detected. You can detect whether SIP is enabled on
Darwin by checking whether (1) there is a binary called 'csrutil' in
the path and, if so, (2) whether the output of executing 'csrutil status'
contains 'System Integrity Protection status: enabled'.
:param platform: name of the platform (returned by sys.platform),
:return: True if library preload will fail by the dynamic linker. """
:return: True if library preload will fail by the dynamic linker."""
if platform in WRAPPER_ONLY_PLATFORMS:
return True
elif platform == 'darwin':
command = ['csrutil', 'status']
pattern = re.compile(r'System Integrity Protection status:\s+enabled')
elif platform == "darwin":
command = ["csrutil", "status"]
pattern = re.compile(r"System Integrity Protection status:\s+enabled")
try:
return any(pattern.match(line) for line in run_command(command))
except:
@ -247,16 +256,16 @@ def is_preload_disabled(platform):
def entry_hash(entry):
""" Implement unique hash method for compilation database entries. """
"""Implement unique hash method for compilation database entries."""
# For faster lookup in set filename is reverted
filename = entry['file'][::-1]
filename = entry["file"][::-1]
# For faster lookup in set directory is reverted
directory = entry['directory'][::-1]
directory = entry["directory"][::-1]
# On OS X the 'cc' and 'c++' compilers are wrappers for
# 'clang' therefore both call would be logged. To avoid
# this the hash does not contain the first word of the
# command.
command = ' '.join(decode(entry['command'])[1:])
command = " ".join(decode(entry["command"])[1:])
return '<>'.join([filename, directory, command])
return "<>".join([filename, directory, command])

459
clang/tools/scan-build-py/lib/libscanbuild/report.py
View File

@ -20,16 +20,16 @@ import datetime
from libscanbuild import duplicate_check
from libscanbuild.clang import get_version
__all__ = ['document']
__all__ = ["document"]
def document(args):
""" Generates cover report and returns the number of bugs/crashes. """
"""Generates cover report and returns the number of bugs/crashes."""
html_reports_available = args.output_format in {'html', 'plist-html', 'sarif-html'}
sarif_reports_available = args.output_format in {'sarif', 'sarif-html'}
html_reports_available = args.output_format in {"html", "plist-html", "sarif-html"}
sarif_reports_available = args.output_format in {"sarif", "sarif-html"}
logging.debug('count crashes and bugs')
logging.debug("count crashes and bugs")
crash_count = sum(1 for _ in read_crashes(args.output))
bug_counter = create_counters()
for bug in read_bugs(args.output, html_reports_available):
@ -39,7 +39,7 @@ def document(args):
if html_reports_available and result:
use_cdb = os.path.exists(args.cdb)
logging.debug('generate index.html file')
logging.debug("generate index.html file")
# common prefix for source files to have sorter path
prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd()
# assemble the cover from multiple fragments
@ -60,24 +60,26 @@ def document(args):
os.remove(fragment)
if sarif_reports_available:
logging.debug('merging sarif files')
logging.debug("merging sarif files")
merge_sarif_files(args.output)
return result
def assemble_cover(args, prefix, fragments):
""" Put together the fragments into a final report. """
"""Put together the fragments into a final report."""
import getpass
import socket
if args.html_title is None:
args.html_title = os.path.basename(prefix) + ' - analyzer results'
args.html_title = os.path.basename(prefix) + " - analyzer results"
with open(os.path.join(args.output, 'index.html'), 'w') as handle:
with open(os.path.join(args.output, "index.html"), "w") as handle:
indent = 0
handle.write(reindent("""
handle.write(
reindent(
"""
|<!DOCTYPE html>
|<html>
| <head>
@ -85,9 +87,14 @@ def assemble_cover(args, prefix, fragments):
| <link type="text/css" rel="stylesheet" href="scanview.css"/>
| <script type='text/javascript' src="sorttable.js"></script>
| <script type='text/javascript' src='selectable.js'></script>
| </head>""", indent).format(html_title=args.html_title))
handle.write(comment('SUMMARYENDHEAD'))
handle.write(reindent("""
| </head>""",
indent,
).format(html_title=args.html_title)
)
handle.write(comment("SUMMARYENDHEAD"))
handle.write(
reindent(
"""
| <body>
| <h1>{html_title}</h1>
| <table>
@ -96,30 +103,41 @@ def assemble_cover(args, prefix, fragments):
| <tr><th>Command Line:</th><td>{cmd_args}</td></tr>
| <tr><th>Clang Version:</th><td>{clang_version}</td></tr>
| <tr><th>Date:</th><td>{date}</td></tr>
| </table>""", indent).format(html_title=args.html_title,
user_name=getpass.getuser(),
host_name=socket.gethostname(),
current_dir=prefix,
cmd_args=' '.join(sys.argv),
clang_version=get_version(args.clang),
date=datetime.datetime.today(
).strftime('%c')))
| </table>""",
indent,
).format(
html_title=args.html_title,
user_name=getpass.getuser(),
host_name=socket.gethostname(),
current_dir=prefix,
cmd_args=" ".join(sys.argv),
clang_version=get_version(args.clang),
date=datetime.datetime.today().strftime("%c"),
)
)
for fragment in fragments:
# copy the content of fragments
with open(fragment, 'r') as input_handle:
with open(fragment, "r") as input_handle:
shutil.copyfileobj(input_handle, handle)
handle.write(reindent("""
handle.write(
reindent(
"""
| </body>
|</html>""", indent))
|</html>""",
indent,
)
)
def bug_summary(output_dir, bug_counter):
""" Bug summary is a HTML table to give a better overview of the bugs. """
"""Bug summary is a HTML table to give a better overview of the bugs."""
name = os.path.join(output_dir, 'summary.html.fragment')
with open(name, 'w') as handle:
name = os.path.join(output_dir, "summary.html.fragment")
with open(name, "w") as handle:
indent = 4
handle.write(reindent("""
handle.write(
reindent(
"""
|<h2>Bug Summary</h2>
|<table>
| <thead>
@ -129,8 +147,13 @@ def bug_summary(output_dir, bug_counter):
| <td class="sorttable_nosort">Display?</td>
| </tr>
| </thead>
| <tbody>""", indent))
handle.write(reindent("""
| <tbody>""",
indent,
)
)
handle.write(
reindent(
"""
| <tr style="font-weight:bold">
| <td class="SUMM_DESC">All Bugs</td>
| <td class="Q">{0}</td>
@ -140,14 +163,24 @@ def bug_summary(output_dir, bug_counter):
| onClick="CopyCheckedStateToCheckButtons(this);"/>
| </center>
| </td>
| </tr>""", indent).format(bug_counter.total))
| </tr>""",
indent,
).format(bug_counter.total)
)
for category, types in bug_counter.categories.items():
handle.write(reindent("""
handle.write(
reindent(
"""
| <tr>
| <th>{0}</th><th colspan=2></th>
| </tr>""", indent).format(category))
| </tr>""",
indent,
).format(category)
)
for bug_type in types.values():
handle.write(reindent("""
handle.write(
reindent(
"""
| <tr>
| <td class="SUMM_DESC">{bug_type}</td>
| <td class="Q">{bug_count}</td>
@ -157,24 +190,34 @@ def bug_summary(output_dir, bug_counter):
| onClick="ToggleDisplay(this,'{bug_type_class}');"/>
| </center>
| </td>
| </tr>""", indent).format(**bug_type))
handle.write(reindent("""
| </tr>""",
indent,
).format(**bug_type)
)
handle.write(
reindent(
"""
| </tbody>
|</table>""", indent))
handle.write(comment('SUMMARYBUGEND'))
|</table>""",
indent,
)
)
handle.write(comment("SUMMARYBUGEND"))
return name
def bug_report(output_dir, prefix):
""" Creates a fragment from the analyzer reports. """
"""Creates a fragment from the analyzer reports."""
pretty = prettify_bug(prefix, output_dir)
bugs = (pretty(bug) for bug in read_bugs(output_dir, True))
name = os.path.join(output_dir, 'bugs.html.fragment')
with open(name, 'w') as handle:
name = os.path.join(output_dir, "bugs.html.fragment")
with open(name, "w") as handle:
indent = 4
handle.write(reindent("""
handle.write(
reindent(
"""
|<h2>Reports</h2>
|<table class="sortable" style="table-layout:automatic">
| <thead>
@ -191,10 +234,15 @@ def bug_report(output_dir, prefix):
| <td class="sorttable_nosort"></td>
| </tr>
| </thead>
| <tbody>""", indent))
handle.write(comment('REPORTBUGCOL'))
| <tbody>""",
indent,
)
)
handle.write(comment("REPORTBUGCOL"))
for current in bugs:
handle.write(reindent("""
handle.write(
reindent(
"""
| <tr class="{bug_type_class}">
| <td class="DESC">{bug_category}</td>
| <td class="DESC">{bug_type}</td>
@ -203,25 +251,35 @@ def bug_report(output_dir, prefix):
| <td class="Q">{bug_line}</td>
| <td class="Q">{bug_path_length}</td>
| <td><a href="{report_file}#EndPath">View Report</a></td>
| </tr>""", indent).format(**current))
handle.write(comment('REPORTBUG', {'id': current['report_file']}))
handle.write(reindent("""
| </tr>""",
indent,
).format(**current)
)
handle.write(comment("REPORTBUG", {"id": current["report_file"]}))
handle.write(
reindent(
"""
| </tbody>
|</table>""", indent))
handle.write(comment('REPORTBUGEND'))
|</table>""",
indent,
)
)
handle.write(comment("REPORTBUGEND"))
return name
def crash_report(output_dir, prefix):
""" Creates a fragment from the compiler crashes. """
"""Creates a fragment from the compiler crashes."""
pretty = prettify_crash(prefix, output_dir)
crashes = (pretty(crash) for crash in read_crashes(output_dir))
name = os.path.join(output_dir, 'crashes.html.fragment')
with open(name, 'w') as handle:
name = os.path.join(output_dir, "crashes.html.fragment")
with open(name, "w") as handle:
indent = 4
handle.write(reindent("""
handle.write(
reindent(
"""
|<h2>Analyzer Failures</h2>
|<p>The analyzer had problems processing the following files:</p>
|<table>
@ -233,49 +291,64 @@ def crash_report(output_dir, prefix):
| <td>STDERR Output</td>
| </tr>
| </thead>
| <tbody>""", indent))
| <tbody>""",
indent,
)
)
for current in crashes:
handle.write(reindent("""
handle.write(
reindent(
"""
| <tr>
| <td>{problem}</td>
| <td>{source}</td>
| <td><a href="{file}">preprocessor output</a></td>
| <td><a href="{stderr}">analyzer std err</a></td>
| </tr>""", indent).format(**current))
handle.write(comment('REPORTPROBLEM', current))
handle.write(reindent("""
| </tr>""",
indent,
).format(**current)
)
handle.write(comment("REPORTPROBLEM", current))
handle.write(
reindent(
"""
| </tbody>
|</table>""", indent))
handle.write(comment('REPORTCRASHES'))
|</table>""",
indent,
)
)
handle.write(comment("REPORTCRASHES"))
return name
def read_crashes(output_dir):
""" Generate a unique sequence of crashes from given output directory. """
"""Generate a unique sequence of crashes from given output directory."""
return (parse_crash(filename)
for filename in glob.iglob(os.path.join(output_dir, 'failures',
'*.info.txt')))
return (
parse_crash(filename)
for filename in glob.iglob(os.path.join(output_dir, "failures", "*.info.txt"))
)
def read_bugs(output_dir, html):
# type: (str, bool) -> Generator[Dict[str, Any], None, None]
""" Generate a unique sequence of bugs from given output directory.
"""Generate a unique sequence of bugs from given output directory.
Duplicates can be in a project if the same module was compiled multiple
times with different compiler options. These would be better to show in
the final report (cover) only once. """
the final report (cover) only once."""
def empty(file_name):
return os.stat(file_name).st_size == 0
duplicate = duplicate_check(
lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug))
lambda bug: "{bug_line}.{bug_path_length}:{bug_file}".format(**bug)
)
# get the right parser for the job.
parser = parse_bug_html if html else parse_bug_plist
# get the input files, which are not empty.
pattern = os.path.join(output_dir, '*.html' if html else '*.plist')
pattern = os.path.join(output_dir, "*.html" if html else "*.plist")
bug_files = (file for file in glob.iglob(pattern) if not empty(file))
for bug_file in bug_files:
@ -283,8 +356,9 @@ def read_bugs(output_dir, html):
if not duplicate(bug):
yield bug
def merge_sarif_files(output_dir, sort_files=False):
""" Reads and merges all .sarif files in the given output directory.
"""Reads and merges all .sarif files in the given output directory.
Each sarif file in the output directory is understood as a single run
and thus appear separate in the top level runs array. This requires
@ -296,43 +370,49 @@ def merge_sarif_files(output_dir, sort_files=False):
def update_sarif_object(sarif_object, runs_count_offset):
"""
Given a SARIF object, checks its dictionary entries for a 'message' property.
If it exists, updates the message index of embedded links in the run index.
Given a SARIF object, checks its dictionary entries for a 'message' property.
If it exists, updates the message index of embedded links in the run index.
Recursively looks through entries in the dictionary.
Recursively looks through entries in the dictionary.
"""
if not isinstance(sarif_object, dict):
return sarif_object
if 'message' in sarif_object:
sarif_object['message'] = match_and_update_run(sarif_object['message'], runs_count_offset)
if "message" in sarif_object:
sarif_object["message"] = match_and_update_run(
sarif_object["message"], runs_count_offset
)
for key in sarif_object:
if isinstance(sarif_object[key], list):
# iterate through subobjects and update it.
arr = [update_sarif_object(entry, runs_count_offset) for entry in sarif_object[key]]
arr = [
update_sarif_object(entry, runs_count_offset)
for entry in sarif_object[key]
]
sarif_object[key] = arr
elif isinstance(sarif_object[key], dict):
sarif_object[key] = update_sarif_object(sarif_object[key], runs_count_offset)
sarif_object[key] = update_sarif_object(
sarif_object[key], runs_count_offset
)
else:
# do nothing
pass
return sarif_object
def match_and_update_run(message, runs_count_offset):
"""
Given a SARIF message object, checks if the text property contains an embedded link and
updates the run index if necessary.
Given a SARIF message object, checks if the text property contains an embedded link and
updates the run index if necessary.
"""
if 'text' not in message:
if "text" not in message:
return message
# we only merge runs, so we only need to update the run index
pattern = re.compile(r'sarif:/runs/(\d+)')
pattern = re.compile(r"sarif:/runs/(\d+)")
text = message['text']
text = message["text"]
matches = re.finditer(pattern, text)
matches_list = list(matches)
@ -340,14 +420,16 @@ def merge_sarif_files(output_dir, sort_files=False):
for idx in range(len(matches_list) - 1, -1, -1):
match = matches_list[idx]
new_run_count = str(runs_count_offset + int(match.group(1)))
text = text[0:match.start(1)] + new_run_count + text[match.end(1):]
text = text[0 : match.start(1)] + new_run_count + text[match.end(1) :]
message['text'] = text
message["text"] = text
return message
sarif_files = (file for file in glob.iglob(os.path.join(output_dir, '*.sarif')) if not empty(file))
sarif_files = (
file
for file in glob.iglob(os.path.join(output_dir, "*.sarif"))
if not empty(file)
)
# exposed for testing since the order of files returned by glob is not guaranteed to be sorted
if sort_files:
sarif_files = list(sarif_files)
@ -358,7 +440,7 @@ def merge_sarif_files(output_dir, sort_files=False):
for sarif_file in sarif_files:
with open(sarif_file) as fp:
sarif = json.load(fp)
if 'runs' not in sarif:
if "runs" not in sarif:
continue
# start with the first file
@ -366,58 +448,60 @@ def merge_sarif_files(output_dir, sort_files=False):
merged = sarif
else:
# extract the run and append it to the merged output
for run in sarif['runs']:
for run in sarif["runs"]:
new_run = update_sarif_object(run, runs_count)
merged['runs'].append(new_run)
merged["runs"].append(new_run)
runs_count += len(sarif['runs'])
runs_count += len(sarif["runs"])
with open(os.path.join(output_dir, 'results-merged.sarif'), 'w') as out:
with open(os.path.join(output_dir, "results-merged.sarif"), "w") as out:
json.dump(merged, out, indent=4, sort_keys=True)
def parse_bug_plist(filename):
""" Returns the generator of bugs from a single .plist file. """
"""Returns the generator of bugs from a single .plist file."""
with open(filename, 'rb') as fp:
content = plistlib.load(fp)
files = content.get('files')
for bug in content.get('diagnostics', []):
if len(files) <= int(bug['location']['file']):
logging.warning('Parsing bug from "%s" failed', filename)
continue
with open(filename, "rb") as fp:
content = plistlib.load(fp)
files = content.get("files")
for bug in content.get("diagnostics", []):
if len(files) <= int(bug["location"]["file"]):
logging.warning('Parsing bug from "%s" failed', filename)
continue
yield {
'result': filename,
'bug_type': bug['type'],
'bug_category': bug['category'],
'bug_line': int(bug['location']['line']),
'bug_path_length': int(bug['location']['col']),
'bug_file': files[int(bug['location']['file'])]
}
yield {
"result": filename,
"bug_type": bug["type"],
"bug_category": bug["category"],
"bug_line": int(bug["location"]["line"]),
"bug_path_length": int(bug["location"]["col"]),
"bug_file": files[int(bug["location"]["file"])],
}
def parse_bug_html(filename):
""" Parse out the bug information from HTML output. """
"""Parse out the bug information from HTML output."""
patterns = [re.compile(r'<!-- BUGTYPE (?P<bug_type>.*) -->$'),
re.compile(r'<!-- BUGFILE (?P<bug_file>.*) -->$'),
re.compile(r'<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$'),
re.compile(r'<!-- BUGLINE (?P<bug_line>.*) -->$'),
re.compile(r'<!-- BUGCATEGORY (?P<bug_category>.*) -->$'),
re.compile(r'<!-- BUGDESC (?P<bug_description>.*) -->$'),
re.compile(r'<!-- FUNCTIONNAME (?P<bug_function>.*) -->$')]
endsign = re.compile(r'<!-- BUGMETAEND -->')
patterns = [
re.compile(r"<!-- BUGTYPE (?P<bug_type>.*) -->$"),
re.compile(r"<!-- BUGFILE (?P<bug_file>.*) -->$"),
re.compile(r"<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$"),
re.compile(r"<!-- BUGLINE (?P<bug_line>.*) -->$"),
re.compile(r"<!-- BUGCATEGORY (?P<bug_category>.*) -->$"),
re.compile(r"<!-- BUGDESC (?P<bug_description>.*) -->$"),
re.compile(r"<!-- FUNCTIONNAME (?P<bug_function>.*) -->$"),
]
endsign = re.compile(r"<!-- BUGMETAEND -->")
bug = {
'report_file': filename,
'bug_function': 'n/a', # compatibility with < clang-3.5
'bug_category': 'Other',
'bug_line': 0,
'bug_path_length': 1
"report_file": filename,
"bug_function": "n/a", # compatibility with < clang-3.5
"bug_category": "Other",
"bug_line": 0,
"bug_path_length": 1,
}
with open(filename, encoding='utf-8') as handler:
with open(filename, encoding="utf-8") as handler:
for line in handler.readlines():
# do not read the file further
if endsign.match(line):
@ -429,61 +513,64 @@ def parse_bug_html(filename):
bug.update(match.groupdict())
break
encode_value(bug, 'bug_line', int)
encode_value(bug, 'bug_path_length', int)
encode_value(bug, "bug_line", int)
encode_value(bug, "bug_path_length", int)
yield bug
def parse_crash(filename):
""" Parse out the crash information from the report file. """
"""Parse out the crash information from the report file."""
match = re.match(r'(.*)\.info\.txt', filename)
match = re.match(r"(.*)\.info\.txt", filename)
name = match.group(1) if match else None
with open(filename, mode='rb') as handler:
with open(filename, mode="rb") as handler:
# this is a workaround to fix windows read '\r\n' as new lines.
lines = [line.decode().rstrip() for line in handler.readlines()]
return {
'source': lines[0],
'problem': lines[1],
'file': name,
'info': name + '.info.txt',
'stderr': name + '.stderr.txt'
"source": lines[0],
"problem": lines[1],
"file": name,
"info": name + ".info.txt",
"stderr": name + ".stderr.txt",
}
def category_type_name(bug):
""" Create a new bug attribute from bug by category and type.
"""Create a new bug attribute from bug by category and type.
The result will be used as CSS class selector in the final report. """
The result will be used as CSS class selector in the final report."""
def smash(key):
""" Make value ready to be HTML attribute value. """
"""Make value ready to be HTML attribute value."""
return bug.get(key, '').lower().replace(' ', '_').replace("'", '')
return bug.get(key, "").lower().replace(" ", "_").replace("'", "")
return escape('bt_' + smash('bug_category') + '_' + smash('bug_type'))
return escape("bt_" + smash("bug_category") + "_" + smash("bug_type"))
def create_counters():
""" Create counters for bug statistics.
"""Create counters for bug statistics.
Two entries are maintained: 'total' is an integer, represents the
number of bugs. The 'categories' is a two level categorisation of bug
counters. The first level is 'bug category' the second is 'bug type'.
Each entry in this classification is a dictionary of 'count', 'type'
and 'label'. """
and 'label'."""
def predicate(bug):
bug_category = bug['bug_category']
bug_type = bug['bug_type']
bug_category = bug["bug_category"]
bug_type = bug["bug_type"]
current_category = predicate.categories.get(bug_category, dict())
current_type = current_category.get(bug_type, {
'bug_type': bug_type,
'bug_type_class': category_type_name(bug),
'bug_count': 0
})
current_type.update({'bug_count': current_type['bug_count'] + 1})
current_type = current_category.get(
bug_type,
{
"bug_type": bug_type,
"bug_type_class": category_type_name(bug),
"bug_count": 0,
},
)
current_type.update({"bug_count": current_type["bug_count"] + 1})
current_category.update({bug_type: current_type})
predicate.categories.update({bug_category: current_category})
predicate.total += 1
@ -495,14 +582,14 @@ def create_counters():
def prettify_bug(prefix, output_dir):
def predicate(bug):
""" Make safe this values to embed into HTML. """
"""Make safe this values to embed into HTML."""
bug['bug_type_class'] = category_type_name(bug)
bug["bug_type_class"] = category_type_name(bug)
encode_value(bug, 'bug_file', lambda x: escape(chop(prefix, x)))
encode_value(bug, 'bug_category', escape)
encode_value(bug, 'bug_type', escape)
encode_value(bug, 'report_file', lambda x: escape(chop(output_dir, x)))
encode_value(bug, "bug_file", lambda x: escape(chop(prefix, x)))
encode_value(bug, "bug_category", escape)
encode_value(bug, "bug_type", escape)
encode_value(bug, "report_file", lambda x: escape(chop(output_dir, x)))
return bug
return predicate
@ -510,28 +597,28 @@ def prettify_bug(prefix, output_dir):
def prettify_crash(prefix, output_dir):
def predicate(crash):
""" Make safe this values to embed into HTML. """
"""Make safe this values to embed into HTML."""
encode_value(crash, 'source', lambda x: escape(chop(prefix, x)))
encode_value(crash, 'problem', escape)
encode_value(crash, 'file', lambda x: escape(chop(output_dir, x)))
encode_value(crash, 'info', lambda x: escape(chop(output_dir, x)))
encode_value(crash, 'stderr', lambda x: escape(chop(output_dir, x)))
encode_value(crash, "source", lambda x: escape(chop(prefix, x)))
encode_value(crash, "problem", escape)
encode_value(crash, "file", lambda x: escape(chop(output_dir, x)))
encode_value(crash, "info", lambda x: escape(chop(output_dir, x)))
encode_value(crash, "stderr", lambda x: escape(chop(output_dir, x)))
return crash
return predicate
def copy_resource_files(output_dir):
""" Copy the javascript and css files to the report directory. """
"""Copy the javascript and css files to the report directory."""
this_dir = os.path.dirname(os.path.realpath(__file__))
for resource in os.listdir(os.path.join(this_dir, 'resources')):
shutil.copy(os.path.join(this_dir, 'resources', resource), output_dir)
for resource in os.listdir(os.path.join(this_dir, "resources")):
shutil.copy(os.path.join(this_dir, "resources", resource), output_dir)
def encode_value(container, key, encode):
""" Run 'encode' on 'container[key]' value and update it. """
"""Run 'encode' on 'container[key]' value and update it."""
if key in container:
value = encode(container[key])
@ -539,56 +626,56 @@ def encode_value(container, key, encode):
def chop(prefix, filename):
""" Create 'filename' from '/prefix/filename' """
"""Create 'filename' from '/prefix/filename'"""
return filename if not len(prefix) else os.path.relpath(filename, prefix)
def escape(text):
""" Paranoid HTML escape method. (Python version independent) """
"""Paranoid HTML escape method. (Python version independent)"""
escape_table = {
'&': '&amp;',
'"': '&quot;',
"'": '&apos;',
'>': '&gt;',
'<': '&lt;'
"&": "&amp;",
'"': "&quot;",
"'": "&apos;",
">": "&gt;",
"<": "&lt;",
}
return ''.join(escape_table.get(c, c) for c in text)
return "".join(escape_table.get(c, c) for c in text)
def reindent(text, indent):
""" Utility function to format html output and keep indentation. """
"""Utility function to format html output and keep indentation."""
result = ''
result = ""
for line in text.splitlines():
if len(line.strip()):
result += ' ' * indent + line.split('|')[1] + os.linesep
result += " " * indent + line.split("|")[1] + os.linesep
return result
def comment(name, opts=dict()):
""" Utility function to format meta information as comment. """
"""Utility function to format meta information as comment."""
attributes = ''
attributes = ""
for key, value in opts.items():
attributes += ' {0}="{1}"'.format(key, value)
return '<!-- {0}{1} -->{2}'.format(name, attributes, os.linesep)
return "<!-- {0}{1} -->{2}".format(name, attributes, os.linesep)
def commonprefix_from(filename):
""" Create file prefix from a compilation database entries. """
"""Create file prefix from a compilation database entries."""
with open(filename, 'r') as handle:
return commonprefix(item['file'] for item in json.load(handle))
with open(filename, "r") as handle:
return commonprefix(item["file"] for item in json.load(handle))
def commonprefix(files):
""" Fixed version of os.path.commonprefix.
"""Fixed version of os.path.commonprefix.
:param files: list of file names.
:return: the longest path prefix that is a prefix of all files. """
:return: the longest path prefix that is a prefix of all files."""
result = None
for current in files:
if result is not None:
@ -597,7 +684,7 @@ def commonprefix(files):
result = current
if result is None:
return ''
return ""
elif not os.path.isdir(result):
return os.path.dirname(result)
else:

47
clang/tools/scan-build-py/lib/libscanbuild/shell.py
View File

@ -7,28 +7,45 @@
import re
import shlex
__all__ = ['encode', 'decode']
__all__ = ["encode", "decode"]
def encode(command):
""" Takes a command as list and returns a string. """
"""Takes a command as list and returns a string."""
def needs_quote(word):
""" Returns true if arguments needs to be protected by quotes.
"""Returns true if arguments needs to be protected by quotes.
Previous implementation was shlex.split method, but that's not good
for this job. Currently is running through the string with a basic
state checking. """
state checking."""
reserved = {' ', '$', '%', '&', '(', ')', '[', ']', '{', '}', '*', '|',
'<', '>', '@', '?', '!'}
reserved = {
" ",
"$",
"%",
"&",
"(",
")",
"[",
"]",
"{",
"}",
"*",
"|",
"<",
">",
"@",
"?",
"!",
}
state = 0
for current in word:
if state == 0 and current in reserved:
return True
elif state == 0 and current == '\\':
elif state == 0 and current == "\\":
state = 1
elif state == 1 and current in reserved | {'\\'}:
elif state == 1 and current in reserved | {"\\"}:
state = 0
elif state == 0 and current == '"':
state = 2
@ -41,10 +58,10 @@ def encode(command):
return state != 0
def escape(word):
""" Do protect argument if that's needed. """
"""Do protect argument if that's needed."""
table = {'\\': '\\\\', '"': '\\"'}
escaped = ''.join([table.get(c, c) for c in word])
table = {"\\": "\\\\", '"': '\\"'}
escaped = "".join([table.get(c, c) for c in word])
return '"' + escaped + '"' if needs_quote(word) else escaped
@ -52,14 +69,14 @@ def encode(command):
def decode(string):
""" Takes a command string and returns as a list. """
"""Takes a command string and returns as a list."""
def unescape(arg):
""" Gets rid of the escaping characters. """
"""Gets rid of the escaping characters."""
if len(arg) >= 2 and arg[0] == arg[-1] and arg[0] == '"':
arg = arg[1:-1]
return re.sub(r'\\(["\\])', r'\1', arg)
return re.sub(r'\\([\\ $%&\(\)\[\]\{\}\*|<>@?!])', r'\1', arg)
return re.sub(r'\\(["\\])', r"\1", arg)
return re.sub(r"\\([\\ $%&\(\)\[\]\{\}\*|<>@?!])", r"\1", arg)
return [unescape(arg) for arg in shlex.split(string)]

Some files were not shown because too many files have changed in this diff Show More