c812ab7312
Extending the python generator: - to generate C-compatibility symbols - to generate macros Differential Revision: https://reviews.llvm.org/D143214
297 lines
9.1 KiB
Python
Executable File
297 lines
9.1 KiB
Python
Executable File
#!/usr/bin/env python
|
||
#===- gen_std.py - ------------------------------------------*- python -*--===#
|
||
#
|
||
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||
# See https://llvm.org/LICENSE.txt for license information.
|
||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||
#
|
||
#===------------------------------------------------------------------------===#
|
||
|
||
"""gen_std.py is a tool to generate a lookup table (from qualified names to
|
||
include headers) for C/C++ Standard Library symbols by parsing archived HTML
|
||
files from cppreference.
|
||
|
||
The generated files are located in clang/include/Tooling/Inclusions.
|
||
|
||
Caveats and FIXMEs:
|
||
- only symbols directly in "std" namespace are added, we should also add std's
|
||
subnamespace symbols (e.g. chrono).
|
||
- symbols with multiple variants or defined in multiple headers aren't added,
|
||
e.g. std::move, std::swap
|
||
|
||
Usage:
|
||
1. Install BeautifulSoup dependency, see instruction:
|
||
https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-beautiful-soup
|
||
2. Download cppreference offline HTML files (html_book_20220730.zip in Unofficial Release) at
|
||
https://en.cppreference.com/w/Cppreference:Archives
|
||
3. Unzip the zip file from step 2 (e.g., to a "cppreference" directory). You should
|
||
get a "cppreference/reference" directory.
|
||
4. Run the command:
|
||
// Generate C++ symbols
|
||
python3 gen_std.py -cppreference cppreference/reference -symbols=cpp > StdSymbolMap.inc
|
||
// Generate C symbols
|
||
python3 gen_std.py -cppreference cppreference/reference -symbols=c > CSymbolMap.inc
|
||
"""
|
||
|
||
|
||
import cppreference_parser
|
||
import argparse
|
||
import datetime
|
||
import os
|
||
import sys
|
||
import re
|
||
|
||
|
||
CODE_PREFIX = """\
|
||
//===-- gen_std.py generated file -------------------------------*- C++ -*-===//
|
||
//
|
||
// Used to build a lookup table (qualified names => include headers) for %s
|
||
// Standard Library symbols.
|
||
//
|
||
// This file was generated automatically by
|
||
// clang/tools/include-mapping/gen_std.py, DO NOT EDIT!
|
||
//
|
||
// Generated from cppreference offline HTML book (modified on %s).
|
||
//===----------------------------------------------------------------------===//
|
||
"""
|
||
|
||
def ParseArg():
|
||
parser = argparse.ArgumentParser(description='Generate StdGen file')
|
||
parser.add_argument('-cppreference', metavar='PATH',
|
||
default='',
|
||
help='path to the cppreference offline HTML directory',
|
||
required=True
|
||
)
|
||
parser.add_argument('-symbols',
|
||
default='cpp',
|
||
help='Generate c or cpp (removed) symbols. One of {cpp, c, cpp_removed}.',
|
||
required=True)
|
||
return parser.parse_args()
|
||
|
||
def AdditionalHeadersForIOSymbols(symbol):
|
||
# IO-related symbols declared in the <iosfwd> header, per C++
|
||
# [iosfwd.syn 31.3.1]:
|
||
iosfwd_symbols = [
|
||
'basic_ios',
|
||
'basic_streambuf',
|
||
'basic_istream',
|
||
'basic_ostream',
|
||
'basic_iostream',
|
||
|
||
'basic_stringbuf',
|
||
'basic_istringstream',
|
||
'basic_ostringstream',
|
||
'basic_stringstream',
|
||
|
||
'basic_spanbuf',
|
||
'basic_ispanstream',
|
||
'basic_ospanstream',
|
||
'basic_spanstream',
|
||
|
||
'basic_filebuf',
|
||
'basic_ifstream',
|
||
'basic_ofstream',
|
||
'basic_fstream',
|
||
|
||
'basic_syncbuf',
|
||
'basic_osyncstream',
|
||
|
||
'istreambuf_iterator',
|
||
'ostreambuf_iterator',
|
||
|
||
'ios',
|
||
'wios',
|
||
|
||
'streambuf',
|
||
'istream',
|
||
'ostream',
|
||
'iostream',
|
||
|
||
'stringbuf',
|
||
'istringstream',
|
||
'ostringstream',
|
||
'stringstream',
|
||
|
||
'spanbuf',
|
||
'ispanstream',
|
||
'ospanstream',
|
||
'spanstream',
|
||
|
||
'filebuf',
|
||
'ifstream',
|
||
'ofstream',
|
||
'fstream',
|
||
|
||
'syncbuf',
|
||
'osyncstream',
|
||
|
||
'wstreambuf',
|
||
'wistream',
|
||
'wostream',
|
||
'wiostream',
|
||
|
||
'wstringbuf',
|
||
'wistringstream',
|
||
'wostringstream',
|
||
'wstringstream',
|
||
|
||
'wspanbuf',
|
||
'wispanstream',
|
||
'wospanstream',
|
||
'wspanstream',
|
||
|
||
'wfilebuf',
|
||
'wifstream',
|
||
'wofstream',
|
||
'wfstream',
|
||
|
||
'wsyncbuf',
|
||
'wosyncstream',
|
||
|
||
'fpos',
|
||
'streampos',
|
||
'wstreampos',
|
||
'u8streampos',
|
||
'u16streampos',
|
||
'u32streampos',
|
||
]
|
||
assert(len(symbol.headers) == 1)
|
||
sym_header = symbol.headers[0]
|
||
headers = []
|
||
# <iostream> is preferred than <iosfwd>
|
||
|
||
# <iostream> is an alternative of <streambuf>, <istream>, <ostream>, <ios>.
|
||
# per C++ [iostream.syn 31.4.1]
|
||
if sym_header in ["<ios>", "<istream>", "<ostream>", "<streambuf>"]:
|
||
headers.append("<iostream>")
|
||
|
||
if symbol.name in iosfwd_symbols:
|
||
headers.append("<iosfwd>")
|
||
|
||
return headers
|
||
|
||
|
||
def GetCCompatibilitySymbols(symbol):
|
||
# C++ form of the C standard headers.
|
||
c_compat_headers = {
|
||
"<cassert>",
|
||
"<cctype>",
|
||
"<cerrno>",
|
||
"<cfenv>",
|
||
"<cfloat>",
|
||
"<cinttypes>",
|
||
"<climits>",
|
||
"<clocale>",
|
||
"<cmath>",
|
||
"<csetjmp>",
|
||
"<csignal>",
|
||
"<cstdarg>",
|
||
"<cstddef>",
|
||
"<cstdint>",
|
||
"<cstdio>",
|
||
"<cstdlib>",
|
||
"<cstring>",
|
||
"<ctime>",
|
||
"<cuchar>",
|
||
"<cwchar>",
|
||
"<cwctype>",
|
||
}
|
||
# C++ [support.c.headers.other] 17.14.7
|
||
# ..., behaves as if each name placed in the standard library namespace by
|
||
# the corresponding <cname> header is placed within the global namespace
|
||
# scope, except for the functions described in [sf.cmath], the
|
||
# std::lerp function overloads ([c.math.lerp]), the declaration of
|
||
# std::byte ([cstddef.syn]), and the functions and function templates
|
||
# described in [support.types.byteops].
|
||
exception_symbols = {
|
||
"(assoc_)?laguerre[f|l]?",
|
||
"(assoc_|sph_)?legendre[f|l]?",
|
||
"beta[f|l]?",
|
||
"(comp_)?ellint_[1-3][f|l]?",
|
||
"(cyl_|sph_)?bessel_[i-k][f|l]?",
|
||
"(cyl_|sph_)?neumann[f|l]?",
|
||
"expint[f|l]?",
|
||
"hermite[f|l]?",
|
||
"riemann_zeta[f|l]?",
|
||
"lerp",
|
||
"byte",
|
||
}
|
||
assert(len(symbol.headers) == 1)
|
||
header = symbol.headers[0]
|
||
if header not in c_compat_headers:
|
||
return []
|
||
if any(re.fullmatch(x, symbol.name) for x in exception_symbols):
|
||
return []
|
||
|
||
# Introduce two more entries, both in the global namespace, one using the
|
||
# C++-compat header and another using the C header.
|
||
results = []
|
||
if symbol.namespace != None:
|
||
# avoid printing duplicated entries, for C macros!
|
||
results.append(cppreference_parser.Symbol(symbol.name, None, [header]))
|
||
c_header = "<" + header[2:-1] + ".h>" # <cstdio> => <stdio.h>
|
||
results.append(cppreference_parser.Symbol(symbol.name, None, [c_header]))
|
||
return results
|
||
|
||
|
||
def main():
|
||
args = ParseArg()
|
||
if args.symbols == 'cpp':
|
||
page_root = os.path.join(args.cppreference, "en", "cpp")
|
||
symbol_index_root = os.path.join(page_root, "symbol_index")
|
||
parse_pages = [
|
||
(page_root, "symbol_index.html", "std::"),
|
||
# std sub-namespace symbols have separated pages.
|
||
# We don't index std literal operators (e.g.
|
||
# std::literals::chrono_literals::operator""d), these symbols can't be
|
||
# accessed by std::<symbol_name>.
|
||
# FIXME: index std::placeholders symbols, placeholders.html page is
|
||
# different (which contains one entry for _1, _2, ..., _N), we need special
|
||
# handling.
|
||
(symbol_index_root, "chrono.html", "std::chrono::"),
|
||
(symbol_index_root, "filesystem.html", "std::filesystem::"),
|
||
(symbol_index_root, "pmr.html", "std::pmr::"),
|
||
(symbol_index_root, "regex_constants.html", "std::regex_constants::"),
|
||
(symbol_index_root, "this_thread.html", "std::this_thread::"),
|
||
# Zombie symbols that were available from the Standard Library, but are
|
||
# removed in the following standards.
|
||
(symbol_index_root, "zombie_names.html", "std::"),
|
||
(symbol_index_root, "macro.html", None),
|
||
]
|
||
elif args.symbols == 'c':
|
||
page_root = os.path.join(args.cppreference, "en", "c")
|
||
symbol_index_root = page_root
|
||
parse_pages = [(page_root, "index.html", None)]
|
||
|
||
if not os.path.exists(symbol_index_root):
|
||
exit("Path %s doesn't exist!" % symbol_index_root)
|
||
|
||
symbols = cppreference_parser.GetSymbols(parse_pages)
|
||
|
||
# We don't have version information from the unzipped offline HTML files.
|
||
# so we use the modified time of the symbol_index.html as the version.
|
||
index_page_path = os.path.join(page_root, "index.html")
|
||
cppreference_modified_date = datetime.datetime.fromtimestamp(
|
||
os.stat(index_page_path).st_mtime).strftime('%Y-%m-%d')
|
||
print(CODE_PREFIX % (args.symbols.upper(), cppreference_modified_date))
|
||
for symbol in symbols:
|
||
if len(symbol.headers) == 1:
|
||
augmented_symbols = [symbol]
|
||
augmented_symbols.extend(GetCCompatibilitySymbols(symbol))
|
||
for s in augmented_symbols:
|
||
s.headers.extend(AdditionalHeadersForIOSymbols(s))
|
||
for header in s.headers:
|
||
# SYMBOL(unqualified_name, namespace, header)
|
||
print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace,
|
||
header))
|
||
elif len(symbol.headers) == 0:
|
||
sys.stderr.write("No header found for symbol %s\n" % symbol.name)
|
||
else:
|
||
# FIXME: support symbols with multiple headers (e.g. std::move).
|
||
sys.stderr.write("Ambiguous header for symbol %s: %s\n" % (
|
||
symbol.name, ', '.join(symbol.headers)))
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|