[DTLTO] Make temporary file handling consistent (#176807)
DTLTO emits temporary files to allow distribution of archive member inputs. It also emits temporary files from the ThinLTO backend, such as the index files needed for each distributed ThinLTO backend compilation. This change brings archive member temporary files into line with those produced by the ThinLTO backend. They are now emitted in the same location, warnings are emitted if they cannot be deleted, and they are cleaned up on abnormal exit (e.g. Ctrl-C). All temporary files are preserved when --save-temps is specified. The existing signal-handling test has been extended to cover the full set of DTLTO temporary files, and a new test has been added to exercise temporary file handling in normal operation. Additionally, a minimal test has been added to show the COFF behaviour. SIE Internal tracker: TOOLCHAIN-21022
This commit is contained in:
Ben Dunbobbin
GitHub
parent
29fd868691
commit
1d74db7e8d
44
cross-project-tests/dtlto/link-savetemps.test
Normal file
44
cross-project-tests/dtlto/link-savetemps.test
Normal file
@ -0,0 +1,44 @@
|
||||
REQUIRES: lld-link
|
||||
|
||||
# Test that DTLTO temporary files are "best-effort" cleaned up unless
|
||||
# --save-temps is specified. We use archives in this test as the handling for
|
||||
# archives requires a superset of the temporary files used for object inputs.
|
||||
|
||||
RUN: rm -rf %t && split-file %s %t && cd %t
|
||||
|
||||
RUN: %clang --target=x86_64-pc-windows-msvc -O2 t1.c -flto=thin -c
|
||||
|
||||
RUN: lld-link /lib /out:t.lib t1.o
|
||||
|
||||
DEFINE: %{tdir} = dummy-to-make-lit-work
|
||||
DEFINE: %{dtlto} = mkdir %{tdir} && \
|
||||
DEFINE: lld-link /subsystem:console /machine:x64 /out:%{tdir}/my.exe \
|
||||
DEFINE: /wholearchive:t.lib \
|
||||
DEFINE: -thinlto-distributor:%python \
|
||||
DEFINE: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \
|
||||
DEFINE: -thinlto-remote-compiler:%clang
|
||||
|
||||
# Check that temporary files are removed normally.
|
||||
REDEFINE: %{tdir} = empty
|
||||
RUN: %{dtlto}
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,ELF
|
||||
|
||||
# Check that --save-temps preserves temporary files.
|
||||
REDEFINE: %{tdir} = savetemps
|
||||
RUN: %{dtlto} /lldsavetemps
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,TEMPS,ELF,OTHER
|
||||
|
||||
# No files are expected before.
|
||||
BOOKEND-NOT: {{.}}
|
||||
TEMPS: {{^}}my.[[#PID:]].dist-file.json{{$}}
|
||||
ELF: {{^}}my.exe{{$}}
|
||||
OTHER: {{^}}my.exe.resolution.txt{{$}}
|
||||
# Filename composition: <archive><member><member offset>.<task>.<pid>.<task>.<pid>.native.o.
|
||||
TEMPS: {{^}}t.libt1.o[[#T1_OFFSET:]].1.[[#%X,HEXPID:]].1.[[#PID]].native.o{{$}}
|
||||
TEMPS: {{^}}t.libt1.o[[#T1_OFFSET]].1.[[#%X,HEXPID]].1.[[#PID]].native.o.thinlto.bc{{$}}
|
||||
TEMPS: {{^}}t.libt1.o[[#T1_OFFSET]].1.[[#%X,HEXPID]].o{{$}}
|
||||
# No files are expected after.
|
||||
BOOKEND-NOT: {{.}}
|
||||
|
||||
#--- t1.c
|
||||
__attribute__((retain)) int mainCRTStartup() { return 0; }
|
||||
20
cross-project-tests/dtlto/local_codegen_and_wait.py
Normal file
20
cross-project-tests/dtlto/local_codegen_and_wait.py
Normal file
@ -0,0 +1,20 @@
|
||||
"""
|
||||
This simple distributor performs code generation locally, creates the
|
||||
"send-signal1" file, and then waits for the "send-signal2" file to appear
|
||||
before exiting. It is intended to be used in tandem with test_temps.py.
|
||||
Please see test_temps.py for more information.
|
||||
"""
|
||||
|
||||
import json, subprocess, sys, time, os, pathlib
|
||||
|
||||
# Load the DTLTO information from the input JSON file.
|
||||
data = json.loads(pathlib.Path(sys.argv[-1]).read_bytes())
|
||||
|
||||
# Iterate over the jobs and execute the codegen tool.
|
||||
for job in data["jobs"]:
|
||||
subprocess.check_call(data["common"]["args"] + job["args"])
|
||||
|
||||
pathlib.Path("send-signal1").touch()
|
||||
|
||||
while not os.path.exists("send-signal2"):
|
||||
time.sleep(0.05)
|
||||
44
cross-project-tests/dtlto/savetemps-lock.test
Normal file
44
cross-project-tests/dtlto/savetemps-lock.test
Normal file
@ -0,0 +1,44 @@
|
||||
# This test relies on locking files which is difficult to do in a way the keeps
|
||||
# a test robust on Linux, so it is restricted to Windows.
|
||||
REQUIRES: ld.lld,system-windows
|
||||
|
||||
# Test that a warning is emitted for each DTLTO temporary file that cannot be
|
||||
# removed. This test uses archives because archive handling exercises a superset
|
||||
# of the temporary files used for object inputs.
|
||||
#
|
||||
# This scenario is logically related to the cases in savetemps.test; however, it
|
||||
# is placed here to maintain coverage, as this behavior can only be tested
|
||||
# effectively on Windows.
|
||||
|
||||
RUN: rm -rf %t && split-file %s %t && cd %t
|
||||
|
||||
RUN: %clang --target=x86_64-linux-gnu -O2 t1.c t2.c -flto=thin -c
|
||||
|
||||
RUN: llvm-ar rcs t.a t1.o t2.o
|
||||
|
||||
# Check that a warning is reported for each temporary file that cannot be
|
||||
# removed. Note that the use of the name "locked" for the output directory
|
||||
# triggers special behaviour in test_temps.py.
|
||||
RUN: mkdir locked && %python %S/test_temps.py locked lock \
|
||||
RUN: %clang --target=x86_64-linux-gnu -nostdlib -O2 -flto=thin \
|
||||
RUN: -fuse-ld=lld -Wl,--whole-archive t.a -o locked/t.elf -shared \
|
||||
RUN: -fthinlto-distributor=%python \
|
||||
RUN: -Xthinlto-distributor=%S/local_codegen_and_wait.py 2>&1 \
|
||||
RUN: | FileCheck %s --implicit-check-not=warning
|
||||
|
||||
# Sanity check for the expected test_temps.py behaviour.
|
||||
CHECK-DAG: Lock any files in the output directory.
|
||||
CHECK-DAG: warning: could not remove the file 'locked{{/|\\}}t.[[#PID:]].dist-file.json': {{.*}}
|
||||
# Filename composition: <archive>(<member> at <offset>).<task>.<pid>.<task>.<pid>.native.o.
|
||||
CHECK-DAG: warning: could not remove the file 'locked{{/|\\}}t.a(t1.o at [[#T1_OFFSET:]]).1.[[#%X,HEXPID:]].1.[[#PID]].native.o': {{.*}}
|
||||
CHECK-DAG: warning: could not remove the file 'locked{{/|\\}}t.a(t1.o at [[#T1_OFFSET]]).1.[[#%X,HEXPID]].1.[[#PID]].native.o.thinlto.bc': {{.*}}
|
||||
CHECK-DAG: warning: could not remove the file 'locked{{/|\\}}t.a(t2.o at [[#T2_OFFSET:]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o': {{.*}}
|
||||
CHECK-DAG: warning: could not remove the file 'locked{{/|\\}}t.a(t2.o at [[#T2_OFFSET]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o.thinlto.bc': {{.*}}
|
||||
CHECK-DAG: warning: could not remove temporary DTLTO input file 'locked{{/|\\}}t.a(t1.o at [[#T1_OFFSET]]).1.[[#%X,HEXPID]].o': {{.*}}
|
||||
CHECK-DAG: warning: could not remove temporary DTLTO input file 'locked{{/|\\}}t.a(t2.o at [[#T2_OFFSET]]).2.[[#%X,HEXPID]].o': {{.*}}
|
||||
|
||||
#--- t1.c
|
||||
__attribute__((retain)) int t1(int x) { return x; }
|
||||
|
||||
#--- t2.c
|
||||
__attribute__((retain)) int t2(int x) { return x; }
|
||||
70
cross-project-tests/dtlto/savetemps.test
Normal file
70
cross-project-tests/dtlto/savetemps.test
Normal file
@ -0,0 +1,70 @@
|
||||
REQUIRES: ld.lld
|
||||
|
||||
# Test that DTLTO temporary files are "best-effort" cleaned up unless
|
||||
# --save-temps is specified. We use archives in this test as the handling for
|
||||
# archives requires a superset of the temporary files used for object inputs.
|
||||
|
||||
RUN: rm -rf %t && split-file %s %t && cd %t
|
||||
|
||||
RUN: %clang --target=x86_64-linux-gnu -O2 t1.c t2.c -flto=thin -c
|
||||
|
||||
RUN: llvm-ar rcs t.a t1.o t2.o
|
||||
|
||||
DEFINE: %{tdir} = dummy-to-make-lit-work
|
||||
DEFINE: %{action} = dummy-to-make-lit-work
|
||||
DEFINE: %{test-temps-dtlto} = \
|
||||
DEFINE: mkdir %{tdir} && %python %S/test_temps.py %{tdir} %{action} \
|
||||
DEFINE: %clang --target=x86_64-linux-gnu -nostdlib -O2 -flto=thin \
|
||||
DEFINE: -fuse-ld=lld -Wl,--whole-archive t.a -o %{tdir}/t.elf -shared \
|
||||
DEFINE: -fthinlto-distributor=%python \
|
||||
DEFINE: -Xthinlto-distributor=%S/local_codegen_and_wait.py
|
||||
|
||||
# Check that all temporary files are removed in normal operation.
|
||||
REDEFINE: %{tdir} = empty
|
||||
REDEFINE: %{action} = none
|
||||
RUN: %{test-temps-dtlto}
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,ELF
|
||||
|
||||
# Check that --save-temps preserves temporary files.
|
||||
REDEFINE: %{tdir} = savetemps
|
||||
REDEFINE: %{action} = none
|
||||
RUN: %{test-temps-dtlto} -Wl,--save-temps
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,TEMPS,INDEX,ELF
|
||||
|
||||
# Check that --thinlto-emit-index-files preserves the index files.
|
||||
REDEFINE: %{tdir} = index
|
||||
REDEFINE: %{action} = none
|
||||
RUN: %{test-temps-dtlto} -Wl,--thinlto-emit-index-files
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,INDEX,ELF
|
||||
|
||||
# No files are expected before.
|
||||
BOOKEND-NOT: {{.}}
|
||||
TEMPS: {{^}}t.[[#PID:]].dist-file.json{{$}}
|
||||
# Filename composition: <archive>(<member> at <offset>).<task>.<pid>.<task>.<pid>.native.o.
|
||||
TEMPS: {{^}}t.a(t1.o at [[#T1_OFFSET:]]).1.[[#%X,HEXPID:]].1.[[#PID]].native.o{{$}}
|
||||
INDEX: {{^}}t.a(t1.o at [[#T1_OFFSET:]]).1.[[#%X,HEXPID:]].1.[[#PID:]].native.o.thinlto.bc{{$}}
|
||||
TEMPS: {{^}}t.a(t1.o at [[#T1_OFFSET]]).1.[[#%X,HEXPID]].o{{$}}
|
||||
TEMPS: {{^}}t.a(t2.o at [[#T2_OFFSET:]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o{{$}}
|
||||
INDEX: {{^}}t.a(t2.o at [[#T2_OFFSET:]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o.thinlto.bc{{$}}
|
||||
TEMPS: {{^}}t.a(t2.o at [[#T2_OFFSET]]).2.[[#%X,HEXPID]].o{{$}}
|
||||
ELF: {{^}}t.elf{{$}}
|
||||
TEMPS: {{^}}t.elf.resolution.txt{{$}}
|
||||
# No files are expected after.
|
||||
BOOKEND-NOT: {{.}}
|
||||
|
||||
# Check that no warnings are produced if temporary files are missing. Note that
|
||||
# the use of the name "removed" for the output directory triggers special
|
||||
# behaviour in test_temps.py.
|
||||
REDEFINE: %{tdir} = removed
|
||||
REDEFINE: %{action} = remove
|
||||
RUN: %{test-temps-dtlto} 2>&1 | FileCheck %s --check-prefix=NOWARN --allow-empty
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,ELF
|
||||
# Sanity check for the expected test_temps.py behaviour.
|
||||
NOWARN: Remove non-essential files in the output directory.
|
||||
NOWARN-NOT: warning
|
||||
|
||||
#--- t1.c
|
||||
__attribute__((retain)) int t1(int x) { return x; }
|
||||
|
||||
#--- t2.c
|
||||
__attribute__((retain)) int t2(int x) { return x; }
|
||||
@ -1,19 +1,22 @@
|
||||
REQUIRES: ld.lld
|
||||
|
||||
# Test that if a link is terminated by a signal (or the equivalent on
|
||||
# Windows), e.g. CTRL-C, DTLTO temporary files are cleaned up.
|
||||
# Windows), e.g. CTRL-C, DTLTO temporary files are cleaned up. We use
|
||||
# archives in this test as the handling for archives requires a superset
|
||||
# of the temporary files used for object inputs.
|
||||
|
||||
RUN: rm -rf %t && split-file %s %t && cd %t
|
||||
|
||||
RUN: %clang --target=x86_64-linux-gnu -O2 t1.c t2.c -flto=thin -c
|
||||
|
||||
RUN: llvm-ar rcs t.a t1.o t2.o
|
||||
|
||||
DEFINE: %{tdir} = dummy-to-make-lit-work
|
||||
DEFINE: %{kill-dtlto} = rm -f send-signal && mkdir %{tdir} && \
|
||||
DEFINE: %python killer.py \
|
||||
DEFINE: %clang --target=x86_64-linux-gnu -nostdlib -O2 -flto=thin \
|
||||
DEFINE: -fuse-ld=lld -Wl,--whole-archive t1.o t2.o -o %{tdir}/t.elf -shared \
|
||||
DEFINE: -fthinlto-distributor=%python \
|
||||
DEFINE: -Xthinlto-distributor=local_codegen_and_wait.py
|
||||
DEFINE: %{kill-dtlto} = mkdir %{tdir} && %python %S/test_temps.py %{tdir} kill \
|
||||
DEFINE: %clang --target=x86_64-linux-gnu -nostdlib -O2 -flto=thin \
|
||||
DEFINE: -fuse-ld=lld -Wl,--whole-archive t.a -o %{tdir}/t.elf -shared \
|
||||
DEFINE: -fthinlto-distributor=%python \
|
||||
DEFINE: -Xthinlto-distributor=%S/local_codegen_and_wait.py
|
||||
|
||||
# Check that all temporary files are removed if the process is aborted.
|
||||
REDEFINE: %{tdir} = empty
|
||||
@ -25,7 +28,7 @@ EMPTY-NOT: {{.}}
|
||||
# Check that --save-temps preserves temporary files if the process is aborted.
|
||||
REDEFINE: %{tdir} = savetemps
|
||||
RUN: %{kill-dtlto} -Wl,--save-temps
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,TEMPS,INDEX
|
||||
RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,TEMPS,INDEX,OTHER
|
||||
|
||||
# Check that --thinlto-emit-index-files preserves the index files if the process
|
||||
# is aborted.
|
||||
@ -36,11 +39,14 @@ RUN: ls %{tdir} | sort | FileCheck %s --check-prefixes=BOOKEND,INDEX
|
||||
# No files are expected before.
|
||||
BOOKEND-NOT: {{.}}
|
||||
TEMPS: {{^}}t.[[#PID:]].dist-file.json{{$}}
|
||||
TEMPS: {{^}}t.elf.{{.+$}}
|
||||
TEMPS: {{^}}t1.1.[[#PID]].native.o{{$}}
|
||||
INDEX: {{^}}t1.1.[[#PID:]].native.o.thinlto.bc{{$}}
|
||||
TEMPS: {{^}}t2.2.[[#PID]].native.o{{$}}
|
||||
INDEX: {{^}}t2.2.[[#PID]].native.o.thinlto.bc{{$}}
|
||||
# Filename composition: <archive>(<member> at <offset>).<task>.<pid>.<task>.<pid>.native.o.
|
||||
TEMPS: {{^}}t.a(t1.o at [[#T1_OFFSET:]]).1.[[#%X,HEXPID:]].1.[[#PID]].native.o{{$}}
|
||||
INDEX: {{^}}t.a(t1.o at [[#T1_OFFSET:]]).1.[[#%X,HEXPID:]].1.[[#PID:]].native.o.thinlto.bc{{$}}
|
||||
TEMPS: {{^}}t.a(t1.o at [[#T1_OFFSET]]).1.[[#%X,HEXPID]].o{{$}}
|
||||
TEMPS: {{^}}t.a(t2.o at [[#T2_OFFSET:]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o{{$}}
|
||||
INDEX: {{^}}t.a(t2.o at [[#T2_OFFSET:]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o.thinlto.bc{{$}}
|
||||
TEMPS: {{^}}t.a(t2.o at [[#T2_OFFSET]]).2.[[#%X,HEXPID]].o{{$}}
|
||||
OTHER: {{^}}t.elf.resolution.txt{{$}}
|
||||
# No files are expected after.
|
||||
BOOKEND-NOT: {{.}}
|
||||
|
||||
@ -49,45 +55,3 @@ __attribute__((retain)) int t1(int x) { return x; }
|
||||
|
||||
#--- t2.c
|
||||
__attribute__((retain)) int t2(int x) { return x; }
|
||||
|
||||
#--- local_codegen_and_wait.py
|
||||
"""Perform codegen locally, create "send-signal" file and wait."""
|
||||
from pathlib import Path
|
||||
import json, subprocess, sys, time
|
||||
|
||||
# Load the DTLTO information from the input JSON file.
|
||||
data = json.loads(Path(sys.argv[-1]).read_bytes())
|
||||
|
||||
# Iterate over the jobs and execute the codegen tool.
|
||||
for job in data["jobs"]:
|
||||
subprocess.check_call(data["common"]["args"] + job["args"])
|
||||
Path("send-signal").touch()
|
||||
while True:
|
||||
time.sleep(1)
|
||||
|
||||
#--- killer.py
|
||||
"""Run command, wait for "send-signal" file to exist, and then send a
|
||||
termination signal."""
|
||||
import os, sys, time, signal, subprocess
|
||||
|
||||
if os.name == "nt":
|
||||
# CREATE_NEW_PROCESS_GROUP is used so that p.send_signal(CTRL_BREAK_EVENT)
|
||||
# does not get sent to the LIT processes that are running the test.
|
||||
kwargs = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
|
||||
else:
|
||||
# Makes the child a process-group leader so os.killpg(p.pid, SIGINT) works.
|
||||
kwargs = {"start_new_session": True}
|
||||
|
||||
p = subprocess.Popen(sys.argv[1:], **kwargs)
|
||||
|
||||
while not os.path.exists("send-signal"):
|
||||
time.sleep(0.05)
|
||||
|
||||
if os.name == "nt":
|
||||
# Note that CTRL_C_EVENT does not appear to work for clang.
|
||||
p.send_signal(signal.CTRL_BREAK_EVENT)
|
||||
else:
|
||||
os.killpg(p.pid, signal.SIGINT)
|
||||
p.wait()
|
||||
|
||||
sys.exit(0 if p.returncode != 0 else 1)
|
||||
|
||||
128
cross-project-tests/dtlto/test_temps.py
Normal file
128
cross-project-tests/dtlto/test_temps.py
Normal file
@ -0,0 +1,128 @@
|
||||
"""
|
||||
This script works in tandem with local_codegen_and_wait.py. By coordinating
|
||||
via the "send-signal*" files, the scripts ensure that the requested action is
|
||||
performed after all DTLTO backend compilations have completed but before
|
||||
DTLTO itself finishes. At this point, DTLTO temporary files have been
|
||||
created but have not yet been cleaned up.
|
||||
|
||||
Usage:
|
||||
%python test_temps.py <output_dir> <action> <command...>
|
||||
|
||||
Run <command>, which must be a ThinLTO link invocation that uses
|
||||
local_codegen_and_wait.py as the ThinLTO distributor. The script waits for
|
||||
the "send-signal1" file to appear, performs <action>, and then creates the
|
||||
"send-signal2" file to allow the link to continue.
|
||||
|
||||
Actions:
|
||||
kill Send an interrupt to cause <command> to terminate after the
|
||||
ThinLTO backend compilations complete.
|
||||
lock (Windows only) Hold open handles to DTLTO temporary files in
|
||||
<output_dir> to prevent their deletion after the ThinLTO backend
|
||||
compilations complete. This action is not supported on Linux, as
|
||||
there is no reliable mechanism to prevent file deletion that is
|
||||
guaranteed to be released when the script exits (AFAICT).
|
||||
remove Delete non-essential files in <output_dir> after the ThinLTO
|
||||
backend compilations complete.
|
||||
"""
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
IS_WIN = os.name == "nt"
|
||||
SIGNAL1 = Path("send-signal1")
|
||||
SIGNAL2 = Path("send-signal2")
|
||||
|
||||
if IS_WIN:
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
CreateFileW = ctypes.WinDLL("kernel32", use_last_error=True).CreateFileW
|
||||
CreateFileW.argtypes = [
|
||||
wintypes.LPCWSTR, # lpFileName
|
||||
wintypes.DWORD, # dwDesiredAccess
|
||||
wintypes.DWORD, # dwShareMode
|
||||
wintypes.LPVOID, # lpSecurityAttributes
|
||||
wintypes.DWORD, # dwCreationDisposition
|
||||
wintypes.DWORD, # dwFlagsAndAttributes
|
||||
wintypes.HANDLE, # hTemplateFile
|
||||
]
|
||||
CreateFileW.restype = wintypes.HANDLE
|
||||
|
||||
|
||||
def lock_no_delete_share(path):
|
||||
# Windows-specific: deny FILE_SHARE_DELETE by omitting it from dwShareMode.
|
||||
h = CreateFileW(
|
||||
path,
|
||||
0x80000000, # GENERIC_READ
|
||||
0x00000003, # FILE_SHARE_READ/WRITE (no FILE_SHARE_DELETE)
|
||||
None, # lpSecurityAttributes
|
||||
3, # OPEN_EXISTING
|
||||
0, # dwFlagsAndAttributes
|
||||
None, # hTemplateFile
|
||||
)
|
||||
if h == wintypes.HANDLE(-1).value:
|
||||
err = ctypes.get_last_error()
|
||||
raise OSError(err, f"CreateFileW failed ({err}) for: {path}")
|
||||
return h
|
||||
|
||||
|
||||
output_dir = Path(sys.argv[1])
|
||||
action = sys.argv[2]
|
||||
|
||||
# "lock" is Windows-only; fail early if invoked elsewhere.
|
||||
if action == "lock" and not IS_WIN:
|
||||
print("error: action 'lock' is only supported on Windows", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Remove any pre-existing signal files from previous script runs.
|
||||
SIGNAL1.unlink(missing_ok=True)
|
||||
SIGNAL2.unlink(missing_ok=True)
|
||||
|
||||
kwargs = {}
|
||||
if action == "kill":
|
||||
if IS_WIN:
|
||||
# CREATE_NEW_PROCESS_GROUP is used so that p.send_signal(CTRL_BREAK_EVENT)
|
||||
# does not get sent to the LIT processes that are running the test.
|
||||
kwargs = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
|
||||
else:
|
||||
# Makes the child a process-group leader so os.killpg(p.pid, SIGINT) works.
|
||||
kwargs = {"start_new_session": True}
|
||||
|
||||
p = subprocess.Popen(sys.argv[3:], **kwargs)
|
||||
|
||||
while not SIGNAL1.exists() and p.poll() is None:
|
||||
time.sleep(0.05)
|
||||
if p.poll() is not None:
|
||||
sys.exit(1)
|
||||
|
||||
if action == "kill":
|
||||
if IS_WIN:
|
||||
# Note that CTRL_C_EVENT does not appear to work for clang.
|
||||
p.send_signal(signal.CTRL_BREAK_EVENT)
|
||||
else:
|
||||
os.killpg(p.pid, signal.SIGINT)
|
||||
|
||||
if action == "lock":
|
||||
print("Lock any files in the output directory.")
|
||||
for f in output_dir.iterdir():
|
||||
if f.is_file():
|
||||
lock_no_delete_share(str(f))
|
||||
|
||||
if action == "remove":
|
||||
print("Remove non-essential files in the output directory.")
|
||||
for f in output_dir.iterdir():
|
||||
if f.is_file() and not f.name.endswith("native.o"):
|
||||
f.unlink()
|
||||
|
||||
SIGNAL2.touch()
|
||||
|
||||
if action == "kill":
|
||||
# Expect termination: succeed if the child fails, fail if it exits cleanly.
|
||||
p.wait()
|
||||
sys.exit(0 if p.returncode != 0 else 1)
|
||||
|
||||
sys.exit(p.wait())
|
||||
@ -142,8 +142,10 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) {
|
||||
ltoObj = std::make_unique<lto::LTO>(createConfig(), backend,
|
||||
ctx.config.ltoPartitions);
|
||||
else
|
||||
ltoObj = std::make_unique<lto::DTLTO>(createConfig(), backend,
|
||||
ctx.config.ltoPartitions);
|
||||
ltoObj = std::make_unique<lto::DTLTO>(
|
||||
createConfig(), backend, ctx.config.ltoPartitions,
|
||||
llvm::lto::LTO::LTOKind::LTOK_Default, ctx.config.outputFile,
|
||||
!ctx.config.saveTempsArgs.empty());
|
||||
}
|
||||
|
||||
BitcodeCompiler::~BitcodeCompiler() = default;
|
||||
|
||||
@ -205,9 +205,10 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
|
||||
ctx.arg.ltoPartitions,
|
||||
ltoModes[ctx.arg.ltoKind]);
|
||||
else
|
||||
ltoObj = std::make_unique<lto::DTLTO>(createConfig(ctx), backend,
|
||||
ctx.arg.ltoPartitions,
|
||||
ltoModes[ctx.arg.ltoKind]);
|
||||
ltoObj = std::make_unique<lto::DTLTO>(
|
||||
createConfig(ctx), backend, ctx.arg.ltoPartitions,
|
||||
ltoModes[ctx.arg.ltoKind], ctx.arg.outputFile,
|
||||
!ctx.arg.saveTempsArgs.empty());
|
||||
// Initialize usedStartStop.
|
||||
if (ctx.bitcodeFiles.empty())
|
||||
return;
|
||||
|
||||
@ -19,9 +19,14 @@ class DTLTO : public LTO {
|
||||
using Base = LTO;
|
||||
|
||||
public:
|
||||
// Inherit constructors.
|
||||
using Base::Base;
|
||||
~DTLTO() override = default;
|
||||
LLVM_ABI DTLTO(Config Conf, ThinBackend Backend,
|
||||
unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode,
|
||||
StringRef LinkerOutputFile, bool SaveTemps)
|
||||
: Base(std::move(Conf), Backend, ParallelCodeGenParallelismLevel,
|
||||
LTOMode),
|
||||
LinkerOutputFile(LinkerOutputFile), SaveTemps(SaveTemps) {
|
||||
assert(!LinkerOutputFile.empty() && "expected a valid linker output file");
|
||||
}
|
||||
|
||||
// Add an input file and prepare it for distribution.
|
||||
LLVM_ABI Expected<std::shared_ptr<InputFile>>
|
||||
@ -37,6 +42,12 @@ private:
|
||||
BumpPtrAllocator PtrAlloc;
|
||||
StringSaver Saver{PtrAlloc};
|
||||
|
||||
/// The output file to which this LTO invocation will contribute.
|
||||
StringRef LinkerOutputFile;
|
||||
|
||||
/// Controls preservation of any created temporary files.
|
||||
bool SaveTemps;
|
||||
|
||||
// Determines if a file at the given path is a thin archive file.
|
||||
Expected<bool> isThinArchive(const StringRef ArchivePath);
|
||||
|
||||
|
||||
@ -25,6 +25,7 @@
|
||||
#include "llvm/Support/MemoryBufferRef.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/Process.h"
|
||||
#include "llvm/Support/Signals.h"
|
||||
#include "llvm/Support/TimeProfiler.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
|
||||
@ -158,7 +159,9 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
|
||||
std::string PID = utohexstr(sys::Process::getProcessId());
|
||||
std::string Seq = std::to_string(InputFiles.size());
|
||||
|
||||
NewModuleId = {sys::path::filename(ModuleId), ".", Seq, ".", PID, ".o"};
|
||||
NewModuleId = sys::path::parent_path(LinkerOutputFile);
|
||||
sys::path::append(NewModuleId, sys::path::filename(ModuleId) + "." + Seq +
|
||||
"." + PID + ".o");
|
||||
}
|
||||
|
||||
// Update the module identifier and save it.
|
||||
@ -174,6 +177,9 @@ Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) {
|
||||
StringRef ModuleId = Input->getName();
|
||||
if (Input->isMemberOfArchive()) {
|
||||
TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId);
|
||||
// Cleanup this file on abnormal process exit.
|
||||
if (!SaveTemps)
|
||||
llvm::sys::RemoveFileOnSignal(ModuleId);
|
||||
MemoryBufferRef MemoryBufferRef = Input->getFileBuffer();
|
||||
if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId))
|
||||
return EC;
|
||||
@ -207,11 +213,18 @@ llvm::Error lto::DTLTO::handleArchiveInputs() {
|
||||
|
||||
// Remove temporary archive member files created to enable distribution.
|
||||
void lto::DTLTO::cleanup() {
|
||||
{
|
||||
if (!SaveTemps) {
|
||||
TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
|
||||
for (auto &Input : InputFiles)
|
||||
if (Input->isMemberOfArchive())
|
||||
sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
|
||||
for (auto &Input : InputFiles) {
|
||||
if (!Input->isMemberOfArchive())
|
||||
continue;
|
||||
std::error_code EC =
|
||||
sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
|
||||
if (EC &&
|
||||
EC != std::make_error_code(std::errc::no_such_file_or_directory))
|
||||
errs() << "warning: could not remove temporary DTLTO input file '"
|
||||
<< Input->getName() << "': " << EC.message() << "\n";
|
||||
}
|
||||
}
|
||||
Base::cleanup();
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user