89031cffd1
The sanitizer-coverage.cpp test case was always failing for me. It turns out the reason for this is that I was building with -DLLVM_INSTALL_BINUTILS_SYMLINKS=ON and sancov.py's grep regex does not handle llvm-objdump's disassembly format (hex immediates have a leading "0x"). While touching those lines also change them to use raw string literals since invalid escape sequnces will become an error in future python versions. Also simplify the code by using subprocess.check_output() instead of Popen(). This also works with python2. Fixes https://bugs.llvm.org/show_bug.cgi?id=44504 Reviewed By: #sanitizers, vitalybuka Differential Revision: https://reviews.llvm.org/D89648
250 lines
7.9 KiB
Python
Executable File
250 lines
7.9 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Merge or print the coverage data collected by asan's coverage.
|
|
# Input files are sequences of 4-byte integers.
|
|
# We need to merge these integers into a set and then
|
|
# either print them (as hex) or dump them into another file.
|
|
import array
|
|
import bisect
|
|
import glob
|
|
import os.path
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
|
|
prog_name = ""
|
|
|
|
def Usage():
|
|
sys.stderr.write(
|
|
"Usage: \n" + \
|
|
" " + prog_name + " merge FILE [FILE...] > OUTPUT\n" \
|
|
" " + prog_name + " print FILE [FILE...]\n" \
|
|
" " + prog_name + " unpack FILE [FILE...]\n" \
|
|
" " + prog_name + " rawunpack FILE [FILE ...]\n" \
|
|
" " + prog_name + " missing BINARY < LIST_OF_PCS\n" \
|
|
"\n")
|
|
exit(1)
|
|
|
|
def CheckBits(bits):
|
|
if bits != 32 and bits != 64:
|
|
raise Exception("Wrong bitness: %d" % bits)
|
|
|
|
def TypeCodeForBits(bits):
|
|
CheckBits(bits)
|
|
return 'L' if bits == 64 else 'I'
|
|
|
|
def TypeCodeForStruct(bits):
|
|
CheckBits(bits)
|
|
return 'Q' if bits == 64 else 'I'
|
|
|
|
kMagic32SecondHalf = 0xFFFFFF32;
|
|
kMagic64SecondHalf = 0xFFFFFF64;
|
|
kMagicFirstHalf = 0xC0BFFFFF;
|
|
|
|
def MagicForBits(bits):
|
|
CheckBits(bits)
|
|
if sys.byteorder == 'little':
|
|
return [kMagic64SecondHalf if bits == 64 else kMagic32SecondHalf, kMagicFirstHalf]
|
|
else:
|
|
return [kMagicFirstHalf, kMagic64SecondHalf if bits == 64 else kMagic32SecondHalf]
|
|
|
|
def ReadMagicAndReturnBitness(f, path):
|
|
magic_bytes = f.read(8)
|
|
magic_words = struct.unpack('II', magic_bytes);
|
|
bits = 0
|
|
idx = 1 if sys.byteorder == 'little' else 0
|
|
if magic_words[idx] == kMagicFirstHalf:
|
|
if magic_words[1-idx] == kMagic64SecondHalf:
|
|
bits = 64
|
|
elif magic_words[1-idx] == kMagic32SecondHalf:
|
|
bits = 32
|
|
if bits == 0:
|
|
raise Exception('Bad magic word in %s' % path)
|
|
return bits
|
|
|
|
def ReadOneFile(path):
|
|
with open(path, mode="rb") as f:
|
|
f.seek(0, 2)
|
|
size = f.tell()
|
|
f.seek(0, 0)
|
|
if size < 8:
|
|
raise Exception('File %s is short (< 8 bytes)' % path)
|
|
bits = ReadMagicAndReturnBitness(f, path)
|
|
size -= 8
|
|
w = size * 8 // bits
|
|
s = struct.unpack_from(TypeCodeForStruct(bits) * (w), f.read(size))
|
|
sys.stderr.write(
|
|
"%s: read %d %d-bit PCs from %s\n" % (prog_name, w, bits, path))
|
|
return s
|
|
|
|
def Merge(files):
|
|
s = set()
|
|
for f in files:
|
|
s = s.union(set(ReadOneFile(f)))
|
|
sys.stderr.write(
|
|
"%s: %d files merged; %d PCs total\n" % (prog_name, len(files), len(s))
|
|
)
|
|
return sorted(s)
|
|
|
|
def PrintFiles(files):
|
|
if len(files) > 1:
|
|
s = Merge(files)
|
|
else: # If there is just on file, print the PCs in order.
|
|
s = ReadOneFile(files[0])
|
|
sys.stderr.write("%s: 1 file merged; %d PCs total\n" % (prog_name, len(s)))
|
|
for i in s:
|
|
print("0x%x" % i)
|
|
|
|
def MergeAndPrint(files):
|
|
if sys.stdout.isatty():
|
|
Usage()
|
|
s = Merge(files)
|
|
bits = 32
|
|
if max(s) > 0xFFFFFFFF:
|
|
bits = 64
|
|
stdout_buf = getattr(sys.stdout, 'buffer', sys.stdout)
|
|
array.array('I', MagicForBits(bits)).tofile(stdout_buf)
|
|
a = struct.pack(TypeCodeForStruct(bits) * len(s), *s)
|
|
stdout_buf.write(a)
|
|
|
|
|
|
def UnpackOneFile(path):
|
|
with open(path, mode="rb") as f:
|
|
sys.stderr.write("%s: unpacking %s\n" % (prog_name, path))
|
|
while True:
|
|
header = f.read(12)
|
|
if not header: return
|
|
if len(header) < 12:
|
|
break
|
|
pid, module_length, blob_size = struct.unpack('iII', header)
|
|
module = f.read(module_length).decode('utf-8')
|
|
blob = f.read(blob_size)
|
|
assert(len(module) == module_length)
|
|
assert(len(blob) == blob_size)
|
|
extracted_file = "%s.%d.sancov" % (module, pid)
|
|
sys.stderr.write("%s: extracting %s\n" % (prog_name, extracted_file))
|
|
# The packed file may contain multiple blobs for the same pid/module
|
|
# pair. Append to the end of the file instead of overwriting.
|
|
with open(extracted_file, 'ab') as f2:
|
|
f2.write(blob)
|
|
# fail
|
|
raise Exception('Error reading file %s' % path)
|
|
|
|
|
|
def Unpack(files):
|
|
for f in files:
|
|
UnpackOneFile(f)
|
|
|
|
def UnpackOneRawFile(path, map_path):
|
|
mem_map = []
|
|
with open(map_path, mode="rt") as f_map:
|
|
sys.stderr.write("%s: reading map %s\n" % (prog_name, map_path))
|
|
bits = int(f_map.readline())
|
|
if bits != 32 and bits != 64:
|
|
raise Exception('Wrong bits size in the map')
|
|
for line in f_map:
|
|
parts = line.rstrip().split()
|
|
mem_map.append((int(parts[0], 16),
|
|
int(parts[1], 16),
|
|
int(parts[2], 16),
|
|
' '.join(parts[3:])))
|
|
mem_map.sort(key=lambda m : m[0])
|
|
mem_map_keys = [m[0] for m in mem_map]
|
|
|
|
with open(path, mode="rb") as f:
|
|
sys.stderr.write("%s: unpacking %s\n" % (prog_name, path))
|
|
|
|
f.seek(0, 2)
|
|
size = f.tell()
|
|
f.seek(0, 0)
|
|
pcs = struct.unpack_from(TypeCodeForStruct(bits) * (size * 8 // bits), f.read(size))
|
|
mem_map_pcs = [[] for i in range(0, len(mem_map))]
|
|
|
|
for pc in pcs:
|
|
if pc == 0: continue
|
|
map_idx = bisect.bisect(mem_map_keys, pc) - 1
|
|
(start, end, base, module_path) = mem_map[map_idx]
|
|
assert pc >= start
|
|
if pc >= end:
|
|
sys.stderr.write("warning: %s: pc %x outside of any known mapping\n" % (prog_name, pc))
|
|
continue
|
|
mem_map_pcs[map_idx].append(pc - base)
|
|
|
|
for ((start, end, base, module_path), pc_list) in zip(mem_map, mem_map_pcs):
|
|
if len(pc_list) == 0: continue
|
|
assert path.endswith('.sancov.raw')
|
|
dst_path = module_path + '.' + os.path.basename(path)[:-4]
|
|
sys.stderr.write("%s: writing %d PCs to %s\n" % (prog_name, len(pc_list), dst_path))
|
|
sorted_pc_list = sorted(pc_list)
|
|
pc_buffer = struct.pack(TypeCodeForStruct(bits) * len(pc_list), *sorted_pc_list)
|
|
with open(dst_path, 'ab+') as f2:
|
|
array.array('I', MagicForBits(bits)).tofile(f2)
|
|
f2.seek(0, 2)
|
|
f2.write(pc_buffer)
|
|
|
|
def RawUnpack(files):
|
|
for f in files:
|
|
if not f.endswith('.sancov.raw'):
|
|
raise Exception('Unexpected raw file name %s' % f)
|
|
f_map = f[:-3] + 'map'
|
|
UnpackOneRawFile(f, f_map)
|
|
|
|
def GetInstrumentedPCs(binary):
|
|
# This looks scary, but all it does is extract all offsets where we call:
|
|
# - __sanitizer_cov() or __sanitizer_cov_with_check(),
|
|
# - with call or callq,
|
|
# - directly or via PLT.
|
|
cmd = r"objdump --no-show-raw-insn -d %s | " \
|
|
r"grep '^\s\+[0-9a-f]\+:\s\+call\(q\|\)\s\+\(0x\|\)[0-9a-f]\+ <__sanitizer_cov\(_with_check\|\|_trace_pc_guard\)\(@plt\|\)>' | " \
|
|
r"grep -o '^\s\+[0-9a-f]\+'" % binary
|
|
lines = subprocess.check_output(cmd, stdin=subprocess.PIPE, shell=True).splitlines()
|
|
# The PCs we get from objdump are off by 4 bytes, as they point to the
|
|
# beginning of the callq instruction. Empirically this is true on x86 and
|
|
# x86_64.
|
|
return set(int(line.strip(), 16) + 4 for line in lines)
|
|
|
|
def PrintMissing(binary):
|
|
if not os.path.isfile(binary):
|
|
raise Exception('File not found: %s' % binary)
|
|
instrumented = GetInstrumentedPCs(binary)
|
|
sys.stderr.write("%s: found %d instrumented PCs in %s\n" % (prog_name,
|
|
len(instrumented),
|
|
binary))
|
|
covered = set(int(line, 16) for line in sys.stdin)
|
|
sys.stderr.write("%s: read %d PCs from stdin\n" % (prog_name, len(covered)))
|
|
missing = instrumented - covered
|
|
sys.stderr.write("%s: %d PCs missing from coverage\n" % (prog_name, len(missing)))
|
|
if (len(missing) > len(instrumented) - len(covered)):
|
|
sys.stderr.write(
|
|
"%s: WARNING: stdin contains PCs not found in binary\n" % prog_name
|
|
)
|
|
for pc in sorted(missing):
|
|
print("0x%x" % pc)
|
|
|
|
if __name__ == '__main__':
|
|
prog_name = sys.argv[0]
|
|
if len(sys.argv) <= 2:
|
|
Usage();
|
|
|
|
if sys.argv[1] == "missing":
|
|
if len(sys.argv) != 3:
|
|
Usage()
|
|
PrintMissing(sys.argv[2])
|
|
exit(0)
|
|
|
|
file_list = []
|
|
for f in sys.argv[2:]:
|
|
file_list += glob.glob(f)
|
|
if not file_list:
|
|
Usage()
|
|
|
|
if sys.argv[1] == "print":
|
|
PrintFiles(file_list)
|
|
elif sys.argv[1] == "merge":
|
|
MergeAndPrint(file_list)
|
|
elif sys.argv[1] == "unpack":
|
|
Unpack(file_list)
|
|
elif sys.argv[1] == "rawunpack":
|
|
RawUnpack(file_list)
|
|
else:
|
|
Usage()
|