shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use TarWriter to create tar archives instead of cpio.

Browse Source 
This is how we use TarWriter in LLD. Now LLD does not append
a file extension, so you need to pass `--reproduce foo.tar`
instead of `--reproduce foo`.

Differential Revision: https://reviews.llvm.org/D28103

llvm-svn: 291210
This commit is contained in:
Rui Ueyama 2017-01-06 02:33:53 +00:00
parent 4bb7883f0c
commit 7f1f912794
16 changed files with 68 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
lld/COFF/Driver.cpp
View File

@ -25,6 +25,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/TarWriter.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
@ -98,9 +99,9 @@ MemoryBufferRef LinkerDriver::takeBuffer(std::unique_ptr<MemoryBuffer> MB) {
MemoryBufferRef MBRef = *MB;
OwningMBs.push_back(std::move(MB));
if (Driver->Cpio)
Driver->Cpio->append(relativeToRoot(MBRef.getBufferIdentifier()),
MBRef.getBuffer());
if (Driver->Tar)
Driver->Tar->append(relativeToRoot(MBRef.getBufferIdentifier()),
MBRef.getBuffer());
return MBRef;
}
@ -458,13 +459,17 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
if (auto *Arg = Args.getLastArg(OPT_linkrepro)) {
SmallString<64> Path = StringRef(Arg->getValue());
sys::path::append(Path, "repro");
ErrorOr<CpioFile *> F = CpioFile::create(Path);
if (F)
Cpio.reset(*F);
else
errs() << "/linkrepro: failed to open " << Path
<< ".cpio: " << F.getError().message() << '\n';
sys::path::append(Path, "repro.tar");
Expected<std::unique_ptr<TarWriter>> ErrOrWriter =
TarWriter::create(Path, "repro");
if (ErrOrWriter) {
Tar = std::move(*ErrOrWriter);
} else {
errs() << "/linkrepro: failed to open " << Path << ": "
<< toString(ErrOrWriter.takeError()) << '\n';
}
}
if (Args.filtered_begin(OPT_INPUT) == Args.filtered_end())
@ -683,10 +688,10 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
if (!Resources.empty())
addBuffer(convertResToCOFF(Resources));
if (Cpio)
Cpio->append("response.txt",
createResponseFile(Args, FilePaths,
ArrayRef<StringRef>(SearchPaths).slice(1)));
if (Tar)
Tar->append("response.txt",
createResponseFile(Args, FilePaths,
ArrayRef<StringRef>(SearchPaths).slice(1)));
// Handle /largeaddressaware
if (Config->is64() || Args.hasArg(OPT_largeaddressaware))

3
lld/COFF/Driver.h
View File

@ -20,6 +20,7 @@
#include "llvm/Object/COFF.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Support/TarWriter.h"
#include <memory>
#include <set>
#include <vector>
@ -74,7 +75,7 @@ private:
ArgParser Parser;
SymbolTable Symtab;
std::unique_ptr<CpioFile> Cpio; // for /linkrepro
std::unique_ptr<llvm::TarWriter> Tar; // for /linkrepro
// Opens a file. Path has to be resolved already.
MemoryBufferRef openFile(StringRef Path);

23
lld/ELF/Driver.cpp
View File

@ -25,6 +25,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdlib>
@ -182,8 +183,8 @@ Optional<MemoryBufferRef> LinkerDriver::readFile(StringRef Path) {
MemoryBufferRef MBRef = MB->getMemBufferRef();
make<std::unique_ptr<MemoryBuffer>>(std::move(MB)); // take MB ownership
if (Cpio)
Cpio->append(relativeToRoot(Path), MBRef.getBuffer());
if (Tar)
Tar->append(relativeToRoot(Path), MBRef.getBuffer());
return MBRef;
}
@ -309,14 +310,16 @@ void LinkerDriver::main(ArrayRef<const char *> ArgsArr, bool CanExitEarly) {
if (const char *Path = getReproduceOption(Args)) {
// Note that --reproduce is a debug option so you can ignore it
// if you are trying to understand the whole picture of the code.
ErrorOr<CpioFile *> F = CpioFile::create(Path);
if (F) {
Cpio.reset(*F);
Cpio->append("response.txt", createResponseFile(Args));
Cpio->append("version.txt", getLLDVersion() + "\n");
} else
error(F.getError(),
Twine("--reproduce: failed to open ") + Path + ".cpio");
Expected<std::unique_ptr<TarWriter>> ErrOrWriter =
TarWriter::create(Path, path::stem(Path));
if (ErrOrWriter) {
Tar = std::move(*ErrOrWriter);
Tar->append("response.txt", createResponseFile(Args));
Tar->append("version.txt", getLLDVersion() + "\n");
} else {
error(Twine("--reproduce: failed to open ") + Path + ": " +
toString(ErrOrWriter.takeError()));
}
}
readConfigs(Args);

3
lld/ELF/Driver.h
View File

@ -17,6 +17,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Support/TarWriter.h"
#include "llvm/Support/raw_ostream.h"
namespace lld {
@ -29,7 +30,7 @@ public:
void main(ArrayRef<const char *> Args, bool CanExitEarly);
void addFile(StringRef Path);
void addLibrary(StringRef Name);
std::unique_ptr<CpioFile> Cpio; // for reproduce
std::unique_ptr<llvm::TarWriter> Tar; // for reproduce
private:
std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef MB);

6
lld/ELF/InputFiles.cpp
View File

@ -525,9 +525,9 @@ ArchiveFile::getMember(const Archive::Symbol *Sym) {
"could not get the buffer for the member defining symbol " +
Sym->getName());
if (C.getParent()->isThin() && Driver->Cpio)
Driver->Cpio->append(relativeToRoot(check(C.getFullName())),
Ret.getBuffer());
if (C.getParent()->isThin() && Driver->Tar)
Driver->Tar->append(relativeToRoot(check(C.getFullName())),
Ret.getBuffer());
if (C.getParent()->isThin())
return {Ret, 0};
return {Ret, C.getChildOffset()};

31
lld/include/lld/Core/Reproduce.h
View File

@ -11,46 +11,15 @@
#define LLD_CORE_REPRODUCE_H
#include "lld/Core/LLVM.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Support/Error.h"
namespace llvm {
class raw_fd_ostream;
namespace opt { class Arg; }
}
namespace lld {
// This class creates a .cpio file for --reproduce (ELF) or /linkrepro (COFF).
//
// If "--reproduce foo" is given, we create a file "foo.cpio" and
// copy all input files to the archive, along with a response file
// to re-run the same command with the same inputs.
// It is useful for reporting issues to LLD developers.
//
// Cpio as a file format is a deliberate choice. It's standardized in
// POSIX and very easy to create. cpio command is available virtually
// on all Unix systems. See
// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_07
// for the format details.
class CpioFile {
public:
static ErrorOr<CpioFile *> create(StringRef OutputPath);
void append(StringRef Path, StringRef Data);
private:
CpioFile(std::unique_ptr<llvm::raw_fd_ostream> OS, StringRef Basename);
std::unique_ptr<llvm::raw_fd_ostream> OS;
llvm::StringSet<> Seen;
std::string Basename;
};
// Makes a given pathname an absolute path first, and then remove
// beginning /. For example, "../foo.o" is converted to "home/john/foo.o",
// assuming that the current directory is "/home/john/bar".

56
lld/lib/Core/Reproduce.cpp
View File

@ -8,66 +8,14 @@
//===----------------------------------------------------------------------===//
#include "lld/Core/Reproduce.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Option/Arg.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/Path.h"
using namespace lld;
using namespace llvm;
using namespace sys;
CpioFile::CpioFile(std::unique_ptr<raw_fd_ostream> OS, StringRef S)
: OS(std::move(OS)), Basename(S) {}
ErrorOr<CpioFile *> CpioFile::create(StringRef OutputPath) {
std::string Path = (OutputPath + ".cpio").str();
std::error_code EC;
auto OS = llvm::make_unique<raw_fd_ostream>(Path, EC, sys::fs::F_None);
if (EC)
return EC;
return new CpioFile(std::move(OS), path::filename(OutputPath));
}
static void writeMember(raw_fd_ostream &OS, StringRef Path, StringRef Data) {
// The c_dev/c_ino pair should be unique according to the spec,
// but no one seems to care.
OS << "070707"; // c_magic
OS << "000000"; // c_dev
OS << "000000"; // c_ino
OS << "100664"; // c_mode: C_ISREG | rw-rw-r--
OS << "000000"; // c_uid
OS << "000000"; // c_gid
OS << "000001"; // c_nlink
OS << "000000"; // c_rdev
OS << "00000000000"; // c_mtime
OS << format("%06o", Path.size() + 1); // c_namesize
OS << format("%011o", Data.size()); // c_filesize
OS << Path << '\0'; // c_name
OS << Data; // c_filedata
}
void CpioFile::append(StringRef Path, StringRef Data) {
if (!Seen.insert(Path).second)
return;
// Construct an in-archive filename so that /home/foo/bar is stored
// as baz/home/foo/bar where baz is the basename of the output file.
// (i.e. in that case we are creating baz.cpio.)
SmallString<128> Fullpath;
path::append(Fullpath, Basename, Path);
writeMember(*OS, convertToUnixPathSeparator(Fullpath), Data);
// Print the trailer and seek back.
// This way we have a valid archive if we crash.
uint64_t Pos = OS->tell();
writeMember(*OS, "TRAILER!!!", "");
OS->seek(Pos);
}
using namespace llvm::sys;
// Makes a given pathname an absolute path first, and then remove
// beginning /. For example, "../foo.o" is converted to "home/john/foo.o",
@ -76,7 +24,7 @@ void CpioFile::append(StringRef Path, StringRef Data) {
// a mess with backslash-as-escape and backslash-as-path-separator.
std::string lld::relativeToRoot(StringRef Path) {
SmallString<128> Abs = Path;
if (sys::fs::make_absolute(Abs))
if (fs::make_absolute(Abs))
return Path;
path::remove_dots(Abs, /*remove_dot_dot=*/true);

9
lld/test/COFF/linkrepro.test
View File

@ -1,6 +1,3 @@
# cpio fails on windows with "Function not implemented".
# REQUIRES: shell
# RUN: rm -rf %t.dir
# RUN: mkdir -p %t.dir/build1 %t.dir/build2 %t.dir/build3
# RUN: yaml2obj < %p/Inputs/hello32.yaml > %t.obj
@ -8,7 +5,7 @@
# RUN: cd %t.dir/build1
# RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
# RUN: /entry:main@0 /linkrepro:. /out:%t.exe
# RUN: cpio -id < repro.cpio
# RUN: tar xf repro.tar
# RUN: diff %t.obj repro/%:t.obj
# RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
@ -16,7 +13,7 @@
# RUN: cd %t.dir/build2
# RUN: lld-link %t.obj /libpath:%p/Inputs /defaultlib:std32 /subsystem:console \
# RUN: /entry:main@0 /linkrepro:. /out:%t.exe
# RUN: cpio -id < repro.cpio
# RUN: tar xf repro.tar
# RUN: diff %t.obj repro/%:t.obj
# RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
@ -24,7 +21,7 @@
# RUN: cd %t.dir/build3
# RUN: env LIB=%p/Inputs lld-link %t.obj /defaultlib:std32 /subsystem:console \
# RUN: /entry:main@0 /linkrepro:. /out:%t.exe
# RUN: cpio -id < repro.cpio
# RUN: tar xf repro.tar
# RUN: diff %t.obj repro/%:t.obj
# RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt

6
lld/test/ELF/reproduce-backslash.s
View File

@ -1,9 +1,9 @@
# REQUIRES: x86, cpio, shell
# REQUIRES: x86, shell
# Test that we don't erroneously replace \ with / on UNIX, as it's
# legal for a filename to contain backslashes.
# RUN: llvm-mc %s -o foo\\.o -filetype=obj -triple=x86_64-pc-linux
# RUN: ld.lld foo\\.o --reproduce repro
# RUN: cpio -it < repro.cpio | FileCheck %s
# RUN: tar tf repro.tar | FileCheck %s
# CHECK: repro/{{.*}}/foo\.o
# CHECK: repro/{{.*}}/foo\\.o

7
lld/test/ELF/reproduce-error.s
View File

@ -1,15 +1,14 @@
# Extracting the cpio archive can get over the path limit on windows.
# Extracting the tar archive can get over the path limit on windows.
# REQUIRES: shell
# RUN: rm -rf %t.dir
# RUN: mkdir -p %t.dir
# RUN: cd %t.dir
# RUN: not ld.lld --reproduce repro abc -o t 2>&1 | FileCheck %s
# RUN: not ld.lld --reproduce repro.tar abc -o t 2>&1 | FileCheck %s
# CHECK: cannot open abc: {{N|n}}o such file or directory
# RUN: grep TRAILER repro.cpio
# RUN: cpio -id < repro.cpio
# RUN: tar xf repro.tar
# RUN: FileCheck --check-prefix=RSP %s < repro/response.txt
# RSP: abc
# RSP: -o t

4
lld/test/ELF/reproduce-linkerscript.s
View File

@ -5,8 +5,8 @@
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build/foo.o
# RUN: echo "INPUT(\"%t.dir/build/foo.o\")" > %t.dir/build/foo.script
# RUN: cd %t.dir
# RUN: ld.lld build/foo.script -o bar --reproduce repro
# RUN: cpio -id < repro.cpio
# RUN: ld.lld build/foo.script -o bar --reproduce repro.tar
# RUN: tar xf repro.tar
# RUN: diff build/foo.script repro/%:t.dir/build/foo.script
# RUN: diff build/foo.o repro/%:t.dir/build/foo.o

4
lld/test/ELF/reproduce-thin-archive.s
View File

@ -5,8 +5,8 @@
# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.dir/foo.o
# RUN: cd %t.dir
# RUN: llvm-ar --format=gnu rcT foo.a foo.o
# RUN: ld.lld -m elf_x86_64 foo.a -o bar --reproduce repro
# RUN: cpio -id < repro.cpio
# RUN: ld.lld -m elf_x86_64 foo.a -o bar --reproduce repro.tar
# RUN: tar xf repro.tar
# RUN: diff foo.a repro/%:t.dir/foo.a
# RUN: diff foo.o repro/%:t.dir/foo.o

6
lld/test/ELF/reproduce-windows.s
View File

@ -1,12 +1,12 @@
# REQUIRES: x86, cpio
# REQUIRES: x86
# Test that a repro archive always uses / instead of \.
# RUN: rm -rf %t.dir
# RUN: mkdir -p %t.dir/build
# RUN: llvm-mc %s -o %t.dir/build/foo.o -filetype=obj -triple=x86_64-pc-linux
# RUN: cd %t.dir
# RUN: ld.lld build/foo.o --reproduce repro
# RUN: cpio -it < repro.cpio | FileCheck %s
# RUN: ld.lld build/foo.o --reproduce repro.tar
# RUN: tar tf repro.tar | FileCheck %s
# CHECK: repro/response.txt
# CHECK: repro/{{.*}}/build/foo.o

5
lld/test/ELF/reproduce-windows2.s
View File

@ -1,4 +1,4 @@
# REQUIRES: system-windows, x86, cpio
# REQUIRES: system-windows, x86
# Test that a response.txt file always uses / instead of \.
# RUN: rm -rf %t.dir
@ -6,6 +6,5 @@
# RUN: llvm-mc %s -o %t.dir/build/foo.o -filetype=obj -triple=x86_64-pc-linux
# RUN: cd %t.dir
# RUN: ld.lld build/foo.o --reproduce repro
# RUN: echo "*response.txt" > list.txt
# RUN: cpio -i --to-stdout --pattern-file=list.txt < repro.cpio | FileCheck %s
# RUN: tar -O -x -f repro.tar repro/response.txt | FileCheck %s
# CHECK: {{.*}}/build/foo.o

18
lld/test/ELF/reproduce.s
View File

@ -1,14 +1,14 @@
# REQUIRES: x86, cpio
# REQUIRES: x86
# Extracting the cpio archive can get over the path limit on windows.
# Extracting the tar archive can get over the path limit on windows.
# REQUIRES: shell
# RUN: rm -rf %t.dir
# RUN: mkdir -p %t.dir/build1
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build1/foo.o
# RUN: cd %t.dir
# RUN: ld.lld --hash-style=gnu build1/foo.o -o bar -shared --as-needed --reproduce repro
# RUN: cpio -id < repro.cpio
# RUN: ld.lld --hash-style=gnu build1/foo.o -o bar -shared --as-needed --reproduce repro.tar
# RUN: tar xf repro.tar
# RUN: diff build1/foo.o repro/%:t.dir/build1/foo.o
# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
@ -25,8 +25,8 @@
# RUN: mkdir -p %t.dir/build2/a/b/c
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build2/foo.o
# RUN: cd %t.dir/build2/a/b/c
# RUN: env LLD_REPRODUCE=repro ld.lld ./../../../foo.o -o bar -shared --as-needed
# RUN: cpio -id < repro.cpio
# RUN: env LLD_REPRODUCE=repro.tar ld.lld ./../../../foo.o -o bar -shared --as-needed
# RUN: tar xf repro.tar
# RUN: diff %t.dir/build2/foo.o repro/%:t.dir/build2/foo.o
# RUN: echo "{ local: *; };" > ver
@ -34,10 +34,10 @@
# RUN: echo > file
# RUN: echo > file2
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o 'foo bar'
# RUN: ld.lld --reproduce repro2 'foo bar' -L"foo bar" -Lfile -Tfile2 \
# RUN: ld.lld --reproduce repro2.tar 'foo bar' -L"foo bar" -Lfile -Tfile2 \
# RUN: --dynamic-list dyn -rpath file --script=file --version-script ver \
# RUN: --dynamic-linker "some unusual/path" -soname 'foo bar' -soname='foo bar'
# RUN: cpio -id < repro2.cpio
# RUN: tar xf repro2.tar
# RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt
# RSP2: "{{.*}}foo bar"
# RSP2-NEXT: -L "{{.*}}foo bar"
@ -51,7 +51,7 @@
# RSP2-NEXT: -soname="foo bar"
# RSP2-NEXT: -soname="foo bar"
# RUN: cpio -it < repro2.cpio | FileCheck %s
# RUN: tar tf repro2.tar | FileCheck %s
# CHECK: repro2/response.txt
# CHECK-NEXT: repro2/version.txt
# CHECK-NEXT: repro2/{{.*}}/dyn

4
lld/test/lit.cfg
View File

@ -258,7 +258,3 @@ cvtres = lit.util.which('cvtres', config.environment['PATH'])
rc = lit.util.which('rc', config.environment['PATH'])
if cvtres and rc:
config.available_features.add('winres')
# Check if "cpio" command exists.
if lit.util.which('cpio', config.environment['PATH']):
config.available_features.add('cpio')