be9c083cf7
Add OnDiskGraphDB and OnDiskKeyValueDB that can be used to implement ObjectStore and ActionCache respectively. Those are on-disk persistent storage that build upon OnDiskTrieHashMap and implements key functions that are required by LLVMCAS interfaces. This abstraction layer defines how the objects are hashed and stored on disk. OnDiskKeyValueDB is a basic OnDiskTrieHashMap while OnDiskGraphDB also defines: * How objects of various size are store on disk and are referenced by the trie nodes. * How to store the references from one stored object to another object that is referenced. In addition to basic APIs for ObjectStore and ActionCache, other advances database configuration features can be implemented in this layer without exposing to the users of the LLVMCAS interface. For example, OnDiskGraphDB has a faulty in function to fetch data from an upstream OnDiskGraphDB if the data is missing.
114 lines
4.2 KiB
C++
114 lines
4.2 KiB
C++
//===- OnDiskKeyValueDB.cpp -------------------------------------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// \file
|
|
/// This file implements OnDiskKeyValueDB, an ondisk key value database.
|
|
///
|
|
/// The KeyValue database file is named `actions.<version>` inside the CAS
|
|
/// directory. The database stores a mapping between a fixed-sized key and a
|
|
/// fixed-sized value, where the size of key and value can be configured when
|
|
/// opening the database.
|
|
///
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/CAS/OnDiskKeyValueDB.h"
|
|
#include "OnDiskCommon.h"
|
|
#include "llvm/ADT/StringExtras.h"
|
|
#include "llvm/Support/Alignment.h"
|
|
#include "llvm/Support/Compiler.h"
|
|
#include "llvm/Support/Errc.h"
|
|
#include "llvm/Support/Path.h"
|
|
|
|
using namespace llvm;
|
|
using namespace llvm::cas;
|
|
using namespace llvm::cas::ondisk;
|
|
|
|
static constexpr StringLiteral ActionCacheFile = "actions.";
|
|
|
|
Expected<ArrayRef<char>> OnDiskKeyValueDB::put(ArrayRef<uint8_t> Key,
|
|
ArrayRef<char> Value) {
|
|
if (LLVM_UNLIKELY(Value.size() != ValueSize))
|
|
return createStringError(errc::invalid_argument,
|
|
"expected value size of " + itostr(ValueSize) +
|
|
", got: " + itostr(Value.size()));
|
|
assert(Value.size() == ValueSize);
|
|
auto ActionP = Cache.insertLazy(
|
|
Key, [&](FileOffset TentativeOffset,
|
|
OnDiskTrieRawHashMap::ValueProxy TentativeValue) {
|
|
assert(TentativeValue.Data.size() == ValueSize);
|
|
llvm::copy(Value, TentativeValue.Data.data());
|
|
});
|
|
if (LLVM_UNLIKELY(!ActionP))
|
|
return ActionP.takeError();
|
|
return (*ActionP)->Data;
|
|
}
|
|
|
|
Expected<std::optional<ArrayRef<char>>>
|
|
OnDiskKeyValueDB::get(ArrayRef<uint8_t> Key) {
|
|
// Check the result cache.
|
|
OnDiskTrieRawHashMap::ConstOnDiskPtr ActionP = Cache.find(Key);
|
|
if (!ActionP)
|
|
return std::nullopt;
|
|
assert(isAddrAligned(Align(8), ActionP->Data.data()));
|
|
return ActionP->Data;
|
|
}
|
|
|
|
Expected<std::unique_ptr<OnDiskKeyValueDB>>
|
|
OnDiskKeyValueDB::open(StringRef Path, StringRef HashName, unsigned KeySize,
|
|
StringRef ValueName, size_t ValueSize) {
|
|
if (std::error_code EC = sys::fs::create_directories(Path))
|
|
return createFileError(Path, EC);
|
|
|
|
SmallString<256> CachePath(Path);
|
|
sys::path::append(CachePath, ActionCacheFile + CASFormatVersion);
|
|
constexpr uint64_t MB = 1024ull * 1024ull;
|
|
constexpr uint64_t GB = 1024ull * 1024ull * 1024ull;
|
|
|
|
uint64_t MaxFileSize = GB;
|
|
auto CustomSize = getOverriddenMaxMappingSize();
|
|
if (!CustomSize)
|
|
return CustomSize.takeError();
|
|
if (*CustomSize)
|
|
MaxFileSize = **CustomSize;
|
|
|
|
std::optional<OnDiskTrieRawHashMap> ActionCache;
|
|
if (Error E = OnDiskTrieRawHashMap::create(
|
|
CachePath,
|
|
"llvm.actioncache[" + HashName + "->" + ValueName + "]",
|
|
KeySize * 8,
|
|
/*DataSize=*/ValueSize, MaxFileSize, /*MinFileSize=*/MB)
|
|
.moveInto(ActionCache))
|
|
return std::move(E);
|
|
|
|
return std::unique_ptr<OnDiskKeyValueDB>(
|
|
new OnDiskKeyValueDB(ValueSize, std::move(*ActionCache)));
|
|
}
|
|
|
|
Error OnDiskKeyValueDB::validate(CheckValueT CheckValue) const {
|
|
return Cache.validate(
|
|
[&](FileOffset Offset,
|
|
OnDiskTrieRawHashMap::ConstValueProxy Record) -> Error {
|
|
auto formatError = [&](Twine Msg) {
|
|
return createStringError(
|
|
llvm::errc::illegal_byte_sequence,
|
|
"bad cache value at 0x" +
|
|
utohexstr((unsigned)Offset.get(), /*LowerCase=*/true) + ": " +
|
|
Msg.str());
|
|
};
|
|
|
|
if (Record.Data.size() != ValueSize)
|
|
return formatError("wrong cache value size");
|
|
if (!isAddrAligned(Align(8), Record.Data.data()))
|
|
return formatError("wrong cache value alignment");
|
|
if (CheckValue)
|
|
return CheckValue(Offset, Record.Data);
|
|
return Error::success();
|
|
});
|
|
}
|