diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index badfb4b1484..fb0fddc457e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" +#include "nix/store/nar-cache.hh" #include #include @@ -26,6 +27,7 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} + , narCache{config.localNarCache.get().empty() ? nullptr : std::make_shared(config.localNarCache.get())} { if (config.secretKeyFile != "") signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile)})); @@ -408,10 +410,36 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); - LengthSink narSize; - TeeSink tee{sink, narSize}; + if (narCache) { + if (auto nar = narCache->getNar(info->narHash)) { + notice("substituted '%s' from local NAR cache", printStorePath(storePath)); + sink(*nar); + stats.narRead++; + stats.narReadBytes += nar->size(); + return; + } + } + + std::unique_ptr narCacheSink; + if (narCache) + narCacheSink = sourceToSink([&](Source & source) { narCache->upsertNar(info->narHash, source); }); + + uint64_t narSize = 0; + + LambdaSink uncompressedSink{ + [&](std::string_view data) { + narSize += data.size(); + if (narCacheSink) + (*narCacheSink)(data); + sink(data); + }, + [&]() { + stats.narRead++; + // stats.narReadCompressedBytes += nar->size(); // FIXME + stats.narReadBytes += narSize; + }}; - auto decompressor = makeDecompressionSink(info->compression, tee); + auto decompressor = makeDecompressionSink(info->compression, uncompressedSink); try { getFile(info->url, *decompressor); @@ -421,9 +449,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) decompressor->finish(); - stats.narRead++; - // stats.narReadCompressedBytes += nar->size(); // FIXME - stats.narReadBytes += narSize.length; + // Note: don't do anything here because it's never reached if we're called as a coroutine. } void BinaryCacheStore::queryPathInfoUncached( @@ -541,7 +567,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, narCache); } ref BinaryCacheStore::getFSAccessor(bool requireValidPath) diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index c316b1199b4..7c6defc94c4 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -12,6 +12,7 @@ namespace nix { struct NarInfo; +class NarCache; class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig @@ -38,7 +39,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig const Setting secretKeyFiles{ this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; - const Setting localNarCache{ + const Setting localNarCache{ this, "", "local-nar-cache", @@ -85,6 +86,8 @@ protected: const std::string cacheInfoFile = "nix-cache-info"; + std::shared_ptr narCache; + BinaryCacheStore(Config &); public: diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 8e88ec51f66..1d6e6e9d47f 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -56,6 +56,7 @@ headers = [ config_pub_h ] + files( 'make-content-addressed.hh', 'names.hh', 'nar-accessor.hh', + 'nar-cache.hh', 'nar-info-disk-cache.hh', 'nar-info.hh', 'outputs-spec.hh', diff --git a/src/libstore/include/nix/store/nar-cache.hh b/src/libstore/include/nix/store/nar-cache.hh new file mode 100644 index 00000000000..00963333bfc --- /dev/null +++ b/src/libstore/include/nix/store/nar-cache.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "nix/util/hash.hh" + +#include + +namespace nix { + +class NarCache +{ + + const std::filesystem::path cacheDir; + + std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext); + +public: + + NarCache(std::filesystem::path cacheDir); + + void upsertNar(const Hash & narHash, Source & source); + + void upsertNarListing(const Hash & narHash, std::string_view narListingData); + + // FIXME: use a sink. + std::optional getNar(const Hash & narHash); + + // FIXME: use a sink. + std::string getNarBytes(const Hash & narHash, uint64_t offset, uint64_t length); + + std::optional getNarListing(const Hash & narHash); +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index 9e1999cc061..bc7c39bfa2a 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -7,6 +7,8 @@ namespace nix { +struct NarCache; + class RemoteFSAccessor : public SourceAccessor { ref store; @@ -15,16 +17,12 @@ class RemoteFSAccessor : public SourceAccessor bool requireValidPath; - Path cacheDir; + std::shared_ptr narCache; std::pair, CanonPath> fetch(const CanonPath & path); friend struct BinaryCacheStore; - Path makeCacheFile(std::string_view hashPart, const std::string & ext); - - ref addToCache(std::string_view hashPart, std::string && nar); - public: /** @@ -32,8 +30,7 @@ public: */ std::shared_ptr accessObject(const StorePath & path); - RemoteFSAccessor( - ref store, bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); + RemoteFSAccessor(ref store, bool requireValidPath = true, std::shared_ptr narCache = {}); std::optional maybeLstat(const CanonPath & path) override; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 5ca54930df6..4bfdab82ab7 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -345,6 +345,7 @@ sources = files( 'misc.cc', 'names.cc', 'nar-accessor.cc', + 'nar-cache.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', 'optimise-store.cc', diff --git a/src/libstore/nar-cache.cc b/src/libstore/nar-cache.cc new file mode 100644 index 00000000000..343038e60eb --- /dev/null +++ b/src/libstore/nar-cache.cc @@ -0,0 +1,81 @@ +#include "nix/store/nar-cache.hh" +#include "nix/util/file-system.hh" + +#include +#include +#include + +namespace nix { + +NarCache::NarCache(std::filesystem::path cacheDir_) + : cacheDir(std::move(cacheDir_)) +{ + assert(!cacheDir.empty()); + createDirs(cacheDir); +} + +std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext) +{ + return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext; +} + +void NarCache::upsertNar(const Hash & narHash, Source & source) +{ + try { + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(narHash, "nar"), source); + } catch (SystemError &) { + ignoreExceptionExceptInterrupt(); + } +} + +void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData) +{ + try { + writeFile(makeCacheFile(narHash, "ls"), narListingData); + } catch (SystemError &) { + ignoreExceptionExceptInterrupt(); + } +} + +std::optional NarCache::getNar(const Hash & narHash) +{ + try { + return nix::readFile(makeCacheFile(narHash, "nar")); + } catch (SystemError &) { + return std::nullopt; + } +} + +std::string NarCache::getNarBytes(const Hash & narHash, uint64_t offset, uint64_t length) +{ + auto cacheFile = makeCacheFile(narHash, "nar"); + + AutoCloseFD fd = toDescriptor(open( + cacheFile.c_str(), + O_RDONLY +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); + if (!fd) + throw SysError("opening NAR cache file %s", cacheFile); + + if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) + throw SysError("seeking in %s", cacheFile); + + std::string buf(length, 0); + readFull(fd.get(), buf.data(), length); + return buf; +} + +std::optional NarCache::getNarListing(const Hash & narHash) +{ + try { + return nix::readFile(makeCacheFile(narHash, "ls")); + } catch (SystemError &) { + return std::nullopt; + } +} + +} // namespace nix diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index e6715cbdfb0..835f424ac98 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,52 +1,15 @@ #include #include "nix/store/remote-fs-accessor.hh" #include "nix/store/nar-accessor.hh" - -#include -#include -#include +#include "nix/store/nar-cache.hh" namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, std::shared_ptr narCache) : store(store) , requireValidPath(requireValidPath) - , cacheDir(cacheDir) -{ - if (cacheDir != "") - createDirs(cacheDir); -} - -Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) + , narCache(std::move(narCache)) { - assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, hashPart, ext); -} - -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) -{ - if (cacheDir != "") { - try { - /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(hashPart, "nar"), nar); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - } - - auto narAccessor = makeNarAccessor(std::move(nar)); - nars.emplace(hashPart, narAccessor); - - if (cacheDir != "") { - try { - nlohmann::json j = listNar(narAccessor, CanonPath::root, true); - writeFile(makeCacheFile(hashPart, "ls"), j.dump()); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - } - - return narAccessor; } std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) @@ -63,51 +26,46 @@ std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & if (i != nars.end()) return i->second; - std::string listing; - Path cacheFile; - - if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { - - try { - listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); - - auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { - AutoCloseFD fd = toDescriptor(open( - cacheFile.c_str(), - O_RDONLY -#ifndef _WIN32 - | O_CLOEXEC -#endif - )); - if (!fd) - throw SysError("opening NAR cache file '%s'", cacheFile); + Hash narHash{HashAlgorithm::SHA256}; - if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) - throw SysError("seeking in '%s'", cacheFile); + if (narCache) { + auto info = store->queryPathInfo(storePath); + narHash = info->narHash; - std::string buf(length, 0); - readFull(fd.get(), buf.data(), length); - - return buf; - }); + if (auto listingData = narCache->getNarListing(narHash)) { + auto narAccessor = + makeLazyNarAccessor(*listingData, [narCache(narCache), narHash](uint64_t offset, uint64_t length) { + return narCache->getNarBytes(narHash, offset, length); + }); nars.emplace(storePath.hashPart(), narAccessor); return narAccessor; - - } catch (SystemError &) { } - try { - auto narAccessor = makeNarAccessor(nix::readFile(cacheFile)); + if (auto nar = narCache->getNar(narHash)) { + auto narAccessor = makeNarAccessor(std::move(*nar)); nars.emplace(storePath.hashPart(), narAccessor); return narAccessor; - } catch (SystemError &) { } } StringSink sink; store->narFromPath(storePath, sink); - return addToCache(storePath.hashPart(), std::move(sink.s)); + + if (narCache) { + StringSource source{sink.s}; + narCache->upsertNar(narHash, source); + } + + auto narAccessor = makeNarAccessor(std::move(sink.s)); + nars.emplace(storePath.hashPart(), narAccessor); + + if (narCache) { + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); + narCache->upsertNarListing(narHash, j.dump()); + } + + return narAccessor; } std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index ca3c08cd9b3..daf8178de69 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -433,6 +433,18 @@ std::string BaseSetting::to_string() const [](const auto & kvpair) { return kvpair.first + "=" + kvpair.second; }); } +template<> +std::filesystem::path BaseSetting::parse(const std::string & str) const +{ + return std::filesystem::path(str).lexically_normal(); +} + +template<> +std::string BaseSetting::to_string() const +{ + return value.string(); +} + template class BaseSetting; template class BaseSetting; template class BaseSetting; @@ -445,6 +457,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting>; +template class BaseSetting; static Path parsePath(const AbstractSetting & s, const std::string & str) { diff --git a/src/libutil/include/nix/util/config-impl.hh b/src/libutil/include/nix/util/config-impl.hh index f407bc86244..6066765c5f1 100644 --- a/src/libutil/include/nix/util/config-impl.hh +++ b/src/libutil/include/nix/util/config-impl.hh @@ -134,6 +134,7 @@ DECLARE_CONFIG_SERIALISER(Strings) DECLARE_CONFIG_SERIALISER(StringSet) DECLARE_CONFIG_SERIALISER(StringMap) DECLARE_CONFIG_SERIALISER(std::set) +DECLARE_CONFIG_SERIALISER(std::filesystem::path) template T BaseSetting::parse(const std::string & str) const diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 5e211703daa..de8390330a3 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -56,9 +56,9 @@ struct LoggerSettings : Config expression evaluation errors. )"}; - Setting jsonLogPath{ + Setting jsonLogPath{ this, - "", + {}, "json-log-path", R"( A file or Unix domain socket to which JSON records of Nix's log output are diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index d6845a494dc..4605f51a237 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -381,18 +381,27 @@ struct LengthSource : Source */ struct LambdaSink : Sink { - typedef std::function lambda_t; + typedef std::function data_t; + typedef std::function cleanup_t; - lambda_t lambda; + data_t dataFun; + cleanup_t cleanupFun; - LambdaSink(const lambda_t & lambda) - : lambda(lambda) + LambdaSink( + const data_t & dataFun, const cleanup_t & cleanupFun = []() {}) + : dataFun(dataFun) + , cleanupFun(cleanupFun) + { + } + + ~LambdaSink() { + cleanupFun(); } void operator()(std::string_view data) override { - lambda(data); + dataFun(data); } }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index b3e69405eb6..189a8b5194d 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -380,7 +380,7 @@ void applyJSONLogger() if (!loggerSettings.jsonLogPath.get().empty()) { try { std::vector> loggers; - loggers.push_back(makeJSONLogger(std::filesystem::path(loggerSettings.jsonLogPath.get()), false)); + loggers.push_back(makeJSONLogger(loggerSettings.jsonLogPath.get(), false)); try { logger = makeTeeLogger(std::move(logger), std::move(loggers)); } catch (...) { diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 2c102df0771..2070969040b 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -227,13 +227,23 @@ mkdir "$narCache" [[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] -rm -rfv "$cacheDir/nar" +mv "$cacheDir/nar" "$cacheDir/nar2" [[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] (! nix store cat --store "file://$cacheDir" "$outPath/foobar") +# Check substitution from the local NAR cache. +clearStore +rm -rf "$narCache" "$cacheDir/nar" +mv "$cacheDir/nar2" "$cacheDir/nar" +nix-store -r --substituters "file://$cacheDir?local-nar-cache=$narCache" --no-require-sigs "$outPath" +mv "$cacheDir/nar" "$cacheDir/nar2" +clearStore +nix-store -r --substituters "file://$cacheDir?local-nar-cache=$narCache" --no-require-sigs "$outPath" + + # Test NAR listing generation. clearCache