Binary cache improvements

doc/manual/source/protocols/nix-cache-info.md

@@ -36,12 +36,44 @@ error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/st
 
 Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority).
 
+### `GetNarInfosV1`
+
+The path (relative to the cache URL) of an endpoint for fetching the
+metadata of multiple store paths in a single request. If this field is
+present, a client may send a `POST` request to this path whose body is
+a list of store path hash parts (one per line). The server responds
+with one JSON object per line (newline-delimited JSON), each being the
+[version 2 JSON representation](@docroot@/protocols/json/store-object-info.md)
+of a store path's metadata, extended with a `"path"` field holding the
+store path it describes. This lets a client fetch the metadata for many
+paths in one request instead of one request per path. A path whose
+object is absent from the response is not available in the cache.
+
+Each object may also contain a `"partialClosure"` field: an array of
+store paths that are (some of) the path's indirect references. This is
+a hint that lets a client start fetching the metadata of an entire
+closure without waiting for the intervening objects; it need not be
+complete and is not covered by the signature.
+
+If the field is absent, the client falls back to fetching each
+`.narinfo` individually.
+
+## Other fields
+
+Any field not listed above is stored verbatim in the client's
+[on-disk cache](#caching-behavior) of `nix-cache-info` but is otherwise
+ignored. This keeps the format forward-compatible: a newer server can
+advertise a capability that an older client persists, and a later
+client version can start using it without the server's metadata having
+to be re-fetched.
+
 ## Example
 
-```
+```text
 StoreDir: /nix/store
 WantMassQuery: 1
 Priority: 30
+GetNarInfosV1: /get-narinfos-v1
 ```
 
 ## Caching Behavior

src/libstore/binary-cache-store.cc

@@ -86,6 +86,8 @@ void BinaryCacheStore::applyCacheInfoFields(const std::map<std::string, std::str
         if (auto priority = string2Int<int>(*value))
             config.priority.setDefault(*priority);
     }
+    if (auto * endpoint = get(fields, "GetNarInfosV1"))
+        getNarInfosV1 = *endpoint;
 }
 
 void BinaryCacheStore::init()

src/libstore/filetransfer.cc

@@ -384,7 +384,8 @@ struct curlFileTransfer : public FileTransfer
                     *logger,
                     lvlTalkative,
                     actFileTransfer,
-                    fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri),
+                    request.activityText ? *request.activityText
+                                         : fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri),
                     Logger::Fields{request.uri.to_string()},
                     request.parentAct);
                 // Reset the start time to when we actually started the download.

src/libstore/http-binary-cache-store.cc

@@ -1,13 +1,17 @@
 #include "nix/store/http-binary-cache-store.hh"
 #include "nix/store/filetransfer.hh"
 #include "nix/store/globals.hh"
+#include "nix/store/nar-info.hh"
 #include "nix/store/nar-info-disk-cache.hh"
 #include "nix/store/sqlite.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/closure.hh"
 #include "nix/store/store-registration.hh"
 #include "nix/store/globals.hh"
 #include "nix/util/topo-sort.hh"
+#include "nix/util/strings.hh"
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -302,6 +306,89 @@ void HttpBinaryCacheStore::getFile(const std::string & path, Callback<std::optio
     }
 }
 
+asio::awaitable<void> HttpBinaryCacheStore::queryPathInfos(
+    const std::set<StorePath> & paths,
+    fun<void(std::vector<std::pair<StorePath, std::shared_ptr<const ValidPathInfo>>>)> callback)
+{
+    using Result = std::pair<StorePath, std::shared_ptr<const ValidPathInfo>>;
+
+    checkEnabled();
+
+    /* Fall back to per-path queries unless the cache advertises an
+       endpoint for fetching multiple narinfos at once (the path to
+       POST to). */
+    if (!getNarInfosV1) {
+        co_await Store::queryPathInfos(paths, std::move(callback));
+        co_return;
+    }
+
+    /* Check the client-side caches first and report the hits; collect
+       the misses to fetch from the server. */
+    std::vector<Result> cached;
+    std::vector<StorePath> misses;
+    for (auto & path : paths) {
+        if (auto r = queryPathInfoFromClientCache(path))
+            cached.emplace_back(path, *r);
+        else
+            misses.push_back(path);
+    }
+    if (!cached.empty())
+        callback(std::move(cached));
+
+    if (misses.empty())
+        co_return;
+
+    /* Fetch the misses in a single request. `body` and `source` live
+       in the coroutine frame, so they stay alive across the
+       `co_await` while the transfer (which holds a raw pointer to
+       `source`) runs. */
+    std::string body;
+    for (auto & path : misses)
+        body += std::string(path.hashPart()) + "\n";
+    StringSource source{body};
+
+    auto request = makeRequest(*getNarInfosV1);
+    request.method = HttpMethod::Post;
+    request.data = {body.size(), source};
+    request.mimeType = "text/plain";
+    request.activityText = fmt("querying info on %d paths from '%s'", misses.size(), request.uri);
+
+    FileTransferResult result;
+    try {
+        result = co_await callbackToAwaitable<FileTransferResult>(
+            [&](Callback<FileTransferResult> cb) { fileTransfer->enqueueFileTransfer(request, std::move(cb)); });
+    } catch (FileTransferError &) {
+        maybeDisable();
+        throw;
+    }
+
+    /* Parse the narinfos (one JSON object per line), indexed by hash
+       part. */
+    std::map<std::string, std::shared_ptr<const NarInfo>, std::less<>> received;
+    for (auto & line : tokenizeString<Strings>(result.data, "\n")) {
+        auto narInfo = std::make_shared<NarInfo>(NarInfo::fromJSON(*this, nlohmann::json::parse(line)));
+        stats.narInfoRead++;
+        received.insert_or_assign(std::string(narInfo->path.hashPart()), std::move(narInfo));
+    }
+
+    /* Match each miss against the received narinfos, cache the result
+       (including negative entries), and report it. */
+    auto cacheKey = config->getReference().render(/*FIXME withParams=*/false);
+
+    std::vector<Result> results;
+    for (auto & path : misses) {
+        std::shared_ptr<const ValidPathInfo> info;
+        if (auto i = received.find(path.hashPart());
+            i != received.end() && (path.name() == MissingName || i->second->path == path))
+            info = i->second;
+        if (diskCache)
+            diskCache->upsertNarInfo(cacheKey, std::string(path.hashPart()), info);
+        pathInfoCache->lock()->upsert(path, PathInfoCacheValue{.value = info});
+        results.emplace_back(path, info);
+    }
+    callback(std::move(results));
+}
+
 std::optional<std::string> HttpBinaryCacheStore::getNixCacheInfo()
 {
     try {

src/libstore/include/nix/store/binary-cache-store.hh

@@ -84,6 +84,14 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */
      */
     Config & config;
 
+    /**
+     * The endpoint (relative to the cache URL) advertised by the
+     * cache's `nix-cache-info` `GetNarInfosV1` field for fetching
+     * multiple `.narinfo` files in one request, if any. Discovered at
+     * `init()` time.
+     */
+    std::optional<std::string> getNarInfosV1;
+
 private:
     std::vector<std::unique_ptr<Signer>> signers;
 

src/libstore/include/nix/store/filetransfer.hh

@@ -184,6 +184,7 @@ struct FileTransferRequest
     HttpMethod method = HttpMethod::Get;
     unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
     ActivityId parentAct;
+    std::optional<std::string> activityText;
     bool decompress = true;
 
     /**

src/libstore/include/nix/store/http-binary-cache-store.hh

@@ -84,6 +84,10 @@ public:
 
     StorePaths topoSortPaths(const StorePathSet & paths) override;
 
+    asio::awaitable<void> queryPathInfos(
+        const std::set<StorePath> & paths,
+        fun<void(std::vector<std::pair<StorePath, std::shared_ptr<const ValidPathInfo>>>)> callback) override;
+
 protected:
 
     std::optional<CompressionAlgo> getCompressionMethod(const std::string & path);

src/libstore/include/nix/store/nar-info.hh

@@ -17,6 +17,17 @@ struct UnkeyedNarInfo : virtual UnkeyedValidPathInfo
     std::optional<Hash> fileHash;
     uint64_t fileSize = 0;
 
+    /**
+     * An optional hint containing (some of) the indirect references
+     * of this path, i.e. a subset of the references closure of the
+     * path, excluding the path itself and its direct references.
+     * This allows substituters to start fetching the NAR info of the
+     * entire closure without waiting for intermediate narinfos. It
+     * is only a hint: it need not be complete and is not covered by
+     * the signature.
+     */
+    StorePathSet partialClosure;
+
     UnkeyedNarInfo(UnkeyedValidPathInfo info)
         : UnkeyedValidPathInfo(std::move(info))
     {
@@ -50,6 +61,19 @@ struct NarInfo : ValidPathInfo, UnkeyedNarInfo
     {
     }
 
+    /**
+     * Combine the unkeyed NAR info with its store path. Used to
+     * reconstruct a `NarInfo` from its JSON representation.
+     */
+    NarInfo(UnkeyedNarInfo info, StorePath path)
+        : UnkeyedValidPathInfo{static_cast<UnkeyedValidPathInfo &&>(info)}
+        /* As in `NarInfo(ValidPathInfo)`, only this most-derived
+           constructor initializes the virtual base. */
+        , ValidPathInfo{std::move(path), static_cast<const UnkeyedValidPathInfo &>(*this)}
+        , UnkeyedNarInfo{std::move(info)}
+    {
+    }
+
     NarInfo(const StoreDirConfig & store, StorePath path, Hash narHash)
         : NarInfo{ValidPathInfo{std::move(path), UnkeyedValidPathInfo{store, narHash}}}
     {
@@ -71,6 +95,21 @@ struct NarInfo : ValidPathInfo, UnkeyedNarInfo
     bool operator==(const NarInfo &) const = default;
 
     std::string to_string(const StoreDirConfig & store) const;
+
+    using UnkeyedNarInfo::toJSON;
+
+    /**
+     * Like `UnkeyedNarInfo::toJSON()`, but also includes the store
+     * path (as a `"path"` field), so the result is self-describing.
+     * Always uses the V2 JSON format.
+     */
+    nlohmann::json toJSON(const StoreDirConfig & store, bool includeImpureInfo) const;
+
+    /**
+     * Inverse of `toJSON()`: reconstruct a keyed `NarInfo` (including
+     * its store path) from JSON. Only the V2 format is supported.
+     */
+    static NarInfo fromJSON(const StoreDirConfig & store, const nlohmann::json & json);
 };
 
 } // namespace nix

src/libstore/include/nix/store/path-info.hh

@@ -43,6 +43,11 @@ struct SubstitutablePathInfo
      * 0 = unknown
      */
     uint64_t narSize;
+    /**
+     * A hint containing (some of) the indirect references of this
+     * path, see `UnkeyedNarInfo::partialClosure`.
+     */
+    StorePathSet partialClosure;
 };
 
 using SubstitutablePathInfos = std::map<StorePath, SubstitutablePathInfo>;

src/libstore/include/nix/store/store-api.hh

@@ -14,6 +14,8 @@
 #include "nix/store/store-dir-config.hh"
 #include "nix/store/store-reference.hh"
 #include "nix/util/source-path.hh"
+#include "nix/util/async.hh"
+#include "nix/util/fun.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>
@@ -430,6 +432,19 @@ public:
      */
     void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept;
 
+    /**
+     * Asynchronously query information about multiple store paths. As
+     * results arrive (possibly in batches from a remote server),
+     * `callback` is invoked one or more times with a vector of
+     * `(path, info)` pairs. A null `info` denotes that the path is
+     * not valid. Every requested path is reported exactly once across
+     * all invocations of `callback`. Unlike `queryPathInfo()`, an
+     * invalid path is not an error.
+     */
+    virtual asio::awaitable<void> queryPathInfos(
+        const std::set<StorePath> & paths,
+        fun<void(std::vector<std::pair<StorePath, std::shared_ptr<const ValidPathInfo>>>)> callback);
+
     /**
      * Version of queryPathInfo() that only queries the local narinfo cache and not
      * the actual store.