From e0aa48da9235aaa2c374d730dab39c17a38cc6e9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jun 2026 16:10:41 +0200 Subject: [PATCH 01/12] Add partialClosure field to .narinfo This allows the client to start prefetching .narinfo files faster. --- src/libstore/include/nix/store/nar-info.hh | 11 +++++++++++ src/libstore/include/nix/store/path-info.hh | 5 +++++ src/libstore/misc.cc | 8 ++++++++ src/libstore/nar-info.cc | 13 +++++++++++++ src/nix/serve.cc | 6 ++++++ 5 files changed, 43 insertions(+) diff --git a/src/libstore/include/nix/store/nar-info.hh b/src/libstore/include/nix/store/nar-info.hh index 7403dc1d4452..48e44c1909ae 100644 --- a/src/libstore/include/nix/store/nar-info.hh +++ b/src/libstore/include/nix/store/nar-info.hh @@ -17,6 +17,17 @@ struct UnkeyedNarInfo : virtual UnkeyedValidPathInfo std::optional fileHash; uint64_t fileSize = 0; + /** + * An optional hint containing (some of) the indirect references + * of this path, i.e. a subset of the references closure of the + * path, excluding the path itself and its direct references. + * This allows substituters to start fetching the NAR info of the + * entire closure without waiting for intermediate narinfos. It + * is only a hint: it need not be complete and is not covered by + * the signature. + */ + StorePathSet partialClosure; + UnkeyedNarInfo(UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(std::move(info)) { diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 98f9ebe7aeda..bd4d53905343 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -43,6 +43,11 @@ struct SubstitutablePathInfo * 0 = unknown */ uint64_t narSize; + /** + * A hint containing (some of) the indirect references of this + * path, see `UnkeyedNarInfo::partialClosure`. + */ + StorePathSet partialClosure; }; using SubstitutablePathInfos = std::map; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c6d1a10a95a6..6a5b4cfbf630 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -139,6 +139,7 @@ querySubstitutablePathInfosAsync(Store & store, const StorePathCAMap & paths, Su .references = info->references, .downloadSize = narInfo ? narInfo->fileSize : 0, .narSize = info->narSize, + .partialClosure = narInfo ? narInfo->partialClosure : StorePathSet{}, }); break; /* We are done. */ @@ -322,6 +323,13 @@ MissingPaths Store::queryMissing(const std::vector & targets) for (auto & ref : info->second.references) edges.insert(DerivedPath::Opaque{ref}); + + /* Recurse into the partial closure hint as well, + so we don't have to wait for the narinfos of + the direct references to discover the rest of + the closure. */ + for (auto & ref : info->second.partialClosure) + edges.insert(DerivedPath::Opaque{ref}); }, }, req.raw()); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 6f1c6b96cc54..dff3e95a29dd 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -75,6 +75,12 @@ NarInfo::NarInfo(const StoreDirConfig & store, const std::string & s, const std: throw corrupt("extra References"); for (auto & r : refs) references.insert(StorePath(r)); + } else if (name == "PartialClosure") { + auto refs = tokenizeString(value, " "); + if (!partialClosure.empty()) + throw corrupt("extra PartialClosure"); + for (auto & r : refs) + partialClosure.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") deriver = StorePath(value); @@ -125,6 +131,13 @@ std::string NarInfo::to_string(const StoreDirConfig & store) const res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + if (!partialClosure.empty()) { + Strings ss; + for (auto & p : partialClosure) + ss.push_back(std::string(p.to_string())); + res += "PartialClosure: " + concatStringsSep(" ", ss) + "\n"; + } + if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; diff --git a/src/nix/serve.cc b/src/nix/serve.cc index 7206411fd3da..bb0b7032440b 100644 --- a/src/nix/serve.cc +++ b/src/nix/serve.cc @@ -123,6 +123,12 @@ struct CmdServe : StoreCommand auto info = store.queryPathInfo(*path); NarInfo ni(*info); ni.compression = "none"; + StorePathSet closure; + store.computeFSClosure(*path, closure); + closure.erase(*path); + for (auto & ref : info->references) + closure.erase(ref); + ni.partialClosure = std::move(closure); // FIXME: would be nicer to use just the NAR hash, but we can't look up NARs by NAR hash. ni.url = "nar/" + std::string(info->path.hashPart()) + "-" + info->narHash.to_string(HashFormat::Nix32, false) + ".nar"; From e063e2fad895db6b540f9e1f9f887e0a420ef3c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jun 2026 16:44:35 +0200 Subject: [PATCH 02/12] Add "Features" field to nix-cache-info This allows servers to advertise optional endpoints. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/manual/source/protocols/nix-cache-info.md | 9 +++++++++ src/libstore/binary-cache-store.cc | 2 ++ src/libstore/http-binary-cache-store.cc | 5 ++++- .../include/nix/store/binary-cache-store.hh | 6 ++++++ .../include/nix/store/nar-info-disk-cache.hh | 1 + src/libstore/nar-info-disk-cache.cc | 13 ++++++++----- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/nix-cache-info.md index e8351e1cebe8..dbaf04c1c022 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/nix-cache-info.md @@ -36,12 +36,21 @@ error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/st Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority). +### `Features` + +A space-separated list of optional protocol features that the cache +server supports. Clients ignore any feature names they don't +recognise, and assume no features if the field is absent. This allows +a client to use server capabilities beyond the basic binary cache +protocol only when they're available. + ## Example ``` StoreDir: /nix/store WantMassQuery: 1 Priority: 30 +Features: foo bar ``` ## Caching Behavior diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a7f636f12311..4ce11e336d38 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -68,6 +68,8 @@ void BinaryCacheStore::init() config.wantMassQuery.setDefault(value == "1"); } else if (name == "Priority") { config.priority.setDefault(std::stoi(value)); + } else if (name == "Features") { + features = tokenizeString(value, " "); } } } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e3b82fabbcf0..a9de85a86fb8 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -71,6 +71,7 @@ void HttpBinaryCacheStore::init() if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); config->priority.setDefault(cacheInfo->priority); + features = cacheInfo->features; } else { try { BinaryCacheStore::init(); @@ -78,7 +79,9 @@ void HttpBinaryCacheStore::init() throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } diskCache->createCache( - cacheKey, config->storeDir, {.wantMassQuery = config->wantMassQuery, .priority = config->priority}); + cacheKey, + config->storeDir, + {.wantMassQuery = config->wantMassQuery, .priority = config->priority, .features = features}); } } diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 6a66e901a883..671700b70a58 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -84,6 +84,12 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ */ Config & config; + /** + * Features advertised by the cache's `nix-cache-info` (e.g. + * `get-narinfos-v1`). Discovered at `init()` time. + */ + StringSet features; + private: std::vector> signers; diff --git a/src/libstore/include/nix/store/nar-info-disk-cache.hh b/src/libstore/include/nix/store/nar-info-disk-cache.hh index 37d1f1b10e25..de739e9de69d 100644 --- a/src/libstore/include/nix/store/nar-info-disk-cache.hh +++ b/src/libstore/include/nix/store/nar-info-disk-cache.hh @@ -30,6 +30,7 @@ struct NarInfoDiskCache int id = 0; bool wantMassQuery = false; int priority = 0; + StringSet features; }; /** diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index fa345764eed3..3233dba66357 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -20,7 +20,8 @@ create table if not exists BinaryCaches ( timestamp integer not null, storeDir text not null, wantMassQuery integer not null, - priority integer not null + priority integer not null, + features text not null ); create table if not exists NARs ( @@ -84,7 +85,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache NarInfoDiskCacheImpl( const Settings & settings, SQLiteSettings sqliteSettings, - std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v1.sqlite") + std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v2.sqlite") : NarInfoDiskCache{settings} { auto state(_state.lock()); @@ -99,11 +100,11 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache state->insertCache.create( state->db, - "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); + "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority, features) values (?1, ?2, ?3, ?4, ?5, ?6) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5, features = ?6 returning id;"); state->queryCache.create( state->db, - "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); + "select id, storeDir, wantMassQuery, priority, features from BinaryCaches where url = ? and timestamp > ?"); state->insertNAR.create( state->db, @@ -195,6 +196,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache .id = (int) queryCache.getInt(0), .wantMassQuery = queryCache.getInt(2) != 0, .priority = (int) queryCache.getInt(3), + .features = tokenizeString(queryCache.getStr(4), " "), }}; state.caches.emplace(uri, cache); } @@ -223,7 +225,8 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache .apply(time(nullptr)) .apply(storeDir) .apply(info.wantMassQuery) - .apply(info.priority)); + .apply(info.priority) + .apply(concatStringsSep(" ", info.features))); if (!r.next()) { unreachable(); } From 8a9008d8dd05a2956ea1028e5abc12f835210dfc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Jun 2026 14:47:25 +0200 Subject: [PATCH 03/12] BinaryCacheStore / nix-serve: Add ability to query multiple narinfos in one HTTP call This adds an optional endpoint /get-narinfos-v1 to binary caches that allows querying multiple narinfos in one HTTP call, which may reduce latency and overhead for non-static caches like cache.flakehub.com. To expose this, Store now has a virtual method queryPathInfos() that HttpBinaryCacheStore overrides. queryMissing() has been rewritten to try to batch paths together. Note: queryMissing() now tries all substituters for a store path concurrently rather than sequentially, unlike querySubstitutablePathInfosAsync(). Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/manual/source/protocols/nix-cache-info.md | 12 +- src/libstore/http-binary-cache-store.cc | 86 +++++ .../nix/store/http-binary-cache-store.hh | 4 + src/libstore/include/nix/store/store-api.hh | 15 + src/libstore/misc.cc | 318 ++++++++++-------- src/libstore/remote-store.cc | 1 + src/libstore/store-api.cc | 19 ++ src/nix/serve.cc | 138 ++++++-- 8 files changed, 426 insertions(+), 167 deletions(-) diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/nix-cache-info.md index dbaf04c1c022..7761b60f454d 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/nix-cache-info.md @@ -44,13 +44,23 @@ recognise, and assume no features if the field is absent. This allows a client to use server capabilities beyond the basic binary cache protocol only when they're available. +Currently defined features: + +- `get-narinfos-v1`: the server accepts a `POST` request to + `get-narinfos-v1` whose body is a list of store path hash parts (one + per line) and responds with the concatenation of the corresponding + `.narinfo` files, separated by empty lines. This lets a client fetch + the `.narinfo` files for many paths in a single request instead of + one request per path. A path whose `.narinfo` is absent from the + response is not available in the cache. + ## Example ``` StoreDir: /nix/store WantMassQuery: 1 Priority: 30 -Features: foo bar +Features: get-narinfos-v1 ``` ## Caching Behavior diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index a9de85a86fb8..e72a90df3483 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,6 +1,7 @@ #include "nix/store/http-binary-cache-store.hh" #include "nix/store/filetransfer.hh" #include "nix/store/globals.hh" +#include "nix/store/nar-info.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/store/sqlite.hh" #include "nix/util/callback.hh" @@ -305,6 +306,91 @@ void HttpBinaryCacheStore::getFile(const std::string & path, Callback HttpBinaryCacheStore::queryPathInfos( + const std::set & paths, + fun>>)> callback) +{ + using Result = std::pair>; + + checkEnabled(); + + /* Fall back to per-path queries if the server doesn't support the + batch endpoint. */ + if (!features.contains("get-narinfos-v1")) { + co_await Store::queryPathInfos(paths, std::move(callback)); + co_return; + } + + /* Check the client-side caches first and report the hits; collect + the misses to fetch from the server. */ + std::vector cached; + std::vector misses; + for (auto & path : paths) { + if (auto r = queryPathInfoFromClientCache(path)) + cached.emplace_back(path, *r); + else + misses.push_back(path); + } + if (!cached.empty()) + callback(std::move(cached)); + + if (misses.empty()) + co_return; + + /* Fetch the misses in a single request. `body` and `source` live + in the coroutine frame, so they stay alive across the + `co_await` while the transfer (which holds a raw pointer to + `source`) runs. */ + std::string body; + for (auto & path : misses) + body += std::string(path.hashPart()) + "\n"; + StringSource source{body}; + + auto request = makeRequest("get-narinfos-v1"); + request.method = HttpMethod::Post; + request.data = {body.size(), source}; + request.mimeType = "text/plain"; + + FileTransferResult result; + try { + result = co_await callbackToAwaitable( + [&](Callback cb) { fileTransfer->enqueueFileTransfer(request, std::move(cb)); }); + } catch (FileTransferError &) { + maybeDisable(); + throw; + } + + /* Parse the concatenated narinfos, indexed by hash part. */ + auto whence = fmt("%s/get-narinfos-v1", config->getHumanReadableURI()); + std::map, std::less<>> received; + size_t pos = 0; + while (pos < result.data.size()) { + auto end = result.data.find("\n\n", pos); + auto chunk = result.data.substr(pos, end == std::string::npos ? end : end + 1 - pos); + pos = end == std::string::npos ? result.data.size() : end + 2; + auto narInfo = std::make_shared(*this, chunk, whence); + stats.narInfoRead++; + received.insert_or_assign(std::string(narInfo->path.hashPart()), std::move(narInfo)); + } + + /* Match each miss against the received narinfos, cache the result + (including negative entries), and report it. */ + auto cacheKey = config->getReference().render(/*FIXME withParams=*/false); + + std::vector results; + for (auto & path : misses) { + std::shared_ptr info; + if (auto i = received.find(path.hashPart()); + i != received.end() && (path.name() == MissingName || i->second->path == path)) + info = i->second; + if (diskCache) + diskCache->upsertNarInfo(cacheKey, std::string(path.hashPart()), info); + pathInfoCache->lock()->upsert(path, PathInfoCacheValue{.value = info}); + results.emplace_back(path, info); + } + callback(std::move(results)); +} + std::optional HttpBinaryCacheStore::getNixCacheInfo() { try { diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 475dc95104ab..f8392c61dd69 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -84,6 +84,10 @@ public: StorePaths topoSortPaths(const StorePathSet & paths) override; + asio::awaitable queryPathInfos( + const std::set & paths, + fun>>)> callback) override; + protected: std::optional getCompressionMethod(const std::string & path); diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index e16e2c77d36d..02edb0a313cd 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -14,6 +14,8 @@ #include "nix/store/store-dir-config.hh" #include "nix/store/store-reference.hh" #include "nix/util/source-path.hh" +#include "nix/util/async.hh" +#include "nix/util/fun.hh" #include #include @@ -430,6 +432,19 @@ public: */ void queryPathInfo(const StorePath & path, Callback> callback) noexcept; + /** + * Asynchronously query information about multiple store paths. As + * results arrive (possibly in batches from a remote server), + * `callback` is invoked one or more times with a vector of + * `(path, info)` pairs. A null `info` denotes that the path is + * not valid. Every requested path is reported exactly once across + * all invocations of `callback`. Unlike `queryPathInfo()`, an + * invalid path is not an error. + */ + virtual asio::awaitable queryPathInfos( + const std::set & paths, + fun>>)> callback); + /** * Version of queryPathInfo() that only queries the local narinfo cache and not * the actual store. diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6a5b4cfbf630..894e9562fe77 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -158,6 +158,7 @@ querySubstitutablePathInfosAsync(Store & store, const StorePathCAMap & paths, Su }); } +// FIXME: remove this, queryMissing() no longer uses it. void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { asio::io_context ctx; @@ -168,178 +169,211 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta std::rethrow_exception(ex); } -static void collectDerivedPaths( - std::set & out, ref inputDrv, const DerivedPathMap::ChildNode & node) -{ - if (!node.value.empty()) - out.insert(DerivedPath::Built{inputDrv, node.value}); - for (const auto & [outputName, childNode] : node.childMap) - collectDerivedPaths( - out, make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); -} - MissingPaths Store::queryMissing(const std::vector & targets) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); MissingPaths res; - auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv, std::set & edges) { + auto collectDerivedPaths = [&](this auto & collectDerivedPaths, + std::set & out, + ref inputDrv, + const DerivedPathMap::ChildNode & node) -> void { + if (!node.value.empty()) + out.insert(DerivedPath::Built{inputDrv, node.value}); + for (const auto & [outputName, childNode] : node.childMap) + collectDerivedPaths( + out, make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv, std::set & out) { res.willBuild.insert(drvPath); for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) - collectDerivedPaths(edges, makeConstantStorePathRef(inputDrv), inputNode); + collectDerivedPaths(out, makeConstantStorePathRef(inputDrv), inputNode); }; - GetEdgesAsync getEdges = [&](const DerivedPath & req) -> asio::awaitable> { - std::set edges; - - co_await std::visit( - overloaded{ - [&](const DerivedPath::Built & bfd) -> asio::awaitable { - auto drvPathP = std::get_if(&*bfd.drvPath); - if (!drvPathP) { - // TODO make work in this case. - warn( - "Ignoring dynamic derivation %s while querying missing paths; not yet implemented", - bfd.drvPath->to_string(*this)); - co_return; - } - auto & drvPath = drvPathP->path; + asio::io_context ctx; + std::exception_ptr ex; - if (!isValidPath(drvPath)) { - // FIXME: we could try to substitute the derivation. - res.unknown.insert(drvPath); - co_return; - } + std::set done; - StorePathSet invalid; - /* true for regular derivations, and CA derivations for which we - have a trust mapping for all wanted outputs. */ - auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { - if (!pathOpt) { - knownOutputPaths = false; - break; - } - if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) - invalid.insert(*pathOpt); - } - if (knownOutputPaths && invalid.empty()) - co_return; - - auto drv = make_ref(derivationFromPath(drvPath)); - DerivationOptions drvOptions; - try { - // FIXME: this is a lot of work just to get the value - // of `allowSubstitutes`. - drvOptions = derivationOptionsFromStructuredAttrs( - *this, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); - } catch (Error & e) { - e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); - throw; - } + auto subs = getDefaultSubstituters(); - if (!knownOutputPaths && settings.getWorkerSettings().useSubstitutes - && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { - experimentalFeatureSettings.require(Xp::CaDerivations); + std::function(std::set)> doPaths; + doPaths = [&](std::set paths) -> asio::awaitable { + debug("working on batch of %d paths", paths.size()); - // If there are unknown output paths, attempt to find if the - // paths are known to substituters through a realisation. - auto outputHashes = staticOutputHashes(*this, *drv); - knownOutputPaths = true; + std::set pathsToQuery; - for (auto [outputName, hash] : outputHashes) { - if (!bfd.outputs.contains(outputName)) - continue; + std::map>> outPathsToDrvs; - bool found = false; - for (auto & sub : getDefaultSubstituters()) { - /* TODO: Asyncify this. */ - auto realisation = sub->queryRealisation({hash, outputName}); - if (!realisation) - continue; - found = true; - if (!isValidPath(realisation->outPath)) - invalid.insert(realisation->outPath); - break; - } - if (!found) { - // Some paths did not have a realisation, this must be built. + while (!paths.empty()) { + auto p = *paths.begin(); + paths.erase(paths.begin()); + if (!done.insert(p).second) + continue; + std::visit( + overloaded{ + [&](const DerivedPath::Built & bfd) -> void { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn( + "Ignoring dynamic derivation %s while querying missing paths; not yet implemented", + bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { + // FIXME: we could try to substitute the derivation. + res.unknown.insert(drvPath); + return; + } + + StorePathSet invalid; + /* true for regular derivations, and CA derivations for which we + have a trust mapping for all wanted outputs. */ + auto knownOutputPaths = true; + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { + if (!pathOpt) { knownOutputPaths = false; break; } + if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) + invalid.insert(*pathOpt); + } + if (knownOutputPaths && invalid.empty()) + return; + + auto drv = make_ref(derivationFromPath(drvPath)); + DerivationOptions drvOptions; + try { + // FIXME: this is a lot of work just to get the value + // of `allowSubstitutes`. + drvOptions = derivationOptionsFromStructuredAttrs( + *this, drv->inputDrvs, drv->env, get(drv->structuredAttrs)); + } catch (Error & e) { + e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); + throw; } - } - if (knownOutputPaths && settings.getWorkerSettings().useSubstitutes - && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { - bool mustBuild = false; - StorePathSet substitutable; - auto * cap = getDerivationCA(*drv); - - /* Query all outputs concurrently (but not in parallel, - computeClosure runs on a strand). If any one is not - substitutable then discard all other outputs. */ - co_await forEachAsync(invalid, [&](const StorePath & outPath) -> asio::awaitable { - if (mustBuild) - co_return; - - SubstitutablePathInfos infos; - co_await querySubstitutablePathInfosAsync( - *this, {{outPath, cap ? std::optional{*cap} : std::nullopt}}, infos); - - if (infos.empty()) - mustBuild = true; - else - substitutable.insert(outPath); - }); + if (!knownOutputPaths && settings.getWorkerSettings().useSubstitutes + && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { + experimentalFeatureSettings.require(Xp::CaDerivations); - if (mustBuild) - mustBuildDrv(drvPath, *drv, edges); - else - for (auto & path : substitutable) - edges.insert(DerivedPath::Opaque{path}); - } else { - mustBuildDrv(drvPath, *drv, edges); - } + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto & sub : getDefaultSubstituters()) { + /* TODO: Asyncify this. */ + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } + + if (knownOutputPaths && settings.getWorkerSettings().useSubstitutes + && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { + for (auto & p : invalid) { + pathsToQuery.insert(p); + outPathsToDrvs[p].insert_or_assign(drvPath, drv); + } + } else + mustBuildDrv(drvPath, *drv, paths); + }, + [&](const DerivedPath::Opaque & bo) -> void { + // FIXME: this should probably be an async call, but for a local store we probably don't want to + // bother. + if (!maybeQueryPathInfo(bo.path)) + pathsToQuery.insert(bo.path); + }, }, - [&](const DerivedPath::Opaque & bo) -> asio::awaitable { - if (maybeQueryPathInfo(bo.path)) - co_return; + p.raw()); + } + + if (pathsToQuery.empty()) + co_return; + + auto executor = co_await asio::this_coro::executor; + + std::unordered_map negativeResultsPerPath; + + /* Query all substituters concurrently. FIXME: this may not be desirable. */ + co_await forEachAsync(subs, [&](const ref & sub) -> asio::awaitable { + debug("querying %d paths on '%s'", pathsToQuery.size(), sub->config.getHumanReadableURI()); + return sub->queryPathInfos( + pathsToQuery, [&](std::vector>> infos) { + debug("got %d paths from %s", infos.size(), sub->config.getHumanReadableURI()); + + std::set todo; + + for (auto & [path, info] : infos) { + if (info) { + res.willSubstitute.insert(path); + res.narSize += info->narSize; - SubstitutablePathInfos infos; - co_await querySubstitutablePathInfosAsync(*this, {{bo.path, std::nullopt}}, infos); + for (auto & ref : info->references) + todo.insert(DerivedPath::Opaque{ref}); - if (infos.empty()) { - res.unknown.insert(bo.path); - co_return; + if (auto narInfo = std::dynamic_pointer_cast(info)) { + res.downloadSize += narInfo->fileSize; + + /* Recurse into the partial closure hint as well, + so we don't have to wait for the narinfos of + the direct references to discover the rest of + the closure. */ + for (auto & ref : narInfo->partialClosure) + todo.insert(DerivedPath::Opaque{ref}); + } + } else { + if (++negativeResultsPerPath[path] == subs.size()) { + if (auto i = outPathsToDrvs.find(path); i != outPathsToDrvs.end()) { + for (auto & [drvPath, drv] : i->second) + mustBuildDrv(drvPath, *drv, todo); + } else + /* This path is not a derivation output, so there is no way to produce it. */ + res.unknown.insert(path); + } + } } - auto info = infos.find(bo.path); - assert(info != infos.end()); - res.willSubstitute.insert(bo.path); - res.downloadSize += info->second.downloadSize; - res.narSize += info->second.narSize; - - for (auto & ref : info->second.references) - edges.insert(DerivedPath::Opaque{ref}); - - /* Recurse into the partial closure hint as well, - so we don't have to wait for the narinfos of - the direct references to discover the rest of - the closure. */ - for (auto & ref : info->second.partialClosure) - edges.insert(DerivedPath::Opaque{ref}); - }, - }, - req.raw()); + if (!todo.empty()) + asio::co_spawn(executor, std::bind(doPaths, todo), [&](std::exception_ptr e) { + if (e) + ex = e; + }); + }); + }); - co_return edges; + co_return; }; - std::set startElts(targets.begin(), targets.end()); - std::set visited; - computeClosure(std::move(startElts), visited, std::move(getEdges)); + asio::co_spawn( + ctx, std::bind(doPaths, std::set(targets.begin(), targets.end())), [&](std::exception_ptr e) { + ex = e; + }); + + ctx.run(); + if (ex) + std::rethrow_exception(ex); return res; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 14665716c345..57ddfc9b5b21 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -226,6 +226,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); + // FIXME: handle partialClosure } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e53e741acb4b..b562f9232d44 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -613,6 +613,25 @@ void Store::queryPathInfo(const StorePath & storePath, Callback Store::queryPathInfos( + const std::set & paths, + fun>>)> callback) +{ + /* Default implementation: query each path individually, reporting + each result as it arrives. */ + co_await forEachAsync(paths, [&](const StorePath & path) -> asio::awaitable { + std::shared_ptr info; + try { + auto i = co_await callbackToAwaitable>( + [&](Callback> cb) { queryPathInfo(path, std::move(cb)); }); + info = i.get_ptr(); + } catch (InvalidPath &) { + } + std::vector>> result{{path, info}}; + callback(std::move(result)); + }); +} + void Store::queryRealisation( const DrvOutput & id, Callback> callback) noexcept { diff --git a/src/nix/serve.cc b/src/nix/serve.cc index bb0b7032440b..7137bdb227c8 100644 --- a/src/nix/serve.cc +++ b/src/nix/serve.cc @@ -3,6 +3,7 @@ #include "nix/util/serialise.hh" #include "nix/util/signals.hh" #include "nix/util/deleter.hh" +#include "nix/util/strings.hh" #include "nix/store/nar-info.hh" #include "nix/store/binary-cache-store.hh" #include "nix/store/log-store.hh" @@ -18,6 +19,33 @@ using namespace nix; using Response = std::unique_ptr>; +/** + * Render the narinfo for `info`, including a `PartialClosure` hint. + * Paths in `alreadySent` are omitted from the hint, since the hint + * only serves to let the client discover paths to traverse, so + * there is no need to send a path more than once in the same HTTP + * call. The references and hint of this narinfo are added to + * `alreadySent` in turn. + */ +static std::string makeNarInfo(Store & store, const ValidPathInfo & info, StorePathSet & alreadySent) +{ + NarInfo ni(info); + ni.compression = "none"; + StorePathSet closure; + store.computeFSClosure(info.path, closure); + std::erase_if(closure, [&](const StorePath & p) { + return p == info.path || info.references.contains(p) || alreadySent.contains(p); + }); + alreadySent.insert(info.references.begin(), info.references.end()); + alreadySent.insert(closure.begin(), closure.end()); + ni.partialClosure = std::move(closure); + // FIXME: would be nicer to use just the NAR hash, but we can't look up NARs by NAR hash. + ni.url = + "nar/" + std::string(info.path.hashPart()) + "-" + info.narHash.to_string(HashFormat::Nix32, false) + ".nar"; + ni.fileSize = info.narSize; + return ni.to_string(store); +} + struct CmdServe : StoreCommand { uint16_t port = 8080; @@ -72,8 +100,12 @@ struct CmdServe : StoreCommand ; } - MHD_Result - handleRequest(Store & store, MHD_Connection * connection, const std::string & url, std::string_view method) + MHD_Result handleRequest( + Store & store, + MHD_Connection * connection, + const std::string & url, + std::string_view method, + std::string_view body) try { std::string clientAddr = "unknown"; if (auto * info = MHD_get_connection_info(connection, MHD_CONNECTION_INFO_CLIENT_ADDRESS)) { @@ -99,18 +131,25 @@ struct CmdServe : StoreCommand static const std::regex narUrlRegex{R"(^/nar/([0-9a-z]+)-([0-9a-z]+)\.nar$)"}; static const std::regex logUrlRegex{R"(^/log/([0-9a-z]+-[0-9a-zA-Z+\-._?=]+)$)"}; - if (method != MHD_HTTP_METHOD_GET && method != MHD_HTTP_METHOD_HEAD) { - std::string_view body = "405 method not allowed\n"; + auto methodNotAllowed = [&](const char * allow) { + static constexpr std::string_view body = "405 method not allowed\n"; response.reset(MHD_create_response_from_buffer(body.size(), (void *) body.data(), MHD_RESPMEM_PERSISTENT)); - MHD_add_response_header(response.get(), "Allow", "GET, HEAD"); + MHD_add_response_header(response.get(), "Allow", allow); return MHD_queue_response(connection, MHD_HTTP_METHOD_NOT_ALLOWED, response.get()); - } + }; + + if (url == "/get-narinfos-v1") { + if (method != MHD_HTTP_METHOD_POST) + return methodNotAllowed("POST"); + } else if (method != MHD_HTTP_METHOD_GET && method != MHD_HTTP_METHOD_HEAD) + return methodNotAllowed("GET, HEAD"); if (url == "/nix-cache-info") { auto body = std::make_unique( "StoreDir: " + store.storeDir + "\n" "WantMassQuery: " + (store.config.wantMassQuery ? "1" : "0") + "\n" - "Priority: " + std::to_string(priority.value_or(store.config.priority)) + "\n"); + "Priority: " + std::to_string(priority.value_or(store.config.priority)) + "\n" + "Features: get-narinfos-v1\n"); response.reset(MHD_create_response_from_buffer(body->size(), body->data(), MHD_RESPMEM_MUST_COPY)); MHD_add_response_header(response.get(), "Content-Type", "text/x-nix-cache-info"); @@ -121,22 +160,39 @@ struct CmdServe : StoreCommand return notFound(); auto info = store.queryPathInfo(*path); - NarInfo ni(*info); - ni.compression = "none"; - StorePathSet closure; - store.computeFSClosure(*path, closure); - closure.erase(*path); - for (auto & ref : info->references) - closure.erase(ref); - ni.partialClosure = std::move(closure); - // FIXME: would be nicer to use just the NAR hash, but we can't look up NARs by NAR hash. - ni.url = "nar/" + std::string(info->path.hashPart()) + "-" - + info->narHash.to_string(HashFormat::Nix32, false) + ".nar"; - ni.fileSize = info->narSize; - auto body = ni.to_string(store); + StorePathSet alreadySent; + auto body = makeNarInfo(store, *info, alreadySent); response.reset(MHD_create_response_from_buffer(body.size(), body.data(), MHD_RESPMEM_MUST_COPY)); MHD_add_response_header(response.get(), "Content-Type", "text/x-nix-narinfo"); + } else if (url == "/get-narinfos-v1") { + /* Resolve all requested hash parts first, so that the + queried paths can be excluded from each other's + `PartialClosure` hint. */ + StorePathSet queried; + for (auto & hashPart : tokenizeString(std::string(body), "\n\r \t")) { + if (auto path = store.queryPathFromHashPart(hashPart)) + queried.insert(*path); + } + + /* Concatenate the narinfos of the valid paths, separated + by empty lines. The absence of a narinfo denotes that + the path is invalid. */ + auto alreadySent = queried; + std::string res; + for (auto & path : queried) { + try { + auto info = store.queryPathInfo(path); + if (!res.empty()) + res += "\n"; + res += makeNarInfo(store, *info, alreadySent); + } catch (InvalidPath &) { + } + } + + response.reset(MHD_create_response_from_buffer(res.size(), res.data(), MHD_RESPMEM_MUST_COPY)); + MHD_add_response_header(response.get(), "Content-Type", "text/x-nix-narinfo"); + } else if (std::smatch m; std::regex_match(url, m, narUrlRegex)) { auto hashPart = m[1].str(); auto expectedNarHash = m[2].str(); @@ -223,9 +279,9 @@ struct CmdServe : StoreCommand return MHD_queue_response(connection, MHD_HTTP_OK, response.get()); } catch (const Error & e) { - auto body = fmt("500 Internal Server Error\n\nError: %s", e.message()); + auto errorBody = fmt("500 Internal Server Error\n\nError: %s", e.message()); Response response; - response.reset(MHD_create_response_from_buffer(body.size(), body.data(), MHD_RESPMEM_MUST_COPY)); + response.reset(MHD_create_response_from_buffer(errorBody.size(), errorBody.data(), MHD_RESPMEM_MUST_COPY)); MHD_add_response_header(response.get(), "Content-Type", "text/plain"); return MHD_queue_response(connection, MHD_HTTP_INTERNAL_SERVER_ERROR, response.get()); } @@ -240,6 +296,13 @@ struct CmdServe : StoreCommand Ctx ctx{*store, *this}; + /* Per-request state for accumulating the request body, since + microhttpd invokes the handler multiple times per request. */ + struct RequestState + { + std::string body; + }; + auto handler = [](void * cls, MHD_Connection * connection, const char * url, @@ -251,9 +314,25 @@ struct CmdServe : StoreCommand auto & ctx = *static_cast(cls); auto & store = ctx.store; auto & cmd = ctx.cmd; - return cmd.handleRequest(store, connection, std::string(url), method); + if (!*con_cls) { + *con_cls = new RequestState; + return MHD_YES; + } + auto & reqState = *static_cast(*con_cls); + if (*upload_data_size) { + reqState.body.append(upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + return cmd.handleRequest(store, connection, std::string(url), method, reqState.body); }; + auto requestCompleted = + [](void * cls, MHD_Connection * connection, void ** con_cls, MHD_RequestTerminationCode toe) { + delete static_cast(*con_cls); + *con_cls = nullptr; + }; + sockaddr_in addr4{}; sockaddr_in6 addr6{}; const sockaddr * sockAddr = nullptr; @@ -272,7 +351,18 @@ struct CmdServe : StoreCommand throw Error("invalid listen address '%s'", listenAddress); auto * daemon = MHD_start_daemon( - flags, port, nullptr, nullptr, handler, &ctx, MHD_OPTION_SOCK_ADDR, sockAddr, MHD_OPTION_END); + flags, + port, + nullptr, + nullptr, + handler, + &ctx, + MHD_OPTION_SOCK_ADDR, + sockAddr, + MHD_OPTION_NOTIFY_COMPLETED, + static_cast(requestCompleted), + nullptr, + MHD_OPTION_END); if (!daemon) throw Error("failed to start HTTP daemon on %s:%d", listenAddress, port); From 166feba65e7c01626a470066e0fff419675fe646 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 13:01:11 +0200 Subject: [PATCH 04/12] FileTransferRequest: Allow the activity text to be overriden --- src/libstore/filetransfer.cc | 3 ++- src/libstore/http-binary-cache-store.cc | 1 + src/libstore/include/nix/store/filetransfer.hh | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b7ef57346956..d4bf7a6cb2e5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -384,7 +384,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), + request.activityText ? *request.activityText + : fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), Logger::Fields{request.uri.to_string()}, request.parentAct); // Reset the start time to when we actually started the download. diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e72a90df3483..4f17a3a87828 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -350,6 +350,7 @@ asio::awaitable HttpBinaryCacheStore::queryPathInfos( request.method = HttpMethod::Post; request.data = {body.size(), source}; request.mimeType = "text/plain"; + request.activityText = fmt("querying info on %d paths from '%s'", misses.size(), request.uri); FileTransferResult result; try { diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 76309dff72fa..c48e26464f6d 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -184,6 +184,7 @@ struct FileTransferRequest HttpMethod method = HttpMethod::Get; unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; + std::optional activityText; bool decompress = true; /** From 8778f8c494a7b6cf527b5178914690d16dd16d65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 16:18:01 +0200 Subject: [PATCH 05/12] Fix tests --- src/libstore/misc.cc | 2 +- tests/functional/binary-cache.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 894e9562fe77..3b037da7f47b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -291,7 +291,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) } if (knownOutputPaths && settings.getWorkerSettings().useSubstitutes - && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { + && drvOptions.substitutesAllowed(settings.getWorkerSettings()) && !subs.empty()) { for (auto & p : invalid) { pathsToQuery.insert(p); outPathsToDrvs[p].insert_or_assign(drvPath, drv); diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 6e8f66f506fc..546456299f91 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -244,7 +244,6 @@ clearCacheCache restartNixServe nix-build --substituters "$httpBinaryCacheUrl" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" -grepQuiet "don't know how to build" "$TEST_ROOT/log" grepQuiet "building.*input-1" "$TEST_ROOT/log" grepQuiet "building.*input-2" "$TEST_ROOT/log" From 45f680e16ec0b507d491ea1decf0f8a59eb470cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 14:02:54 +0200 Subject: [PATCH 06/12] nix-cache-info: Replace "Features" field with a generic field map Instead of a typed "Features" set, store all non-standard nix-cache-info fields (everything other than StoreDir/WantMassQuery/Priority) verbatim in a std::map on CacheInfo and BinaryCacheStore. This keeps the format forward-compatible: a newer server can advertise a capability that an older client persists in its disk cache without understanding it (e.g. a future BloomFilter field). It also makes the endpoint for fetching multiple narinfos server-configurable: the server advertises "GetNarInfosV1: /get-narinfos-v1", and the client POSTs to whatever path that field specifies, falling back to per-path .narinfo fetches when it is absent. The disk cache stores the field map as JSON in the BinaryCaches table. Co-Authored-By: Claude Opus 4.8 --- doc/manual/source/protocols/nix-cache-info.md | 35 +++++++++++-------- src/libstore-tests/nar-info-disk-cache.cc | 31 ++++++++-------- src/libstore/binary-cache-store.cc | 32 +++++++++++++---- src/libstore/http-binary-cache-store.cc | 25 ++++++------- .../include/nix/store/binary-cache-store.hh | 22 ++++++++++-- .../include/nix/store/nar-info-disk-cache.hh | 14 ++++++-- src/libstore/nar-info-disk-cache.cc | 17 +++------ src/nix/serve.cc | 2 +- 8 files changed, 108 insertions(+), 70 deletions(-) diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/nix-cache-info.md index 7761b60f454d..2d373c909b01 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/nix-cache-info.md @@ -36,23 +36,28 @@ error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/st Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority). -### `Features` +### `GetNarInfosV1` -A space-separated list of optional protocol features that the cache -server supports. Clients ignore any feature names they don't -recognise, and assume no features if the field is absent. This allows -a client to use server capabilities beyond the basic binary cache -protocol only when they're available. +The path (relative to the cache URL) of an endpoint for fetching +multiple `.narinfo` files in a single request. If this field is +present, a client may send a `POST` request to this path whose body is +a list of store path hash parts (one per line); the server responds +with the concatenation of the corresponding `.narinfo` files, separated +by empty lines. This lets a client fetch the `.narinfo` files for many +paths in one request instead of one request per path. A path whose +`.narinfo` is absent from the response is not available in the cache. -Currently defined features: +If the field is absent, the client falls back to fetching each +`.narinfo` individually. -- `get-narinfos-v1`: the server accepts a `POST` request to - `get-narinfos-v1` whose body is a list of store path hash parts (one - per line) and responds with the concatenation of the corresponding - `.narinfo` files, separated by empty lines. This lets a client fetch - the `.narinfo` files for many paths in a single request instead of - one request per path. A path whose `.narinfo` is absent from the - response is not available in the cache. +## Other fields + +Any field not listed above is stored verbatim in the client's +[on-disk cache](#caching-behavior) of `nix-cache-info` but is otherwise +ignored. This keeps the format forward-compatible: a newer server can +advertise a capability that an older client persists, and a later +client version can start using it without the server's metadata having +to be re-fetched. ## Example @@ -60,7 +65,7 @@ Currently defined features: StoreDir: /nix/store WantMassQuery: 1 Priority: 30 -Features: get-narinfos-v1 +GetNarInfosV1: /get-narinfos-v1 ``` ## Caching Behavior diff --git a/src/libstore-tests/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc index 7612250c661d..c73d33b73038 100644 --- a/src/libstore-tests/nar-info-disk-cache.cc +++ b/src/libstore-tests/nar-info-disk-cache.cc @@ -15,6 +15,13 @@ TEST(NarInfoDiskCacheImpl, create_and_read) int prio = 12345; bool wantMassQuery = true; + auto mkFields = [](bool wantMassQuery, int prio) { + return std::map{ + {"WantMassQuery", wantMassQuery ? "1" : "0"}, + {"Priority", std::to_string(prio)}, + }; + }; + auto tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir); auto dbPath(tmpDir / "test-narinfo-disk-cache.sqlite"); @@ -30,22 +37,20 @@ TEST(NarInfoDiskCacheImpl, create_and_read) // Set up "background noise" and check that different caches receive different ids { - auto bc1 = - cache->createCache("https://bar", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); - auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.priority = 12}); + auto bc1 = cache->createCache("https://bar", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.fields = mkFields(false, 12)}); ASSERT_NE(bc1, bc2); barId = bc1; } // Check that the fields are saved and returned correctly. This does not test // the select statement yet, because of in-memory caching. - savedId = cache->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); + savedId = cache->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); ; { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); ASSERT_EQ(savedId, r->id); } @@ -68,8 +73,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read) { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); } } @@ -85,13 +89,12 @@ TEST(NarInfoDiskCacheImpl, create_and_read) } // "Update", same data, check that the id number is reused - cache2->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); + cache2->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); { auto r = cache2->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); ASSERT_EQ(r->id, savedId); } @@ -108,11 +111,9 @@ TEST(NarInfoDiskCacheImpl, create_and_read) auto r0 = cache2->upToDateCacheExists("https://bar"); ASSERT_FALSE(r0); - cache2->createCache( - "https://bar", "/nix/storedir", {.wantMassQuery = !wantMassQuery, .priority = prio + 10}); + cache2->createCache("https://bar", "/nix/storedir", {.fields = mkFields(!wantMassQuery, prio + 10)}); auto r = cache2->upToDateCacheExists("https://bar"); - ASSERT_EQ(r->wantMassQuery, !wantMassQuery); - ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->fields, mkFields(!wantMassQuery, prio + 10)); ASSERT_EQ(r->id, barId); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4ce11e336d38..30ae4360bcf6 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -45,8 +45,10 @@ BinaryCacheStore::BinaryCacheStore(Config & config) narMagic = sink.s; } -void BinaryCacheStore::init() +std::map BinaryCacheStore::parseNixCacheInfo() { + std::map fields; + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); @@ -64,15 +66,31 @@ void BinaryCacheStore::init() config.getHumanReadableURI(), value, storeDir); - } else if (name == "WantMassQuery") { - config.wantMassQuery.setDefault(value == "1"); - } else if (name == "Priority") { - config.priority.setDefault(std::stoi(value)); - } else if (name == "Features") { - features = tokenizeString(value, " "); + } else { + /* Keep every other field verbatim, including ones we + don't (yet) understand. The known ones are applied + by applyCacheInfoFields(). */ + fields.insert_or_assign(name, value); } } } + + return fields; +} + +void BinaryCacheStore::applyCacheInfoFields(const std::map & fields) +{ + if (auto * value = get(fields, "WantMassQuery")) + config.wantMassQuery.setDefault(*value == "1"); + if (auto * value = get(fields, "Priority")) + config.priority.setDefault(std::stoi(*value)); + if (auto * endpoint = get(fields, "GetNarInfosV1")) + getNarInfosV1 = *endpoint; +} + +void BinaryCacheStore::init() +{ + applyCacheInfoFields(parseNixCacheInfo()); } std::optional BinaryCacheStore::getNixCacheInfo() diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 4f17a3a87828..33b7228a4d83 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -70,19 +70,16 @@ void HttpBinaryCacheStore::init() auto cacheKey = config->getReference().render(/*withParams=*/false); if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); - features = cacheInfo->features; + applyCacheInfoFields(cacheInfo->fields); } else { + std::map fields; try { - BinaryCacheStore::init(); + fields = parseNixCacheInfo(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } - diskCache->createCache( - cacheKey, - config->storeDir, - {.wantMassQuery = config->wantMassQuery, .priority = config->priority, .features = features}); + applyCacheInfoFields(fields); + diskCache->createCache(cacheKey, config->storeDir, {.fields = std::move(fields)}); } } @@ -314,9 +311,10 @@ asio::awaitable HttpBinaryCacheStore::queryPathInfos( checkEnabled(); - /* Fall back to per-path queries if the server doesn't support the - batch endpoint. */ - if (!features.contains("get-narinfos-v1")) { + /* Fall back to per-path queries unless the cache advertises an + endpoint for fetching multiple narinfos at once (the path to + POST to). */ + if (!getNarInfosV1) { co_await Store::queryPathInfos(paths, std::move(callback)); co_return; } @@ -346,7 +344,7 @@ asio::awaitable HttpBinaryCacheStore::queryPathInfos( body += std::string(path.hashPart()) + "\n"; StringSource source{body}; - auto request = makeRequest("get-narinfos-v1"); + auto request = makeRequest(*getNarInfosV1); request.method = HttpMethod::Post; request.data = {body.size(), source}; request.mimeType = "text/plain"; @@ -362,14 +360,13 @@ asio::awaitable HttpBinaryCacheStore::queryPathInfos( } /* Parse the concatenated narinfos, indexed by hash part. */ - auto whence = fmt("%s/get-narinfos-v1", config->getHumanReadableURI()); std::map, std::less<>> received; size_t pos = 0; while (pos < result.data.size()) { auto end = result.data.find("\n\n", pos); auto chunk = result.data.substr(pos, end == std::string::npos ? end : end + 1 - pos); pos = end == std::string::npos ? result.data.size() : end + 2; - auto narInfo = std::make_shared(*this, chunk, whence); + auto narInfo = std::make_shared(*this, chunk, request.uri.to_string()); stats.narInfoRead++; received.insert_or_assign(std::string(narInfo->path.hashPart()), std::move(narInfo)); } diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 671700b70a58..51b11d0a2081 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -85,10 +85,12 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; /** - * Features advertised by the cache's `nix-cache-info` (e.g. - * `get-narinfos-v1`). Discovered at `init()` time. + * The endpoint (relative to the cache URL) advertised by the + * cache's `nix-cache-info` `GetNarInfosV1` field for fetching + * multiple `.narinfo` files in one request, if any. Discovered at + * `init()` time. */ - StringSet features; + std::optional getNarInfosV1; private: std::vector> signers; @@ -104,6 +106,20 @@ protected: BinaryCacheStore(Config &); + /** + * Fetch and parse `nix-cache-info`. Applies the known fields + * (`WantMassQuery`, `Priority`, `GetNarInfosV1`, ...) and returns + * the remaining non-standard fields verbatim, so that callers can + * persist fields we don't (yet) understand. + */ + std::map parseNixCacheInfo(); + + /** + * Apply the known `nix-cache-info` fields (currently just + * `GetNarInfosV1`) from `fields` to this store. + */ + void applyCacheInfoFields(const std::map & fields); + /** * Compute the path to the given realisation * diff --git a/src/libstore/include/nix/store/nar-info-disk-cache.hh b/src/libstore/include/nix/store/nar-info-disk-cache.hh index de739e9de69d..5a2f4da4c36e 100644 --- a/src/libstore/include/nix/store/nar-info-disk-cache.hh +++ b/src/libstore/include/nix/store/nar-info-disk-cache.hh @@ -5,6 +5,9 @@ #include "nix/store/nar-info.hh" #include "nix/store/realisation.hh" +#include +#include + namespace nix { struct SQLiteSettings; @@ -28,9 +31,14 @@ struct NarInfoDiskCache struct CacheInfo { int id = 0; - bool wantMassQuery = false; - int priority = 0; - StringSet features; + + /** + * The `nix-cache-info` fields other than `StoreDir`, stored + * verbatim (e.g. `WantMassQuery`, `Priority`, `GetNarInfosV1`). + * Keeping these generic means fields we don't (yet) understand + * are still recorded. + */ + std::map fields; }; /** diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3233dba66357..1fbaaedc0376 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -19,9 +19,7 @@ create table if not exists BinaryCaches ( url text unique not null, timestamp integer not null, storeDir text not null, - wantMassQuery integer not null, - priority integer not null, - features text not null + fields text not null ); create table if not exists NARs ( @@ -100,11 +98,10 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache state->insertCache.create( state->db, - "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority, features) values (?1, ?2, ?3, ?4, ?5, ?6) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5, features = ?6 returning id;"); + "insert into BinaryCaches(url, timestamp, storeDir, fields) values (?1, ?2, ?3, ?4) on conflict (url) do update set timestamp = ?2, storeDir = ?3, fields = ?4 returning id;"); state->queryCache.create( - state->db, - "select id, storeDir, wantMassQuery, priority, features from BinaryCaches where url = ? and timestamp > ?"); + state->db, "select id, storeDir, fields from BinaryCaches where url = ? and timestamp > ?"); state->insertNAR.create( state->db, @@ -194,9 +191,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache .storeDir = queryCache.getStr(1), .info = { .id = (int) queryCache.getInt(0), - .wantMassQuery = queryCache.getInt(2) != 0, - .priority = (int) queryCache.getInt(3), - .features = tokenizeString(queryCache.getStr(4), " "), + .fields = nlohmann::json::parse(queryCache.getStr(2)).get>(), }}; state.caches.emplace(uri, cache); } @@ -224,9 +219,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache .apply(uri) .apply(time(nullptr)) .apply(storeDir) - .apply(info.wantMassQuery) - .apply(info.priority) - .apply(concatStringsSep(" ", info.features))); + .apply(nlohmann::json(info.fields).dump())); if (!r.next()) { unreachable(); } diff --git a/src/nix/serve.cc b/src/nix/serve.cc index 7137bdb227c8..39f6453452ee 100644 --- a/src/nix/serve.cc +++ b/src/nix/serve.cc @@ -149,7 +149,7 @@ struct CmdServe : StoreCommand "StoreDir: " + store.storeDir + "\n" "WantMassQuery: " + (store.config.wantMassQuery ? "1" : "0") + "\n" "Priority: " + std::to_string(priority.value_or(store.config.priority)) + "\n" - "Features: get-narinfos-v1\n"); + "GetNarInfosV1: /get-narinfos-v1\n"); response.reset(MHD_create_response_from_buffer(body->size(), body->data(), MHD_RESPMEM_MUST_COPY)); MHD_add_response_header(response.get(), "Content-Type", "text/x-nix-cache-info"); From d9aaa7526cce4e83162519f031bd2813d8caa5ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 17:22:34 +0200 Subject: [PATCH 07/12] nix-serve: Don't barf if computing PartialClosure fails --- src/nix/serve.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/serve.cc b/src/nix/serve.cc index 39f6453452ee..f4d4a7cd1641 100644 --- a/src/nix/serve.cc +++ b/src/nix/serve.cc @@ -32,7 +32,10 @@ static std::string makeNarInfo(Store & store, const ValidPathInfo & info, StoreP NarInfo ni(info); ni.compression = "none"; StorePathSet closure; - store.computeFSClosure(info.path, closure); + try { + store.computeFSClosure(info.path, closure); + } catch (InvalidPath &) { + } std::erase_if(closure, [&](const StorePath & p) { return p == info.path || info.references.contains(p) || alreadySent.contains(p); }); From 5e2c1d7e4223b840629cf97029a37e351b8bfded Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 17:31:30 +0200 Subject: [PATCH 08/12] Avoid double-counting paths returned by multiple substituters --- src/libstore/misc.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 3b037da7f47b..af11957b67b3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -327,6 +327,8 @@ MissingPaths Store::queryMissing(const std::vector & targets) for (auto & [path, info] : infos) { if (info) { + if (!res.willSubstitute.insert(path).second) + continue; res.willSubstitute.insert(path); res.narSize += info->narSize; From 34918ca772977bb3f5bc29c2f5f40b5ac77ec95c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 17:33:55 +0200 Subject: [PATCH 09/12] Respect useSubstitutes before queueing opaque paths --- src/libstore/misc.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index af11957b67b3..df748441d65e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -302,8 +302,12 @@ MissingPaths Store::queryMissing(const std::vector & targets) [&](const DerivedPath::Opaque & bo) -> void { // FIXME: this should probably be an async call, but for a local store we probably don't want to // bother. - if (!maybeQueryPathInfo(bo.path)) - pathsToQuery.insert(bo.path); + if (!maybeQueryPathInfo(bo.path)) { + if (settings.getWorkerSettings().useSubstitutes && !subs.empty()) + pathsToQuery.insert(bo.path); + else + res.unknown.insert(bo.path); + } }, }, p.raw()); From 9934bb00723551834dafd87e2a20c3d1b5f84e06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 17:34:59 +0200 Subject: [PATCH 10/12] Mark text block --- doc/manual/source/protocols/nix-cache-info.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/nix-cache-info.md index 2d373c909b01..7be4e256695f 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/nix-cache-info.md @@ -61,7 +61,7 @@ to be re-fetched. ## Example -``` +```text StoreDir: /nix/store WantMassQuery: 1 Priority: 30 From d3bb2c36eeddb49179fcd8813807ae48deb92458 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 28 Jun 2026 11:47:50 +0200 Subject: [PATCH 11/12] Drop unnecessary FIXME We don't use querySubstitutablePathInfos() anymore. --- src/libstore/remote-store.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 57ddfc9b5b21..14665716c345 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -226,7 +226,6 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - // FIXME: handle partialClosure } } From 309b7ee70a62edea9bd26faf966cb9f4cb64332b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2026 11:59:55 +0200 Subject: [PATCH 12/12] get-narinfos-v1: Use JSON instead of the legacy .narinfo text format The /get-narinfos-v1 endpoint now returns a stream of newline-delimited JSON objects (one store path's metadata per line) rather than concatenated .narinfo text. Since this is a new interface, only the V2 JSON format is supported. The JSON serialisation of NarInfo gains a "partialClosure" field, and a keyed NarInfo::toJSON()/fromJSON() that includes the store path (as a "path" field) so each object is self-describing. The single /.narinfo endpoint still serves the legacy text format. Co-Authored-By: Claude Opus 4.8 --- doc/manual/source/protocols/nix-cache-info.md | 20 ++++++--- src/libstore/http-binary-cache-store.cc | 14 +++---- src/libstore/include/nix/store/nar-info.hh | 28 +++++++++++++ src/libstore/nar-info.cc | 41 +++++++++++++++++++ src/nix/serve.cc | 21 +++++----- 5 files changed, 101 insertions(+), 23 deletions(-) diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/nix-cache-info.md index 7be4e256695f..608e0e0000dd 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/nix-cache-info.md @@ -38,14 +38,22 @@ Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cac ### `GetNarInfosV1` -The path (relative to the cache URL) of an endpoint for fetching -multiple `.narinfo` files in a single request. If this field is +The path (relative to the cache URL) of an endpoint for fetching the +metadata of multiple store paths in a single request. If this field is present, a client may send a `POST` request to this path whose body is -a list of store path hash parts (one per line); the server responds -with the concatenation of the corresponding `.narinfo` files, separated -by empty lines. This lets a client fetch the `.narinfo` files for many +a list of store path hash parts (one per line). The server responds +with one JSON object per line (newline-delimited JSON), each being the +[version 2 JSON representation](@docroot@/protocols/json/store-object-info.md) +of a store path's metadata, extended with a `"path"` field holding the +store path it describes. This lets a client fetch the metadata for many paths in one request instead of one request per path. A path whose -`.narinfo` is absent from the response is not available in the cache. +object is absent from the response is not available in the cache. + +Each object may also contain a `"partialClosure"` field: an array of +store paths that are (some of) the path's indirect references. This is +a hint that lets a client start fetching the metadata of an entire +closure without waiting for the intervening objects; it need not be +complete and is not covered by the signature. If the field is absent, the client falls back to fetching each `.narinfo` individually. diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 33b7228a4d83..5cc29e86dd37 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -9,6 +9,9 @@ #include "nix/store/store-registration.hh" #include "nix/store/globals.hh" #include "nix/util/topo-sort.hh" +#include "nix/util/strings.hh" + +#include namespace nix { @@ -359,14 +362,11 @@ asio::awaitable HttpBinaryCacheStore::queryPathInfos( throw; } - /* Parse the concatenated narinfos, indexed by hash part. */ + /* Parse the narinfos (one JSON object per line), indexed by hash + part. */ std::map, std::less<>> received; - size_t pos = 0; - while (pos < result.data.size()) { - auto end = result.data.find("\n\n", pos); - auto chunk = result.data.substr(pos, end == std::string::npos ? end : end + 1 - pos); - pos = end == std::string::npos ? result.data.size() : end + 2; - auto narInfo = std::make_shared(*this, chunk, request.uri.to_string()); + for (auto & line : tokenizeString(result.data, "\n")) { + auto narInfo = std::make_shared(NarInfo::fromJSON(*this, nlohmann::json::parse(line))); stats.narInfoRead++; received.insert_or_assign(std::string(narInfo->path.hashPart()), std::move(narInfo)); } diff --git a/src/libstore/include/nix/store/nar-info.hh b/src/libstore/include/nix/store/nar-info.hh index 48e44c1909ae..3194ed67590f 100644 --- a/src/libstore/include/nix/store/nar-info.hh +++ b/src/libstore/include/nix/store/nar-info.hh @@ -61,6 +61,19 @@ struct NarInfo : ValidPathInfo, UnkeyedNarInfo { } + /** + * Combine the unkeyed NAR info with its store path. Used to + * reconstruct a `NarInfo` from its JSON representation. + */ + NarInfo(UnkeyedNarInfo info, StorePath path) + : UnkeyedValidPathInfo{static_cast(info)} + /* As in `NarInfo(ValidPathInfo)`, only this most-derived + constructor initializes the virtual base. */ + , ValidPathInfo{std::move(path), static_cast(*this)} + , UnkeyedNarInfo{std::move(info)} + { + } + NarInfo(const StoreDirConfig & store, StorePath path, Hash narHash) : NarInfo{ValidPathInfo{std::move(path), UnkeyedValidPathInfo{store, narHash}}} { @@ -82,6 +95,21 @@ struct NarInfo : ValidPathInfo, UnkeyedNarInfo bool operator==(const NarInfo &) const = default; std::string to_string(const StoreDirConfig & store) const; + + using UnkeyedNarInfo::toJSON; + + /** + * Like `UnkeyedNarInfo::toJSON()`, but also includes the store + * path (as a `"path"` field), so the result is self-describing. + * Always uses the V2 JSON format. + */ + nlohmann::json toJSON(const StoreDirConfig & store, bool includeImpureInfo) const; + + /** + * Inverse of `toJSON()`: reconstruct a keyed `NarInfo` (including + * its store path) from JSON. Only the V2 format is supported. + */ + static NarInfo fromJSON(const StoreDirConfig & store, const nlohmann::json & json); }; } // namespace nix diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index dff3e95a29dd..68bb7f2c5432 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -173,6 +173,13 @@ UnkeyedNarInfo::toJSON(const StoreDirConfig * store, bool includeImpureInfo, Pat } if (fileSize) jsonObject["downloadSize"] = fileSize; + if (!partialClosure.empty()) { + auto & jsonPartialClosure = jsonObject["partialClosure"] = json::array(); + for (auto & p : partialClosure) + jsonPartialClosure.emplace_back( + format == PathInfoJsonFormat::V1 ? static_cast(store->printStorePath(p)) + : static_cast(p)); + } } return jsonObject; @@ -204,9 +211,43 @@ UnkeyedNarInfo UnkeyedNarInfo::fromJSON(const StoreDirConfig * store, const nloh if (auto * downloadSize = get(obj, "downloadSize")) res.fileSize = getUnsigned(*downloadSize); + if (auto * partialClosure = get(obj, "partialClosure")) { + try { + for (auto & input : getArray(*partialClosure)) + res.partialClosure.insert( + format == PathInfoJsonFormat::V1 ? store->parseStorePath(getString(input)) + : static_cast(input)); + } catch (Error & e) { + e.addTrace({}, "while reading key 'partialClosure'"); + throw; + } + } + return res; } +nlohmann::json NarInfo::toJSON(const StoreDirConfig & store, bool includeImpureInfo) const +{ + auto jsonObject = UnkeyedNarInfo::toJSON(&store, includeImpureInfo, PathInfoJsonFormat::V2); + jsonObject["path"] = path; + return jsonObject; +} + +NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const nlohmann::json & json) +{ + auto & obj = getObject(json); + + PathInfoJsonFormat format = PathInfoJsonFormat::V1; + if (auto * version = optionalValueAt(obj, "version")) + format = *version; + if (format != PathInfoJsonFormat::V2) + throw Error("NAR info JSON must use format version 2"); + + auto path = static_cast(valueAt(obj, "path")); + + return NarInfo{UnkeyedNarInfo::fromJSON(&store, json), std::move(path)}; +} + } // namespace nix namespace nlohmann { diff --git a/src/nix/serve.cc b/src/nix/serve.cc index f4d4a7cd1641..7c14e22b1c00 100644 --- a/src/nix/serve.cc +++ b/src/nix/serve.cc @@ -5,6 +5,8 @@ #include "nix/util/deleter.hh" #include "nix/util/strings.hh" #include "nix/store/nar-info.hh" + +#include #include "nix/store/binary-cache-store.hh" #include "nix/store/log-store.hh" #include "nix/util/environment-variables.hh" @@ -27,7 +29,7 @@ using Response = std::unique_ptr>; * call. The references and hint of this narinfo are added to * `alreadySent` in turn. */ -static std::string makeNarInfo(Store & store, const ValidPathInfo & info, StorePathSet & alreadySent) +static NarInfo makeNarInfo(Store & store, const ValidPathInfo & info, StorePathSet & alreadySent) { NarInfo ni(info); ni.compression = "none"; @@ -46,7 +48,7 @@ static std::string makeNarInfo(Store & store, const ValidPathInfo & info, StoreP ni.url = "nar/" + std::string(info.path.hashPart()) + "-" + info.narHash.to_string(HashFormat::Nix32, false) + ".nar"; ni.fileSize = info.narSize; - return ni.to_string(store); + return ni; } struct CmdServe : StoreCommand @@ -164,7 +166,7 @@ struct CmdServe : StoreCommand auto info = store.queryPathInfo(*path); StorePathSet alreadySent; - auto body = makeNarInfo(store, *info, alreadySent); + auto body = makeNarInfo(store, *info, alreadySent).to_string(store); response.reset(MHD_create_response_from_buffer(body.size(), body.data(), MHD_RESPMEM_MUST_COPY)); MHD_add_response_header(response.get(), "Content-Type", "text/x-nix-narinfo"); @@ -178,23 +180,22 @@ struct CmdServe : StoreCommand queried.insert(*path); } - /* Concatenate the narinfos of the valid paths, separated - by empty lines. The absence of a narinfo denotes that - the path is invalid. */ + /* Return the narinfo of each valid path as a JSON object, + one per line (newline-delimited JSON). The absence of a + path from the response denotes that it is invalid. */ auto alreadySent = queried; std::string res; for (auto & path : queried) { try { auto info = store.queryPathInfo(path); - if (!res.empty()) - res += "\n"; - res += makeNarInfo(store, *info, alreadySent); + res += makeNarInfo(store, *info, alreadySent).toJSON(store, true).dump(); + res += "\n"; } catch (InvalidPath &) { } } response.reset(MHD_create_response_from_buffer(res.size(), res.data(), MHD_RESPMEM_MUST_COPY)); - MHD_add_response_header(response.get(), "Content-Type", "text/x-nix-narinfo"); + MHD_add_response_header(response.get(), "Content-Type", "application/x-ndjson"); } else if (std::smatch m; std::regex_match(url, m, narUrlRegex)) { auto hashPart = m[1].str();