From 1cc66f658a77ef6023a58c8f6958b64a0e12d06a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 20:02:17 +0200 Subject: [PATCH 1/2] NarInfoDiskCache: Support arbitrary nix-cache-info fields This replaces the WantMassQuery and Priority SQLite columns with a generic JSON field storing arbitrary nix-cache-info fields. This allows fields to be added in the future without having to change the database schema. Co-Authored-By: Claude Opus 4.8 --- src/libstore-tests/nar-info-disk-cache.cc | 31 ++++++++++--------- src/libstore/binary-cache-store.cc | 28 ++++++++++++++--- src/libstore/http-binary-cache-store.cc | 10 +++--- .../include/nix/store/binary-cache-store.hh | 10 ++++++ .../include/nix/store/nar-info-disk-cache.hh | 12 +++++-- src/libstore/nar-info-disk-cache.cc | 16 ++++------ 6 files changed, 70 insertions(+), 37 deletions(-) diff --git a/src/libstore-tests/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc index 7612250c661d..c73d33b73038 100644 --- a/src/libstore-tests/nar-info-disk-cache.cc +++ b/src/libstore-tests/nar-info-disk-cache.cc @@ -15,6 +15,13 @@ TEST(NarInfoDiskCacheImpl, create_and_read) int prio = 12345; bool wantMassQuery = true; + auto mkFields = [](bool wantMassQuery, int prio) { + return std::map{ + {"WantMassQuery", wantMassQuery ? "1" : "0"}, + {"Priority", std::to_string(prio)}, + }; + }; + auto tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir); auto dbPath(tmpDir / "test-narinfo-disk-cache.sqlite"); @@ -30,22 +37,20 @@ TEST(NarInfoDiskCacheImpl, create_and_read) // Set up "background noise" and check that different caches receive different ids { - auto bc1 = - cache->createCache("https://bar", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); - auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.priority = 12}); + auto bc1 = cache->createCache("https://bar", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.fields = mkFields(false, 12)}); ASSERT_NE(bc1, bc2); barId = bc1; } // Check that the fields are saved and returned correctly. This does not test // the select statement yet, because of in-memory caching. - savedId = cache->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); + savedId = cache->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); ; { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); ASSERT_EQ(savedId, r->id); } @@ -68,8 +73,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read) { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); } } @@ -85,13 +89,12 @@ TEST(NarInfoDiskCacheImpl, create_and_read) } // "Update", same data, check that the id number is reused - cache2->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio}); + cache2->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); { auto r = cache2->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); ASSERT_EQ(r->id, savedId); } @@ -108,11 +111,9 @@ TEST(NarInfoDiskCacheImpl, create_and_read) auto r0 = cache2->upToDateCacheExists("https://bar"); ASSERT_FALSE(r0); - cache2->createCache( - "https://bar", "/nix/storedir", {.wantMassQuery = !wantMassQuery, .priority = prio + 10}); + cache2->createCache("https://bar", "/nix/storedir", {.fields = mkFields(!wantMassQuery, prio + 10)}); auto r = cache2->upToDateCacheExists("https://bar"); - ASSERT_EQ(r->wantMassQuery, !wantMassQuery); - ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->fields, mkFields(!wantMassQuery, prio + 10)); ASSERT_EQ(r->id, barId); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a7f636f12311..54df5c3466f6 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -45,8 +45,10 @@ BinaryCacheStore::BinaryCacheStore(Config & config) narMagic = sink.s; } -void BinaryCacheStore::init() +std::map BinaryCacheStore::parseNixCacheInfo() { + std::map fields; + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); @@ -64,13 +66,29 @@ void BinaryCacheStore::init() config.getHumanReadableURI(), value, storeDir); - } else if (name == "WantMassQuery") { - config.wantMassQuery.setDefault(value == "1"); - } else if (name == "Priority") { - config.priority.setDefault(std::stoi(value)); + } else { + /* Keep every other field verbatim, including ones we + don't (yet) understand. The known ones are applied + by applyCacheInfoFields(). */ + fields.insert_or_assign(name, value); } } } + + return fields; +} + +void BinaryCacheStore::applyCacheInfoFields(const std::map & fields) +{ + if (auto * value = get(fields, "WantMassQuery")) + config.wantMassQuery.setDefault(*value == "1"); + if (auto * value = get(fields, "Priority")) + config.priority.setDefault(std::stoi(*value)); +} + +void BinaryCacheStore::init() +{ + applyCacheInfoFields(parseNixCacheInfo()); } std::optional BinaryCacheStore::getNixCacheInfo() diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e3b82fabbcf0..7a2f8b8b6d56 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -69,16 +69,16 @@ void HttpBinaryCacheStore::init() auto cacheKey = config->getReference().render(/*withParams=*/false); if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + applyCacheInfoFields(cacheInfo->fields); } else { + std::map fields; try { - BinaryCacheStore::init(); + fields = parseNixCacheInfo(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } - diskCache->createCache( - cacheKey, config->storeDir, {.wantMassQuery = config->wantMassQuery, .priority = config->priority}); + applyCacheInfoFields(fields); + diskCache->createCache(cacheKey, config->storeDir, {.fields = std::move(fields)}); } } diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 6a66e901a883..1f600d8fbc8f 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -98,6 +98,16 @@ protected: BinaryCacheStore(Config &); + /** + * Fetch and parse `nix-cache-info`. + */ + std::map parseNixCacheInfo(); + + /** + * Apply the known `nix-cache-info` fields from `fields` to this store. + */ + void applyCacheInfoFields(const std::map & fields); + /** * Compute the path to the given realisation * diff --git a/src/libstore/include/nix/store/nar-info-disk-cache.hh b/src/libstore/include/nix/store/nar-info-disk-cache.hh index 37d1f1b10e25..ebc8977d26bb 100644 --- a/src/libstore/include/nix/store/nar-info-disk-cache.hh +++ b/src/libstore/include/nix/store/nar-info-disk-cache.hh @@ -5,6 +5,9 @@ #include "nix/store/nar-info.hh" #include "nix/store/realisation.hh" +#include +#include + namespace nix { struct SQLiteSettings; @@ -28,8 +31,13 @@ struct NarInfoDiskCache struct CacheInfo { int id = 0; - bool wantMassQuery = false; - int priority = 0; + + /** + * The `nix-cache-info` fields other than `StoreDir`, stored + * verbatim (e.g. `WantMassQuery`). Keeping these generic + * means fields we don't (yet) understand are still recorded. + */ + std::map fields; }; /** diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index fa345764eed3..1fbaaedc0376 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -19,8 +19,7 @@ create table if not exists BinaryCaches ( url text unique not null, timestamp integer not null, storeDir text not null, - wantMassQuery integer not null, - priority integer not null + fields text not null ); create table if not exists NARs ( @@ -84,7 +83,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache NarInfoDiskCacheImpl( const Settings & settings, SQLiteSettings sqliteSettings, - std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v1.sqlite") + std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v2.sqlite") : NarInfoDiskCache{settings} { auto state(_state.lock()); @@ -99,11 +98,10 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache state->insertCache.create( state->db, - "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); + "insert into BinaryCaches(url, timestamp, storeDir, fields) values (?1, ?2, ?3, ?4) on conflict (url) do update set timestamp = ?2, storeDir = ?3, fields = ?4 returning id;"); state->queryCache.create( - state->db, - "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); + state->db, "select id, storeDir, fields from BinaryCaches where url = ? and timestamp > ?"); state->insertNAR.create( state->db, @@ -193,8 +191,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache .storeDir = queryCache.getStr(1), .info = { .id = (int) queryCache.getInt(0), - .wantMassQuery = queryCache.getInt(2) != 0, - .priority = (int) queryCache.getInt(3), + .fields = nlohmann::json::parse(queryCache.getStr(2)).get>(), }}; state.caches.emplace(uri, cache); } @@ -222,8 +219,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache .apply(uri) .apply(time(nullptr)) .apply(storeDir) - .apply(info.wantMassQuery) - .apply(info.priority)); + .apply(nlohmann::json(info.fields).dump())); if (!r.next()) { unreachable(); } From 6108e4dd00ffab6f8f4e2f26f2b25cedc34e3930 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jun 2026 20:21:12 +0200 Subject: [PATCH 2/2] BinaryCacheStore: Don't throw on malformed Priority in nix-cache-info Co-Authored-By: Claude Opus 4.8 (1M context) --- src/libstore/binary-cache-store.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 54df5c3466f6..92d743a16439 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -82,8 +82,10 @@ void BinaryCacheStore::applyCacheInfoFields(const std::map(*value)) + config.priority.setDefault(*priority); + } } void BinaryCacheStore::init()