From c28eab561b378153ff032f98d673bda34f2f97e0 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Mon, 29 Sep 2025 16:57:55 +0400 Subject: [PATCH] Allow to substitute FODs with different storeDir We already handle `sub->storeDir` being different from `storeDir` when substituting or copying; in particular, if the store path being copied is a content-addressed derivation with no references (which all FODs are), we can safely substitute. In such cases, the store paths are different only because of different `storeDir`'s; the content will be exactly the same. The only nitpick is that FODs are technically allowed to "reference" `$NIX_STORE_DIR` or similar in the output (which would make the output different on different nix stores), but I could not find any FODs in the wild actually doing this, and in any case this is a bug since it would just break when building with a different store dir. What this commit does is to pass through the `StorePath` field of the `nix-cache-info` file to the `nix::Store` corresponding to the substituter. It also handles restoring this field from the local narinfo cache, however this was also mostly done already and the only change necessary is to pass through the field from the cache to the store. Finally, this also adds the test for this behaviour, both checking that FODs can be substituted from different storeDirs, and that non-FODs can't. The main reason to do this is to allow using substituters as/instead of "hashed mirrors" [1], while using different Nix stores on different machines. The benefits of using substituters for this task are: * They do not require external dependencies like curl, speeding up your first world rebuild; * They are more generic, working with any fetcher rather than just fetchurl; * They work for both `outputHashMode`s rather than just `flat`; * They do not require somewhat hacky tooling [2] that tarballs.nixos.org relies on, instead reusing an already existing mechanism; * They remove unnecessary duplication, since cache.nixos.org stores all redistributable tarballs already. To see this change in action for yourself, run $ NIX_STORE=/tmp/nix/store nix build --store /tmp -j 0 --substituters https://cache.nixos.org nixpkgs#hello.src And observe how the archive is substituted instead of being fetched. [1]: https://github.com/NixOS/nixpkgs/blob/3ba94658f039035d17ff6b963e6fb4c66093ccf0/pkgs/build-support/fetchurl/mirrors.nix#L4 [2]: https://github.com/NixOS/nixpkgs/blob/3ba94658f039035d17ff6b963e6fb4c66093ccf0/maintainers/scripts/copy-tarballs.pl --- src/libstore/binary-cache-store.cc | 3 +- src/libstore/http-binary-cache-store.cc | 1 + .../include/nix/store/nar-info-disk-cache.hh | 1 + src/libstore/include/nix/store/store-api.hh | 5 +- .../include/nix/store/store-dir-config.hh | 2 +- src/libstore/nar-info-disk-cache.cc | 6 +- tests/functional/binary-cache.sh | 65 +++++++++++++++++++ 7 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index badfb4b1484..daa6e809c0f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -57,11 +57,12 @@ void BinaryCacheStore::init() auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) - throw Error( + warn( "binary cache '%s' is for Nix stores with prefix '%s', not '%s'", config.getHumanReadableURI(), value, storeDir); + config.storeDir = value; } else if (name == "WantMassQuery") { config.wantMassQuery.setDefault(value == "1"); } else if (name == "Priority") { diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 6922c0f69d5..74635d56412 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -80,6 +80,7 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri.to_string())) { config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); config->priority.setDefault(cacheInfo->priority); + config->storeDir = cacheInfo->storeDir; } else { try { BinaryCacheStore::init(); diff --git a/src/libstore/include/nix/store/nar-info-disk-cache.hh b/src/libstore/include/nix/store/nar-info-disk-cache.hh index 253487b3033..0d2683b388e 100644 --- a/src/libstore/include/nix/store/nar-info-disk-cache.hh +++ b/src/libstore/include/nix/store/nar-info-disk-cache.hh @@ -21,6 +21,7 @@ public: int id; bool wantMassQuery; int priority; + Path storeDir; }; virtual std::optional upToDateCacheExists(const std::string & uri) = 0; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 6d3f6b8d0df..0c5d40d9d19 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -88,14 +88,15 @@ private: public: - const PathSetting storeDir_{ + PathSetting storeDir_{ this, getDefaultNixStoreDir(), "store", R"( Logical location of the Nix store, usually `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. + between stores if they have the same `store` setting, + with the exception of fixed-output derivation outputs. )"}; }; diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index 07cda5c12af..6c0a84b175e 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -29,7 +29,7 @@ MakeError(BadStorePathName, BadStorePath); */ struct StoreDirConfig { - const Path & storeDir; + Path & storeDir; // pure methods diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 11608a667b3..ba395616aaf 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -239,7 +239,11 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache auto cache(queryCacheRaw(*state, uri)); if (!cache) return std::nullopt; - return CacheInfo{.id = cache->id, .wantMassQuery = cache->wantMassQuery, .priority = cache->priority}; + return CacheInfo{ + .id = cache->id, + .wantMassQuery = cache->wantMassQuery, + .priority = cache->priority, + .storeDir = cache->storeDir}; }); } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 2c102df0771..94f3fd1cb08 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -306,3 +306,68 @@ nix-store --delete "$outPath" "$docPath" # -vvv is the level that logs during the loop timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs + +# Check that we can substitute FODs from a substituter with different StoreDir + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 +expr=' + with import '"${config_nix}"'; + mkDerivation { + name = "fod"; + buildCommand = "mkdir $out; echo Foo > $out/foo"; + outputs = [ "out" ]; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "0sfsas2ngwycqq0f712dbbjzcva2ld9yb39hkvfhlv11lvbd78wp"; + } +' +outPath=$(nix-build --no-out-link -E "$expr") +nix copy --to "file://$cacheDir" "$outPath" + +other_store="$TEST_ROOT/other_store" +rm -rf "$other_store" +otherOutPath=$(NIX_STORE_DIR="$other_store/nix/store" NIX_STORE="$other_store/nix/store" nix-build --no-out-link -E "$expr" \ + --option max-jobs 0 \ + --option store "$other_store" \ + --option substituters "file://$cacheDir" \ + --option trusted-binary-caches "file://$cacheDir" \ + --no-require-sigs) + +if ! [[ "$(basename "$otherOutPath")" != "$(basename "$outPath")" ]]; then + fail "Store hashes for $otherOutPath and $outPath are the same despite different store dirs" +fi +if ! [[ -d "$otherOutPath" ]]; then + fail "Could not substitute FOD path from a substituter with a different store dir: $otherOutPath does not exist" +fi +if ! diff -r "$otherOutPath" "$outPath"; then + fail "The contents of $otherOutPath and $outPath are different even though they should be the same" +fi + +# Check that substituting a non-FOD from a substituter with different StoreDir fails + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 +expr=' + with import '"${config_nix}"'; + mkDerivation { + name = "fod"; + buildCommand = "mkdir $out; echo Foo > $out/foo"; + outputs = [ "out" ]; + } +' + +outPath=$(nix-build --no-out-link -E "$expr") +nix copy --to "file://$cacheDir" "$outPath" + +other_store="$TEST_ROOT/other_store" + +if NIX_STORE_DIR="$other_store/nix/store" NIX_STORE="$other_store/nix/store" nix-build --no-out-link -E "$expr" \ + --option max-jobs 0 \ + --option store "$other_store" \ + --option substituters "file://$cacheDir" \ + --option trusted-binary-caches "file://$cacheDir" \ + --no-require-sigs; +then + fail "Substituting a non-FOD path from a substituter with a different StoreDir should fail" +fi