diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6cf902e358c..2808f87f214 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -236,24 +236,17 @@ EvalState::EvalState( {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)}, })) , rootFS([&] { - auto accessor = [&]() -> decltype(rootFS) { - /* In pure eval mode, we provide a filesystem that only - contains the Nix store. */ - if (settings.pureEval) - return storeFS; - - /* If we have a chroot store and pure eval is not enabled, - use a union accessor to make the chroot store available - at its logical location while still having the underlying - directory available. This is necessary for instance if - we're evaluating a file from the physical /nix/store - while using a chroot store. */ - auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy)); - if (store->storeDir != realStoreDir) - return makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); - - return getFSSourceAccessor(); - }(); + /* In pure eval mode, we provide a filesystem that only + contains the Nix store. + + Otherwise, use a union accessor to make the augmented store + available at its logical location while still having the + underlying directory available. This is necessary for + instance if we're evaluating a file from the physical + /nix/store while using a chroot store, and also for lazy + mounted fetchTree. */ + auto accessor = settings.pureEval ? storeFS.cast() + : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) @@ -3133,6 +3126,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ auto res = (r / CanonPath(suffix)).resolveSymlinks(); if (res.pathExists()) return res; + + // Backward compatibility hack: throw an exception if access + // to this path is not allowed. + if (auto accessor = res.accessor.dynamic_pointer_cast()) + accessor->checkAccess(res.path); } if (hasPrefix(path, "nix/")) @@ -3199,6 +3197,11 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat if (path.resolveSymlinks().pathExists()) return finish(std::move(path)); else { + // Backward compatibility hack: throw an exception if access + // to this path is not allowed. + if (auto accessor = path.accessor.dynamic_pointer_cast()) + accessor->checkAccess(path.path); + logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)}); } } diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index f61dab3a82c..29d9db2afca 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -42,6 +42,7 @@ class Store; namespace fetchers { struct Settings; struct InputCache; +struct Input; } // namespace fetchers struct EvalSettings; class EvalState; @@ -570,6 +571,11 @@ public: void checkURI(const std::string & uri); + /** + * Mount an input on the Nix store. + */ + StorePath mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor); + /** * Parse a Nix expression from the specified file. */ diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index f90bc37df0a..8622ab20885 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,5 +1,7 @@ #include "nix/store/store-api.hh" #include "nix/expr/eval.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/fetchers/fetch-to-store.hh" namespace nix { @@ -18,4 +20,27 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +StorePath +EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) +{ + auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + + allowPath(storePath); // FIXME: should just whitelist the entire virtual store + + storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); + + auto narHash = store->queryPathInfo(storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + + if (originalInput.getNarHash() && narHash != *originalInput.getNarHash()) + throw Error( + (unsigned int) 102, + "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + originalInput.to_string(), + narHash.to_string(HashFormat::SRI, true), + originalInput.getNarHash()->to_string(HashFormat::SRI, true)); + + return storePath; +} + } // namespace nix diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e673e55a012..e76e39f7dcd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -10,6 +10,7 @@ #include "nix/util/url.hh" #include "nix/expr/value-to-json.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/input-cache.hh" #include @@ -218,11 +219,11 @@ static void fetchTree( throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); } - auto [storePath, input2] = input.fetchToStore(state.store); + auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No); - state.allowPath(storePath); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); - emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 6ce78e115be..b1e8b9d72bb 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/environment-variables.hh" namespace nix { @@ -27,14 +28,22 @@ StorePath fetchToStore( std::optional cacheKey; - if (!filter && path.accessor->fingerprint) { - cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); + auto [subpath, fingerprint] = filter ? std::pair>{path.path, std::nullopt} + : path.accessor->getFingerprint(path.path); + + if (fingerprint) { + cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs()); if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) { debug("store path cache hit for '%s'", path); return res->storePath; } - } else + } else { + static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; + if (barf && !filter) + throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); + // FIXME: could still provide in-memory caching keyed on `SourcePath`. debug("source path '%s' is uncacheable", path); + } Activity act( *logger, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 045aafdcb4b..f697ec6f514 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -356,8 +356,10 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); - assert(!accessor->fingerprint); - accessor->fingerprint = result.getFingerprint(store); + if (!accessor->fingerprint) + accessor->fingerprint = result.getFingerprint(store); + else + result.cachedFingerprint = accessor->fingerprint; return {accessor, std::move(result)}; } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index a99ecacef0b..8f1b50eb937 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -16,15 +16,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path) return next->readFile(prefix / path); } +void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) +{ + checkAccess(path); + return next->readFile(prefix / path, sink, sizeCallback); +} + bool FilteringSourceAccessor::pathExists(const CanonPath & path) { return isAllowed(path) && next->pathExists(prefix / path); } std::optional FilteringSourceAccessor::maybeLstat(const CanonPath & path) +{ + return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt; +} + +SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path) { checkAccess(path); - return next->maybeLstat(prefix / path); + return next->lstat(prefix / path); } SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path) @@ -49,6 +60,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path) return displayPrefix + next->showPath(prefix / path) + displaySuffix; } +std::pair> FilteringSourceAccessor::getFingerprint(const CanonPath & path) +{ + if (fingerprint) + return {path, fingerprint}; + return next->getFingerprint(prefix / path); +} + void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f6f5c30ee90..7c16301675f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -893,8 +893,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (auto repoPath = repoInfo.getPath(); - repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -907,7 +906,7 @@ struct GitInputScheme : InputScheme writeString("deleted:", hashSink); writeString(file.abs(), hashSink); } - return makeFingerprint(*repoInfo.workdirInfo.headRev) + return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false); } return std::nullopt; diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index f8a57bfb366..5e98caa5816 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -36,8 +36,12 @@ struct FilteringSourceAccessor : SourceAccessor std::string readFile(const CanonPath & path) override; + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override; + bool pathExists(const CanonPath & path) override; + Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; @@ -46,6 +50,8 @@ struct FilteringSourceAccessor : SourceAccessor std::string showPath(const CanonPath & path) override; + std::pair> getFingerprint(const CanonPath & path) override; + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 3c4b9c06dc5..aa0411ff9a3 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -123,8 +123,6 @@ struct PathInputScheme : InputScheme auto absPath = getAbsPath(input); - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); - // FIXME: check whether access to 'path' is allowed. auto storePath = store->maybeParseStorePath(absPath.string()); @@ -133,43 +131,33 @@ struct PathInputScheme : InputScheme time_t mtime = 0; if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); // FIXME: try to substitute storePath. auto src = sinkToSource( [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } - // To avoid copying the path again to the /nix/store, we need to add a cache entry. - ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; - auto fp = getFingerprint(store, input); - if (fp) { - auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath); - } + auto accessor = ref{store->getFSAccessor(*storePath)}; + + // To prevent `fetchToStore()` copying the path again to Nix + // store, pre-create an entry in the fetcher cache. + auto info = store->queryPathInfo(*storePath); + accessor->fingerprint = + fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); + input.settings->getCache()->upsert( + makeFetchToStoreCacheKey( + input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + *store, + {}, + *storePath); /* Trust the lastModified value supplied by the user, if any. It's not a "secure" attribute so we don't care. */ if (!input.getLastModified()) input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {ref{store->getFSAccessor(*storePath)}, std::move(input)}; - } - - std::optional getFingerprint(ref store, const Input & input) const override - { - if (isRelative(input)) - return std::nullopt; - - /* If this path is in the Nix store, use the hash of the - store object and the subpath. */ - auto path = getAbsPath(input); - try { - auto [storePath, subPath] = store->toStorePath(path.string()); - auto info = store->queryPathInfo(storePath); - return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); - } catch (Error &) { - return std::nullopt; - } + return {accessor, std::move(input)}; } std::optional experimentalFeature() const override diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 3acf589a582..48611896301 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -24,21 +24,6 @@ using namespace flake; namespace flake { -static StorePath copyInputToStore( - EvalState & state, fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) -{ - auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName()); - - state.allowPath(storePath); - - auto narHash = state.store->queryPathInfo(storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - - assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store)); - - return storePath; -} - static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) { if (value.isThunk() && value.isTrivial()) @@ -360,11 +345,14 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } - // Copy the tree to the store. - auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor); - // Re-parse flake.nix from the store. - return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath); + return readFlake( + state, + originalRef, + resolvedRef, + lockedRef, + state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)), + lockRootAttrPath); } Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) @@ -721,11 +709,10 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); - // FIXME: allow input to be lazy. - auto storePath = copyInputToStore( - state, lockedRef.input, input.ref->input, cachedInput.accessor); - - return {state.storePath(storePath), lockedRef}; + return { + state.storePath( + state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor)), + lockedRef}; } }(); diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 7419ef392c7..671444e6f37 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -121,7 +121,7 @@ struct SourceAccessor : std::enable_shared_from_this std::string typeString(); }; - Stat lstat(const CanonPath & path); + virtual Stat lstat(const CanonPath & path); virtual std::optional maybeLstat(const CanonPath & path) = 0; @@ -180,6 +180,27 @@ struct SourceAccessor : std::enable_shared_from_this */ std::optional fingerprint; + /** + * Return the fingerprint for `path`. This is usually the + * fingerprint of the current accessor, but for composite + * accessors (like `MountedSourceAccessor`), we want to return the + * fingerprint of the "inner" accessor if the current one lacks a + * fingerprint. + * + * So this method is intended to return the most-outer accessor + * that has a fingerprint for `path`. It also returns the path that `path` + * corresponds to in that accessor. + * + * For example: in a `MountedSourceAccessor` that has + * `/nix/store/foo` mounted, + * `getFingerprint("/nix/store/foo/bar")` will return the path + * `/bar` and the fingerprint of the `/nix/store/foo` accessor. + */ + virtual std::pair> getFingerprint(const CanonPath & path) + { + return {path, fingerprint}; + } + /** * Return the maximum last-modified time of the files in this * tree, if available. diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 5c0ecc1ff40..d9398045cc5 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -27,6 +27,12 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor return accessor->readFile(subpath); } + Stat lstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->lstat(subpath); + } + std::optional maybeLstat(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); @@ -85,6 +91,14 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor else return nullptr; } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + auto [accessor, subpath] = resolve(path); + return accessor->getFingerprint(subpath); + } }; ref makeMountedSourceAccessor(std::map> mounts) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index 96b6a643a22..e3b39f14ed2 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor } return std::nullopt; } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + for (auto & accessor : accessors) { + auto [subpath, fingerprint] = accessor->getFingerprint(path); + if (fingerprint) + return {subpath, fingerprint}; + } + return {path, std::nullopt}; + } }; ref makeUnionSourceAccessor(std::vector> && accessors) diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 422cab96cc2..77bc030605f 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -2,6 +2,8 @@ source ../common.sh +export _NIX_TEST_BARF_ON_UNCACHEABLE=1 + # shellcheck disable=SC2034 # this variable is used by tests that source this file registry=$TEST_ROOT/registry.json diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index fe5acf26dec..a7d86698de8 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Check that dirtying a submodule makes the entire thing dirty. [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] echo '"foo"' > "$rootRepo"/submodule/sub.nix -[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] -[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] # Test that `nix flake metadata` parses `submodule` correctly. cat > "$rootRepo"/flake.nix <&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep '/flakeB.*does not exist' +expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "'flakeB' is too short to be a valid store path" # Test relative non-flake inputs. cat > "$flakeFollowsA"/flake.nix <