Add option to include untracked files in git and mercurial fetchers

It can be confusing (especially for new users) that the git and
mercurial fetchers do not include untracked files. This commit adds an
option for the fetchers to include untracked files. The option is
disabled by default.

Author: jfroche

src/libfetchers/fetch-settings.hh

@@ -70,6 +70,9 @@ struct FetchSettings : public Config
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
 
+    Setting<bool> includeUntrackedFiles{this, false, "include-untracked-files",
+        "Whether to include untracked files in Git/Mercurial trees."};
+
     Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
         R"(
           Path or URI of the global flake registry.

src/libfetchers/git-utils.cc

@@ -257,7 +257,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
     }
 
-    WorkdirInfo getWorkdirInfo() override
+    WorkdirInfo getWorkdirInfo(bool includeUntrackedFiles) override
     {
         WorkdirInfo info;
 
@@ -284,6 +284,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         git_status_options options = GIT_STATUS_OPTIONS_INIT;
         options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
         options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
+        if (includeUntrackedFiles) {
+            options.flags |=  GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+                              GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS;
+        }
         if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
             throw Error("getting working directory status: %s", git_error_last()->message);
 

src/libfetchers/git-utils.hh

@@ -48,7 +48,7 @@ struct GitRepo
         std::vector<Submodule> submodules;
     };
 
-    virtual WorkdirInfo getWorkdirInfo() = 0;
+    virtual WorkdirInfo getWorkdirInfo(bool includeUntrackedFiles) = 0;
 
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;

src/libfetchers/git.cc

@@ -386,7 +386,7 @@ struct GitInputScheme : InputScheme
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
         if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
-            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
+            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(fetchSettings.includeUntrackedFiles);
 
         return repoInfo;
     }

src/libfetchers/mercurial.cc

@@ -175,7 +175,10 @@ struct MercurialInputScheme : InputScheme
 
         if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
 
-            bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+            Strings hgStatusOptions = { "status", "-R", actualUrl, "--modified", "--added", "--removed" };
+            if (fetchSettings.includeUntrackedFiles)
+                hgStatusOptions.push_back("--unknown");
+            bool clean = runHg(hgStatusOptions) == "";
 
             if (!clean) {
 
@@ -190,8 +193,11 @@ struct MercurialInputScheme : InputScheme
 
                 input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
 
+                hgStatusOptions = { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" };
+                if (fetchSettings.includeUntrackedFiles)
+                    hgStatusOptions.push_back("--unknown");
                 auto files = tokenizeString<std::set<std::string>>(
-                    runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+                    runHg(hgStatusOptions), "\0"s);
 
                 Path actualPath(absPath(actualUrl));
 

tests/functional/fetchGit.sh

@@ -106,6 +106,25 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/.git ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
+# Using an unclean tree with --include-git-untracked-files option should yield the tracked and uncommitted changes
+path21=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[ ! -e $path21/hello ]
+[ -e $path21/bar ]
+[ -e $path21/dir2/bar ]
+[ ! -e $path21/.git ]
+[[ $(cat $path21/dir1/foo) = foo ]]
+
+# Using an unclean tree with --include-git-untracked-files option should take into account .gitignore files
+echo "/bar" >> $repo/.gitignore
+echo "bar" >> $repo/dir2/.gitignore
+path22=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[ ! -e $path22/hello ]
+[ ! -e $path22/bar ]
+[ ! -e $path22/dir2/bar ]
+[ ! -e $path22/.git ]
+git -C $repo checkout -- $repo/.gitignore
+git -C $repo clean -fd
+
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]]

tests/functional/fetchMercurial.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-[[ $(type -p hq) ]] || skipTest "Mercurial not installed"
+[[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
 clearStore
 
@@ -88,6 +88,22 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath"
 [ ! -e $path2/.hg ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
+path21=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path21/hello ]
+[ -e $path21/bar ]
+[ -e $path21/dir2/bar ]
+[ ! -e $path21/.hg ]
+[[ $(cat $path21/dir1/foo) = foo ]]
+
+echo "bar" >> $repo/.hgignore
+path22=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path22/hello ]
+[ ! -e $path22/bar ]
+[ ! -e $path22/dir2/bar ]
+[ ! -e $path22/.git ]
+sed -i '/bar/d' $repo/.hgignore
+hg status --cwd $repo
+
 [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
 
 # ... unless we're using an explicit ref.

tests/functional/flakes/mercurial.sh

@@ -1,6 +1,6 @@
 source ./common.sh
 
-[[ $(type -p hq) ]] || skipTest "Mercurial not installed"
+[[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
 flake1Dir=$TEST_ROOT/flake-hg1
 mkdir -p $flake1Dir