External derivation builders

src/libstore/globals.cc

@@ -341,10 +341,15 @@ PathsInChroot BaseSetting<PathsInChroot>::parse(const std::string & str) const
             i.pop_back();
         }
         size_t p = i.find('=');
-        if (p == std::string::npos)
-            pathsInChroot[i] = {.source = i, .optional = optional};
-        else
-            pathsInChroot[i.substr(0, p)] = {.source = i.substr(p + 1), .optional = optional};
+        std::string inside, outside;
+        if (p == std::string::npos) {
+            inside = i;
+            outside = i;
+        } else {
+            inside = i.substr(0, p);
+            outside = i.substr(p + 1);
+        }
+        pathsInChroot[inside] = {.source = outside, .optional = optional};
     }
     return pathsInChroot;
 }
@@ -374,6 +379,24 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const
     }
 }
 
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Settings::ExternalBuilder, systems, program, args);
+
+template<>
+Settings::ExternalBuilders BaseSetting<Settings::ExternalBuilders>::parse(const std::string & str) const
+{
+    try {
+        return nlohmann::json::parse(str).template get<Settings::ExternalBuilders>();
+    } catch (std::exception & e) {
+        throw UsageError("parsing setting '%s': %s", name, e.what());
+    }
+}
+
+template<>
+std::string BaseSetting<Settings::ExternalBuilders>::to_string() const
+{
+    return nlohmann::json(value).dump();
+}
+
 template<>
 void BaseSetting<PathsInChroot>::appendOrSet(PathsInChroot newValue, bool append)
 {

src/libstore/include/nix/store/globals.hh

@@ -1372,6 +1372,77 @@ public:
           Default is 0, which disables the warning.
           Set it to 1 to warn on all paths.
         )"};
+
+    struct ExternalBuilder
+    {
+        std::vector<std::string> systems;
+        Path program;
+        std::vector<std::string> args;
+    };
+
+    using ExternalBuilders = std::vector<ExternalBuilder>;
+
+    Setting<ExternalBuilders> externalBuilders{
+        this,
+        {},
+        "external-builders",
+        R"(
+          Helper programs that execute derivations.
+
+          The program is passed a JSON document that describes the build environment as the final argument.
+          The JSON document looks like this:
+
+            {
+              "args": [
+                "-e",
+                "/nix/store/vj1c3wf9…-source-stdenv.sh",
+                "/nix/store/shkw4qm9…-default-builder.sh"
+              ],
+              "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash",
+              "env": {
+                "HOME": "/homeless-shelter",
+                "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash",
+                "nativeBuildInputs": "/nix/store/l31j72f1…-version-check-hook",
+                "out": "/nix/store/2yx2prgx…-hello-2.12.2"
+                …
+              },
+              "inputPaths": [
+                "/nix/store/14dciax3…-glibc-2.32-54-dev",
+                "/nix/store/1azs5s8z…-gettext-0.21",
+                …
+              ],
+              "outputs": {
+                "out": "/nix/store/2yx2prgx…-hello-2.12.2"
+              },
+              "realStoreDir": "/nix/store",
+              "storeDir": "/nix/store",
+              "system": "aarch64-linux",
+              "tmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0/build",
+              "tmpDirInSandbox": "/build",
+              "topTmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0",
+              "version": 1
+            }
+        )",
+        {},   // aliases
+        true, // document default
+        // NOTE(cole-h): even though we can make the experimental feature required here, the errors
+        // are not as good (it just becomes a warning if you try to use this setting without the
+        // experimental feature)
+        //
+        // With this commented out:
+        //
+        // error: experimental Nix feature 'external-builders' is disabled; add '--extra-experimental-features
+        // external-builders' to enable it
+        //
+        // With this uncommented:
+        //
+        // warning: Ignoring setting 'external-builders' because experimental feature 'external-builders' is not enabled
+        // error: Cannot build '/nix/store/vwsp4qd8…-opentofu-1.10.2.drv'.
+        //        Reason: required system or feature not available
+        //        Required system: 'aarch64-linux' with features {}
+        //        Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test}
+        // Xp::ExternalBuilders
+    };
 };
 
 // FIXME: don't use a global variable.

src/libstore/unix/build/derivation-builder.cc

@@ -229,6 +229,12 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder
         return acquireUserLock(1, false);
     }
 
+    /**
+     * Throw an exception if we can't do this derivation because of
+     * missing system features.
+     */
+    virtual void checkSystem();
+
     /**
      * Return the paths that should be made available in the sandbox.
      * This includes:
@@ -666,21 +672,8 @@ static bool checkNotWorldWritable(std::filesystem::path path)
     return true;
 }
 
-std::optional<Descriptor> DerivationBuilderImpl::startBuild()
+void DerivationBuilderImpl::checkSystem()
 {
-    if (useBuildUsers()) {
-        if (!buildUser)
-            buildUser = getBuildUser();
-
-        if (!buildUser)
-            return std::nullopt;
-    }
-
-    /* Make sure that no other processes are executing under the
-       sandbox uids. This must be done before any chownToBuilder()
-       calls. */
-    prepareUser();
-
     /* Right platform? */
     if (!drvOptions.canBuildLocally(store, drv)) {
         auto msg =
@@ -704,6 +697,24 @@ std::optional<Descriptor> DerivationBuilderImpl::startBuild()
 
         throw BuildError(BuildResult::Failure::InputRejected, msg);
     }
+}
+
+std::optional<Descriptor> DerivationBuilderImpl::startBuild()
+{
+    if (useBuildUsers()) {
+        if (!buildUser)
+            buildUser = getBuildUser();
+
+        if (!buildUser)
+            return std::nullopt;
+    }
+
+    checkSystem();
+
+    /* Make sure that no other processes are executing under the
+       sandbox uids. This must be done before any chownToBuilder()
+       calls. */
+    prepareUser();
 
     auto buildDir = store.config->getBuildDir();
 
@@ -1904,12 +1915,16 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
 #include "chroot-derivation-builder.cc"
 #include "linux-derivation-builder.cc"
 #include "darwin-derivation-builder.cc"
+#include "external-derivation-builder.cc"
 
 namespace nix {
 
 std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
     LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
 {
+    if (auto builder = ExternalDerivationBuilder::newIfSupported(store, miscMethods, params))
+        return builder;
+
     bool useSandbox = false;
 
     /* Are we doing a sandboxed build? */

src/libstore/unix/build/external-derivation-builder.cc

@@ -0,0 +1,123 @@
+namespace nix {
+
+struct ExternalDerivationBuilder : DerivationBuilderImpl
+{
+    Settings::ExternalBuilder externalBuilder;
+
+    ExternalDerivationBuilder(
+        LocalStore & store,
+        std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
+        DerivationBuilderParams params,
+        Settings::ExternalBuilder externalBuilder)
+        : DerivationBuilderImpl(store, std::move(miscMethods), std::move(params))
+        , externalBuilder(std::move(externalBuilder))
+    {
+        experimentalFeatureSettings.require(Xp::ExternalBuilders);
+    }
+
+    static std::unique_ptr<ExternalDerivationBuilder> newIfSupported(
+        LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> & miscMethods, DerivationBuilderParams & params)
+    {
+        for (auto & handler : settings.externalBuilders.get()) {
+            for (auto & system : handler.systems)
+                if (params.drv.platform == system)
+                    return std::make_unique<ExternalDerivationBuilder>(
+                        store, std::move(miscMethods), std::move(params), handler);
+        }
+        return {};
+    }
+
+    Path tmpDirInSandbox() override
+    {
+        /* In a sandbox, for determinism, always use the same temporary
+           directory. */
+        return "/build";
+    }
+
+    void setBuildTmpDir() override
+    {
+        tmpDir = topTmpDir + "/build";
+        createDir(tmpDir, 0700);
+    }
+
+    void checkSystem() override {}
+
+    void startChild() override
+    {
+        if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix"))
+            throw Error("'recursive-nix' is not supported yet by external derivation builders");
+
+        auto json = nlohmann::json::object();
+
+        json.emplace("version", 1);
+        json.emplace("builder", drv.builder);
+        {
+            auto l = nlohmann::json::array();
+            for (auto & i : drv.args)
+                l.push_back(rewriteStrings(i, inputRewrites));
+            json.emplace("args", std::move(l));
+        }
+        {
+            auto j = nlohmann::json::object();
+            for (auto & [name, value] : env)
+                j.emplace(name, rewriteStrings(value, inputRewrites));
+            json.emplace("env", std::move(j));
+        }
+        json.emplace("topTmpDir", topTmpDir);
+        json.emplace("tmpDir", tmpDir);
+        json.emplace("tmpDirInSandbox", tmpDirInSandbox());
+        json.emplace("storeDir", store.storeDir);
+        json.emplace("realStoreDir", store.config->realStoreDir.get());
+        json.emplace("system", drv.platform);
+        {
+            auto l = nlohmann::json::array();
+            for (auto & i : inputPaths)
+                l.push_back(store.printStorePath(i));
+            json.emplace("inputPaths", std::move(l));
+        }
+        {
+            auto l = nlohmann::json::object();
+            for (auto & i : scratchOutputs)
+                l.emplace(i.first, store.printStorePath(i.second));
+            json.emplace("outputs", std::move(l));
+        }
+
+        // TODO(cole-h): writing this to stdin is too much effort right now, if we want to revisit
+        // that, see this comment by Eelco about how to make it not suck:
+        // https://github.com/DeterminateSystems/nix-src/pull/141#discussion_r2205493257
+        auto jsonFile = std::filesystem::path{topTmpDir} / "build.json";
+        writeFile(jsonFile, json.dump());
+
+        pid = startProcess([&]() {
+            openSlave();
+            try {
+                commonChildInit();
+
+                Strings args = {externalBuilder.program};
+
+                if (!externalBuilder.args.empty()) {
+                    args.insert(args.end(), externalBuilder.args.begin(), externalBuilder.args.end());
+                }
+
+                args.insert(args.end(), jsonFile);
+
+                if (chdir(tmpDir.c_str()) == -1)
+                    throw SysError("changing into '%1%'", tmpDir);
+
+                chownToBuilder(topTmpDir);
+
+                setUser();
+
+                debug("executing external builder: %s", concatStringsSep(" ", args));
+                execv(externalBuilder.program.c_str(), stringsToCharPtrs(args).data());
+
+                throw SysError("executing '%s'", externalBuilder.program);
+            } catch (...) {
+                handleChildException(true);
+                _exit(1);
+            }
+        });
+    }
+};
+
+} // namespace nix

src/libutil/experimental-features.cc

@@ -304,6 +304,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/55",
     },
+    {
+        .tag = Xp::ExternalBuilders,
+        .name = "external-builders",
+        .description = R"(
+            Enables support for external builders / sandbox providers.
+        )",
+        .trackingUrl = "",
+    },
     {
         .tag = Xp::BLAKE3Hashes,
         .name = "blake3-hashes",

src/libutil/include/nix/util/experimental-features.hh

@@ -37,6 +37,7 @@ enum struct ExperimentalFeature {
     MountedSSHStore,
     VerifiedFetches,
     PipeOperators,
+    ExternalBuilders,
     BLAKE3Hashes,
 };
 

tests/functional/external-builders.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+TODO_NixOS
+
+needLocalStore "'--external-builders' can’t be used with the daemon"
+
+expr="$TEST_ROOT/expr.nix"
+cat > "$expr" <<EOF
+with import ${config_nix};
+mkDerivation {
+  name = "external";
+  system = "x68_46-xunil";
+  buildCommand = ''
+    echo xyzzy
+    printf foo > \$out
+  '';
+}
+EOF
+
+external_builder="$TEST_ROOT/external-builder.sh"
+cat > "$external_builder" <<EOF
+#! $SHELL -e
+
+PATH=$PATH
+
+[[ "\$1" = bla ]]
+
+system="\$(jq -r .system < "\$2")"
+builder="\$(jq -r .builder < "\$2")"
+args="\$(jq -r '.args | join(" ")' < "\$2")"
+export buildCommand="\$(jq -r .env.buildCommand < "\$2")"
+export out="\$(jq -r .env.out < "\$2")"
+[[ \$system = x68_46-xunil ]]
+
+printf "\2\n"
+
+# In a real external builder, we would now call something like qemu to emulate the system.
+"\$builder" \$args
+
+printf bar >> \$out
+EOF
+chmod +x "$external_builder"
+
+nix build -L --file "$expr" --out-link "$TEST_ROOT/result" \
+  --extra-experimental-features external-builders \
+  --external-builders "[{\"systems\": [\"x68_46-xunil\"], \"args\": [\"bla\"], \"program\": \"$external_builder\"}]"
+
+[[ $(cat "$TEST_ROOT/result") = foobar ]]