diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcf0814d8f0..00a80895158 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,18 +65,42 @@ jobs: instrumented: false primary: true stdenv: stdenv + withAWS: true + withCurlS3: false + # TODO: remove once curl-based-s3 fully lands + - scenario: on ubuntu (no s3) + runs-on: ubuntu-24.04 + os: linux + instrumented: false + primary: false + stdenv: stdenv + withAWS: false + withCurlS3: false + # TODO: remove once curl-based-s3 fully lands + - scenario: on ubuntu (curl s3) + runs-on: ubuntu-24.04 + os: linux + instrumented: false + primary: false + stdenv: stdenv + withAWS: false + withCurlS3: true - scenario: on macos runs-on: macos-14 os: darwin instrumented: false primary: true stdenv: stdenv + withAWS: true + withCurlS3: false - scenario: on ubuntu (with sanitizers / coverage) runs-on: ubuntu-24.04 os: linux instrumented: true primary: false stdenv: clangStdenv + withAWS: true + withCurlS3: false name: tests ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 @@ -99,7 +123,9 @@ jobs: run: | nix build --file ci/gha/tests/wrapper.nix componentTests -L \ --arg withInstrumentation ${{ matrix.instrumented }} \ - --argstr stdenv "${{ matrix.stdenv }}" + --argstr stdenv "${{ matrix.stdenv }}" \ + ${{ format('--arg withAWS {0}', matrix.withAWS) }} \ + ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} - name: Run flake checks and prepare the installer tarball run: | ci/gha/tests/build-checks @@ -110,6 +136,8 @@ jobs: nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \ --arg withInstrumentation ${{ matrix.instrumented }} \ --argstr stdenv "${{ matrix.stdenv }}" \ + ${{ format('--arg withAWS {0}', matrix.withAWS) }} \ + ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} \ --out-link coverage-reports cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY if: ${{ matrix.instrumented }} @@ -240,6 +268,18 @@ jobs: vm_tests: needs: basic-checks + strategy: + fail-fast: false + matrix: + include: + # TODO: remove once curl-based-s3 fully lands + - scenario: legacy s3 + withAWS: true + withCurlS3: false + - scenario: curl s3 + withAWS: false + withCurlS3: true + name: vm_tests (${{ matrix.scenario }}) runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v5 @@ -250,13 +290,16 @@ jobs: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} - uses: DeterminateSystems/magic-nix-cache-action@main - - run: | + - name: Build VM tests + run: | nix build -L \ - .#hydraJobs.tests.functional_user \ - .#hydraJobs.tests.githubFlakes \ - .#hydraJobs.tests.nix-docker \ - .#hydraJobs.tests.tarballFlakes \ - ; + --file ci/gha/vm-tests/wrapper.nix \ + ${{ format('--arg withAWS {0}', matrix.withAWS) }} \ + ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} \ + functional_user \ + githubFlakes \ + nix-docker \ + tarballFlakes flake_regressions: needs: vm_tests diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index b89d51c76c1..bbcd7e6b79b 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -12,6 +12,8 @@ componentTestsPrefix ? "", withSanitizers ? false, withCoverage ? false, + withAWS ? null, + withCurlS3 ? null, ... }: @@ -65,6 +67,12 @@ rec { # Boehm is incompatible with ASAN. nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; + # Override AWS configuration if specified + nix-store = prev.nix-store.override ( + lib.optionalAttrs (withAWS != null) { inherit withAWS; } + // lib.optionalAttrs (withCurlS3 != null) { inherit withCurlS3; } + ); + mesonComponentOverrides = lib.composeManyExtensions componentOverrides; # Unclear how to make Perl bindings work with a dynamically linked ASAN. nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; diff --git a/ci/gha/tests/wrapper.nix b/ci/gha/tests/wrapper.nix index dc280ebbbe0..c1655f8c064 100644 --- a/ci/gha/tests/wrapper.nix +++ b/ci/gha/tests/wrapper.nix @@ -5,6 +5,8 @@ stdenv ? "stdenv", componentTestsPrefix ? "", withInstrumentation ? false, + withAWS ? null, + withCurlS3 ? null, }@args: import ./. ( args @@ -12,5 +14,6 @@ import ./. ( getStdenv = p: p.${stdenv}; withSanitizers = withInstrumentation; withCoverage = withInstrumentation; + inherit withAWS withCurlS3; } ) diff --git a/ci/gha/vm-tests/wrapper.nix b/ci/gha/vm-tests/wrapper.nix new file mode 100644 index 00000000000..2ca80974c61 --- /dev/null +++ b/ci/gha/vm-tests/wrapper.nix @@ -0,0 +1,45 @@ +{ + nixFlake ? builtins.getFlake ("git+file://" + toString ../../..), + system ? "x86_64-linux", + withAWS ? null, + withCurlS3 ? null, +}: + +let + pkgs = nixFlake.inputs.nixpkgs.legacyPackages.${system}; + lib = pkgs.lib; + + # Create base nixComponents using the flake's makeComponents + baseNixComponents = nixFlake.lib.makeComponents { + inherit pkgs; + }; + + # Override nixComponents if AWS parameters are specified + nixComponents = + if (withAWS == null && withCurlS3 == null) then + baseNixComponents + else + baseNixComponents.overrideScope ( + final: prev: { + nix-store = prev.nix-store.override ( + lib.optionalAttrs (withAWS != null) { inherit withAWS; } + // lib.optionalAttrs (withCurlS3 != null) { inherit withCurlS3; } + ); + } + ); + + # Import NixOS tests with the overridden nixComponents + tests = import ../../../tests/nixos { + inherit lib pkgs nixComponents; + nixpkgs = nixFlake.inputs.nixpkgs; + inherit (nixFlake.inputs) nixpkgs-23-11; + }; +in +{ + inherit (tests) + functional_user + githubFlakes + nix-docker + tarballFlakes + ; +} diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 1c08e466e7b..0eb8e36875a 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -34,6 +34,9 @@ let inherit (lib) fileset; in +assert lib.assertMsg (!withAWS || !withCurlS3) + "withAWS and withCurlS3 are mutually exclusive - cannot enable both S3 implementations simultaneously"; + mkMesonLibrary (finalAttrs: { pname = "nix-store"; inherit version;