diff --git a/.circleci/config.yml b/.circleci/config.yml index b6410196c4485..d04c65f66a26b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,15 +61,15 @@ jobs: - run: name: check go version command: | - inv -e check-go-version + deva inv -e check-go-version - run: name: grab go deps command: | - inv -e deps + deva inv -e deps - run: name: install go tooling command: | - inv -e install-tools + deva inv -e install-tools - save_cache: <<: *save_deps paths: @@ -87,7 +87,7 @@ jobs: - setup_remote_docker - run: name: run integration tests - command: inv -e integration-tests --race --remote-docker + command: deva inv -e integration-tests --race --remote-docker docker_tests: <<: *job_template @@ -97,10 +97,10 @@ jobs: - setup_remote_docker - run: name: run docker image tests - command: inv -e docker.test + command: deva inv -e docker.test - run: name: run docker image integration tests - command: inv -e docker.integration-tests + command: deva inv -e docker.integration-tests workflows: version: 2 diff --git a/.copyright-overrides.yml b/.copyright-overrides.yml index 8b2abfaad8be4..39ead57ec3728 100644 --- a/.copyright-overrides.yml +++ b/.copyright-overrides.yml @@ -295,7 +295,7 @@ go.opentelemetry.io/otel/semconv/internal: Copyright The OpenTelemetry Authors go.opentelemetry.io/otel/semconv/v1.12.0: Copyright The OpenTelemetry Authors golang.org/x/crypto/chacha20poly1305: Copyright (c) 2009 The Go Authors. All rights reserved -# Not added by inv generate-licenses but spotted by inv-e lint-licenses +# Not added by `deva inv generate-licenses` but spotted by `deva inv -e lint-licenses` github.com/AdaLogics/go-fuzz-headers: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com>|AdamKorcz |Sebastiaan van Stijn |AdaLogics <48351493+AdaLogics@users.noreply.github.com>|Kazuyoshi Kato github.com/AdamKorcz/go-118-fuzz-build/testing: AdamKorcz |AdamKorcz <44787359+AdamKorcz@users.noreply.github.com>|John Howard |Kazuyoshi Kato |Khaled Yakdan |AdamKorcz |Sebastiaan van Stijn github.com/Microsoft/go-winio/pkg/bindfilter: Copyright (c) 2015 Microsoft @@ -389,4 +389,4 @@ github.com/transparency-dev/merkle/proof: - Copyright 2022 Google LLC. All Rights Reserved. github.com/transparency-dev/merkle/rfc6962: Copyright 2016 Google LLC. All Rights Reserved. -gopkg.in/go-jose/go-jose.v2: Copyright 2014 Square Inc. \ No newline at end of file +gopkg.in/go-jose/go-jose.v2: Copyright 2014 Square Inc. diff --git a/.deva/version b/.deva/version new file mode 100644 index 0000000000000..17b2ccd9bf905 --- /dev/null +++ b/.deva/version @@ -0,0 +1 @@ +0.4.3 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4a36139da9e6d..beae2f708a3bf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -288,7 +288,7 @@ /omnibus/config/software/snmp-traps.rb @DataDog/ndm-core /omnibus/resources/*/msi/ @DataDog/windows-agent -# The following is managed by `inv lint-components` -- DO NOT EDIT +# The following is managed by `deva inv lint-components` -- DO NOT EDIT # BEGIN COMPONENTS /comp @DataDog/agent-runtimes /comp/agent @DataDog/agent-runtimes diff --git a/.github/actions/install-deva/action.yml b/.github/actions/install-deva/action.yml new file mode 100644 index 0000000000000..34c7416ed5093 --- /dev/null +++ b/.github/actions/install-deva/action.yml @@ -0,0 +1,25 @@ +name: Install deva +description: Installs the defined version of deva +inputs: + version: + description: The version of deva to install + required: false + default: "" + features: + description: A space-separated list of features to install + required: false + default: "" + +runs: + using: composite + steps: + - name: Set version + id: set-version + run: echo "version=$(cat .deva/version)" >> $GITHUB_OUTPUT + shell: bash + + - name: Install deva + uses: DataDog/datadog-agent-dev@5ba7488629dc306a7002cda678ea46db39e54a17 + with: + version: ${{ inputs.version || steps.set-version.outputs.version }} + features: ${{ inputs.features }} diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index c90f579c43877..952bc2f8b63f7 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -18,7 +18,7 @@ updates: - dependency-name: github.com/DataDog/datadog-agent/* # Ignore golang.org/x/... deps to avoid noise, they are updated together, pretty regularly - dependency-name: golang.org/x/* - # OpenTelemetry collector packages need to be updated with inv rather than dependabot + # OpenTelemetry collector packages need to be updated with `deva inv` rather than dependabot - dependency-name: go.opentelemetry.io/collector/* - dependency-name: github.com/open-telemetry/opentelemetry-collector-contrib/* schedule: @@ -53,7 +53,7 @@ updates: - dependency-name: github.com/mailru/easyjson # Ignore golang.org/x/... deps to avoid noise, they are updated together, pretty regularly - dependency-name: golang.org/x/* - # OpenTelemetry collector packages need to be updated with inv rather than dependabot + # OpenTelemetry collector packages need to be updated with `deva inv` rather than dependabot - dependency-name: go.opentelemetry.io/collector/* - dependency-name: github.com/open-telemetry/opentelemetry-collector-contrib/* schedule: diff --git a/.github/workflows/assess_permissions.yml b/.github/workflows/assess_permissions.yml index c3bc57e9e8fed..e1368cd2d9963 100644 --- a/.github/workflows/assess_permissions.yml +++ b/.github/workflows/assess_permissions.yml @@ -28,14 +28,13 @@ jobs: python-version-file: .python-version cache: "pip" - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tasks/libs/requirements-github.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks legacy-github - name: Assess Repository Permissions env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SLACK_DATADOG_AGENT_BOT_TOKEN: ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }} - run: inv -e github.check-permissions --repo ${{ matrix.value }} + run: deva inv -e github.check-permissions --repo ${{ matrix.value }} diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index 18d287bc5086a..20f505811ffe7 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -8,7 +8,7 @@ on: jobs: auto_assign_issue: runs-on: ubuntu-latest - container: + container: image: ghcr.io/datadog/agent-issue-auto-assign:latest credentials: username: ${{ github.actor }} @@ -22,10 +22,12 @@ jobs: - name: Install dependencies # Dependencies are installed at runtime. Otherwise it would create a huge image see https://hub.docker.com/r/pytorch/pytorch/tags run: | - pip install --upgrade pip && pip install --no-compile --no-cache-dir torch transformers invoke codeowners slack-sdk PyGithub python-gitlab semver + pip install --upgrade pip + pip install --no-compile --no-cache-dir torch transformers datadog-agent-dev=="$(cat .deva/version)" + deva self dep sync -f legacy-tasks - name: Assign issue env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SLACK_DATADOG_AGENT_BOT_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }} run: | - inv -e issue.assign-owner -i ${{ github.event.issue.number }} + deva inv -e issue.assign-owner -i ${{ github.event.issue.number }} diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index 6fc97c57e5167..e38e79d176dd7 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -71,19 +71,20 @@ jobs: # in case it's a Go update PR go-version: ${{ inputs.go_version }} - - name: Install python dependencies - run: | - python3 -m pip install -r requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Get current Go version id: current_go_version run: | - echo "GO_VERSION=$(inv go-version)" >> $GITHUB_OUTPUT + echo "GO_VERSION=$(deva inv go-version)" >> $GITHUB_OUTPUT - name: Get current buildimage tag id: current_buildimage_tag run: | - echo "BUILDIMAGE_TAG=$(inv buildimages.get-tag)" >> $GITHUB_OUTPUT + echo "BUILDIMAGE_TAG=$(deva inv buildimages.get-tag)" >> $GITHUB_OUTPUT - name: Update buildimages IDs and Go version id: update_build_images @@ -96,10 +97,10 @@ jobs: IMAGES_ID: ${{ inputs.images_id }} run: | if [ "$CURRENT_GO_VERSION" = "$INPUT_GO_VERSION" ]; then - inv -e buildimages.update --tag "$IMAGES_ID" "$TEST_VERSION_FLAG" + deva inv -e buildimages.update --tag "$IMAGES_ID" "$TEST_VERSION_FLAG" echo "MESSAGE=Update buildimages ID to $IMAGES_ID" >> $GITHUB_OUTPUT else - inv -e update-go --image-tag "$IMAGES_ID" "$TEST_VERSION_FLAG" $INCLUDE_OTEL_MODULES -v "$INPUT_GO_VERSION" + deva inv -e update-go --image-tag "$IMAGES_ID" "$TEST_VERSION_FLAG" $INCLUDE_OTEL_MODULES -v "$INPUT_GO_VERSION" echo "MESSAGE=Update Go version to $INPUT_GO_VERSION" >> $GITHUB_OUTPUT fi @@ -151,7 +152,7 @@ jobs: GITHUB_REF: ${{ github.ref }} run: | # Generate the PR description - inv -e buildimages.generate-pr-body \ + deva inv -e buildimages.generate-pr-body \ "$CURRENT_BUILDIMAGE_TAG" \ "$IMAGES_ID" \ "$CURRENT_GO_VERSION" \ diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml index bcbdf235ebd18..aee9a43c299bd 100644 --- a/.github/workflows/chase_release_managers.yml +++ b/.github/workflows/chase_release_managers.yml @@ -24,10 +24,10 @@ jobs: with: python-version-file: .python-version cache: "pip" - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt -r tasks/libs/requirements-github.txt -r tasks/requirements_release_tasks.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Chase release managers env: ATLASSIAN_USERNAME : ${{ secrets.ATLASSIAN_USERNAME }} @@ -35,4 +35,4 @@ jobs: SLACK_DATADOG_AGENT_BOT_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }} VERSION: ${{ github.event.inputs.version }} run: | - inv -e release.chase-release-managers --version "$VERSION" + deva inv -e release.chase-release-managers --version "$VERSION" diff --git a/.github/workflows/code_review_complexity.yml b/.github/workflows/code_review_complexity.yml index 1d371046ac55f..277ae78feb34b 100644 --- a/.github/workflows/code_review_complexity.yml +++ b/.github/workflows/code_review_complexity.yml @@ -29,10 +29,12 @@ jobs: with: python-version-file: .python-version cache: 'pip' - cache-dependency-path: '**/requirements*.txt' - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Check code review complexity env: PR_NUMBER: ${{ github.event.pull_request.number }} - run: inv -e github.assign-codereview-label --pr-id="$PR_NUMBER" + run: deva inv -e github.assign-codereview-label --pr-id="$PR_NUMBER" diff --git a/.github/workflows/collector-generate-and-update.yml b/.github/workflows/collector-generate-and-update.yml index 684db0a3a9954..5680fbe80b45f 100644 --- a/.github/workflows/collector-generate-and-update.yml +++ b/.github/workflows/collector-generate-and-update.yml @@ -26,17 +26,17 @@ jobs: with: go-version-file: .go-version - - name: Install Dependencies - run: | - python3 -m pip install --upgrade pip - pip install -r requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Run Collector Update Script env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - inv -e install-tools - inv -e collector.update - inv -e collector.generate - inv -e generate-licenses - inv -e collector.pull-request + deva inv -e install-tools + deva inv -e collector.update + deva inv -e collector.generate + deva inv -e generate-licenses + deva inv -e collector.pull-request diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 6571f1937b3d7..44515841bfaa2 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -32,12 +32,10 @@ jobs: python-version-file: .python-version cache: "pip" - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tasks/libs/requirements-github.txt - pip install -r tasks/requirements_release_tasks.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Check previous agent 6 RC status if: ${{ env.IS_AGENT6_RELEASE == 'true' }} @@ -47,7 +45,7 @@ jobs: DD_APP_KEY: ${{ secrets.DD_APP_KEY }} SLACK_DATADOG_AGENT_CI_WEBHOOK: ${{ secrets.SLACK_DATADOG_AGENT_CI_WEBHOOK }} run: | - inv -e release.check-previous-agent6-rc + deva inv -e release.check-previous-agent6-rc - name: Determine the release active branches id: branches @@ -55,7 +53,7 @@ jobs: if ${{ env.IS_AGENT6_RELEASE == 'true' }}; then echo "value=[\"$AGENT6_RELEASE_BRANCH\"]" >> $GITHUB_OUTPUT else - echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT + echo "value=$(deva inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT fi - name: Set the warning option @@ -86,12 +84,10 @@ jobs: python-version-file: .python-version cache: "pip" - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tasks/libs/requirements-github.txt - pip install -r tasks/requirements_release_tasks.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Check for changes since last RC id: check_for_changes @@ -103,9 +99,9 @@ jobs: WARNING: ${{ needs.find_release_branches.outputs.warning }} run: | if [ -n "${{ needs.find_release_branches.outputs.warning }}" ]; then - echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX" "$WARNING")" >> $GITHUB_OUTPUT + echo "CHANGES=$(deva inv -e release.check-for-changes -r "$MATRIX" "$WARNING")" >> $GITHUB_OUTPUT else - echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX")" >> $GITHUB_OUTPUT + echo "CHANGES=$(deva inv -e release.check-for-changes -r "$MATRIX")" >> $GITHUB_OUTPUT fi - name: Check if agent 6 is in qualification phase @@ -123,7 +119,7 @@ jobs: MATRIX: ${{ matrix.value }} run: | if ${{ env.IS_AGENT6_RELEASE == 'true' }}; then - inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT6_RELEASE_SLACK_WEBHOOK }} --patch-version + deva inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT6_RELEASE_SLACK_WEBHOOK }} --patch-version else - inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }} + deva inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }} fi diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml index c8bcc356d0387..69765db35ec44 100644 --- a/.github/workflows/create_release_schedule.yml +++ b/.github/workflows/create_release_schedule.yml @@ -29,10 +29,10 @@ jobs: with: python-version-file: .python-version cache: "pip" - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt -r tasks/libs/requirements-github.txt -r tasks/requirements_release_tasks.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Create release schedule env: ATLASSIAN_USERNAME : ${{ secrets.ATLASSIAN_USERNAME }} @@ -40,4 +40,4 @@ jobs: INPUT_VERSION: ${{ github.event.inputs.version }} CUTOFF_DATE: ${{ github.event.inputs.cutoff_date }} run: | - inv -e release.create-schedule --version "$INPUT_VERSION" --cutoff-date "$CUTOFF_DATE" + deva inv -e release.create-schedule --version "$INPUT_VERSION" --cutoff-date "$CUTOFF_DATE" diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index 49d9d6c02811b..00d26c8f25f81 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -65,7 +65,10 @@ jobs: with: python-version-file: .python-version cache: 'pip' - - run: pip install -r requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Install go uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 @@ -81,7 +84,7 @@ jobs: env: ARTIFACT_NAME: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }} run: | - inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json + deva inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json - name: Upload artifact uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 @@ -106,7 +109,10 @@ jobs: with: python-version-file: .python-version cache: 'pip' - - run: pip install -r requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Install go uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 @@ -122,8 +128,8 @@ jobs: - name: Combine constants run: | - inv -e security-agent.combine-btfhub-constants --archive-path=./dev/dist/constants - inv -e security-agent.split-btfhub-constants + deva inv -e security-agent.combine-btfhub-constants --archive-path=./dev/dist/constants + deva inv -e security-agent.split-btfhub-constants - name: Compute branch name id: branch-name diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 2e115c89e08ff..bc8567d3d6e23 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -36,8 +36,10 @@ jobs: with: python-version-file: .python-version - - name: Install dependencies - run: pip install -r tasks/requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Configure Git for GitHub Actions bot run: | @@ -45,10 +47,10 @@ jobs: git config --local user.email 'github-actions[bot]@users.noreply.github.com' - name: Validate documentation - run: invoke docs.build --validate + run: deva inv docs.build --validate - name: Build documentation - run: invoke docs.build + run: deva inv docs.build - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: diff --git a/.github/workflows/external-contributor.yml b/.github/workflows/external-contributor.yml index 88e6aec370cd9..002fb10d0d352 100644 --- a/.github/workflows/external-contributor.yml +++ b/.github/workflows/external-contributor.yml @@ -27,12 +27,14 @@ jobs: with: python-version-file: .python-version cache: 'pip' - cache-dependency-path: '**/requirements*.txt' - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Set label on external contributor PRs run: | - inv -e github.handle-community-pr --repo="$GH_REPO" --pr-id="$NUMBER" --labels="$LABELS" + deva inv -e github.handle-community-pr --repo="$GH_REPO" --pr-id="$NUMBER" --labels="$LABELS" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index 5cf95e13d5070..c226ec22ce38d 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -23,18 +23,20 @@ jobs: with: python-version-file: .python-version cache: "pip" - - name: Install python requirements.txt - run: python3 -m pip install -r requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Go mod tidy - run: inv -e tidy + run: deva inv -e tidy - name: Update LICENSE-3rdparty.csv if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependencies-go-tools') }} run: | - inv -e install-tools - inv -e generate-licenses + deva inv -e install-tools + deva inv -e generate-licenses - name: Update mocks if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependencies-go-tools') }} - run: inv -e security-agent.gen-mocks # generate both security agent and process mocks + run: deva inv -e security-agent.gen-mocks # generate both security agent and process mocks - uses: stefanzweifel/git-auto-commit-action@e348103e9026cc0eee72ae06630dbe30c8bf7a79 # v5.1.0 id: autocommit with: diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml index 50a88ddcbdf76..68eaaf7cd9b7b 100644 --- a/.github/workflows/label-analysis.yml +++ b/.github/workflows/label-analysis.yml @@ -31,13 +31,15 @@ jobs: with: python-version-file: .python-version cache: 'pip' - cache-dependency-path: '**/requirements*.txt' - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Auto assign team label env: PR_ID: ${{ github.event.pull_request.number }} - run: inv -e github.assign-team-label --pr-id="$PR_ID" + run: deva inv -e github.assign-team-label --pr-id="$PR_ID" release-note-check: if: github.triggering_actor != 'dd-devflow[bot]' runs-on: ubuntu-latest @@ -52,14 +54,16 @@ jobs: with: python-version-file: .python-version cache: 'pip' - cache-dependency-path: '**/requirements*.txt' - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Check release note env: BRANCH_NAME: ${{ github.head_ref }} PR_ID: ${{ github.event.pull_request.number }} - run: inv -e linter.releasenote + run: deva inv -e linter.releasenote fetch-labels: needs: assign-team-label if: github.triggering_actor != 'dd-devflow[bot]' @@ -111,14 +115,16 @@ jobs: with: python-version-file: .python-version cache: "pip" - cache-dependency-path: '**/requirements*.txt' - - name: Install python dependencies - run: pip3 install -r requirements.txt + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Check qa/[done|no-code-change] labels are not set together env: LABELS: ${{ needs.fetch-labels.outputs.LABELS}} run: | - inv -e github.check-qa-labels --labels "${LABELS[@]}" + deva inv -e github.check-qa-labels --labels "${LABELS[@]}" agenttelemetry-list-change-ack-check: if: github.triggering_actor != 'dd-devflow[bot]' @@ -133,11 +139,13 @@ jobs: with: python-version-file: .python-version cache: 'pip' - cache-dependency-path: '**/requirements*.txt' - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Check agent telemetry metric list - run: inv -e github.agenttelemetry-list-change-ack-check --pr-id=${{ github.event.pull_request.number }} + run: deva inv -e github.agenttelemetry-list-change-ack-check --pr-id=${{ github.event.pull_request.number }} ask-reviews: if: github.triggering_actor != 'dd-devflow[bot]' && github.event.action == 'labeled' && github.event.label.name == 'ask-review' @@ -152,10 +160,12 @@ jobs: with: python-version-file: .python-version cache: 'pip' - cache-dependency-path: '**/requirements*.txt' - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt --break-system-packages + cache-dependency-path: '.deva/version' + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Ask for code reviews env: SLACK_DATADOG_AGENT_BOT_TOKEN : ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }} - run: inv -e issue.ask-reviews -p ${{ github.event.pull_request.number }} + run: deva inv -e issue.ask-reviews -p ${{ github.event.pull_request.number }} diff --git a/.github/workflows/report-merged-pr.yml b/.github/workflows/report-merged-pr.yml index bb6e1b2197a94..90008a967a484 100644 --- a/.github/workflows/report-merged-pr.yml +++ b/.github/workflows/report-merged-pr.yml @@ -29,13 +29,15 @@ jobs: with: python-version-file: .python-version cache: "pip" - cache-dependency-path: '**/requirements*.txt' + cache-dependency-path: '.deva/version' - - name: Install python dependencies - run: pip3 install -r requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Send merge event to Datadog env: PR_NUMBER: ${{ github.event.pull_request.number }} run: | - invoke -e github.pr-merge-dd-event-sender -p "$PR_NUMBER" + deva inv -e github.pr-merge-dd-event-sender -p "$PR_NUMBER" diff --git a/.github/workflows/update_dependencies.yml b/.github/workflows/update_dependencies.yml index a5546d9bf4ec7..d20a7c5ab5e97 100644 --- a/.github/workflows/update_dependencies.yml +++ b/.github/workflows/update_dependencies.yml @@ -33,8 +33,10 @@ jobs: - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: cache: "pip" - - name: Install dependencies - run: pip install -r requirements.txt -r tasks/requirements.txt + - name: Install deva + uses: ./.github/actions/install-deva + with: + features: legacy-tasks - name: Update every golang.org/x/... package run: | go get -u golang.org/x/arch golang.org/x/crypto \ @@ -46,8 +48,8 @@ jobs: golang.org/x/exp # run tidy twice because the first one doesn't always clean everything # depending on module order - inv -e tidy - inv -e tidy + deva inv -e tidy + deva inv -e tidy - uses: stefanzweifel/git-auto-commit-action@e348103e9026cc0eee72ae06630dbe30c8bf7a79 # v5.1.0 id: autocommit with: diff --git a/.gitlab/.pre/cancel-prev-pipelines.yml b/.gitlab/.pre/cancel-prev-pipelines.yml index ab2f55afd4596..e5b30d2c237e0 100644 --- a/.gitlab/.pre/cancel-prev-pipelines.yml +++ b/.gitlab/.pre/cancel-prev-pipelines.yml @@ -15,4 +15,4 @@ cancel-prev-pipelines: - when: on_success script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - - inv pipeline.auto-cancel-previous-pipelines + - deva inv pipeline.auto-cancel-previous-pipelines diff --git a/.gitlab/.pre/create_release_qa_cards.yml b/.gitlab/.pre/create_release_qa_cards.yml index 5bc59073212a2..c6b4ef0c178db 100644 --- a/.gitlab/.pre/create_release_qa_cards.yml +++ b/.gitlab/.pre/create_release_qa_cards.yml @@ -10,8 +10,8 @@ create_release_qa_cards: - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is + # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - pip install ddqa --break-system-packages - - inv release.create-qa-cards -t ${CI_COMMIT_REF_NAME} + - deva inv release.create-qa-cards -t ${CI_COMMIT_REF_NAME} allow_failure: true diff --git a/.gitlab/.pre/gitlab_configuration.yml b/.gitlab/.pre/gitlab_configuration.yml index 10ea5a2d3817c..ea210df11ee92 100644 --- a/.gitlab/.pre/gitlab_configuration.yml +++ b/.gitlab/.pre/gitlab_configuration.yml @@ -7,10 +7,10 @@ test_gitlab_configuration: - !reference [.on_gitlab_changes] script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - - inv -e linter.gitlab-ci - - inv -e linter.job-change-path - - inv -e linter.gitlab-change-paths - - inv -e linter.ssm-parameters + - deva inv -e linter.gitlab-ci + - deva inv -e linter.job-change-path + - deva inv -e linter.gitlab-change-paths + - deva inv -e linter.ssm-parameters test_gitlab_compare_to: stage: .pre @@ -25,8 +25,9 @@ test_gitlab_compare_to: # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - pip install -r tasks/requirements.txt --break-system-packages - - inv pipeline.compare-to-itself + - pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - deva self dep sync -f legacy-tasks + - deva inv pipeline.compare-to-itself # Computes and uploads the GitLab CI configuration diff as an artifact compute_gitlab_ci_config: @@ -42,7 +43,7 @@ compute_gitlab_ci_config: script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - mkdir -p artifacts - - inv -e gitlab.compute-gitlab-ci-config --before-file artifacts/before.gitlab-ci.yml --after-file artifacts/after.gitlab-ci.yml --diff-file artifacts/diff.gitlab-ci.yml + - deva inv -e gitlab.compute-gitlab-ci-config --before-file artifacts/before.gitlab-ci.yml --after-file artifacts/after.gitlab-ci.yml --diff-file artifacts/diff.gitlab-ci.yml artifacts: when: always paths: @@ -56,8 +57,8 @@ lint_gitlab_ci: needs: [compute_gitlab_ci_config] script: - status=0 - - inv -e linter.gitlab-ci-jobs-needs-rules --config-file artifacts/after.gitlab-ci.yml || status=1 - - inv -e linter.gitlab-ci-jobs-owners --config-file artifacts/after.gitlab-ci.yml || status=1 + - deva inv -e linter.gitlab-ci-jobs-needs-rules --config-file artifacts/after.gitlab-ci.yml || status=1 + - deva inv -e linter.gitlab-ci-jobs-owners --config-file artifacts/after.gitlab-ci.yml || status=1 - | if [ $status != 0 ]; then echo "At least one linter failed, exiting..." >& 2 @@ -70,4 +71,4 @@ lint_gitlab_ci_jobs_codeowners: tags: ["arch:arm64"] needs: [] script: - - inv -e linter.gitlab-ci-jobs-codeowners --all-files + - deva inv -e linter.gitlab-ci-jobs-codeowners --all-files diff --git a/.gitlab/benchmarks/benchmarks.yml b/.gitlab/benchmarks/benchmarks.yml index f93c20016900d..3b4323c1aaebb 100644 --- a/.gitlab/benchmarks/benchmarks.yml +++ b/.gitlab/benchmarks/benchmarks.yml @@ -10,7 +10,8 @@ benchmark: tags: ["team:apm-k8s-tweaked-metal-datadog-agent", "specific:true"] script: - export ARTIFACTS_DIR="$(pwd)/artifacts" && mkdir -p $ARTIFACTS_DIR - - pip install -r requirements.txt + - pip install datadog-agent-dev=="$(cat .deva/version)" + - deva self dep sync -f legacy-tasks - ./test/benchmarks/apm_scripts/capture-hardware-software-info.sh - ./test/benchmarks/apm_scripts/run-benchmarks.sh - ./test/benchmarks/apm_scripts/analyze-results.sh diff --git a/.gitlab/binary_build/cluster_agent.yml b/.gitlab/binary_build/cluster_agent.yml index 8362d1e6e72a1..77223d3be087f 100644 --- a/.gitlab/binary_build/cluster_agent.yml +++ b/.gitlab/binary_build/cluster_agent.yml @@ -7,8 +7,8 @@ before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e cluster-agent.build --release-version "$RELEASE_VERSION_7" + - deva inv check-go-version + - deva inv -e cluster-agent.build --release-version "$RELEASE_VERSION_7" artifacts: paths: - $CLUSTER_AGENT_BINARIES_DIR/datadog-cluster-agent diff --git a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml index 1bfce1afa4923..4de39de475c87 100644 --- a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml +++ b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml @@ -17,9 +17,9 @@ cluster_agent_cloudfoundry-build_amd64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e cluster-agent-cloudfoundry.build + - deva inv check-go-version + - deva inv -e cluster-agent-cloudfoundry.build - cd $CI_PROJECT_DIR/$CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR - mkdir -p $OMNIBUS_PACKAGE_DIR - - PACKAGE_VERSION=$(inv agent.version --url-safe) || exit $? + - PACKAGE_VERSION=$(deva inv agent.version --url-safe) || exit $? - tar cf $OMNIBUS_PACKAGE_DIR/datadog-cluster-agent-cloudfoundry-$PACKAGE_VERSION-$ARCH.tar.xz datadog-cluster-agent-cloudfoundry diff --git a/.gitlab/binary_build/cws_instrumentation.yml b/.gitlab/binary_build/cws_instrumentation.yml index 40a89abaab900..aeb7bba4480d1 100644 --- a/.gitlab/binary_build/cws_instrumentation.yml +++ b/.gitlab/binary_build/cws_instrumentation.yml @@ -3,8 +3,8 @@ stage: binary_build needs: ["go_mod_tidy_check"] script: - - inv check-go-version - - inv -e cws-instrumentation.build --arch-suffix + - deva inv check-go-version + - deva inv -e cws-instrumentation.build --arch-suffix artifacts: paths: - $CWS_INSTRUMENTATION_BINARIES_DIR/cws-instrumentation.$ARCH diff --git a/.gitlab/binary_build/fakeintake.yml b/.gitlab/binary_build/fakeintake.yml index 01f751199274d..fe9fb4e0f1796 100644 --- a/.gitlab/binary_build/fakeintake.yml +++ b/.gitlab/binary_build/fakeintake.yml @@ -8,5 +8,4 @@ build_fakeintake: image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] script: - - inv fakeintake.build - + - deva inv fakeintake.build diff --git a/.gitlab/binary_build/linux.yml b/.gitlab/binary_build/linux.yml index 6973d6b9d4517..5cb7752ccec79 100644 --- a/.gitlab/binary_build/linux.yml +++ b/.gitlab/binary_build/linux.yml @@ -9,8 +9,8 @@ build_dogstatsd_static-binary_x64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e dogstatsd.build --static + - deva inv check-go-version + - deva inv -e dogstatsd.build --static - $S3_CP_CMD $CI_PROJECT_DIR/$STATIC_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/static/dogstatsd.$ARCH build_dogstatsd_static-binary_arm64: @@ -26,8 +26,8 @@ build_dogstatsd_static-binary_arm64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e dogstatsd.build --static + - deva inv check-go-version + - deva inv -e dogstatsd.build --static - $S3_CP_CMD $CI_PROJECT_DIR/$STATIC_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/static/dogstatsd.$ARCH build_dogstatsd-binary_x64: @@ -41,8 +41,8 @@ build_dogstatsd-binary_x64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e dogstatsd.build + - deva inv check-go-version + - deva inv -e dogstatsd.build - $S3_CP_CMD $CI_PROJECT_DIR/$DOGSTATSD_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/dogstatsd/dogstatsd build_dogstatsd-binary_arm64: @@ -58,8 +58,8 @@ build_dogstatsd-binary_arm64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e dogstatsd.build + - deva inv check-go-version + - deva inv -e dogstatsd.build - $S3_CP_CMD $CI_PROJECT_DIR/$DOGSTATSD_BINARIES_DIR/dogstatsd $S3_ARTIFACTS_URI/dogstatsd/dogstatsd.$ARCH # IoT Agent builds to make sure the build is not broken because of build flags @@ -76,8 +76,8 @@ build_iot_agent-binary_x64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e agent.build --flavor iot + - deva inv check-go-version + - deva inv -e agent.build --flavor iot - $S3_CP_CMD $CI_PROJECT_DIR/$AGENT_BINARIES_DIR/agent $S3_ARTIFACTS_URI/iot/agent build_iot_agent-binary_arm64: @@ -92,6 +92,5 @@ build_iot_agent-binary_arm64: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version - - inv -e agent.build --flavor iot - + - deva inv check-go-version + - deva inv -e agent.build --flavor iot diff --git a/.gitlab/binary_build/otel_agent.yml b/.gitlab/binary_build/otel_agent.yml index ca44c46fd78c7..0d7e3acf19721 100644 --- a/.gitlab/binary_build/otel_agent.yml +++ b/.gitlab/binary_build/otel_agent.yml @@ -7,7 +7,7 @@ - when: on_success script: - !reference [.retrieve_linux_go_deps] - - inv -e otel-agent.build + - deva inv -e otel-agent.build needs: ["go_deps"] variables: KUBERNETES_MEMORY_REQUEST: "16Gi" diff --git a/.gitlab/binary_build/serverless.yml b/.gitlab/binary_build/serverless.yml index 392d20d94b45f..3291c6b51db4d 100644 --- a/.gitlab/binary_build/serverless.yml +++ b/.gitlab/binary_build/serverless.yml @@ -7,7 +7,7 @@ before_script: - !reference [.retrieve_linux_go_deps] script: - - inv check-go-version + - deva inv check-go-version - cd cmd/serverless && go build -ldflags="-w -s" -a -v -tags "serverless otlp" -o $BINARY_NAME build_serverless-deb_x64: diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml index dc7ffc511b4aa..97b238ea22ab3 100644 --- a/.gitlab/binary_build/system_probe.yml +++ b/.gitlab/binary_build/system_probe.yml @@ -11,9 +11,9 @@ - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -depth # -delete implies -depth - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -delete || true # Allow failure, we can't remove parent folders of datadog-agent script: - - inv check-go-version - - inv -e system-probe.build-object-files --strip-object-files - - inv -e system-probe.save-build-outputs $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz + - deva inv check-go-version + - deva inv -e system-probe.build-object-files --strip-object-files + - deva inv -e system-probe.save-build-outputs $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz variables: KUBERNETES_MEMORY_REQUEST: "6Gi" KUBERNETES_MEMORY_LIMIT: "12Gi" diff --git a/.gitlab/check_merge/do_not_merge.yml b/.gitlab/check_merge/do_not_merge.yml index 214e051a7d00e..579140d993aef 100644 --- a/.gitlab/check_merge/do_not_merge.yml +++ b/.gitlab/check_merge/do_not_merge.yml @@ -22,5 +22,5 @@ do-not-merge: echo "This workflow fails so that the pull request cannot be merged" exit 1 fi - - inv -e release.check-omnibus-branches --no-worktree || exit 1 + - deva inv -e release.check-omnibus-branches --no-worktree || exit 1 - exit 0 diff --git a/.gitlab/common/container_publish_job_templates.yml b/.gitlab/common/container_publish_job_templates.yml index 600b32655894b..cc1ec39a46d13 100644 --- a/.gitlab/common/container_publish_job_templates.yml +++ b/.gitlab/common/container_publish_job_templates.yml @@ -22,7 +22,7 @@ fi - IMG_VARIABLES="$(sed -E "s#(${SRC_AGENT}|${SRC_DSD}|${SRC_DCA}|${SRC_CWS_INSTRUMENTATION})#\1${ECR_RELEASE_SUFFIX}#g" <<<"$IMG_VARIABLES")" - IMG_SOURCES="$(sed -E "s#(${SRC_AGENT}|${SRC_DSD}|${SRC_DCA}|${SRC_CWS_INSTRUMENTATION})#\1${ECR_RELEASE_SUFFIX}#g" <<<"$IMG_SOURCES")" - - "inv pipeline.trigger-child-pipeline --project-name DataDog/public-images --git-ref main --timeout 1800 + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/public-images --git-ref main --timeout 1800 --variable IMG_VARIABLES --variable IMG_REGISTRIES --variable IMG_SOURCES diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml index 5dcf60ab2e130..6abed6f0edc08 100644 --- a/.gitlab/common/macos.yml +++ b/.gitlab/common/macos.yml @@ -53,8 +53,9 @@ # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-github.txt --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks - pyenv rehash - - inv -e rtloader.make - - inv -e rtloader.install - - inv -e install-tools + - deva inv -e rtloader.make + - deva inv -e rtloader.install + - deva inv -e install-tools diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index e0d98900b6340..2b59a012c3017 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -1,6 +1,6 @@ -# File generated by inv buildimages.update-test-infra-definitions +# File generated by deva inv buildimages.update-test-infra-definitions # Please do not edit this file manually -# To update the test-infra-definitions version, run `inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] +# To update the test-infra-definitions version, run `deva inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] --- diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index e42f7c10d16cb..8abe95c18e061 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -16,7 +16,7 @@ - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO token | docker login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi # Build image, use target none label to avoid replication - - docker buildx build --no-cache --push --pull --platform linux/$ARCH --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} $BUILD_ARG --build-arg DD_GIT_REPOSITORY_URL=https://github.com/DataDog/datadog-agent --build-arg DD_GIT_COMMIT_SHA=${CI_COMMIT_SHA} --file $BUILD_CONTEXT/Dockerfile --tag ${TARGET_TAG} --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label "org.opencontainers.image.authors=Datadog " --label "org.opencontainers.image.source=https://github.com/DataDog/datadog-agent" --label "org.opencontainers.image.version=$(inv agent.version)" --label "org.opencontainers.image.revision=${CI_COMMIT_SHA}" --label "org.opencontainers.image.vendor=Datadog, Inc." --label "target=none" $BUILD_CONTEXT + - docker buildx build --no-cache --push --pull --platform linux/$ARCH --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} $BUILD_ARG --build-arg DD_GIT_REPOSITORY_URL=https://github.com/DataDog/datadog-agent --build-arg DD_GIT_COMMIT_SHA=${CI_COMMIT_SHA} --file $BUILD_CONTEXT/Dockerfile --tag ${TARGET_TAG} --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label "org.opencontainers.image.authors=Datadog " --label "org.opencontainers.image.source=https://github.com/DataDog/datadog-agent" --label "org.opencontainers.image.version=$(deva inv agent.version)" --label "org.opencontainers.image.revision=${CI_COMMIT_SHA}" --label "org.opencontainers.image.vendor=Datadog, Inc." --label "target=none" $BUILD_CONTEXT # Squash image - crane flatten -t ${TARGET_TAG} ${TARGET_TAG} # Workaround for temporary network failures diff --git a/.gitlab/deploy_containers/deploy_containers_a7.yml b/.gitlab/deploy_containers/deploy_containers_a7.yml index a2601485e5e25..7c79a14f47a8b 100644 --- a/.gitlab/deploy_containers/deploy_containers_a7.yml +++ b/.gitlab/deploy_containers/deploy_containers_a7.yml @@ -25,7 +25,7 @@ include: stage: deploy_containers dependencies: [] before_script: - - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(deva inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi - export IMG_BASE_SRC="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7${JMX}-amd64,${IMG_BASE_SRC}-7${JMX}-arm64" - export IMG_WINDOWS_SOURCES="${IMG_BASE_SRC}-7${JMX}-win1809${FLAVOR}-amd64,${IMG_BASE_SRC}-7${JMX}-winltsc2022${FLAVOR}-amd64" @@ -39,7 +39,7 @@ include: before_script: - OS=$(echo $OS_LTSC_MAPPING | cut -d ':' -f 1) - LTSC_VERSION=$(echo $OS_LTSC_MAPPING | cut -d ':' -f 2) - - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(deva inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi - IMG_BASE_SRC="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_SOURCES="${IMG_BASE_SRC}-7${JMX}-win${OS}${FLAVOR}-amd64" - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${FLAVOR}-ltsc${LTSC_VERSION}${JMX}" @@ -75,7 +75,7 @@ include: stage: deploy_containers dependencies: [] before_script: - - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)-ot-beta" || exit $?; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(deva inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)-ot-beta" || exit $?; fi - export IMG_SOURCES="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta${JMX}-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta${JMX}-arm64" - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${JMX}" @@ -163,7 +163,7 @@ deploy_containers-dogstatsd: !reference [.manual_on_deploy_auto_on_rc] dependencies: [] before_script: - - VERSION="$(inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $? + - VERSION="$(deva inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $? - export IMG_SOURCES="${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64" - export IMG_DESTINATIONS="${DSD_REPOSITORY}:${VERSION}" @@ -314,4 +314,3 @@ deploy_containers_latest-a7-fips: - JMX: - "" - "-jmx" - diff --git a/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml b/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml index 71c74a817b347..fac976a593f8c 100644 --- a/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml +++ b/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml @@ -15,7 +15,7 @@ include: - job: "docker_build_cws_instrumentation_arm64" artifacts: false before_script: - - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --url-safe)" || exit $?; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(deva inv agent.version --url-safe)" || exit $?; fi - if [[ "$CWS_INSTRUMENTATION_REPOSITORY" == "" ]]; then export CWS_INSTRUMENTATION_REPOSITORY="cws-instrumentation"; fi - export IMG_BASE_SRC="${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64" diff --git a/.gitlab/deploy_dca/deploy_dca.yml b/.gitlab/deploy_dca/deploy_dca.yml index ab02012e6092d..512e7d07c354b 100644 --- a/.gitlab/deploy_dca/deploy_dca.yml +++ b/.gitlab/deploy_dca/deploy_dca.yml @@ -15,7 +15,7 @@ include: - job: "docker_build_cluster_agent_arm64" artifacts: false before_script: - - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --url-safe)" || exit $?; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(deva inv agent.version --url-safe)" || exit $?; fi - if [[ "$CLUSTER_AGENT_REPOSITORY" == "" ]]; then export CLUSTER_AGENT_REPOSITORY="cluster-agent"; fi - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64" diff --git a/.gitlab/deploy_packages/nix.yml b/.gitlab/deploy_packages/nix.yml index 314fcb4961fe7..1ea9c095fbc55 100644 --- a/.gitlab/deploy_packages/nix.yml +++ b/.gitlab/deploy_packages/nix.yml @@ -168,7 +168,7 @@ deploy_staging_dsd: needs: ["build_dogstatsd-binary_x64"] script: - $S3_CP_CMD $S3_ARTIFACTS_URI/dogstatsd/dogstatsd ./dogstatsd - - PACKAGE_VERSION=$(inv agent.version --url-safe) || exit $? + - PACKAGE_VERSION=$(deva inv agent.version --url-safe) || exit $? - $S3_CP_CMD ./dogstatsd $S3_DSD6_URI/linux/dogstatsd-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 # deploy iot-agent x64 binary to staging bucket @@ -181,7 +181,7 @@ deploy_staging_iot_agent: needs: ["build_iot_agent-binary_x64"] script: - $S3_CP_CMD $S3_ARTIFACTS_URI/iot/agent ./agent - - PACKAGE_VERSION=$(inv agent.version --url-safe) || exit $? + - PACKAGE_VERSION=$(deva inv agent.version --url-safe) || exit $? - $S3_CP_CMD ./agent $S3_DSD6_URI/linux/iot/agent-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 # Datadog Installer diff --git a/.gitlab/deploy_packages/oci.yml b/.gitlab/deploy_packages/oci.yml index 4d8ecadebfdff..9c75a0b59255c 100644 --- a/.gitlab/deploy_packages/oci.yml +++ b/.gitlab/deploy_packages/oci.yml @@ -16,7 +16,7 @@ include: script: - set +x - !reference [.retrieve_linux_go_tools_deps] - - VERSION="$(inv agent.version --url-safe)-1" || exit $? + - VERSION="$(deva inv agent.version --url-safe)-1" || exit $? - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - go env -w GOPRIVATE="github.com/DataDog/*" - export PATH=$PATH:$(go env GOPATH)/bin @@ -28,7 +28,7 @@ include: - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${VERSION} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${CI_COMMIT_SHA} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} pipeline-${CI_PIPELINE_ID} - # necessary for ddstaging until we deprecate `-dev` packaging + # necessary for ddstaging until we deprecate `-dev` packaging - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} ${VERSION} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} ${CI_COMMIT_SHA} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} pipeline-${CI_PIPELINE_ID} diff --git a/.gitlab/deps_build/deps_build.yml b/.gitlab/deps_build/deps_build.yml index 0a1722882fa57..31cf6a35c7088 100644 --- a/.gitlab/deps_build/deps_build.yml +++ b/.gitlab/deps_build/deps_build.yml @@ -91,6 +91,6 @@ build_processed_btfhub_archive: variables: KUBERNETES_CPU_REQUEST: 32 script: - - inv -e system-probe.process-btfhub-archive --branch $BTFHUB_ARCHIVE_BRANCH + - deva inv -e system-probe.process-btfhub-archive --branch $BTFHUB_ARCHIVE_BRANCH - $S3_CP_CMD btfs-x86_64.tar $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-x86_64.tar --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - $S3_CP_CMD btfs-arm64.tar $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-arm64.tar --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers diff --git a/.gitlab/deps_fetch/deps_fetch.yml b/.gitlab/deps_fetch/deps_fetch.yml index 89c38920434d0..b4ff5436ba04e 100644 --- a/.gitlab/deps_fetch/deps_fetch.yml +++ b/.gitlab/deps_fetch/deps_fetch.yml @@ -41,8 +41,8 @@ go_deps: # If the cache already contains the dependencies, don't redownload them # but still provide the artifact that's expected for the other jobs to run - if [ -f modcache.tar.xz ]; then exit 0; fi - - inv -e deps --verbose - - inv -e install-tools + - deva inv -e deps --verbose + - deva inv -e install-tools - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache.tar.xz . artifacts: expire_in: 1 day @@ -68,7 +68,7 @@ go_tools_deps: extends: .cache script: - if [ -f modcache_tools.tar.xz ]; then exit 0; fi - - inv -e install-tools + - deva inv -e install-tools # Partial mitigation for #incident-30779. It will cache datadog-packages , but if the upstream version is updated without the key cache changing, the cache will not work until the key cache changes. # Long term solution will be to provide the datadog-packages as a binary hosted internally - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" @@ -92,7 +92,7 @@ go_e2e_deps: extends: .cache script: - if [ -f modcache_e2e.tar.xz ]; then exit 0; fi - - inv -e new-e2e-tests.deps + - deva inv -e new-e2e-tests.deps - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz . artifacts: expire_in: 1 day diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml index 93d7dd84b6394..b695cefef3b0d 100644 --- a/.gitlab/e2e/e2e.yml +++ b/.gitlab/e2e/e2e.yml @@ -37,7 +37,7 @@ - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_GCP credentials_json > ~/gcp-credentials.json || exit $? - export GOOGLE_APPLICATION_CREDENTIALS=~/gcp-credentials.json # Generate external links to CI VISIBILITY, used by artifacts:reports:annotations - - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH + - deva inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH variables: SHOULD_RUN_IN_FLAKES_FINDER: "true" KUBERNETES_MEMORY_REQUEST: 12Gi @@ -59,7 +59,7 @@ EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json E2E_LOGS_PROCESSING_TEST_DEPTH: 1 script: - - inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer --logs-folder=$E2E_OUTPUT_DIR/logs --logs-post-processing --logs-post-processing-test-depth=$E2E_LOGS_PROCESSING_TEST_DEPTH + - deva inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer --logs-folder=$E2E_OUTPUT_DIR/logs --logs-post-processing --logs-post-processing-test-depth=$E2E_LOGS_PROCESSING_TEST_DEPTH after_script: - $CI_PROJECT_DIR/tools/ci/junit_upload.sh artifacts: @@ -502,7 +502,7 @@ new-e2e-installer-windows: # Must run before new_e2e_template changes the aws profile # Note: this is similar to the WINDOWS_AGENT_VERSION in new-e2e_windows_msi but this job is running cross platforms # Note 2: new_e2e_template does not define AGENT_MAJOR_VERSION, so define it as 7 below. - - CURRENT_AGENT_VERSION=$(invoke agent.version) || exit $?; export CURRENT_AGENT_VERSION + - CURRENT_AGENT_VERSION=$(deva inv agent.version) || exit $?; export CURRENT_AGENT_VERSION - export STABLE_AGENT_VERSION_PACKAGE=$(curl --retry 10 --retry-all-errors -sS https://hub.docker.com/v2/namespaces/datadog/repositories/agent-package/tags | jq -r '.results[] | .name' | sort | tail -n 2 | head -n 1) - export STABLE_INSTALLER_VERSION_PACKAGE=$(curl --retry 10 --retry-all-errors -sS https://hub.docker.com/v2/namespaces/datadog/repositories/installer-package/tags | jq -r '.results[] | .name' | sort | tail -n 2 | head -n 1) - !reference [.new_e2e_template, before_script] @@ -741,7 +741,7 @@ generate-flakes-finder-pipeline: - tests_windows_sysprobe_x64 tags: ["arch:amd64"] script: - - inv -e testwasher.generate-flake-finder-pipeline + - deva inv -e testwasher.generate-flake-finder-pipeline artifacts: paths: - $CI_PROJECT_DIR/flake-finder-gitlab-ci.yml @@ -779,7 +779,7 @@ new-e2e-eks-cleanup-on-failure: # Use S3 backend - PULUMI_CONFIG_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E pulumi_passphrase) || exit $?; export PULUMI_CONFIG_PASSPHRASE - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" - - inv -e new-e2e-tests.cleanup-remote-stacks --stack-regex "ci-init-$CI_PIPELINE_ID.*eks.*" --pulumi-backend=dd-pulumi-state + - deva inv -e new-e2e-tests.cleanup-remote-stacks --stack-regex "ci-init-$CI_PIPELINE_ID.*eks.*" --pulumi-backend=dd-pulumi-state variables: E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key diff --git a/.gitlab/e2e_install_packages/common.yml b/.gitlab/e2e_install_packages/common.yml index 965f5c67ce6f8..177c196a2438d 100644 --- a/.gitlab/e2e_install_packages/common.yml +++ b/.gitlab/e2e_install_packages/common.yml @@ -32,7 +32,7 @@ END_MAJOR_VERSION: [7] script: - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token ) || exit $?; export DATADOG_AGENT_API_KEY - - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION --test-washer + - deva inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION --test-washer .new-e2e_script_upgrade_persisting_integrations: stage: e2e_install_packages @@ -43,7 +43,7 @@ E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest script: - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY - - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version 7 --test-washer + - deva inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version 7 --test-washer .new-e2e_rpm: stage: e2e_install_packages @@ -54,4 +54,4 @@ E2E_LOGS_PROCESSING_TEST_DEPTH: 2 # We use a single test suite and run all the platforms test as subtest script: - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY - - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --test-washer + - deva inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --test-washer diff --git a/.gitlab/e2e_install_packages/windows.yml b/.gitlab/e2e_install_packages/windows.yml index a911d61abbfc1..0d24b51b7ac05 100644 --- a/.gitlab/e2e_install_packages/windows.yml +++ b/.gitlab/e2e_install_packages/windows.yml @@ -9,11 +9,11 @@ before_script: # WINDOWS_AGENT_VERSION is used to verify the installed agent version # Must run before new_e2e_template changes the aws profile - - WINDOWS_AGENT_VERSION=$(invoke agent.version) || exit $?; export WINDOWS_AGENT_VERSION + - WINDOWS_AGENT_VERSION=$(deva inv agent.version) || exit $?; export WINDOWS_AGENT_VERSION - !reference [.new_e2e_template, before_script] script: # LAST_STABLE_VERSION is used for upgrade test - - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7" --no-worktree) + - export LAST_STABLE_VERSION=$(deva inv release.get-release-json-value "last_stable::7" --no-worktree) - !reference [.new_e2e_template, script] .new-e2e_windows_msi: diff --git a/.gitlab/e2e_pre_test/e2e_pre_test.yml b/.gitlab/e2e_pre_test/e2e_pre_test.yml index 35af4a004bebe..1f56b4934f00c 100644 --- a/.gitlab/e2e_pre_test/e2e_pre_test.yml +++ b/.gitlab/e2e_pre_test/e2e_pre_test.yml @@ -10,7 +10,7 @@ e2e_pre_test: - job: publish_fakeintake optional: true script: - - inv -e new-e2e-tests.run --targets ./test-infra-definition --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer + - deva inv -e new-e2e-tests.run --targets ./test-infra-definition --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer after_script: - $CI_PROJECT_DIR/tools/ci/junit_upload.sh variables: diff --git a/.gitlab/functional_test/oracle.yml b/.gitlab/functional_test/oracle.yml index 571564b4b3e22..299b1b51040ee 100644 --- a/.gitlab/functional_test/oracle.yml +++ b/.gitlab/functional_test/oracle.yml @@ -19,4 +19,4 @@ oracle: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv oracle.test \ No newline at end of file + - deva inv oracle.test diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml index e41014c56de96..31ed02eb67448 100644 --- a/.gitlab/functional_test/regression_detector.yml +++ b/.gitlab/functional_test/regression_detector.yml @@ -37,7 +37,7 @@ single-machine-performance-regression_detector: - mkdir outputs # Also needed for smp job sync step # Compute merge base of current commit and `main` - git fetch origin - - SMP_BASE_BRANCH=$(inv release.get-release-json-value base_branch --no-worktree) + - SMP_BASE_BRANCH=$(deva inv release.get-release-json-value base_branch --no-worktree) - echo "Looking for merge base for branch ${SMP_BASE_BRANCH}" - SMP_MERGE_BASE=$(git merge-base ${CI_COMMIT_SHA} origin/${SMP_BASE_BRANCH}) - echo "Merge base is ${SMP_MERGE_BASE}" diff --git a/.gitlab/install_script_testing/install_script_testing.yml b/.gitlab/install_script_testing/install_script_testing.yml index a05c75e5f5d1c..9c85b83ca1dd4 100644 --- a/.gitlab/install_script_testing/install_script_testing.yml +++ b/.gitlab/install_script_testing/install_script_testing.yml @@ -9,7 +9,7 @@ test_install_script: - export TESTING_APT_URL=$DEB_TESTING_S3_BUCKET - export TESTING_YUM_URL=$RPM_TESTING_S3_BUCKET - export TEST_PIPELINE_ID=$CI_PIPELINE_ID - - "inv pipeline.trigger-child-pipeline --project-name DataDog/agent-linux-install-script --git-ref main --timeout 5400 + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/agent-linux-install-script --git-ref main --timeout 5400 --variable TESTING_APT_URL --variable TESTING_YUM_URL --variable TEST_PIPELINE_ID" @@ -17,4 +17,3 @@ test_install_script: rules: - !reference [.except_no_a7] - !reference [.on_deploy] - diff --git a/.gitlab/integration_test/dogstatsd.yml b/.gitlab/integration_test/dogstatsd.yml index d8b7ef4470419..4ebd5477c000e 100644 --- a/.gitlab/integration_test/dogstatsd.yml +++ b/.gitlab/integration_test/dogstatsd.yml @@ -11,4 +11,4 @@ dogstatsd_x64_size_test: - mkdir -p $STATIC_BINARIES_DIR - $S3_CP_CMD $S3_ARTIFACTS_URI/static/dogstatsd.amd64 $STATIC_BINARIES_DIR/dogstatsd script: - - inv -e dogstatsd.size-test --skip-build + - deva inv -e dogstatsd.size-test --skip-build diff --git a/.gitlab/integration_test/linux.yml b/.gitlab/integration_test/linux.yml index bbd009004f0a2..6372640c095af 100644 --- a/.gitlab/integration_test/linux.yml +++ b/.gitlab/integration_test/linux.yml @@ -25,7 +25,7 @@ agent_integration_tests: - export DOCKER_REGISTRY_URL # DockerHub login not to be rate-limited during the docker compose - echo "$DOCKER_TOKEN" | docker login --username "$DOCKER_USER" --password-stdin "$DOCKER_REGISTRY_URL" - - inv -e integration-tests --race --remote-docker + - deva inv -e integration-tests --race --remote-docker docker_integration_tests: extends: .integration_tests_deb @@ -33,8 +33,8 @@ docker_integration_tests: # This job is not stable yet because of rate limit issues and micro vms beta status. allow_failure: true script: - - inv -e docker.test - - inv -e docker.integration-tests + - deva inv -e docker.test + - deva inv -e docker.integration-tests trace_agent_integration_tests: extends: .integration_tests_deb @@ -42,4 +42,4 @@ trace_agent_integration_tests: timeout: 50m retry: 2 script: - - inv -e integration-tests --race --only="Trace Agent" + - deva inv -e integration-tests --race --only="Trace Agent" diff --git a/.gitlab/integration_test/otel.yml b/.gitlab/integration_test/otel.yml index 4aaef3d4a6132..e9902544cf055 100644 --- a/.gitlab/integration_test/otel.yml +++ b/.gitlab/integration_test/otel.yml @@ -9,9 +9,9 @@ integration_tests_otel: needs: ["go_deps"] script: - !reference [.retrieve_linux_go_deps] - - inv check-otel-build - - inv check-otel-module-versions - - inv otel-agent.integration-test + - deva inv check-otel-build + - deva inv check-otel-module-versions + - deva inv otel-agent.integration-test rules: - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/ when: never diff --git a/.gitlab/internal_image_deploy/internal_image_deploy.yml b/.gitlab/internal_image_deploy/internal_image_deploy.yml index 3418e7cb872d5..44588fcb37f7e 100644 --- a/.gitlab/internal_image_deploy/internal_image_deploy.yml +++ b/.gitlab/internal_image_deploy/internal_image_deploy.yml @@ -30,7 +30,7 @@ docker_trigger_internal: TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly" fi - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 --variable IMAGE_VERSION --variable IMAGE_NAME --variable RELEASE_TAG @@ -76,7 +76,7 @@ docker_trigger_internal-fips: TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly" fi - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 --variable IMAGE_VERSION --variable IMAGE_NAME --variable RELEASE_TAG @@ -122,7 +122,7 @@ docker_trigger_internal-ot: TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly" fi - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 --variable IMAGE_VERSION --variable IMAGE_NAME --variable RELEASE_TAG @@ -169,7 +169,7 @@ docker_trigger_cluster_agent_internal: TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly" fi - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi - - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600 --variable IMAGE_VERSION --variable IMAGE_NAME --variable RELEASE_TAG diff --git a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml index d50af13dc029c..f13a994d0cfaa 100644 --- a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml +++ b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml @@ -50,7 +50,7 @@ internal_kubernetes_deploy_experimental: BUNDLE_VERSION_OVERRIDE: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - - "inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main --variable OPTION_AUTOMATIC_ROLLOUT --variable EXPLICIT_WORKFLOWS --variable OPTION_PRE_SCRIPT @@ -80,6 +80,7 @@ notify-slack: needs: ["internal_kubernetes_deploy_experimental"] script: - export SDM_JWT=$(vault read -field=token identity/oidc/token/sdm) - - python3 -m pip install -r tasks/requirements.txt -r tasks/libs/requirements-notifications.txt + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" + - deva self dep sync -f legacy-tasks -f legacy-notifications - SLACK_DATADOG_AGENT_BOT_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT token) || exit $?; export SLACK_DATADOG_AGENT_BOT_TOKEN - - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} || exit $? + - deva inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} || exit $? diff --git a/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml index 14d861983fe24..6817c58afbbeb 100644 --- a/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml +++ b/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml @@ -26,7 +26,7 @@ rc_kubernetes_deploy: AGENT_IMAGE_TAG: $CI_COMMIT_REF_NAME script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - - "inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main + - "deva inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main --variable OPTION_AUTOMATIC_ROLLOUT --variable EXPLICIT_WORKFLOWS --variable OPTION_PRE_SCRIPT diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index b3ac2f1379654..aee0242965815 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -100,7 +100,7 @@ # - In KMT we change the AWS_PROFILE environment variable, and that might cause the ssm wrapper to fail getting the keys. Export # the DD_API_KEY variable before changing the AWS_PROFILE variable. .tag_kmt_ci_job: - - inv -e kmt.tag-ci-job + - deva inv -e kmt.tag-ci-job # -- Test dependencies @@ -156,14 +156,14 @@ - !reference [.retrieve_linux_go_deps] - !reference [.kmt_new_profile] - !reference [.write_ssh_key_file] - - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH || true + - deva inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH || true script: - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR - pulumi login $(cat $STACK_DIR | tr -d '\n') # Each VM gets 12GB of memory. We have set 8GB as base for the system, and upto 4GB can be used # by a ram backed filesystem holding the system-probe test packages - - inv -e kmt.gen-config --ci --arch=$ARCH --output-file=$VMCONFIG_FILE --vmconfig-template=$TEST_COMPONENT --memory=12288 - - inv -e system-probe.start-microvms --provision-instance --provision-microvms --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-${TEST_COMPONENT}-${ARCH}-${CI_PIPELINE_ID} --run-agent + - deva inv -e kmt.gen-config --ci --arch=$ARCH --output-file=$VMCONFIG_FILE --vmconfig-template=$TEST_COMPONENT --memory=12288 + - deva inv -e system-probe.start-microvms --provision-instance --provision-microvms --vmconfig=$VMCONFIG_FILE $INSTANCE_TYPE_ARG $AMI_ID_ARG --ssh-key-name=$AWS_EC2_SSH_KEY_NAME --ssh-key-path=$AWS_EC2_SSH_KEY_FILE --infra-env=$INFRA_ENV --stack-name=kernel-matrix-testing-${TEST_COMPONENT}-${ARCH}-${CI_PIPELINE_ID} --run-agent - jq "." $CI_PROJECT_DIR/stack.output - pulumi logout after_script: @@ -216,7 +216,7 @@ # and Gitlab thinks it can simplify the dependency graph. This leads to a problem where, if any other dependency of the run test # jobs fail (e.g., building the system probe tests), the run tests will be skipped, and the cleanup job will run even though the # setup_env job hasn't finished. This causes instances to be leftover for more time than necessary. - - inv kmt.wait-for-setup-job --pipeline-id $CI_PIPELINE_ID --arch $ARCH --component $TEST_COMPONENT + - deva inv kmt.wait-for-setup-job --pipeline-id $CI_PIPELINE_ID --arch $ARCH --component $TEST_COMPONENT - aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_PIPELINE $FILTER_ARCH $FILTER_INSTANCE_TYPE $FILTER_TEST_COMPONENT --output json --query $QUERY_INSTANCE_IDS | tee -a instance.json - cat instance.json | jq -r 'map(.[]) | .[]' | grep -v "null" | xargs --no-run-if-empty -n 1 -t aws ec2 terminate-instances --instance-ids after_script: @@ -257,7 +257,7 @@ - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt - - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH + - deva inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH - !reference [.define_if_collect_complexity] script: - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output) @@ -270,7 +270,7 @@ fi - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET))) | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output) - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET))) | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output) - - GO_VERSION=$(inv go-version) + - GO_VERSION=$(deva inv go-version) - !reference [.setup_ssh_config] # ssh into each micro-vm and run initialization script. This script will also run the tests. - scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt" @@ -337,9 +337,12 @@ notify_ebpf_complexity_changes: # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/kernel_matrix_testing/requirements-ci.txt --break-system-packages # Required for printing the tables - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + # TODO: Remove this override once the standard build image supports building system probe + - echo "libvirt-python; sys_platform == 'never'" >> /tmp/uv-overrides.txt + - export UV_OVERRIDE=/tmp/uv-overrides.txt + - python3 -m deva self dep sync -f legacy-tasks -f legacy-kernel-matrix-testing - !reference [.setup_agent_github_app] - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN script: - - inv -e ebpf.generate-complexity-summary-for-pr + - deva inv -e ebpf.generate-complexity-summary-for-pr diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index 39f3fe8f3c7b1..c65fdbf9795a9 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -33,7 +33,7 @@ upload_dependencies_sysprobe_arm64: - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi # Pull base images - mkdir $KMT_DOCKERS - - inv -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH + - deva inv -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH after_script: - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] diff --git a/.gitlab/lint/linux.yml b/.gitlab/lint/linux.yml index 802e430b468db..84e617a1d3fa4 100644 --- a/.gitlab/lint/linux.yml +++ b/.gitlab/lint/linux.yml @@ -10,10 +10,10 @@ script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev - - inv -e rtloader.install - - inv -e install-tools - - inv -e linter.go --cpus $KUBERNETES_CPU_REQUEST --debug $FLAVORS $EXTRA_OPTS + - deva inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev + - deva inv -e rtloader.install + - deva inv -e install-tools + - deva inv -e linter.go --cpus $KUBERNETES_CPU_REQUEST --debug $FLAVORS $EXTRA_OPTS .linux_x64: image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -53,4 +53,3 @@ lint_flavor_heroku_linux-x64: - .linux_x64 variables: FLAVORS: '--flavor heroku' - diff --git a/.gitlab/lint/macos.yml b/.gitlab/lint/macos.yml index d4da06b2a518d..eb9d06ae1a70a 100644 --- a/.gitlab/lint/macos.yml +++ b/.gitlab/lint/macos.yml @@ -9,7 +9,7 @@ include: script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - inv -e linter.go --cpus 12 --debug --timeout 60 + - deva inv -e linter.go --cpus 12 --debug --timeout 60 lint_macos_gitlab_amd64: extends: .lint_macos_gitlab diff --git a/.gitlab/lint/technical_linters.yml b/.gitlab/lint/technical_linters.yml index b84b1ece24671..8f3185fcd13e8 100644 --- a/.gitlab/lint/technical_linters.yml +++ b/.gitlab/lint/technical_linters.yml @@ -10,14 +10,14 @@ lint_licenses: script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - inv -e install-tools - - inv -e lint-licenses + - deva inv -e install-tools + - deva inv -e lint-licenses needs: ["go_tools_deps", "go_deps"] lint_shell: extends: .lint script: - - inv -e install-shellcheck + - deva inv -e install-shellcheck - shellcheck --version #Excludes: #SC2028: echo may not expand escape sequences. Use printf. @@ -27,36 +27,36 @@ lint_shell: lint_filename: extends: .lint script: - - inv -e linter.filenames + - deva inv -e linter.filenames lint_copyrights: extends: .lint script: - - inv -e linter.copyrights + - deva inv -e linter.copyrights lint_codeowners: extends: .lint script: - - inv -e github.lint-codeowner + - deva inv -e github.lint-codeowner lint_components: extends: .lint script: - - inv -e lint-components lint-fxutil-oneshot-test + - deva inv -e lint-components lint-fxutil-oneshot-test lint_python: extends: .lint script: - - inv -e linter.python + - deva inv -e linter.python lint_update_go: extends: .lint script: - - inv -e linter.update-go + - deva inv -e linter.update-go validate_modules: extends: .lint script: - - inv -e modules.validate - - inv -e modules.validate-used-by-otel + - deva inv -e modules.validate + - deva inv -e modules.validate-used-by-otel diff --git a/.gitlab/lint/windows.yml b/.gitlab/lint/windows.yml index f532e79e2feb6..da31fda47af2c 100644 --- a/.gitlab/lint/windows.yml +++ b/.gitlab/lint/windows.yml @@ -20,6 +20,7 @@ -e CI_PIPELINE_ID=${CI_PIPELINE_ID} -e CI_PROJECT_NAME=${CI_PROJECT_NAME} -e GOMODCACHE="c:\modcache" + -e RUST_LOG="uv=trace" ${WINBUILDIMAGE} powershell.exe -c "c:\mnt\tasks\winbuildscripts\Invoke-Linters.ps1 -BuildOutOfSource 1 -CheckGoVersion 1 -InstallDeps 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index 7e456a0330a91..885d2c9a5f4ec 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -6,7 +6,8 @@ - SLACK_DATADOG_AGENT_BOT_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT token) || exit $?; export SLACK_DATADOG_AGENT_BOT_TOKEN - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" + - deva self dep sync -f legacy-tasks -f legacy-notifications # Notify jobs are allowed to fail but are monitored by https://app.datadoghq.com/monitors/132367692 .notify-job: @@ -23,9 +24,10 @@ notify-on-tagged-success: script: | MESSAGE_TEXT=":host-green: Tagged build <$CI_PIPELINE_URL|$CI_PIPELINE_ID> succeeded. *$CI_COMMIT_REF_NAME* is available in the staging repositories." - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt + python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" + deva self dep sync -f legacy-tasks -f legacy-notifications SLACK_DATADOG_AGENT_BOT_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT token) || exit $?; export SLACK_DATADOG_AGENT_BOT_TOKEN - invoke notify.post-message -c "#agent-release-sync" -m "$MESSAGE_TEXT" + deva inv notify.post-message -c "#agent-release-sync" -m "$MESSAGE_TEXT" notify: extends: .notify-job @@ -38,8 +40,8 @@ notify: timeout: 15 minutes # Added to prevent a stuck job blocking the resource_group defined above script: - !reference [.notify_setup] - - invoke -e notify.send-message -p $CI_PIPELINE_ID - - invoke -e notify.check-consistent-failures -p $CI_PIPELINE_ID + - deva inv -e notify.send-message -p $CI_PIPELINE_ID + - deva inv -e notify.check-consistent-failures -p $CI_PIPELINE_ID send_pipeline_stats: extends: .notify-job @@ -49,7 +51,7 @@ send_pipeline_stats: dependencies: [] script: - !reference [.notify_setup] - - invoke -e notify.send-stats + - deva inv -e notify.send-stats notify_github: extends: .notify-job @@ -75,14 +77,15 @@ notify_github: # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks - messagefile="$(mktemp)" - echo "Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:" >> "$messagefile" - echo '```sh' >> "$messagefile" - - echo "inv aws.create-vm --pipeline-id=$CI_PIPELINE_ID --os-family=ubuntu" >> "$messagefile" + - echo "deva inv aws.create-vm --pipeline-id=$CI_PIPELINE_ID --os-family=ubuntu" >> "$messagefile" - echo '```' >> "$messagefile" - 'echo "Note: This applies to commit **$CI_COMMIT_SHORT_SHA**" >> "$messagefile"' - - inv -e github.pr-commenter --title "Test changes on VM" --body "$(cat "$messagefile")" --echo + - deva inv -e github.pr-commenter --title "Test changes on VM" --body "$(cat "$messagefile")" --echo - rm "$messagefile" notify_gitlab_ci_changes: @@ -101,9 +104,10 @@ notify_gitlab_ci_changes: # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks - !reference [.setup_agent_github_app] - - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment + - deva inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment .failure_summary_job: image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -123,7 +127,7 @@ notify_failure_summary_on_pipeline: - !reference [.on_main_always] script: - !reference [.notify_setup] - - inv -e notify.failure-summary-upload-pipeline-data + - deva inv -e notify.failure-summary-upload-pipeline-data # Send failure summary notifications daily and weekly notify_failure_summary_daily: @@ -141,12 +145,12 @@ notify_failure_summary_daily: exit fi # Daily - - inv -e notify.failure-summary-send-notifications --daily-summary + - deva inv -e notify.failure-summary-send-notifications --daily-summary # Send weekly if necessary (note that this workflow is usually triggered early in the morning) - | if [ "$weekday" = "Friday" ]; then echo 'Sending weekly summary' - inv -e notify.failure-summary-send-notifications --weekly-summary + deva inv -e notify.failure-summary-send-notifications --weekly-summary fi close_failing_tests_stale_issues: @@ -171,5 +175,6 @@ close_failing_tests_stale_issues: - DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_APP_KEY_ORG2 token) || exit $?; export DD_APP_KEY - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME - - python3 -m pip install -r requirements.txt -r tasks/requirements_release_tasks.txt --break-system-packages # For Atlassian / Jira dependencies - - inv -e notify.close-failing-tests-stale-issues + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks + - deva inv -e notify.close-failing-tests-stale-issues diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml index fd800a910ab65..9bd00e9a03882 100644 --- a/.gitlab/package_build/dmg.yml +++ b/.gitlab/package_build/dmg.yml @@ -8,8 +8,9 @@ - !reference [.setup_macos_github_app] - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - - python3 -m pip install -r tasks/libs/requirements-github.txt - - inv -e github.trigger-macos --workflow-type "build" --datadog-agent-ref "$CI_COMMIT_SHA" --release-version "$RELEASE_VERSION" --destination "$OMNIBUS_PACKAGE_DIR" --version-cache "$VERSION_CACHE_CONTENT" --integrations-core-ref "$INTEGRATIONS_CORE_VERSION" + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" + - python3 -m deva self dep sync -f legacy-tasks + - deva inv -e github.trigger-macos --workflow-type "build" --datadog-agent-ref "$CI_COMMIT_SHA" --release-version "$RELEASE_VERSION" --destination "$OMNIBUS_PACKAGE_DIR" --version-cache "$VERSION_CACHE_CONTENT" --integrations-core-ref "$INTEGRATIONS_CORE_VERSION" - !reference [.upload_sbom_artifacts] timeout: 3h # MacOS builds can take 1h~2h, increase the timeout to avoid timeout flakes artifacts: diff --git a/.gitlab/package_build/heroku.yml b/.gitlab/package_build/heroku.yml index 07fa1ad5c77b5..6f2399e839260 100644 --- a/.gitlab/package_build/heroku.yml +++ b/.gitlab/package_build/heroku.yml @@ -28,7 +28,7 @@ - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor heroku + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor heroku - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.lint_linux_packages] - !reference [.upload_sbom_artifacts] diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 5210e76f41957..804b5315f783b 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -5,7 +5,7 @@ .common_build_oci: script: - echo "About to build for $RELEASE_VERSION" - - AGENT_VERSION="$(inv agent.version -u)-1" || exit $? + - AGENT_VERSION="$(deva inv agent.version -u)-1" || exit $? - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/"$AGENT_VERSION" - !reference [.retrieve_linux_go_deps] - !reference [.cache_omnibus_ruby_deps, setup] @@ -21,7 +21,7 @@ - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf # NOTE: for now, we consider "ociru" to be a "redhat_target" in omnibus/lib/ostools.rb # if we ever start building on a different platform, that might need to change - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru --install-directory="$INSTALL_DIR" + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --host-distribution=ociru --install-directory="$INSTALL_DIR" - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.upload_sbom_artifacts] variables: @@ -101,13 +101,13 @@ installer-install-scripts: RELEASE_VERSION: "$RELEASE_VERSION_7" script: - !reference [.retrieve_linux_go_deps] - - VERSION="$(inv agent.version --url-safe)-1" || exit $? + - VERSION="$(deva inv agent.version --url-safe)-1" || exit $? - echo "About to build for $VERSION" - mkdir -p $OMNIBUS_PACKAGE_DIR - - inv -e installer.build-linux-script "default" "$VERSION" - - inv -e installer.build-linux-script "databricks" "$VERSION" - - inv -e installer.build-linux-script "emr" "$VERSION" - - inv -e installer.build-linux-script "dataproc" "$VERSION" + - deva inv -e installer.build-linux-script "default" "$VERSION" + - deva inv -e installer.build-linux-script "databricks" "$VERSION" + - deva inv -e installer.build-linux-script "emr" "$VERSION" + - deva inv -e installer.build-linux-script "dataproc" "$VERSION" - mv ./bin/installer/install*.sh $OMNIBUS_PACKAGE_DIR/ - ls -la $OMNIBUS_PACKAGE_DIR artifacts: @@ -148,7 +148,7 @@ powershell_script_signing: - rm -rf $OMNIBUS_PACKAGE_DIR/* # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" ${INSTALL_DIR_PARAM} + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project="installer" ${INSTALL_DIR_PARAM} - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.upload_sbom_artifacts] variables: @@ -202,7 +202,7 @@ installer-amd64-oci: variables: DESTINATION_FILE: "datadog-updater_7-amd64-oci.tar.xz" before_script: - - AGENT_VERSION="$(inv agent.version -u)-1" || exit $? + - AGENT_VERSION="$(deva inv agent.version -u)-1" || exit $? - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/"$AGENT_VERSION" - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR" @@ -211,7 +211,7 @@ installer-arm64-oci: variables: DESTINATION_FILE: "datadog-updater_7-arm64-oci.tar.xz" before_script: - - AGENT_VERSION="$(inv agent.version -u)-1" || exit $? + - AGENT_VERSION="$(deva inv agent.version -u)-1" || exit $? - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/"$AGENT_VERSION" - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR" diff --git a/.gitlab/package_build/linux.yml b/.gitlab/package_build/linux.yml index cb7cf752ebb5a..d517f4bd02664 100644 --- a/.gitlab/package_build/linux.yml +++ b/.gitlab/package_build/linux.yml @@ -12,7 +12,7 @@ - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR" --config-directory "$CONFIG_DIR" --install-directory "$INSTALL_DIR" + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR" --config-directory "$CONFIG_DIR" --install-directory "$INSTALL_DIR" - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.upload_sbom_artifacts] @@ -106,7 +106,7 @@ datadog-agent-7-arm64-fips: - !reference [.cache_omnibus_ruby_deps, setup] # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR/* - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor iot + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor iot - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.upload_sbom_artifacts] before_script: @@ -151,7 +151,7 @@ iot-agent-armhf: - !reference [.cache_omnibus_ruby_deps, setup] # remove artifacts from previous pipelines that may come from the cache - rm -rf $OMNIBUS_PACKAGE_DIR/* - - inv -e omnibus.build --release-version $RELEASE_VERSION_7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project dogstatsd + - deva inv -e omnibus.build --release-version $RELEASE_VERSION_7 --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project dogstatsd - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.upload_sbom_artifacts] variables: diff --git a/.gitlab/package_deps_build/package_deps_build.yml b/.gitlab/package_deps_build/package_deps_build.yml index 218efb9d8d62e..18dad6a3ec07d 100644 --- a/.gitlab/package_deps_build/package_deps_build.yml +++ b/.gitlab/package_deps_build/package_deps_build.yml @@ -27,7 +27,7 @@ - $S3_CP_CMD $S3_DD_AGENT_OMNIBUS_BTFS_URI/$BTFHUB_ARCHIVE_BRANCH/btfs-$ARCH.tar . - tar -xf btfs-$ARCH.tar - tar -xf sysprobe-build-outputs.tar.xz - - inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/${ARCH}/co-re" + - deva inv -e system-probe.generate-minimized-btfs --source-dir "$CI_PROJECT_DIR/btfs-$ARCH" --output-dir "$CI_PROJECT_DIR/minimized-btfs" --bpf-programs "$CI_PROJECT_DIR/pkg/ebpf/bytecode/build/${ARCH}/co-re" - cd minimized-btfs - tar -cJf $CI_PROJECT_DIR/minimized-btfs.tar.xz * - $S3_CP_CMD $CI_PROJECT_DIR/minimized-btfs.tar.xz $S3_PROJECT_ARTIFACTS_URI/btfs/$MIN_BTFS_FILENAME diff --git a/.gitlab/packaging/deb.yml b/.gitlab/packaging/deb.yml index 277897c6060f6..b1e19114ad6f1 100644 --- a/.gitlab/packaging/deb.yml +++ b/.gitlab/packaging/deb.yml @@ -5,7 +5,7 @@ - !reference [.cache_omnibus_ruby_deps, setup] - echo "About to package for $RELEASE_VERSION" - !reference [.setup_deb_signing_key] - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project ${DD_PROJECT} ${OMNIBUS_EXTRA_ARGS} + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project ${DD_PROJECT} ${OMNIBUS_EXTRA_ARGS} - !reference [.lint_linux_packages] artifacts: expire_in: 2 weeks @@ -82,7 +82,7 @@ agent_deb-arm64-a7-fips: - !reference [.cache_omnibus_ruby_deps, setup] - echo "About to package for $RELEASE_VERSION" - !reference [.setup_deb_signing_key] - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project ${DD_PROJECT} --flavor ot ${OMNIBUS_EXTRA_ARGS} + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project ${DD_PROJECT} --flavor ot ${OMNIBUS_EXTRA_ARGS} - !reference [.lint_linux_packages] ot_agent_deb-x64-a7: @@ -135,7 +135,7 @@ installer_deb-arm64: - !reference [.cache_omnibus_ruby_deps, setup] - echo "About to package for $RELEASE_VERSION" - !reference [.setup_deb_signing_key] - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --flavor iot + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --flavor iot - !reference [.lint_linux_packages] artifacts: expire_in: 2 weeks @@ -187,4 +187,3 @@ dogstatsd_deb-arm64: variables: DD_PROJECT: dogstatsd PACKAGE_REQUIRED_FILES_LIST: "test/required_files/dogstatsd-deb.txt" - diff --git a/.gitlab/packaging/oci.yml b/.gitlab/packaging/oci.yml index e96ece68316fc..2163afe04dea1 100644 --- a/.gitlab/packaging/oci.yml +++ b/.gitlab/packaging/oci.yml @@ -6,7 +6,7 @@ image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] before_script: - - PACKAGE_VERSION="$(inv agent.version --url-safe)-1" || exit $? + - PACKAGE_VERSION="$(deva inv agent.version --url-safe)-1" || exit $? - export INSTALL_DIR=/opt/datadog-packages/${OCI_PRODUCT}/${PACKAGE_VERSION} variables: KUBERNETES_CPU_REQUEST: 16 @@ -19,7 +19,8 @@ # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks - set +x - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - go env -w GOPRIVATE="github.com/DataDog/*" diff --git a/.gitlab/packaging/rpm.yml b/.gitlab/packaging/rpm.yml index 2bb7a0bb32015..41d6e6e1451ca 100644 --- a/.gitlab/packaging/rpm.yml +++ b/.gitlab/packaging/rpm.yml @@ -11,7 +11,7 @@ - printf -- "$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $RPM_GPG_KEY)" | gpg --import --batch - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi - RPM_SIGNING_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $RPM_SIGNING_PASSPHRASE) || exit $?; export RPM_SIGNING_PASSPHRASE - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project=${DD_PROJECT} ${OMNIBUS_EXTRA_ARGS} + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project=${DD_PROJECT} ${OMNIBUS_EXTRA_ARGS} - ls -la $OMNIBUS_PACKAGE_DIR/ - !reference [.lint_linux_packages] stage: packaging @@ -152,7 +152,7 @@ installer_suse_rpm-arm64: - printf -- "$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $RPM_GPG_KEY)" | gpg --import --batch - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi - RPM_SIGNING_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $RPM_SIGNING_PASSPHRASE) || exit $?; export RPM_SIGNING_PASSPHRASE - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --flavor=iot ${OMNIBUS_EXTRA_ARGS} + - deva inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --flavor=iot ${OMNIBUS_EXTRA_ARGS} - ls -la $OMNIBUS_PACKAGE_DIR/ - !reference [.lint_linux_packages] stage: packaging diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index 5e0e78ee5527f..96a2084e16a3b 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -54,27 +54,27 @@ send_pkg_size: - ls -l $OMNIBUS_PACKAGE_DIR_SUSE # Agent 7 - - inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "iot-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "dogstatsd" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "heroku-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-heroku-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "iot-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "dogstatsd" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "heroku-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-heroku-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_arm64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - inv package.send-size --flavor "iot-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_arm64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - inv package.send-size --flavor "dogstatsd" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_arm64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 + - deva inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_arm64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 + - deva inv package.send-size --flavor "iot-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_arm64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 + - deva inv package.send-size --flavor "dogstatsd" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_arm64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - inv package.send-size --flavor "agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "iot-agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "dogstatsd" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "iot-agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "dogstatsd" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - inv package.send-size --flavor "iot-agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 + - deva inv package.send-size --flavor "agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 + - deva inv package.send-size --flavor "iot-agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "iot-agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-iot-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "dogstatsd" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-dogstatsd-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "iot-agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-iot-agent-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 + - deva inv package.send-size --flavor "dogstatsd" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-dogstatsd-7.*.x86_64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 + - deva inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 check_pkg_size: @@ -106,4 +106,4 @@ check_pkg_size: - iot_agent_rpm-arm64 script: - !reference [.setup_agent_github_app] - - inv package.check-size + - deva inv package.check-size diff --git a/.gitlab/post_rc_build/post_rc_tasks.yml b/.gitlab/post_rc_build/post_rc_tasks.yml index 2522d4e48dd56..946be1ba8049b 100644 --- a/.gitlab/post_rc_build/post_rc_tasks.yml +++ b/.gitlab/post_rc_build/post_rc_tasks.yml @@ -15,8 +15,9 @@ update_rc_build_links: - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/requirements_release_tasks.txt --break-system-packages + # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks - PATCH=$(echo "$CI_COMMIT_REF_NAME" | cut -d'.' -f3 | cut -c1) - if [[ "$PATCH" == "0" ]]; then PATCH_OPTION=""; else PATCH_OPTION="-p"; fi - - inv -e release.update-build-links ${CI_COMMIT_REF_NAME} ${PATCH_OPTION} + - deva inv -e release.update-build-links ${CI_COMMIT_REF_NAME} ${PATCH_OPTION} diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml index 023d0add7e1c1..086a597af5781 100644 --- a/.gitlab/setup/setup.yml +++ b/.gitlab/setup/setup.yml @@ -4,7 +4,7 @@ setup_agent_version: image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] script: - - inv -e agent.version --cache-version || exit $? + - deva inv -e agent.version --cache-version || exit $? - $S3_CP_CMD $CI_PROJECT_DIR/agent-version.cache $S3_ARTIFACTS_URI/agent-version.cache needs: [] @@ -19,17 +19,18 @@ github_rate_limit_info: # Python 3.12 changes default behavior how packages are installed. # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt datadog_api_client --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks # Send stats for app 1 - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 1 + - deva inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 1 # Send stats for app 2 - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 2 + - deva inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 2 allow_failure: true diff --git a/.gitlab/source_test/common.yml b/.gitlab/source_test/common.yml index 35ad4187c3333..f41d29093f5c7 100644 --- a/.gitlab/source_test/common.yml +++ b/.gitlab/source_test/common.yml @@ -5,4 +5,4 @@ .upload_coverage: # Upload coverage files to Codecov. Never fail on coverage upload. - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV token) || exit $?; export CODECOV_TOKEN - - inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true + - deva inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true diff --git a/.gitlab/source_test/ebpf.yml b/.gitlab/source_test/ebpf.yml index 4637997de80a3..8f08a4822d327 100644 --- a/.gitlab/source_test/ebpf.yml +++ b/.gitlab/source_test/ebpf.yml @@ -17,11 +17,11 @@ - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] script: - - inv -e install-tools - - inv -e system-probe.object-files - - invoke -e linter.go --build system-probe-unit-tests --cpus 4 --targets ./pkg - - invoke -e security-agent.run-ebpf-unit-tests --verbose - - invoke -e linter.go --targets=./pkg/security/tests --cpus 4 --build-tags="functionaltests stresstests trivy containerd linux_bpf ebpf_bindata" + - deva inv -e install-tools + - deva inv -e system-probe.object-files + - deva inv -e linter.go --build system-probe-unit-tests --cpus 4 --targets ./pkg + - deva inv -e security-agent.run-ebpf-unit-tests --verbose + - deva inv -e linter.go --targets=./pkg/security/tests --cpus 4 --build-tags="functionaltests stresstests trivy containerd linux_bpf ebpf_bindata" tests_ebpf_x64: extends: .tests_linux_ebpf @@ -54,10 +54,10 @@ tests_ebpf_arm64: before_script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - inv -e install-tools + - deva inv -e install-tools - !reference [.retrieve_sysprobe_deps] script: - - inv -e kmt.prepare --ci --component="system-probe" + - deva inv -e kmt.prepare --ci --component="system-probe" prepare_sysprobe_ebpf_functional_tests_arm64: extends: .prepare_sysprobe_ebpf_functional_tests @@ -86,10 +86,10 @@ prepare_sysprobe_ebpf_functional_tests_x64: before_script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - inv -e install-tools + - deva inv -e install-tools - !reference [.retrieve_sysprobe_deps] script: - - inv -e kmt.prepare --ci --component="security-agent" + - deva inv -e kmt.prepare --ci --component="security-agent" - mkdir -p /opt/datadog-agent/embedded/bin - cp /tmp/clang-bpf /opt/datadog-agent/embedded/bin/clang-bpf - cp /tmp/llc-bpf /opt/datadog-agent/embedded/bin/llc-bpf diff --git a/.gitlab/source_test/go_generate_check.yml b/.gitlab/source_test/go_generate_check.yml index 7dd7ada2f2292..85dc790173af0 100644 --- a/.gitlab/source_test/go_generate_check.yml +++ b/.gitlab/source_test/go_generate_check.yml @@ -10,8 +10,9 @@ security_go_generate_check: before_script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - pip3 install wheel + - pip3 install wheel datadog-agent-dev=="$(cat .deva/version)" - pip3 install -r docs/cloud-workload-security/scripts/requirements-docs.txt - - inv -e install-tools + - deva self dep sync -f legacy-tasks + - deva inv -e install-tools script: - - inv -e security-agent.go-generate-check + - deva inv -e security-agent.go-generate-check diff --git a/.gitlab/source_test/golang_deps_diff.yml b/.gitlab/source_test/golang_deps_diff.yml index 3454f19a58476..61856c8553264 100644 --- a/.gitlab/source_test/golang_deps_diff.yml +++ b/.gitlab/source_test/golang_deps_diff.yml @@ -16,7 +16,7 @@ golang_deps_diff: script: # Get API key to send metrics - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - - inv -e diff.go-deps --report-file=deps-report.md --report-metrics --git-ref "${CI_COMMIT_REF_NAME}" + - deva inv -e diff.go-deps --report-file=deps-report.md --report-metrics --git-ref "${CI_COMMIT_REF_NAME}" artifacts: paths: - deps-report.md @@ -65,4 +65,4 @@ golang_deps_send_count_metrics: script: # Get API key to send metrics - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - - inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}" + - deva inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}" diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index aed975dccc481..3e4137ac2b449 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -5,10 +5,10 @@ before_script: - source /root/.bashrc && conda activate $CONDA_ENV - !reference [.retrieve_linux_go_deps] - - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev - - inv -e rtloader.install - - inv -e rtloader.format --raise-if-changed - - inv -e rtloader.test + - deva inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev + - deva inv -e rtloader.install + - deva inv -e rtloader.format --raise-if-changed + - deva inv -e rtloader.test # Placeholder script, overridden by .linux_tests when running all go tests script: ["# Skipping go tests"] @@ -27,13 +27,13 @@ EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json script: - !reference [.retrieve_linux_go_tools_deps] - - inv -e install-tools - - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH + - deva inv -e install-tools + - deva inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH - FAST_TESTS_FLAG="" - if [[ "$FAST_TESTS" == "true" ]]; then FAST_TESTS_FLAG="--only-impacted-packages"; fi - - inv -e sds.build-library - - inv -e agent.build - - inv -e test $FLAVORS --include-sds --race --profile --rerun-fails=2 --coverage --cpus $KUBERNETES_CPU_REQUEST $EXTRA_OPTS --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" --build-stdlib $FAST_TESTS_FLAG --test-washer + - deva inv -e sds.build-library + - deva inv -e agent.build + - deva inv -e test $FLAVORS --include-sds --race --profile --rerun-fails=2 --coverage --cpus $KUBERNETES_CPU_REQUEST $EXTRA_OPTS --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" --build-stdlib $FAST_TESTS_FLAG --test-washer artifacts: expire_in: 2 weeks when: always @@ -138,8 +138,8 @@ go_mod_tidy_check: before_script: - !reference [.retrieve_linux_go_deps] script: - - inv -e check-mod-tidy - - inv -e check-go-mod-replaces + - deva inv -e check-mod-tidy + - deva inv -e check-go-mod-replaces variables: KUBERNETES_MEMORY_REQUEST: "16Gi" KUBERNETES_MEMORY_LIMIT: "16Gi" @@ -164,8 +164,8 @@ new-e2e-unit-tests: # Use S3 backend - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" script: - - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH - - inv -e new-e2e-tests.run --targets ./pkg/utils --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} + - deva inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH + - deva inv -e new-e2e-tests.run --targets ./pkg/utils --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} after_script: - !reference [.upload_junit_source] variables: diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index 0da65270b98fb..1998672a91ffb 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -13,11 +13,11 @@ include: script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] - - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH + - deva inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH - FAST_TESTS_FLAG="" - if [[ "$FAST_TESTS" == "true" ]]; then FAST_TESTS_FLAG="--only-impacted-packages"; fi - - inv -e test --rerun-fails=2 --race --profile --cpus 12 --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG --test-washer --coverage - - inv -e invoke-unit-tests + - deva inv -e test --rerun-fails=2 --race --profile --cpus 12 --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG --test-washer --coverage + - deva inv -e invoke-unit-tests artifacts: expire_in: 2 weeks when: always diff --git a/.gitlab/source_test/notify.yml b/.gitlab/source_test/notify.yml index 05fc53bc0829e..85f4ba502dc1a 100644 --- a/.gitlab/source_test/notify.yml +++ b/.gitlab/source_test/notify.yml @@ -8,11 +8,12 @@ unit_tests_notify: - when: always script: # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is + # In particular, --break-system-packages command line option is # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks - !reference [.setup_agent_github_app] - - inv notify.unit-tests --pipeline-id $CI_PIPELINE_ID --pipeline-url $CI_PIPELINE_URL --branch-name $CI_COMMIT_REF_NAME + - deva inv notify.unit-tests --pipeline-id $CI_PIPELINE_ID --pipeline-url $CI_PIPELINE_URL --branch-name $CI_COMMIT_REF_NAME needs: - tests_deb-x64-py3 - tests_deb-arm64-py3 diff --git a/.gitlab/source_test/protobuf.yml b/.gitlab/source_test/protobuf.yml index 2353f8dd66576..143d6b848e1ec 100644 --- a/.gitlab/source_test/protobuf.yml +++ b/.gitlab/source_test/protobuf.yml @@ -4,5 +4,5 @@ protobuf_test: tags: ["arch:amd64"] needs: [] script: - - inv install-tools - - inv -e protobuf.generate + - deva inv install-tools + - deva inv -e protobuf.generate diff --git a/.gitlab/source_test/slack.yml b/.gitlab/source_test/slack.yml index 0912810a05c57..58e82038b43a8 100644 --- a/.gitlab/source_test/slack.yml +++ b/.gitlab/source_test/slack.yml @@ -9,4 +9,4 @@ slack_teams_channels_check: - !reference [.except_mergequeue] - when: on_success script: - - inv -e notify.check-teams + - deva inv -e notify.check-teams diff --git a/.gitlab/source_test/tooling_unit_tests.yml b/.gitlab/source_test/tooling_unit_tests.yml index 80c164d53550b..e8eacb1d871c9 100644 --- a/.gitlab/source_test/tooling_unit_tests.yml +++ b/.gitlab/source_test/tooling_unit_tests.yml @@ -8,5 +8,6 @@ invoke_unit_tests: rules: - !reference [.on_invoke_tasks_changes] script: - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - - inv -e invoke-unit-tests.run + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" --break-system-packages + - python3 -m deva self dep sync -f legacy-tasks + - deva inv -e invoke-unit-tests.run diff --git a/.gitlab/trigger_release/trigger_release.yml b/.gitlab/trigger_release/trigger_release.yml index 0cdb00f5a64ff..23307dfbffcbe 100644 --- a/.gitlab/trigger_release/trigger_release.yml +++ b/.gitlab/trigger_release/trigger_release.yml @@ -18,9 +18,9 @@ script: # agent-release-management creates pipeline for both Agent 6 and Agent 7 # when triggered with major version 7 - - RELEASE_VERSION="$(inv agent.version --url-safe --omnibus-format)-1" || exit $?; export RELEASE_VERSION + - RELEASE_VERSION="$(deva inv agent.version --url-safe --omnibus-format)-1" || exit $?; export RELEASE_VERSION - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - - 'inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-release-management" --git-ref "main" + - 'deva inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-release-management" --git-ref "main" --variable ACTION --variable AUTO_RELEASE --variable BUILD_PIPELINE_ID @@ -100,10 +100,11 @@ generate_windows_gitlab_runner_bump_pr: script: # We are using the agent platform auto PR github app to access the buildenv repository (already used for macOS builds) - !reference [.setup_github_app_agent_platform_auto_pr] - - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" + - python3 -m deva self dep sync -f legacy-tasks - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - SLACK_DATADOG_AGENT_BOT_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT token) || exit $?; export SLACK_DATADOG_AGENT_BOT_TOKEN - - inv -e github.update-windows-runner-version + - deva inv -e github.update-windows-runner-version # Manual job to generate the gitlab bump pr on buildenv if trigger_auto_staging_release fails generate_windows_gitlab_runner_bump_pr_manual: @@ -121,7 +122,8 @@ generate_windows_gitlab_runner_bump_pr_manual: script: # We are using the agent platform auto PR github app to access the buildenv repository (already used for macOS builds) - !reference [.setup_github_app_agent_platform_auto_pr] - - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt + - python3 -m pip install datadog-agent-dev=="$(cat .deva/version)" + - python3 -m deva self dep sync -f legacy-tasks - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - SLACK_DATADOG_AGENT_BOT_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT token) || exit $?; export SLACK_DATADOG_AGENT_BOT_TOKEN - - inv -e github.update-windows-runner-version + - deva inv -e github.update-windows-runner-version diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c5e547faae66..519b8e2a42fc0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,14 +54,14 @@ repos: - id: protected-branches name: protected-branches description: checks that the commit isn't created on a protected branch - entry: 'inv git.check-protected-branch' + entry: 'deva inv git.check-protected-branch' language: system pass_filenames: false stages: [pre-commit, pre-push] - id: copyright name: copyright description: copyright headers - entry: 'inv linter.copyrights --only-staged-files' + entry: 'deva inv linter.copyrights --only-staged-files' language: system require_serial: true files: \.go$ @@ -69,7 +69,7 @@ repos: - id: win-clang-format name: win-clang-format description: clang-format - entry: 'inv pre-commit.check-winclang-format' + entry: 'deva inv pre-commit.check-winclang-format' language: system types_or: [c, c++] pass_filenames: false @@ -84,7 +84,7 @@ repos: - id: go-mod-tidy name: go-mod-tidy description: check that all go.mod files are tidy - entry: 'inv tidy' + entry: 'deva inv tidy' language: system require_serial: true files: (\.go|^go\.mod|^go\.sum)$ @@ -93,7 +93,7 @@ repos: - id: go-test name: go-test description: run go test on modified packages - entry: 'inv test --only-modified-packages' + entry: 'deva inv test --only-modified-packages' language: system require_serial: true always_run: true @@ -102,7 +102,7 @@ repos: - id: go-linter name: go-linter description: run go linter on modified packages - entry: 'inv linter.go --only-modified-packages' + entry: 'deva inv linter.go --only-modified-packages' language: system require_serial: true always_run: true @@ -111,7 +111,7 @@ repos: - id: shell-check-no-set-x name: shell-check-no-set-x description: check that no `set -x` is set within github / gitlab workflows - entry: 'inv pre-commit.check-set-x' + entry: 'deva inv pre-commit.check-set-x' language: system require_serial: true files: (\.yaml|\.yml|\.sh|Dockerfile)$ @@ -119,7 +119,7 @@ repos: - id: gitlab-configuration name: gitlab-configuration description: test the gitlab configuration on main - entry: 'inv linter.gitlab-ci -t main' + entry: 'deva inv linter.gitlab-ci -t main' language: system require_serial: true files: .*gitlab.*\.yml$ @@ -128,7 +128,7 @@ repos: - id: gitlab-lint-jobs-codeowners name: gitlab-lint-jobs-codeowners description: lint the gitlab configuration to verify jobs codeowners - entry: 'inv linter.gitlab-ci-jobs-codeowners' + entry: 'deva inv linter.gitlab-ci-jobs-codeowners' language: system require_serial: true files: .*gitlab.*\.yml$ @@ -136,20 +136,20 @@ repos: - id: update-go name: update-go description: test formatting of files will allow go update - entry: 'inv linter.update-go' + entry: 'deva inv linter.update-go' language: system pass_filenames: false - id: check-go-modules-in-python name: check-go-modules-in-python description: Validate all go modules are declared in Invoke tasks - entry: 'inv modules.validate --fix-format' + entry: 'deva inv modules.validate --fix-format' language: system pass_filenames: false files: (.*go\.mod|modules\.yml|.*gomodules\.py|.*modules\.py)$ - id: protocol-buffer name: validate-protocol-buffer-generated-files description: Validate that the generated protocol buffer files are up to date - entry: 'inv protobuf.generate --no-do-mockgen' + entry: 'deva inv protobuf.generate --no-do-mockgen' language: system pass_filenames: false files: (.*go\.mod|modules\.yml|.*gomodules\.py|.*modules\.py|.*.proto)$ diff --git a/.run/Build agent.run.xml b/.run/Build agent.run.xml index 66434c2aac5b7..ffd013c68d467 100644 --- a/.run/Build agent.run.xml +++ b/.run/Build agent.run.xml @@ -13,7 +13,7 @@