2025 06 03 parallel credentials

.github/workflows/pipelines-drift-detection.yml

@@ -30,14 +30,31 @@ on:
       api_base_url:
         type: string
         default: "https://api.prod.app.gruntwork.io/api/v1"
+      pipelines_binary_url:
+        type: string
+        default: ""
+        description: "Override where we fetch pipelines from, used for internal testing"
+      pipelines_cli_version:
+        type: string
+        default: "v0.39.0"
+        description: "For Gruntwork internal testing - the version of the pipelines CLI to use"
+      pipelines_actions_ref:
+        type: string
+        default: "v3.6.4"
+        description: "For Gruntwork internal testing - the ref of the pipelines actions to use"
+      pipelines_credentials_ref:
+        type: string
+        default: "v1"
+        description: "For Gruntwork internal testing - the ref of the pipelines credentials to use"
     secrets:
         PIPELINES_READ_TOKEN:
           required: false
         PR_CREATE_TOKEN:
           required: false
 env:
-  PIPELINES_CLI_VERSION: v0.39.0
-  PIPELINES_ACTIONS_VERSION: v3.6.4
+  PIPELINES_CLI_VERSION: ${{ inputs.pipelines_cli_version }}
+  PIPELINES_ACTIONS_REF: ${{ inputs.pipelines_actions_ref }}
+  PIPELINES_CREDENTIALS_REF: ${{ inputs.pipelines_credentials_ref }}
   BOILERPLATE_VERSION: v0.5.16
   GRUNTWORK_INSTALLER_VERSION: v0.0.40
 
@@ -48,37 +65,59 @@ jobs:
     outputs:
       units: ${{ steps.determine-units.outputs.units }}
     steps:
-      - name: Fetch Gruntwork Read Token
-        id: pipelines-gruntwork-read-token
-        uses: gruntwork-io/pipelines-credentials@v1
-        with:
-          PIPELINES_TOKEN_PATH: "pipelines-read/gruntwork-io"
-          FALLBACK_TOKEN: ${{ secrets.PIPELINES_READ_TOKEN }}
-          api_base_url: ${{ inputs.api_base_url }}
+      - name: Record workflow env vars
+        env:
+          PIPELINES_BINARY_URL: ${{ inputs.pipelines_binary_url }}
+        run: |
+          time_now=$(date -u +"%s")
+          echo "PIPELINES_JOB_START_TIME=$time_now" >> $GITHUB_ENV
+          echo "PIPELINES_BINARY_URL=$PIPELINES_BINARY_URL" >> $GITHUB_ENV
 
+      - name: Checkout Pipelines Credentials
+        uses: actions/checkout@v4
+        with:
+          path: pipelines-credentials
+          repository: gruntwork-io/pipelines-credentials
+          ref: ${{ env.PIPELINES_CREDENTIALS_REF }}
 
-      - name: Fetch Org Read Token
-        id: pipelines-customer-org-read-token
-        uses: gruntwork-io/pipelines-credentials@v1
+      - name: Fetch Pipelines Tokens
+        id: pipelines-tokens
+        uses: ./pipelines-credentials
         with:
-          PIPELINES_TOKEN_PATH: pipelines-read/${{ github.repository_owner }}
-          FALLBACK_TOKEN: ${{ secrets.PIPELINES_READ_TOKEN }}
+          PIPELINES_TOKEN_PATHS: |
+            [
+              {
+                "name": "gruntwork_read",
+                "path": "pipelines-read/gruntwork-io",
+                "fallback_secret": "${{ secrets.PIPELINES_READ_TOKEN }}"
+              },
+              {
+                "name": "customer_org_read",
+                "path": "pipelines-read/${{ github.repository_owner }}",
+                "fallback_secret": "${{ secrets.PIPELINES_READ_TOKEN }}"
+              },
+              {
+                "name": "pr_create",
+                "path": "propose-infra-change/${{ github.repository_owner }}",
+                "fallback_secret": "${{ secrets.PR_CREATE_TOKEN }}"
+              }
+            ]
           api_base_url: ${{ inputs.api_base_url }}
 
       - name: Checkout Pipelines Actions
         uses: actions/checkout@v4
         with:
           path: pipelines-actions
           repository: gruntwork-io/pipelines-actions
-          ref: ${{ env.PIPELINES_ACTIONS_VERSION }}
-          token: ${{ steps.pipelines-gruntwork-read-token.outputs.PIPELINES_TOKEN }}
+          ref: ${{ env.PIPELINES_ACTIONS_REF }}
+          token: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).gruntwork_read }}
 
       - name: Check out repo code
         uses: actions/checkout@v4
         with:
           path: infra-live-repo
           fetch-depth: 0
-          token: ${{ steps.pipelines-customer-org-read-token.outputs.PIPELINES_TOKEN }}
+          token: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).customer_org_read }}
 
       - name: Determine Units
         id: determine-units
@@ -99,97 +138,119 @@ jobs:
       JOB_NAME: Detect Drift in ${{ matrix.unit.path }}
     name: Detect Drift in ${{ matrix.unit.path }}
     steps:
-      - name: Fetch Gruntwork Read Token
-        id: pipelines-gruntwork-read-token
-        uses: gruntwork-io/pipelines-credentials@v1
+      - name: Checkout Pipelines Credentials
+        uses: actions/checkout@v4
         with:
-          PIPELINES_TOKEN_PATH: "pipelines-read/gruntwork-io"
-          FALLBACK_TOKEN: ${{ secrets.PIPELINES_READ_TOKEN }}
-          api_base_url: ${{ inputs.api_base_url }}
+          path: pipelines-credentials
+          repository: gruntwork-io/pipelines-credentials
+          ref: ${{ env.PIPELINES_CREDENTIALS_REF }}
 
-      - name: Fetch Org Read Token
-        id: pipelines-customer-org-read-token
-        uses: gruntwork-io/pipelines-credentials@v1
+      - name: Fetch Pipelines Tokens
+        id: pipelines-tokens
+        uses: ./pipelines-credentials
         with:
-          PIPELINES_TOKEN_PATH: pipelines-read/${{ github.repository_owner }}
-          FALLBACK_TOKEN: ${{ secrets.PIPELINES_READ_TOKEN }}
+          PIPELINES_TOKEN_PATHS: |
+            [
+              {
+                "name": "gruntwork_read",
+                "path": "pipelines-read/gruntwork-io",
+                "fallback_secret": "${{ secrets.PIPELINES_READ_TOKEN }}"
+              },
+              {
+                "name": "customer_org_read",
+                "path": "pipelines-read/${{ github.repository_owner }}",
+                "fallback_secret": "${{ secrets.PIPELINES_READ_TOKEN }}"
+              },
+              {
+                "name": "pr_create",
+                "path": "propose-infra-change/${{ github.repository_owner }}",
+                "fallback_secret": "${{ secrets.PR_CREATE_TOKEN }}"
+              }
+            ]
           api_base_url: ${{ inputs.api_base_url }}
 
       - name: Checkout Pipelines Actions
         uses: actions/checkout@v4
         with:
           path: pipelines-actions
           repository: gruntwork-io/pipelines-actions
-          ref: ${{ env.PIPELINES_ACTIONS_VERSION }}
-          token: ${{ steps.pipelines-gruntwork-read-token.outputs.PIPELINES_TOKEN }}
+          ref: ${{ env.PIPELINES_ACTIONS_REF }}
+          token: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).gruntwork_read }}
 
       - name: Check out repo code
         uses: actions/checkout@v4
         with:
           path: infra-live-repo
           fetch-depth: 0
-          token: ${{ steps.pipelines-customer-org-read-token.outputs.PIPELINES_TOKEN }}
+          token: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).customer_org_read }}
 
-      - name: Execute Drift Detection
-        id: determine-drift
-        uses: ./pipelines-actions/.github/actions/pipelines-drift-detection-determine-drift
+      - name: Pipelines Drift Detector
+        id: detect_drift
+        uses: ./pipelines-actions/.github/actions/pipelines-drift-detect
         with:
-          PIPELINES_GRUNTWORK_READ_TOKEN: ${{ steps.pipelines-gruntwork-read-token.outputs.PIPELINES_TOKEN }}
-          PIPELINES_CUSTOMER_ORG_READ_TOKEN: ${{ steps.pipelines-customer-org-read-token.outputs.PIPELINES_TOKEN }}
-          UNIT_ID: ${{ matrix.unit.id }}
-          UNIT_PATH: ${{ matrix.unit.path }}
-          JOB_NAME: ${{ env.JOB_NAME }}
-          STEP_NAME: "Execute Drift Detection"
+          PIPELINES_GRUNTWORK_READ_TOKEN: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).gruntwork_read }}
+          PIPELINES_CUSTOMER_ORG_READ_TOKEN: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).customer_org_read }}
+          PR_COMMENT_WRITE_TOKEN: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).pr_create }}
+          terragrunt_command: "run-all plan"
+          # Disable automatic plan for drift detection
+          TF_INPUT: 0
 
   consolidate_jobs:
     name: Consolidate Jobs
     runs-on: ${{ fromJSON(inputs.runner) }}
     needs: pipelines_drift_detection
     steps:
-      - name: Fetch Gruntwork Read Token
-        id: pipelines-gruntwork-read-token
-        uses: gruntwork-io/pipelines-credentials@v1
-        with:
-          PIPELINES_TOKEN_PATH: "pipelines-read/gruntwork-io"
-          FALLBACK_TOKEN: ${{ secrets.PIPELINES_READ_TOKEN }}
-          api_base_url: ${{ inputs.api_base_url }}
-
-      - name: Fetch Org Read Token
-        id: pipelines-customer-org-read-token
-        uses: gruntwork-io/pipelines-credentials@v1
+      - name: Checkout Pipelines Credentials
+        uses: actions/checkout@v4
         with:
-          PIPELINES_TOKEN_PATH: pipelines-read/${{ github.repository_owner }}
-          FALLBACK_TOKEN: ${{ secrets.PIPELINES_READ_TOKEN }}
-          api_base_url: ${{ inputs.api_base_url }}
+          path: pipelines-credentials
+          repository: gruntwork-io/pipelines-credentials
+          ref: ${{ env.PIPELINES_CREDENTIALS_REF }}
 
-      - name: Fetch Create PR Token
-        id: pipelines-propose-infra-change-token
-        uses: gruntwork-io/pipelines-credentials@v1
+      - name: Fetch Pipelines Tokens
+        id: pipelines-tokens
+        uses: ./pipelines-credentials
         with:
-          PIPELINES_TOKEN_PATH: propose-infra-change/${{ github.repository_owner }}
-          FALLBACK_TOKEN: ${{ secrets.PR_CREATE_TOKEN }}
+          PIPELINES_TOKEN_PATHS: |
+            [
+              {
+                "name": "gruntwork_read",
+                "path": "pipelines-read/gruntwork-io",
+                "fallback_secret": "${{ secrets.PIPELINES_READ_TOKEN }}"
+              },
+              {
+                "name": "customer_org_read",
+                "path": "pipelines-read/${{ github.repository_owner }}",
+                "fallback_secret": "${{ secrets.PIPELINES_READ_TOKEN }}"
+              },
+              {
+                "name": "pr_create",
+                "path": "propose-infra-change/${{ github.repository_owner }}",
+                "fallback_secret": "${{ secrets.PR_CREATE_TOKEN }}"
+              }
+            ]
           api_base_url: ${{ inputs.api_base_url }}
 
       - name: Checkout Pipelines Actions
         uses: actions/checkout@v4
         with:
           path: pipelines-actions
           repository: gruntwork-io/pipelines-actions
-          ref: ${{ env.PIPELINES_ACTIONS_VERSION }}
-          token: ${{ steps.pipelines-gruntwork-read-token.outputs.PIPELINES_TOKEN }}
+          ref: ${{ env.PIPELINES_ACTIONS_REF }}
+          token: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).gruntwork_read }}
 
       - name: Check out repo code
         uses: actions/checkout@v4
         with:
           path: infra-live-repo
           fetch-depth: 0
-          token: ${{ steps.pipelines-customer-org-read-token.outputs.PIPELINES_TOKEN }}
+          token: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).customer_org_read }}
 
       - name: "Consolidate Jobs"
         id: consolidate-jobs
         uses: ./pipelines-actions/.github/actions/pipelines-drift-detection-consolidate-jobs
         with:
           PATH_FILTER: ${{ inputs.path }}
-          PIPELINES_GRUNTWORK_READ_TOKEN: ${{ steps.pipelines-gruntwork-read-token.outputs.PIPELINES_TOKEN }}
-          PR_CREATE_TOKEN: ${{ steps.pipelines-propose-infra-change-token.outputs.PIPELINES_TOKEN }}
+          PIPELINES_GRUNTWORK_READ_TOKEN: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).gruntwork_read }}
+          PR_CREATE_TOKEN: ${{ fromJSON(steps.pipelines-tokens.outputs.PIPELINES_TOKENS).pr_create }}
           BRANCH_NAME: ${{ inputs.branch-name }}