main.yml

name: CI/CD Pipeline
on:
  repository_dispatch:
    types: [notebooks-updated]
jobs:
  commit-checks:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: YAML Validity check
        uses: ibiqlik/action-yamllint@v3
        with:
          file_or_dir: .
          config_data: |
            extends: relaxed
            rules:
              line-length: disable
              trailing-spaces: disable
              new-line-at-end-of-file: disable
              document-start: disable
              indentation: disable
              truthy: disable

      - name: Run GitLeaks secret scanner
        run: |
          GITLEAKS_VERSION=$(curl -s "https://api.github.com/repos/gitleaks/gitleaks/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+')
          curl -sSfL "https://github.com/gitleaks/gitleaks/releases/latest/download/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" -o gitleaks.tar.gz
          tar -xzf gitleaks.tar.gz
          ./gitleaks detect --source . --verbose

      - name: Run Ruff
        uses: astral-sh/ruff-action@v3
        with:
          src: "./src"
          args: "format --check"

  build:
    name: Build Docker Image
    needs: commit-checks
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: read
      packages: write
      id-token: write
    steps:
      - uses: actions/checkout@v2

      # This is used for buildx to use multiple architectures
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      # Checkout the repo that triggered the workflow
      - name: Checkout notebook repo
        uses: actions/checkout@v2
        with:
          repository: dprRoberta/Notebooks
          token: ${{ secrets.PAT_TOKEN }}
          path: notebooks-repo

      # Filter the notebooks and no other files to create images
      - name: Filter changed notebooks
        run: |
          mkdir -p changed_notebooks
          FILES="${{ github.event.client_payload.changed_files }}"
          echo "$FILES" | sed -E 's/(\.[^ ]+)/\1\n/g' | while IFS= read -r file; do
            file="$(echo "$file" | xargs)"
            [[ -z "$file" ]] && continue
            echo "Copying: $file"
            cp -v "notebooks-repo/$file" changed_notebooks/
          done
          echo "Files actually copied:"
          ls -l changed_notebooks

      - name: Run generator (create Dockerfile, main.py and requirements.txt)
        run: |
          pip install nbformat
          rm -rf src/services/
          cd src/
          python3 generator.py ../changed_notebooks/
          echo "Generated services:"
          ls -la services/ || echo "No services directory"
      
      - name: Install Test Dependencies
        run: |
          pip install pytest pytest-cov httpx nbformat fastapi papermill uvicorn pandas lxml

      - name: Run Tests on Generated Service
        run: |
          export PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/src
          python -m pytest --cov=src --cov-report=term-missing -s

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: eu-west-1

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2

      # Build per service directory instead of repo root
      - name: Build, tag, and push images to ECR
        id: build-push
        env:
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          ECR_REPOSITORY: notebook2rest-services
          IMAGE_TAG: ${{ github.event.client_payload.notebook_sha || github.sha }}
        run: |
          SERVICE_LIST=""

          if [ ! -d "src/services" ] || [ -z "$(ls -A src/services 2>/dev/null)" ]; then
            echo "No services to build."
            echo "services_built=false" >> $GITHUB_OUTPUT
            exit 0
          fi

          for service_dir in src/services/*/; do
            SERVICE_NAME=$(basename "$service_dir")

            # Tag format: <service-name>-<full-git-sha>
            FULL_TAG="${SERVICE_NAME}-${IMAGE_TAG}"
            FULL_IMAGE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${FULL_TAG}"

            echo "============================================"
            echo "Processing: ${SERVICE_NAME}"
            echo "Image:      ${FULL_IMAGE}"
            echo "============================================"

            # Build from the generated service directory (not repo root)
            docker build \
              -t "${FULL_IMAGE}" \
              "${service_dir}"

            docker push "${FULL_IMAGE}"

            echo "Successfully pushed: ${FULL_IMAGE}"

            # Collect service info for the GitOps update step
            # Format: service_name|ecr_repo_uri|tag
            SERVICE_LIST="${SERVICE_LIST}${SERVICE_NAME}|${ECR_REGISTRY}/${ECR_REPOSITORY}|${FULL_TAG}\n"
          done

          echo "services_built=true" >> $GITHUB_OUTPUT

          # Write the list to a file so the next step can read it
          echo -e "$SERVICE_LIST" > /tmp/built-services.txt
          echo "Built services:"
          cat /tmp/built-services.txt

      # Update the GitOps repo with new image tags
      - name: Update GitOps repo
        if: steps.build-push.outputs.services_built == 'true'
        env:
          GITOPS_REPO: dprRoberta/notebook2rest-gitops
        run: |
          # --- Git identity for the automated commit ---
          git config --global user.name "notebook2rest-ci"
          git config --global user.email "ci@notebook2rest.dev"

          # --- Clone the GitOps repo ---
          git clone https://x-access-token:${{ secrets.GITOPS_PAT }}@github.com/${GITOPS_REPO}.git gitops-repo
          cd gitops-repo

          # --- Read the list of built services ---
          SERVICES_FILE="/tmp/built-services.txt"

          if [ ! -f "$SERVICES_FILE" ]; then
            echo "No services file found - nothing to update."
            exit 0
          fi

          COMMIT_MSG="deploy:"

          while IFS='|' read -r SERVICE_NAME IMAGE_REPO IMAGE_TAG; do
            # Skip empty lines
            [ -z "$SERVICE_NAME" ] && continue

            echo "Updating values for: ${SERVICE_NAME}"
            echo "  Image: ${IMAGE_REPO}:${IMAGE_TAG}"

            # Create the app directory if it doesn't exist
            mkdir -p "apps/${SERVICE_NAME}"

            # Derive the ingress path from the service name
            INGRESS_PATH="/api/${SERVICE_NAME}"

            # Write the values.yaml for this service
            {
              echo "# Auto-generated by CI/CD pipeline"
              echo "# Source commit: ${{ github.event.client_payload.notebook_sha }}"
              echo "# Generated at: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
              echo ""
              echo "fullnameOverride: \"${SERVICE_NAME}\""
              echo ""
              echo "image:"
              echo "  repository: \"${IMAGE_REPO}\""
              echo "  tag: \"${IMAGE_TAG}\""
              echo ""
              echo "ingress:"
              echo "  path: \"${INGRESS_PATH}\""
              echo ""
              echo "env:"
              echo "  API_PREFIX: \"${INGRESS_PATH}\""
            } > "apps/${SERVICE_NAME}/values.yaml"

            COMMIT_MSG="${COMMIT_MSG} ${SERVICE_NAME}@${IMAGE_TAG::12}"
          done < "$SERVICES_FILE"

          # --- Commit and push if there are changes ---
          git add apps/
          if git diff --cached --quiet; then
            echo "No changes to commit."
          else
            # Pull-rebase in case another pipeline run pushed in the meantime
            git pull --rebase origin main || true
            git commit -m "${COMMIT_MSG}"
            git push origin main
            echo "Successfully pushed GitOps update."
          fi