feat: Add automatic backup to Google Drive and local disk #9392
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| push: | |
| branches: [ 'main', 'rollback/**' ] | |
| # אל תתעלם משינויים ב-.cursorrules כדי שה-CI ירוץ גם על PRים שמעדכנים אותו | |
| pull_request: | |
| branches: [ '**' ] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| # אל תתעלם מ-.cursorrules גם כאן | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| security-events: write | |
| statuses: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| code-quality: | |
| name: "🔍 Code Quality & Security" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: 🐍 Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements/*.txt | |
| - name: 📦 Install Linters & Tools | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install flake8 black isort mypy bandit safety ruff pip-audit | |
| - name: 📋 Lint (Flake8) | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: 🎨 Format Check (Black) | |
| run: black --check --diff . || true | |
| - name: 🔡 Import Sort Check (isort) | |
| run: isort --check-only --diff . || true | |
| - name: 🏷️ Type Check (MyPy) | |
| run: | | |
| # Run focused strict checks for modules we hardened | |
| mypy services/code_service.py services/__init__.py code_preview.py autocomplete_manager.py --config-file mypy.ini --show-error-codes | |
| # Run broad, permissive pass for the rest (non-blocking for now) | |
| mypy . --ignore-missing-imports --show-error-codes | tee mypy-output.txt || true | |
| if grep -E '\[(attr-defined|return-value)\]' mypy-output.txt; then | |
| echo "Found mypy attr-defined/return-value errors above" | |
| exit 1 | |
| fi | |
| - name: 🔒 Security Scan (Bandit) | |
| run: bandit -q -r . || true | |
| - name: 🛡️ Dependency Check (Safety) | |
| run: safety check --full-report || true | |
| - name: 📦 Dependency Audit (pip-audit) | |
| run: | | |
| pip-audit -r requirements/production.txt -f json -o pip-audit.json || true | |
| - name: 📊 Upload dependency audit | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dependency-audit | |
| path: pip-audit.json | |
| if-no-files-found: ignore | |
| - name: 🐶 Ruff (optional) | |
| run: ruff check . || true | |
| - name: Report required status (pending) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'pending', | |
| context: 'Code Quality & Security', | |
| description: 'Job queued/running', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| - name: Report required status (success) | |
| if: ${{ success() }} | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'success', | |
| context: 'Code Quality & Security', | |
| description: 'Checks passed', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| - name: Report required status (failure) | |
| if: ${{ failure() }} | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'failure', | |
| context: 'Code Quality & Security', | |
| description: 'Checks failed', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| - name: Report required status (cancelled) | |
| if: ${{ cancelled() }} | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'failure', | |
| context: 'Code Quality & Security', | |
| description: 'Job cancelled', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| unit-tests: | |
| name: "Unit Tests (${{ matrix.python-version }})" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ '3.11', '3.12' ] | |
| services: | |
| mongodb: | |
| image: mongo:6.0 | |
| env: | |
| MONGO_INITDB_ROOT_USERNAME: test | |
| MONGO_INITDB_ROOT_PASSWORD: test123 | |
| # Avoid host port binding to prevent conflicts; job connects via service hostname | |
| options: >- | |
| --health-cmd "echo 'db.runCommand(\"ping\").ok' | mongosh localhost:27017/test --quiet" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| redis: | |
| image: redis:7-alpine | |
| # Avoid host port binding; job connects via service hostname | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: 📥 Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements/*.txt | |
| - name: 📦 Install Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| # התקנה ראשונית (לצורך יצירת constraints) | |
| pip install -r requirements/development.txt | |
| # הפקת constraints אוטומטי ושימוש בו | |
| pip freeze | sort > constraints.txt | |
| pip install -r requirements/development.txt -c constraints.txt | |
| - name: 🔎 Dependency integrity (pip check) | |
| run: pip check || true | |
| - name: Report required status (pending) | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| const ver = process.env.PYV || '${{ matrix.python-version }}'; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'pending', | |
| context: `Unit Tests (${ver})`, | |
| description: 'Job queued/running', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| env: | |
| PYV: ${{ matrix.python-version }} | |
| - name: 🧪 Smoke compile (exclude .restore) | |
| run: | | |
| find . -type f -name "*.py" -not -path "./.restore/*" -print0 | xargs -0 -n1 -P 4 python -m py_compile | |
| - name: 🧪 Run Tests | |
| env: | |
| BOT_TOKEN: ${{ secrets.TEST_BOT_TOKEN || '1234567890:TEST_TOKEN_FOR_TESTING' }} | |
| MONGODB_URL: mongodb://test:test123@mongodb:27017/test_db?authSource=admin | |
| REDIS_URL: redis://redis:6379/0 | |
| GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} | |
| PASTEBIN_API_KEY: ${{ secrets.TEST_PASTEBIN_KEY }} | |
| FEATURE_MY_COLLECTIONS: '1' | |
| # Minimize retry/backoff delays during tests to avoid long waits | |
| HTTP_RESILIENCE_MAX_ATTEMPTS: '1' | |
| HTTP_RESILIENCE_BACKOFF_BASE: '0' | |
| HTTP_RESILIENCE_BACKOFF_MAX: '0' | |
| HTTP_RESILIENCE_JITTER: '0' | |
| REQUESTS_RETRIES: '0' | |
| REQUESTS_RETRY_BACKOFF: '0' | |
| AIOHTTP_TIMEOUT_TOTAL: '6' | |
| run: | | |
| pytest -n auto --dist=loadscope -v -o addopts="" --cov=. --cov-report=xml --cov-report=term \ | |
| --durations=0 --json-report --json-report-file=unit-durations.json | |
| - name: Upload coverage to Codecov | |
| if: always() | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: coverage.xml | |
| fail_ci_if_error: false | |
| verbose: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload unit test durations artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unit-durations-${{ matrix.python-version }}-${{ github.run_id }} | |
| path: | | |
| unit-durations.json | |
| if-no-files-found: ignore | |
| - name: Report required status (success) | |
| if: ${{ success() }} | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| const ver = process.env.PYV || '${{ matrix.python-version }}'; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'success', | |
| context: `Unit Tests (${ver})`, | |
| description: 'Tests passed', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| env: | |
| PYV: ${{ matrix.python-version }} | |
| - name: Report required status (failure) | |
| if: ${{ failure() }} | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| const ver = process.env.PYV || '${{ matrix.python-version }}'; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'failure', | |
| context: `Unit Tests (${ver})`, | |
| description: 'Tests failed', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| env: | |
| PYV: ${{ matrix.python-version }} | |
| - name: Report required status (cancelled) | |
| if: ${{ cancelled() }} | |
| uses: actions/github-script@v7 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| const ver = process.env.PYV || '${{ matrix.python-version }}'; | |
| await github.request('POST /repos/{owner}/{repo}/statuses/{sha}', { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| state: 'failure', | |
| context: `Unit Tests (${ver})`, | |
| description: 'Job cancelled', | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| env: | |
| PYV: ${{ matrix.python-version }} | |
| hadolint: | |
| name: "Dockerfile lint (hadolint)" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run hadolint | |
| uses: hadolint/hadolint-action@v3.1.0 | |
| with: | |
| dockerfile: Dockerfile | |
| failure-threshold: error | |
| gitleaks: | |
| name: "Secrets scan (gitleaks)" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Run gitleaks | |
| uses: gitleaks/gitleaks-action@v2 | |
| with: | |
| args: detect --no-banner --redact --source . | |
| continue-on-error: true | |
| semgrep: | |
| name: "SAST (Semgrep)" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Semgrep scan | |
| uses: returntocorp/semgrep-action@v1 | |
| with: | |
| config: p/ci | |
| generateSarif: true | |
| env: | |
| SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} | |
| continue-on-error: true | |
| yamllint: | |
| name: "YAML lint" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run yamllint | |
| uses: ibiqlik/action-yamllint@v3 | |
| with: | |
| file_or_dir: . | |
| config_file: .yamllint.yaml | |
| continue-on-error: true | |
| lychee: | |
| name: "Link checker (lychee)" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check links | |
| uses: lycheeverse/lychee-action@v1 | |
| with: | |
| args: --no-progress README.md docs/ | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| continue-on-error: true | |
| alembic-migrations: | |
| name: "Alembic migrations check" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| services: | |
| postgres: | |
| image: postgres:16 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: app | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd="pg_isready -U postgres" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check if alembic.ini exists | |
| id: has_alembic | |
| run: | | |
| if [ -f alembic.ini ]; then | |
| echo "exists=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "exists=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Skip if missing | |
| if: steps.has_alembic.outputs.exists == 'false' | |
| run: echo "alembic.ini not found; skipping migration check." | |
| - uses: actions/setup-python@v5 | |
| if: steps.has_alembic.outputs.exists == 'true' | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| if: steps.has_alembic.outputs.exists == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements/production.txt | |
| pip freeze | sort > constraints.txt | |
| pip install -r requirements/production.txt -c constraints.txt | |
| pip install alembic psycopg2-binary | |
| - name: Wait for DB | |
| if: steps.has_alembic.outputs.exists == 'true' | |
| run: | | |
| for i in {1..30}; do | |
| if pg_isready -h localhost -p 5432 -U postgres; then | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| - name: Run Alembic migrations | |
| if: steps.has_alembic.outputs.exists == 'true' | |
| env: | |
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/app | |
| run: | | |
| alembic upgrade head | |