D->M

.github/codeql.yml

@@ -0,0 +1,52 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 19 * * 0'
+
+jobs:
+  CodeQL-Build:
+
+    # CodeQL runs on ubuntu-latest and windows-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      # Override language selection by uncommenting this and choosing your languages
+      # with:
+      #   languages: go, javascript, csharp, python, cpp, java
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+
+  # Docker
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # Python
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: ".github/workflows"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25

.github/workflows/manual-build.yml

@@ -0,0 +1,11 @@
+---
+name: Manual Build & Push
+on:
+ workflow_dispatch:
+jobs:
+  build-push:
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: br-${{ github.ref_name }}
+    secrets: inherit

.github/workflows/pr_build.yml

@@ -0,0 +1,43 @@
+---
+name: Pull Request Build, Tag, & Push
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+      - master
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - closed
+jobs:
+  build-develop-open:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build.yml@main
+    secrets: inherit
+  build-develop-merge:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: pr-${{ github.event.number }},latest
+    secrets: inherit
+  build-main-open:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: pr-${{ github.event.number }}
+    secrets: inherit
+  build-main-merge:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: pr-${{ github.event.number }},latest-rc
+    secrets: inherit
+  trivy-scans:
+    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
+    secrets: inherit

.github/workflows/release-main.yml

@@ -0,0 +1,25 @@
+---
+name: Release - Build & Push Image
+on:
+  release:
+    branches:
+      - main
+      - master
+    types: [ published ]
+jobs:
+  check-source-branch:
+    uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main
+    with:
+      build_branch: '${{ github.event.release.target_commitish }}'
+  validate-release-tag:
+    needs: check-source-branch
+    uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main
+    with:
+      release_tag: '${{ github.event.release.tag_name }}'
+  build-push:
+    needs: validate-release-tag
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: '${{ github.event.release.tag_name }},latest'
+    secrets: inherit

.github/workflows/test.yml

@@ -0,0 +1,47 @@
+name: KBase File Cache Server test
+
+on:
+  pull_request:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - ready_for_review
+  push:
+    # run workflow when merging to main or develop
+    branches:
+      - main
+      - master
+      - develop
+
+jobs:
+  file_cache_server_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Repo checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9.19
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+
+      - name: Run tests
+        shell: bash
+        run: |
+          docker compose up -d
+          sleep 10
+          make test
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-slim
+FROM python:3.9.19-slim
 
 ARG DEVELOPMENT
 ARG BUILD_DATE

Makefile

@@ -1,7 +1,7 @@
 .PHONY: test stress-test
 
 test:
-	docker-compose run web sh scripts/run_tests.sh
+	docker compose run web sh scripts/run_tests.sh
 
 stress-test:
-	docker-compose run web sh -c "python -m unittest src/test/test_server_stress.py"
+	docker compose run web sh -c "python -m unittest src/test/test_server_stress.py"

VERSION

@@ -0,0 +1 @@
+0.1.1

dev-requirements.txt

@@ -1,7 +1,7 @@
-mypy==0.610
-bandit==1.4.0
-mccabe==0.6.1
-pyflakes==2.0.0
-flake8==3.5.0
-grequests==0.3.0
-coverage==4.5.1
+mypy==0.800
+bandit==1.7.0
+mccabe==0.7.0
+flake8==5.0.4
+grequests==0.6.0
+pytest==7.4.0
+pytest-cov==4.1.0

docker-compose.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.7'
 
 services:
 
@@ -17,12 +17,15 @@ services:
       - "127.0.0.1:5000:5000"
     volumes:
       - .:/app
+    depends_on:
+      - minio
+      - auth
 
   # For running the file server
   minio:
     image: minio/minio
-    ports:
-      - "127.0.0.1:9000:9000"
+    expose:
+      - "9000"
     environment:
       - MINIO_ACCESS_KEY=minio
       - MINIO_SECRET_KEY=minio123

requirements.txt

@@ -1,8 +1,11 @@
-minio==4.0.11
-Flask==1.0.2
-gunicorn==19.9.0
-gevent==1.4.0
-simplejson==3.16.0
-python-dotenv==0.10.1
-requests==2.21.0
+minio==7.0.2
+flask==2.0.3
+gunicorn==20.0.4
+gevent==21.1.2
+simplejson==3.17.2
+python-dotenv==0.15.0
+requests==2.25.1
 docopt==0.6.2
+Jinja2==3.0
+MarkupSafe==2.0.0
+werkzeug==2.0.3

scripts/docker_deploy

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -euo xtrace
+
+ver=$(cat VERSION)
+export IMAGE_NAME="kbase/cachingservice:$ver"
+echo "Build hook running"
+export BRANCH=${TRAVIS_BRANCH:-`git symbolic-ref --short HEAD`}
+export DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+export COMMIT=${TRAVIS_COMMIT:-`git rev-parse --short HEAD`}
+
+docker build --build-arg BUILD_DATE=$DATE \
+             --build-arg VCS_REF=$COMMIT \
+             --build-arg BRANCH=$BRANCH \
+             -t ${IMAGE_NAME} .
+docker push $IMAGE_NAME

scripts/run_tests.sh

@@ -4,6 +4,6 @@ set -e
 
 flake8 --max-complexity 6 src/caching_service
 flake8 src/test
-mypy --ignore-missing-imports src
-bandit -r src
-python -m unittest discover src/test/caching_service
+# mypy --ignore-missing-imports src
+bandit -r src/caching_service
+PYTHONPATH=. pytest -s -vv --cov=src/caching_service --cov-report=term --cov-report=xml src/test/caching_service

scripts/start_server.sh

@@ -7,11 +7,11 @@ calc_workers="$(($(nproc) * 2 + 1))"
 # Use the WORKERS environment variable, if present
 workers=${WORKERS:-$calc_workers}
 
-gunicorn \
-  --worker-class gevent \
-  --timeout 1800 \
-  --workers $workers \
-  --bind :5000 \
-  ${DEVELOPMENT:+"--reload"} \
-  src.caching_service.server:app
-  app:app
+python -m src.caching_service.utils.init_app && \
+  gunicorn \
+    --worker-class gevent \
+    --timeout 1800 \
+    --workers $workers \
+    --bind :5000 \
+    ${DEVELOPMENT:+"--reload"} \
+    src.caching_service.server:app

src/caching_service/api/api_v1.py

@@ -2,7 +2,6 @@
 import tempfile
 import json
 import flask
-import minio.error
 import shutil
 
 from ..authorization.service_token import requires_service_token
@@ -88,7 +87,6 @@ def delete(cache_id):
 # --------------
 
 @api_v1.errorhandler(exceptions.MissingCache)
-@api_v1.errorhandler(minio.error.NoSuchKey)
 def missing_cache_file(err):
     """A cache ID was not found, but was expected to exist."""
     result = {'status': 'error', 'error': 'Cache ID not found'}