From e27bcd5c6d2a9533a73b040bb9b65aeed923c887 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Tue, 23 Sep 2025 13:37:49 +0930 Subject: [PATCH 01/12] Initial attempt at helm chart --- .github/ct-config.yaml | 11 + .github/workflows/helm-chart.yml | 292 ++++++++ helm-chart/DEPLOYMENT.md | 635 ++++++++++++++++++ helm-chart/stac-fastapi/Chart.yaml | 32 + helm-chart/stac-fastapi/README.md | 402 +++++++++++ helm-chart/stac-fastapi/templates/NOTES.txt | 84 +++ .../stac-fastapi/templates/_helpers.tpl | 149 ++++ .../stac-fastapi/templates/deployment.yaml | 59 ++ helm-chart/stac-fastapi/templates/hpa.yaml | 28 + .../stac-fastapi/templates/ingress.yaml | 59 ++ .../stac-fastapi/templates/networkpolicy.yaml | 27 + .../templates/poddisruptionbudget.yaml | 18 + .../stac-fastapi/templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 10 + .../templates/servicemonitor.yaml | 20 + .../stac-fastapi/values-elasticsearch.yaml | 155 +++++ helm-chart/stac-fastapi/values-external.yaml | 52 ++ .../stac-fastapi/values-opensearch.yaml | 164 +++++ helm-chart/stac-fastapi/values.yaml | 359 ++++++++++ helm-chart/test-chart.sh | 452 +++++++++++++ 20 files changed, 3023 insertions(+) create mode 100644 .github/ct-config.yaml create mode 100644 .github/workflows/helm-chart.yml create mode 100644 helm-chart/DEPLOYMENT.md create mode 100644 helm-chart/stac-fastapi/Chart.yaml create mode 100644 helm-chart/stac-fastapi/README.md create mode 100644 helm-chart/stac-fastapi/templates/NOTES.txt create mode 100644 helm-chart/stac-fastapi/templates/_helpers.tpl create mode 100644 helm-chart/stac-fastapi/templates/deployment.yaml create mode 100644 helm-chart/stac-fastapi/templates/hpa.yaml create mode 100644 helm-chart/stac-fastapi/templates/ingress.yaml create mode 100644 helm-chart/stac-fastapi/templates/networkpolicy.yaml create mode 100644 helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml create mode 100644 helm-chart/stac-fastapi/templates/service.yaml create mode 100644 helm-chart/stac-fastapi/templates/serviceaccount.yaml create mode 100644 helm-chart/stac-fastapi/templates/servicemonitor.yaml create mode 100644 helm-chart/stac-fastapi/values-elasticsearch.yaml create mode 100644 helm-chart/stac-fastapi/values-external.yaml create mode 100644 helm-chart/stac-fastapi/values-opensearch.yaml create mode 100644 helm-chart/stac-fastapi/values.yaml create mode 100755 helm-chart/test-chart.sh diff --git a/.github/ct-config.yaml b/.github/ct-config.yaml new file mode 100644 index 000000000..7a0653bd4 --- /dev/null +++ b/.github/ct-config.yaml @@ -0,0 +1,11 @@ +# Configuration for chart-testing +chart-dirs: + - helm-chart + +chart-repos: + - elastic=https://helm.elastic.co + - opensearch=https://opensearch-project.github.io/helm-charts/ + +helm-extra-args: --timeout 10m + +validate-maintainers: false \ No newline at end of file diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml new file mode 100644 index 000000000..6f014ba14 --- /dev/null +++ b/.github/workflows/helm-chart.yml @@ -0,0 +1,292 @@ +name: Helm Chart CI + +on: + push: + paths: + - 'helm-chart/**' + - '.github/workflows/helm-chart.yml' + pull_request: + paths: + - 'helm-chart/**' + - '.github/workflows/helm-chart.yml' + +env: + HELM_VERSION: v3.13.0 + KUBECTL_VERSION: v1.28.0 + +jobs: + lint-and-test: + runs-on: ubuntu-latest + strategy: + matrix: + backend: [elasticsearch, opensearch] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBECTL_VERSION }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.6.1 + + - name: Add Helm repositories + run: | + helm repo add elastic https://helm.elastic.co + helm repo add opensearch https://opensearch-project.github.io/helm-charts/ + helm repo update + + - name: Lint Helm chart + run: | + cd helm-chart/stac-fastapi + helm dependency update + helm lint . + + - name: Template Helm chart + run: | + cd helm-chart/stac-fastapi + helm template test-release . \ + --set backend=${{ matrix.backend }} \ + --set ${{ matrix.backend }}.enabled=true \ + --set app.image.tag=latest \ + --output-dir /tmp/helm-test-${{ matrix.backend }} + + - name: Validate templated manifests + run: | + # Check that all required resources are created + ls -la /tmp/helm-test-${{ matrix.backend }}/stac-fastapi/templates/ + + # Validate YAML syntax + find /tmp/helm-test-${{ matrix.backend }} -name "*.yaml" -exec kubectl apply --dry-run=client -f {} \; + + - name: Run chart-testing (lint) + run: | + ct lint --config .github/ct-config.yaml --charts helm-chart/stac-fastapi + + integration-test: + runs-on: ubuntu-latest + needs: lint-and-test + strategy: + matrix: + backend: [elasticsearch, opensearch] + k8s-version: ['1.26.6', '1.27.3', '1.28.0'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBECTL_VERSION }} + + - name: Create kind cluster + uses: helm/kind-action@v1.8.0 + with: + node_image: kindest/node:v${{ matrix.k8s-version }} + cluster_name: kind + config: | + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + + - name: Add Helm repositories + run: | + helm repo add elastic https://helm.elastic.co + helm repo add opensearch https://opensearch-project.github.io/helm-charts/ + helm repo update + + - name: Install STAC FastAPI chart + run: | + cd helm-chart/stac-fastapi + helm dependency update + + # Install with specific backend + helm install stac-fastapi-test . \ + --namespace stac-fastapi \ + --create-namespace \ + --set backend=${{ matrix.backend }} \ + --set ${{ matrix.backend }}.enabled=true \ + --set app.image.tag=latest \ + --set app.service.type=ClusterIP \ + --wait \ + --timeout=10m + + - name: Wait for deployment + run: | + kubectl wait --for=condition=ready pod \ + -l "app.kubernetes.io/name=stac-fastapi" \ + -n stac-fastapi \ + --timeout=300s + + - name: Check deployment status + run: | + kubectl get all -n stac-fastapi + helm status stac-fastapi-test -n stac-fastapi + + - name: Test API endpoints + run: | + # Port forward to the service + kubectl port-forward -n stac-fastapi service/stac-fastapi-test 8080:80 & + sleep 10 + + # Test root endpoint + curl -f http://localhost:8080/ || exit 1 + + # Test collections endpoint + curl -f http://localhost:8080/collections || exit 1 + + # Test search endpoint + curl -f -X POST http://localhost:8080/search \ + -H "Content-Type: application/json" \ + -d '{}' || exit 1 + + echo "All API endpoints are working!" + + - name: Test database connectivity + run: | + # Check if database is responding + DB_SERVICE=$(kubectl get svc -n stac-fastapi -l "app=stac-fastapi-test-${{ matrix.backend }}-master" -o jsonpath="{.items[0].metadata.name}" || echo "") + + if [[ -n "$DB_SERVICE" ]]; then + kubectl port-forward -n stac-fastapi service/$DB_SERVICE 9200:9200 & + sleep 5 + curl -f http://localhost:9200/_health || echo "Database health check failed" + else + echo "Database service not found or using external database" + fi + + - name: Load test data + run: | + # Port forward to the API service + kubectl port-forward -n stac-fastapi service/stac-fastapi-test 8080:80 & + sleep 5 + + # Create test collection + curl -X POST http://localhost:8080/collections \ + -H "Content-Type: application/json" \ + -d '{ + "id": "test-collection", + "title": "Test Collection", + "description": "A test collection", + "extent": { + "spatial": {"bbox": [[-180, -90, 180, 90]]}, + "temporal": {"interval": [["2020-01-01T00:00:00Z", "2024-12-31T23:59:59Z"]]} + }, + "license": "public-domain" + }' || echo "Collection creation failed" + + # Create test item + curl -X POST http://localhost:8080/collections/test-collection/items \ + -H "Content-Type: application/json" \ + -d '{ + "id": "test-item", + "type": "Feature", + "stac_version": "1.0.0", + "collection": "test-collection", + "geometry": { + "type": "Polygon", + "coordinates": [[[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]] + }, + "bbox": [-1, -1, 1, 1], + "properties": {"datetime": "2023-06-15T12:00:00Z"}, + "assets": { + "thumbnail": { + "href": "https://example.com/thumbnail.jpg", + "type": "image/jpeg" + } + } + }' || echo "Item creation failed" + + # Test search + sleep 5 # Allow time for indexing + RESULT=$(curl -s -X POST http://localhost:8080/search \ + -H "Content-Type: application/json" \ + -d '{"collections": ["test-collection"]}') + + echo "Search result: $RESULT" + + - name: Cleanup + if: always() + run: | + helm uninstall stac-fastapi-test -n stac-fastapi || true + kubectl delete namespace stac-fastapi || true + + security-scan: + runs-on: ubuntu-latest + needs: lint-and-test + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Add Helm repositories + run: | + helm repo add elastic https://helm.elastic.co + helm repo add opensearch https://opensearch-project.github.io/helm-charts/ + helm repo update + + - name: Template chart for security scan + run: | + cd helm-chart/stac-fastapi + helm dependency update + helm template security-scan . \ + --set backend=elasticsearch \ + --set elasticsearch.enabled=true \ + --output-dir /tmp/security-scan + + - name: Run Checkov security scan + uses: bridgecrewio/checkov-action@master + with: + directory: /tmp/security-scan + framework: kubernetes + soft_fail: true + output_format: sarif + output_file_path: checkov-results.sarif + + - name: Upload Checkov results + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: checkov-results.sarif \ No newline at end of file diff --git a/helm-chart/DEPLOYMENT.md b/helm-chart/DEPLOYMENT.md new file mode 100644 index 000000000..27ed93648 --- /dev/null +++ b/helm-chart/DEPLOYMENT.md @@ -0,0 +1,635 @@ +# STAC FastAPI Kubernetes Deployment Guide + +This guide provides comprehensive instructions for deploying STAC FastAPI on Kubernetes using the provided Helm chart. + +## Table of Contents + +1. [Overview](#overview) +2. [Prerequisites](#prerequisites) +3. [Quick Start](#quick-start) +4. [Deployment Options](#deployment-options) +5. [Configuration](#configuration) +6. [Production Deployment](#production-deployment) +7. [Monitoring and Observability](#monitoring-and-observability) +8. [Troubleshooting](#troubleshooting) +9. [Maintenance](#maintenance) + +## Overview + +The STAC FastAPI Helm chart supports multiple deployment configurations: + +- **Elasticsearch Backend**: Deploy with bundled or external Elasticsearch +- **OpenSearch Backend**: Deploy with bundled or external OpenSearch +- **High Availability**: Multi-replica deployments with proper load balancing +- **Auto-scaling**: Horizontal Pod Autoscaler based on CPU/memory metrics +- **Security**: Network policies, RBAC, and pod security contexts +- **Monitoring**: Prometheus metrics and Grafana dashboards + +## Prerequisites + +### Required + +- Kubernetes cluster (v1.16+) +- Helm 3.0+ +- kubectl configured to access your cluster +- Sufficient cluster resources (see resource requirements below) + +### Optional + +- NGINX Ingress Controller (for ingress) +- cert-manager (for TLS certificates) +- Prometheus Operator (for monitoring) +- Persistent storage class (for database persistence) + +### Resource Requirements + +**Minimum (Development)**: +- 2 CPU cores +- 4 GB RAM +- 20 GB storage + +**Recommended (Production)**: +- 8 CPU cores +- 16 GB RAM +- 200 GB SSD storage + +## Quick Start + +### 1. Clone and Prepare + +```bash +git clone https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch.git +cd stac-fastapi-elasticsearch-opensearch/helm-chart +``` + +### 2. Update Dependencies + +```bash +helm dependency update stac-fastapi +``` + +### 3. Install with Elasticsearch + +```bash +helm install my-stac-api stac-fastapi \ + --set backend=elasticsearch \ + --set elasticsearch.enabled=true \ + --set opensearch.enabled=false \ + --create-namespace \ + --namespace stac-fastapi +``` + +### 4. Install with OpenSearch + +```bash +helm install my-stac-api stac-fastapi \ + --set backend=opensearch \ + --set elasticsearch.enabled=false \ + --set opensearch.enabled=true \ + --create-namespace \ + --namespace stac-fastapi +``` + +### 5. Access the API + +```bash +# Port forward to access locally +kubectl port-forward -n stac-fastapi service/my-stac-api 8080:80 + +# Test the API +curl http://localhost:8080/ +curl http://localhost:8080/collections +``` + +## Deployment Options + +### 1. Development Deployment + +**Use case**: Local development, testing, minimal resources + +```bash +helm install dev-stac stac-fastapi \ + --set backend=elasticsearch \ + --set app.replicaCount=1 \ + --set elasticsearch.replicas=1 \ + --set elasticsearch.minimumMasterNodes=1 \ + --set elasticsearch.resources.requests.memory=1Gi \ + --set elasticsearch.volumeClaimTemplate.resources.requests.storage=10Gi +``` + +### 2. Production Deployment with Elasticsearch + +**Use case**: High-availability production environment + +```bash +helm install prod-stac stac-fastapi \ + --values stac-fastapi/values-elasticsearch.yaml \ + --set app.ingress.enabled=true \ + --set app.ingress.hosts[0].host=stac-api.yourdomain.com \ + --set elasticsearch.volumeClaimTemplate.storageClassName=fast-ssd +``` + +### 3. Production Deployment with OpenSearch + +**Use case**: Production environment with OpenSearch preference + +```bash +helm install prod-stac stac-fastapi \ + --values stac-fastapi/values-opensearch.yaml \ + --set app.ingress.enabled=true \ + --set app.ingress.hosts[0].host=stac-api.yourdomain.com \ + --set opensearch.persistence.storageClass=fast-ssd +``` + +### 4. External Database Deployment + +**Use case**: Connect to existing Elasticsearch/OpenSearch cluster + +```bash +# First create secret for API key +kubectl create secret generic elasticsearch-credentials \ + --from-literal=api-key="your-api-key-here" + +# Deploy +helm install external-stac stac-fastapi \ + --values stac-fastapi/values-external.yaml \ + --set externalDatabase.host=elasticsearch.example.com \ + --set externalDatabase.port=9200 \ + --set externalDatabase.ssl=true +``` + +### 5. Multi-Environment Deployment + +**Use case**: Deploy multiple environments (dev, staging, prod) + +```bash +# Development +helm install dev-stac stac-fastapi \ + --namespace stac-dev \ + --create-namespace \ + --set backend=elasticsearch \ + --set app.env.ENVIRONMENT=development + +# Staging +helm install staging-stac stac-fastapi \ + --namespace stac-staging \ + --create-namespace \ + --values stac-fastapi/values-elasticsearch.yaml \ + --set app.env.ENVIRONMENT=staging + +# Production +helm install prod-stac stac-fastapi \ + --namespace stac-production \ + --create-namespace \ + --values stac-fastapi/values-elasticsearch.yaml \ + --set app.env.ENVIRONMENT=production +``` + +## Configuration + +### Application Configuration + +```yaml +app: + env: + # API Configuration + STAC_FASTAPI_TITLE: "My STAC API" + STAC_FASTAPI_DESCRIPTION: "A production STAC API" + STAC_FASTAPI_VERSION: "6.0.0" + + # Performance Tuning + WEB_CONCURRENCY: "8" # Number of worker processes + ENABLE_DIRECT_RESPONSE: "true" # Maximum performance mode + DATABASE_REFRESH: "false" # Better bulk performance + + # Large Dataset Optimization + ENABLE_DATETIME_INDEX_FILTERING: "true" # Temporal partitioning + DATETIME_INDEX_MAX_SIZE_GB: "50" # Index size limit + + # Rate Limiting + STAC_FASTAPI_RATE_LIMIT: "1000/minute" # API rate limit + + # Feature Toggles + ENABLE_TRANSACTIONS_EXTENSIONS: "true" # Enable POST operations + STAC_INDEX_ASSETS: "true" # Index asset metadata +``` + +### Database Configuration + +#### Elasticsearch Production Settings + +```yaml +elasticsearch: + replicas: 3 + minimumMasterNodes: 2 + + esConfig: + elasticsearch.yml: | + # Cluster settings + cluster.name: "stac-elasticsearch-prod" + action.destructive_requires_name: true + + # Performance tuning + indices.memory.index_buffer_size: 20% + thread_pool.write.queue_size: 1000 + thread_pool.search.queue_size: 1000 + + # Disk usage thresholds + cluster.routing.allocation.disk.threshold_enabled: true + cluster.routing.allocation.disk.watermark.low: 85% + cluster.routing.allocation.disk.watermark.high: 90% + cluster.routing.allocation.disk.watermark.flood_stage: 95% + + resources: + requests: + cpu: "2000m" + memory: "8Gi" + limits: + cpu: "4000m" + memory: "8Gi" + + esJavaOpts: "-Xmx4g -Xms4g" +``` + +#### OpenSearch Production Settings + +```yaml +opensearch: + replicas: 3 + + config: + opensearch.yml: | + # Cluster settings + cluster.name: stac-opensearch-prod + action.destructive_requires_name: true + + # Performance tuning + indices.memory.index_buffer_size: 20% + thread_pool.write.queue_size: 1000 + thread_pool.search.queue_size: 1000 + + # Disk usage thresholds + cluster.routing.allocation.disk.threshold_enabled: true + cluster.routing.allocation.disk.watermark.low: 85% + cluster.routing.allocation.disk.watermark.high: 90% + cluster.routing.allocation.disk.watermark.flood_stage: 95% + + resources: + requests: + cpu: "2000m" + memory: "8Gi" + limits: + cpu: "4000m" + memory: "8Gi" + + opensearchJavaOpts: "-Xmx4g -Xms4g" +``` + +### Ingress Configuration + +```yaml +app: + ingress: + enabled: true + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-body-size: "100m" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + cert-manager.io/cluster-issuer: "letsencrypt-prod" + + hosts: + - host: stac-api.yourdomain.com + paths: + - path: / + pathType: Prefix + + tls: + - secretName: stac-api-tls + hosts: + - stac-api.yourdomain.com +``` + +## Production Deployment + +### 1. Pre-deployment Checklist + +- [ ] Kubernetes cluster properly sized +- [ ] Storage classes configured +- [ ] Ingress controller installed +- [ ] DNS records configured +- [ ] SSL certificates ready +- [ ] Monitoring stack deployed +- [ ] Backup strategy defined + +### 2. Production Values File + +Create a production-specific values file: + +```yaml +# values-production.yaml +backend: elasticsearch + +app: + replicaCount: 3 + + image: + tag: "v6.0.0" # Use specific version + pullPolicy: IfNotPresent + + resources: + requests: + cpu: "1000m" + memory: "2Gi" + limits: + cpu: "2000m" + memory: "4Gi" + + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 20 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + + env: + ENVIRONMENT: "production" + WEB_CONCURRENCY: "8" + ENABLE_DIRECT_RESPONSE: "true" + DATABASE_REFRESH: "false" + ENABLE_DATETIME_INDEX_FILTERING: "true" + DATETIME_INDEX_MAX_SIZE_GB: "100" + STAC_FASTAPI_RATE_LIMIT: "2000/minute" + +elasticsearch: + enabled: true + replicas: 5 + minimumMasterNodes: 3 + + resources: + requests: + cpu: "2000m" + memory: "8Gi" + limits: + cpu: "4000m" + memory: "8Gi" + + volumeClaimTemplate: + storageClassName: "fast-ssd" + resources: + requests: + storage: 500Gi + +podDisruptionBudget: + enabled: true + minAvailable: 2 + +monitoring: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + +networkPolicy: + enabled: true +``` + +### 3. Deploy to Production + +```bash +# Deploy +helm install stac-prod stac-fastapi \ + --namespace stac-production \ + --create-namespace \ + --values values-production.yaml \ + --wait \ + --timeout=15m + +# Verify deployment +kubectl get all -n stac-production +helm status stac-prod -n stac-production +``` + +### 4. Post-deployment Verification + +```bash +# Test API endpoints +curl https://stac-api.yourdomain.com/ +curl https://stac-api.yourdomain.com/collections +curl -X POST https://stac-api.yourdomain.com/search \ + -H "Content-Type: application/json" \ + -d '{}' + +# Check database health +kubectl port-forward -n stac-production \ + service/stac-prod-elasticsearch-master 9200:9200 & +curl http://localhost:9200/_cluster/health +``` + +## Monitoring and Observability + +### 1. Enable Prometheus Monitoring + +```yaml +monitoring: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s +``` + +### 2. Grafana Dashboards + +Create custom dashboards for: +- API request rates and latency +- Database performance metrics +- Resource utilization +- Error rates and patterns + +### 3. Alerting Rules + +Example Prometheus alerting rules: + +```yaml +# stac-fastapi-alerts.yaml +groups: + - name: stac-fastapi + rules: + - alert: STACAPIHighErrorRate + expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "High error rate detected" + + - alert: STACAPIHighLatency + expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "High latency detected" +``` + +### 4. Log Aggregation + +Configure log shipping to your preferred logging solution: + +```yaml +app: + podAnnotations: + fluentd.io/include: "true" + fluentd.io/exclude: "false" +``` + +## Troubleshooting + +### Common Issues + +#### 1. Pod Startup Issues + +```bash +# Check pod status +kubectl get pods -n stac-fastapi + +# View pod logs +kubectl logs -f deployment/my-stac-api -n stac-fastapi + +# Describe pod for events +kubectl describe pod -n stac-fastapi +``` + +#### 2. Database Connectivity Issues + +```bash +# Check database service +kubectl get svc -n stac-fastapi + +# Test database connectivity +kubectl exec -it deployment/my-stac-api -n stac-fastapi -- \ + curl http://elasticsearch:9200/_health + +# Check database logs +kubectl logs -f statefulset/my-stac-api-elasticsearch-master -n stac-fastapi +``` + +#### 3. Performance Issues + +```bash +# Check resource usage +kubectl top pods -n stac-fastapi +kubectl top nodes + +# Check HPA status +kubectl get hpa -n stac-fastapi + +# View detailed metrics +kubectl describe hpa my-stac-api -n stac-fastapi +``` + +#### 4. Ingress Issues + +```bash +# Check ingress status +kubectl get ingress -n stac-fastapi + +# Check ingress controller logs +kubectl logs -f deployment/nginx-controller -n nginx-ingress + +# Test DNS resolution +nslookup stac-api.yourdomain.com +``` + +### Debugging Tools + +Use the provided test script for comprehensive debugging: + +```bash +# Validate deployment +./test-chart.sh validate + +# Load test data +./test-chart.sh load-data + +# Full cleanup +./test-chart.sh cleanup +``` + +## Maintenance + +### 1. Upgrades + +```bash +# Update chart dependencies +helm dependency update stac-fastapi + +# Upgrade deployment +helm upgrade stac-prod stac-fastapi \ + --namespace stac-production \ + --values values-production.yaml \ + --wait + +# Rollback if needed +helm rollback stac-prod 1 -n stac-production +``` + +### 2. Scaling + +```bash +# Manual scaling +kubectl scale deployment stac-prod \ + --replicas=5 -n stac-production + +# Update HPA limits +helm upgrade stac-prod stac-fastapi \ + --set app.autoscaling.maxReplicas=30 \ + --reuse-values +``` + +### 3. Backup and Restore + +#### Database Snapshots + +```bash +# Create snapshot +kubectl exec -it stac-prod-elasticsearch-master-0 -n stac-production -- \ + curl -X PUT "localhost:9200/_snapshot/backup/snapshot_$(date +%Y%m%d)" \ + -H 'Content-Type: application/json' \ + -d '{"indices": "*", "ignore_unavailable": true}' + +# List snapshots +kubectl exec -it stac-prod-elasticsearch-master-0 -n stac-production -- \ + curl "localhost:9200/_snapshot/backup/_all" +``` + +#### Configuration Backup + +```bash +# Backup Helm values +helm get values stac-prod -n stac-production > backup-values.yaml + +# Backup Kubernetes resources +kubectl get all -n stac-production -o yaml > backup-resources.yaml +``` + +### 4. Security Updates + +```bash +# Update to latest images +helm upgrade stac-prod stac-fastapi \ + --set app.image.tag=latest \ + --set elasticsearch.imageTag=8.11.0 \ + --reuse-values + +# Apply security patches +kubectl patch deployment stac-prod \ + -n stac-production \ + -p '{"spec":{"template":{"metadata":{"annotations":{"kubectl.kubernetes.io/restartedAt":"'$(date +%Y-%m-%dT%H:%M:%S%z)'"}}}}}' +``` + +This deployment guide provides comprehensive instructions for deploying and maintaining STAC FastAPI in production Kubernetes environments. Adapt the configurations based on your specific requirements and infrastructure constraints. \ No newline at end of file diff --git a/helm-chart/stac-fastapi/Chart.yaml b/helm-chart/stac-fastapi/Chart.yaml new file mode 100644 index 000000000..b3e96f1b7 --- /dev/null +++ b/helm-chart/stac-fastapi/Chart.yaml @@ -0,0 +1,32 @@ +apiVersion: v2 +name: stac-fastapi +description: A Helm chart for STAC FastAPI with Elasticsearch and OpenSearch backends +type: application +version: 0.1.0 +appVersion: "6.0.0" + +keywords: + - stac + - fastapi + - elasticsearch + - opensearch + - geospatial + - api + +home: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch +sources: + - https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch + +maintainers: + - name: STAC Utils + url: https://github.com/stac-utils + +dependencies: + - name: elasticsearch + version: 19.19.4 + repository: https://helm.elastic.co + condition: elasticsearch.enabled + - name: opensearch + version: 2.18.0 + repository: https://opensearch-project.github.io/helm-charts/ + condition: opensearch.enabled \ No newline at end of file diff --git a/helm-chart/stac-fastapi/README.md b/helm-chart/stac-fastapi/README.md new file mode 100644 index 000000000..08589837e --- /dev/null +++ b/helm-chart/stac-fastapi/README.md @@ -0,0 +1,402 @@ +# STAC FastAPI Helm Chart + +This Helm chart deploys the STAC FastAPI application with support for both Elasticsearch and OpenSearch backends. + +## Overview + +The chart provides a flexible deployment solution for STAC FastAPI with the following features: + +- **Dual Backend Support**: Choose between Elasticsearch or OpenSearch +- **Bundled or External Database**: Deploy with bundled database or connect to external clusters +- **Production Ready**: Includes monitoring, autoscaling, and security configurations +- **High Availability**: Support for multi-replica deployments with proper disruption budgets +- **Performance Optimized**: Configurable performance settings for large-scale deployments + +## Prerequisites + +- Kubernetes 1.16+ +- Helm 3.0+ +- Storage class for persistent volumes (if using bundled databases) + +## Quick Start + +### Deploy with Elasticsearch + +```bash +# Add dependencies +helm dependency update ./helm-chart/stac-fastapi + +# Install with Elasticsearch backend +helm install my-stac-api ./helm-chart/stac-fastapi \ + --set backend=elasticsearch \ + --set elasticsearch.enabled=true \ + --set opensearch.enabled=false +``` + +### Deploy with OpenSearch + +```bash +# Add dependencies +helm dependency update ./helm-chart/stac-fastapi + +# Install with OpenSearch backend +helm install my-stac-api ./helm-chart/stac-fastapi \ + --set backend=opensearch \ + --set elasticsearch.enabled=false \ + --set opensearch.enabled=true +``` + +### Deploy with External Database + +```bash +# Create secret for API key (if needed) +kubectl create secret generic elasticsearch-api-key \ + --from-literal=api-key="your-api-key-here" + +# Install with external database +helm install my-stac-api ./helm-chart/stac-fastapi \ + --values ./helm-chart/stac-fastapi/values-external.yaml \ + --set externalDatabase.host="your-database-host" \ + --set externalDatabase.port=9200 +``` + +## Configuration + +### Backend Selection + +The chart supports two backends: + +- `elasticsearch`: Uses Elasticsearch as the backend database +- `opensearch`: Uses OpenSearch as the backend database + +Set the backend using: + +```yaml +backend: elasticsearch # or opensearch +``` + +### Application Configuration + +Key application settings: + +```yaml +app: + replicaCount: 2 + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + env: + STAC_FASTAPI_TITLE: "STAC API" + STAC_FASTAPI_DESCRIPTION: "A STAC FastAPI implementation" + ENVIRONMENT: "production" + WEB_CONCURRENCY: "10" + ENABLE_DIRECT_RESPONSE: "false" + DATABASE_REFRESH: "false" + ENABLE_DATETIME_INDEX_FILTERING: "false" + STAC_FASTAPI_RATE_LIMIT: "200/minute" +``` + +### Database Configuration + +#### Bundled Elasticsearch + +```yaml +elasticsearch: + enabled: true + clusterName: "stac-elasticsearch" + replicas: 3 + minimumMasterNodes: 2 + + resources: + requests: + cpu: "1000m" + memory: "4Gi" + limits: + cpu: "2000m" + memory: "4Gi" + + volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 100Gi +``` + +#### Bundled OpenSearch + +```yaml +opensearch: + enabled: true + clusterName: "stac-opensearch" + replicas: 3 + + resources: + requests: + cpu: "1000m" + memory: "4Gi" + limits: + cpu: "2000m" + memory: "4Gi" + + persistence: + enabled: true + size: 100Gi + storageClass: "fast-ssd" +``` + +#### External Database + +```yaml +externalDatabase: + enabled: true + host: "elasticsearch.example.com" + port: 9200 + ssl: true + verifyCerts: true + apiKeySecret: "elasticsearch-credentials" + apiKeySecretKey: "api-key" +``` + +### Ingress Configuration + +```yaml +app: + ingress: + enabled: true + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + cert-manager.io/cluster-issuer: "letsencrypt-prod" + hosts: + - host: stac-api.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: stac-fastapi-tls + hosts: + - stac-api.example.com +``` + +### Autoscaling + +```yaml +app: + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 +``` + +### Monitoring + +```yaml +monitoring: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s +``` + +## Performance Tuning + +### For Large Datasets + +Enable datetime-based index filtering for better performance with temporal queries: + +```yaml +app: + env: + ENABLE_DATETIME_INDEX_FILTERING: "true" + DATETIME_INDEX_MAX_SIZE_GB: "50" +``` + +### For Maximum Performance + +Enable direct response mode (disables FastAPI dependencies): + +```yaml +app: + env: + ENABLE_DIRECT_RESPONSE: "true" + DATABASE_REFRESH: "false" +``` + +### For High Throughput + +Increase worker processes and rate limits: + +```yaml +app: + env: + WEB_CONCURRENCY: "8" + STAC_FASTAPI_RATE_LIMIT: "1000/minute" +``` + +## Security + +### Network Policies + +Enable network policies for additional security: + +```yaml +networkPolicy: + enabled: true + ingress: + - from: + - namespaceSelector: + matchLabels: + name: nginx-ingress + ports: + - protocol: TCP + port: 8080 +``` + +### Pod Security Context + +Configure security contexts: + +```yaml +app: + podSecurityContext: + fsGroup: 2000 + + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 +``` + +## High Availability + +### Pod Disruption Budget + +Ensure availability during maintenance: + +```yaml +podDisruptionBudget: + enabled: true + minAvailable: 1 +``` + +### Anti-Affinity + +Spread pods across nodes: + +```yaml +app: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - stac-fastapi + topologyKey: kubernetes.io/hostname +``` + +## Examples + +The chart includes several example values files: + +- `values-elasticsearch.yaml`: Production-ready Elasticsearch deployment +- `values-opensearch.yaml`: Production-ready OpenSearch deployment +- `values-external.yaml`: External database configuration + +Use them as starting points: + +```bash +helm install my-stac-api ./helm-chart/stac-fastapi \ + --values ./helm-chart/stac-fastapi/values-elasticsearch.yaml +``` + +## Upgrading + +To upgrade an existing deployment: + +```bash +helm upgrade my-stac-api ./helm-chart/stac-fastapi \ + --values your-values.yaml +``` + +## Uninstalling + +To remove the deployment: + +```bash +helm uninstall my-stac-api +``` + +**Note**: Persistent volumes for databases may need to be manually deleted. + +## Troubleshooting + +### Check Pod Status + +```bash +kubectl get pods -l app.kubernetes.io/name=stac-fastapi +``` + +### View Logs + +```bash +kubectl logs -l app.kubernetes.io/name=stac-fastapi +``` + +### Check Database Connectivity + +```bash +kubectl exec -it deployment/my-stac-api -- curl http://elasticsearch:9200/_health +``` + +### Port Forward for Local Testing + +```bash +kubectl port-forward service/my-stac-api 8080:80 +``` + +Then visit http://localhost:8080 + +## Configuration Reference + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `backend` | Database backend (elasticsearch/opensearch) | `elasticsearch` | +| `app.replicaCount` | Number of application replicas | `2` | +| `app.image.repository` | Application image repository | `ghcr.io/stac-utils/stac-fastapi` | +| `app.image.tag` | Application image tag | `""` (uses chart appVersion) | +| `app.service.type` | Kubernetes service type | `ClusterIP` | +| `app.service.port` | Service port | `80` | +| `app.ingress.enabled` | Enable ingress | `false` | +| `app.autoscaling.enabled` | Enable horizontal pod autoscaler | `false` | +| `elasticsearch.enabled` | Deploy Elasticsearch | `true` | +| `opensearch.enabled` | Deploy OpenSearch | `false` | +| `externalDatabase.enabled` | Use external database | `false` | +| `monitoring.enabled` | Enable monitoring | `false` | +| `networkPolicy.enabled` | Enable network policies | `false` | +| `podDisruptionBudget.enabled` | Enable pod disruption budget | `false` | + +For a complete list of configuration options, see the `values.yaml` file. + +## Contributing + +Contributions are welcome! Please ensure any changes maintain compatibility with both Elasticsearch and OpenSearch backends. + +## License + +This chart is released under the same license as the STAC FastAPI project. \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/NOTES.txt b/helm-chart/stac-fastapi/templates/NOTES.txt new file mode 100644 index 000000000..9314411c6 --- /dev/null +++ b/helm-chart/stac-fastapi/templates/NOTES.txt @@ -0,0 +1,84 @@ +1. Get the application URL by running these commands: +{{- if .Values.app.ingress.enabled }} +{{- range $host := .Values.app.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.app.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.app.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "stac-fastapi.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.app.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "stac-fastapi.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "stac-fastapi.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.app.service.port }} +{{- else if contains "ClusterIP" .Values.app.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "{{ include "stac-fastapi.selectorLabels" . }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} + +2. Backend Configuration: + - Backend: {{ .Values.backend }} + {{- if .Values.externalDatabase.enabled }} + - Database: External ({{ .Values.externalDatabase.host }}:{{ .Values.externalDatabase.port }}) + {{- else if eq .Values.backend "elasticsearch" }} + - Database: Bundled Elasticsearch + {{- else if eq .Values.backend "opensearch" }} + - Database: Bundled OpenSearch + {{- end }} + +3. API Documentation: + Once the application is running, you can access: + - OpenAPI documentation: /docs + - STAC API landing page: / + - Collections endpoint: /collections + - Search endpoint: /search + +4. Health Checks: + - Liveness probe: GET / + - Readiness probe: GET / + +{{- if and (eq .Values.backend "elasticsearch") .Values.elasticsearch.enabled }} + +5. Elasticsearch Access: + export ES_POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ .Release.Name }}-elasticsearch-master" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ .Release.Namespace }} port-forward $ES_POD_NAME 9200:9200 + # Then access Elasticsearch at http://localhost:9200 +{{- end }} + +{{- if and (eq .Values.backend "opensearch") .Values.opensearch.enabled }} + +5. OpenSearch Access: + export OS_POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name=opensearch" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ .Release.Namespace }} port-forward $OS_POD_NAME 9200:9200 + # Then access OpenSearch at http://localhost:9200 +{{- end }} + +{{- if .Values.app.autoscaling.enabled }} + +6. Autoscaling: + Horizontal Pod Autoscaler is enabled with: + - Min replicas: {{ .Values.app.autoscaling.minReplicas }} + - Max replicas: {{ .Values.app.autoscaling.maxReplicas }} + {{- if .Values.app.autoscaling.targetCPUUtilizationPercentage }} + - CPU target: {{ .Values.app.autoscaling.targetCPUUtilizationPercentage }}% + {{- end }} + {{- if .Values.app.autoscaling.targetMemoryUtilizationPercentage }} + - Memory target: {{ .Values.app.autoscaling.targetMemoryUtilizationPercentage }}% + {{- end }} +{{- end }} + +{{- if .Values.monitoring.enabled }} + +7. Monitoring: + {{- if .Values.monitoring.prometheus.enabled }} + - Prometheus metrics: /metrics + {{- if .Values.monitoring.prometheus.serviceMonitor.enabled }} + - ServiceMonitor created for Prometheus Operator + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/_helpers.tpl b/helm-chart/stac-fastapi/templates/_helpers.tpl new file mode 100644 index 000000000..87c3ad307 --- /dev/null +++ b/helm-chart/stac-fastapi/templates/_helpers.tpl @@ -0,0 +1,149 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "stac-fastapi.name" -}} +{{- default .Chart.Name .Values.app.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "stac-fastapi.fullname" -}} +{{- if .Values.app.fullnameOverride }} +{{- .Values.app.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.app.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "stac-fastapi.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "stac-fastapi.labels" -}} +helm.sh/chart: {{ include "stac-fastapi.chart" . }} +{{ include "stac-fastapi.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "stac-fastapi.selectorLabels" -}} +app.kubernetes.io/name: {{ include "stac-fastapi.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "stac-fastapi.serviceAccountName" -}} +{{- if .Values.app.serviceAccount.create }} +{{- default (include "stac-fastapi.fullname" .) .Values.app.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.app.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the database host based on backend selection +*/}} +{{- define "stac-fastapi.databaseHost" -}} +{{- if .Values.externalDatabase.enabled }} +{{- .Values.externalDatabase.host }} +{{- else if eq .Values.backend "elasticsearch" }} +{{- if .Values.elasticsearch.enabled }} +{{- printf "%s-%s" .Release.Name "elasticsearch-master" }} +{{- else }} +{{- fail "Elasticsearch is not enabled but backend is set to elasticsearch" }} +{{- end }} +{{- else if eq .Values.backend "opensearch" }} +{{- if .Values.opensearch.enabled }} +{{- printf "%s-%s" .Release.Name "opensearch-cluster-master" }} +{{- else }} +{{- fail "OpenSearch is not enabled but backend is set to opensearch" }} +{{- end }} +{{- else }} +{{- fail "Invalid backend specified. Must be 'elasticsearch' or 'opensearch'" }} +{{- end }} +{{- end }} + +{{/* +Create the database port based on backend selection +*/}} +{{- define "stac-fastapi.databasePort" -}} +{{- if .Values.externalDatabase.enabled }} +{{- .Values.externalDatabase.port }} +{{- else if eq .Values.backend "elasticsearch" }} +{{- .Values.elasticsearch.service.httpPort | default 9200 }} +{{- else if eq .Values.backend "opensearch" }} +{{- .Values.opensearch.service.httpPort | default 9200 }} +{{- end }} +{{- end }} + +{{/* +Create the image repository with tag +*/}} +{{- define "stac-fastapi.image" -}} +{{- $registry := .Values.global.imageRegistry | default .Values.app.image.repository }} +{{- $tag := .Values.app.image.tag | default .Chart.AppVersion }} +{{- if eq .Values.backend "elasticsearch" }} +{{- printf "%s-es:%s" $registry $tag }} +{{- else if eq .Values.backend "opensearch" }} +{{- printf "%s-os:%s" $registry $tag }} +{{- end }} +{{- end }} + +{{/* +Create environment variables for the application +*/}} +{{- define "stac-fastapi.environment" -}} +- name: BACKEND + value: {{ .Values.backend | quote }} +- name: ES_HOST + value: {{ include "stac-fastapi.databaseHost" . | quote }} +- name: ES_PORT + value: {{ include "stac-fastapi.databasePort" . | quote }} +{{- if .Values.externalDatabase.enabled }} +- name: ES_USE_SSL + value: {{ .Values.externalDatabase.ssl | quote }} +- name: ES_VERIFY_CERTS + value: {{ .Values.externalDatabase.verifyCerts | quote }} +- name: ES_TIMEOUT + value: {{ .Values.externalDatabase.timeout | quote }} +{{- if .Values.externalDatabase.apiKeySecret }} +- name: ES_API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.externalDatabase.apiKeySecret }} + key: {{ .Values.externalDatabase.apiKeySecretKey }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.app.env }} +- name: {{ $key }} + value: {{ $value | quote }} +{{- end }} +{{- range $key, $secretName := .Values.app.envFromSecret }} +- name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ $secretName }} + key: {{ $key }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/deployment.yaml b/helm-chart/stac-fastapi/templates/deployment.yaml new file mode 100644 index 000000000..4d94b8dee --- /dev/null +++ b/helm-chart/stac-fastapi/templates/deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "stac-fastapi.fullname" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} +spec: + {{- if not .Values.app.autoscaling.enabled }} + replicas: {{ .Values.app.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "stac-fastapi.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.app.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "stac-fastapi.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.app.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "stac-fastapi.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.app.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.app.securityContext | nindent 12 }} + image: {{ include "stac-fastapi.image" . }} + imagePullPolicy: {{ .Values.app.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.app.service.targetPort }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.app.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.app.readinessProbe | nindent 12 }} + env: + {{- include "stac-fastapi.environment" . | nindent 12 }} + resources: + {{- toYaml .Values.app.resources | nindent 12 }} + {{- with .Values.app.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.app.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.app.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/hpa.yaml b/helm-chart/stac-fastapi/templates/hpa.yaml new file mode 100644 index 000000000..5e00c217e --- /dev/null +++ b/helm-chart/stac-fastapi/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.app.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "stac-fastapi.fullname" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "stac-fastapi.fullname" . }} + minReplicas: {{ .Values.app.autoscaling.minReplicas }} + maxReplicas: {{ .Values.app.autoscaling.maxReplicas }} + metrics: + {{- if .Values.app.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.app.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.app.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.app.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/ingress.yaml b/helm-chart/stac-fastapi/templates/ingress.yaml new file mode 100644 index 000000000..3a31feb3d --- /dev/null +++ b/helm-chart/stac-fastapi/templates/ingress.yaml @@ -0,0 +1,59 @@ +{{- if .Values.app.ingress.enabled -}} +{{- $fullName := include "stac-fastapi.fullname" . -}} +{{- $svcPort := .Values.app.service.port -}} +{{- if and .Values.app.ingress.className (not (hasKey .Values.app.ingress.annotations "kubernetes.io/ingress.class")) }} + {{- $_ := set .Values.app.ingress.annotations "kubernetes.io/ingress.class" .Values.app.ingress.className}} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} + {{- with .Values.app.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.app.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.app.ingress.className }} + {{- end }} + {{- if .Values.app.ingress.tls }} + tls: + {{- range .Values.app.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.app.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/networkpolicy.yaml b/helm-chart/stac-fastapi/templates/networkpolicy.yaml new file mode 100644 index 000000000..10f9bd9c7 --- /dev/null +++ b/helm-chart/stac-fastapi/templates/networkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "stac-fastapi.fullname" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "stac-fastapi.selectorLabels" . | nindent 6 }} + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress }} + - Egress + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- end }} + {{- if .Values.networkPolicy.egress }} + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml b/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..d27ada320 --- /dev/null +++ b/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ include "stac-fastapi.fullname" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "stac-fastapi.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/service.yaml b/helm-chart/stac-fastapi/templates/service.yaml new file mode 100644 index 000000000..c44d3fee5 --- /dev/null +++ b/helm-chart/stac-fastapi/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "stac-fastapi.fullname" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} +spec: + type: {{ .Values.app.service.type }} + ports: + - port: {{ .Values.app.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "stac-fastapi.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/serviceaccount.yaml b/helm-chart/stac-fastapi/templates/serviceaccount.yaml new file mode 100644 index 000000000..bf517ca9c --- /dev/null +++ b/helm-chart/stac-fastapi/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "stac-fastapi.serviceAccountName" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} + {{- with .Values.app.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/servicemonitor.yaml b/helm-chart/stac-fastapi/templates/servicemonitor.yaml new file mode 100644 index 000000000..f4da1405c --- /dev/null +++ b/helm-chart/stac-fastapi/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.monitoring.enabled .Values.monitoring.prometheus.enabled .Values.monitoring.prometheus.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "stac-fastapi.fullname" . }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} + {{- with .Values.monitoring.prometheus.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "stac-fastapi.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + interval: {{ .Values.monitoring.prometheus.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.monitoring.prometheus.serviceMonitor.scrapeTimeout }} + path: /metrics +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-elasticsearch.yaml b/helm-chart/stac-fastapi/values-elasticsearch.yaml new file mode 100644 index 000000000..bd914c514 --- /dev/null +++ b/helm-chart/stac-fastapi/values-elasticsearch.yaml @@ -0,0 +1,155 @@ +# Example values for deploying STAC FastAPI with Elasticsearch backend +# Override the default values.yaml with these settings + +# Set backend to elasticsearch +backend: elasticsearch + +# STAC FastAPI application configuration +app: + replicaCount: 2 + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + # Service configuration + service: + type: LoadBalancer + port: 80 + targetPort: 8080 + + # Ingress configuration + ingress: + enabled: true + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + cert-manager.io/cluster-issuer: "letsencrypt-prod" + hosts: + - host: stac-api.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: stac-fastapi-tls + hosts: + - stac-api.example.com + + # Resource limits + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + + # Autoscaling + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + + # Environment variables + env: + STAC_FASTAPI_TITLE: "Production STAC API with Elasticsearch" + STAC_FASTAPI_DESCRIPTION: "High-performance STAC API for geospatial data discovery" + ENVIRONMENT: "production" + WEB_CONCURRENCY: "4" + ENABLE_DIRECT_RESPONSE: "true" # Enable for maximum performance + DATABASE_REFRESH: "false" # Better performance for bulk operations + ENABLE_DATETIME_INDEX_FILTERING: "true" # Enable for large datasets + DATETIME_INDEX_MAX_SIZE_GB: "50" + STAC_FASTAPI_RATE_LIMIT: "1000/minute" + +# Elasticsearch configuration +elasticsearch: + enabled: true + + # Cluster configuration + clusterName: "stac-elasticsearch-prod" + replicas: 3 + minimumMasterNodes: 2 + + # Resource configuration for production + resources: + requests: + cpu: "1000m" + memory: "4Gi" + limits: + cpu: "2000m" + memory: "4Gi" + + # JVM heap size (should be 50% of memory limit) + esJavaOpts: "-Xmx2g -Xms2g" + + # Production-ready Elasticsearch configuration + esConfig: + elasticsearch.yml: | + cluster.name: "stac-elasticsearch-prod" + network.host: 0.0.0.0 + discovery.seed_hosts: ["stac-elasticsearch-prod-master-headless"] + cluster.initial_master_nodes: ["stac-elasticsearch-prod-master-0", "stac-elasticsearch-prod-master-1", "stac-elasticsearch-prod-master-2"] + action.destructive_requires_name: true + xpack.security.enabled: false + xpack.security.transport.ssl.enabled: false + xpack.security.http.ssl.enabled: false + indices.memory.index_buffer_size: 20% + thread_pool.write.queue_size: 1000 + thread_pool.search.queue_size: 1000 + + # Persistent storage + volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + storageClassName: "fast-ssd" # Use high-performance storage class + resources: + requests: + storage: 100Gi + +# Disable OpenSearch since we're using Elasticsearch +opensearch: + enabled: false + +# External database disabled since we're using bundled Elasticsearch +externalDatabase: + enabled: false + +# Monitoring configuration +monitoring: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: + monitoring: "prometheus" + +# Pod disruption budget for high availability +podDisruptionBudget: + enabled: true + minAvailable: 1 + +# Network policies for security +networkPolicy: + enabled: true + ingress: + - from: + - namespaceSelector: + matchLabels: + name: nginx-ingress + ports: + - protocol: TCP + port: 8080 + egress: + - to: + - podSelector: + matchLabels: + app: stac-elasticsearch-prod-master + ports: + - protocol: TCP + port: 9200 \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-external.yaml b/helm-chart/stac-fastapi/values-external.yaml new file mode 100644 index 000000000..8a5780907 --- /dev/null +++ b/helm-chart/stac-fastapi/values-external.yaml @@ -0,0 +1,52 @@ +# Example values for using external Elasticsearch/OpenSearch +# Use this when you have an existing database cluster + +# Choose your backend +backend: elasticsearch # or opensearch + +# STAC FastAPI application configuration +app: + replicaCount: 3 + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + # Resource configuration + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + + # Environment variables + env: + STAC_FASTAPI_TITLE: "STAC API with External Database" + STAC_FASTAPI_DESCRIPTION: "STAC API connected to external Elasticsearch/OpenSearch cluster" + ENVIRONMENT: "production" + WEB_CONCURRENCY: "4" + +# Disable bundled databases +elasticsearch: + enabled: false + +opensearch: + enabled: false + +# External database configuration +externalDatabase: + enabled: true + host: "elasticsearch.database.svc.cluster.local" # Your external database host + port: 9200 + ssl: true + verifyCerts: true + timeout: 30 + # Reference to secret containing API key + apiKeySecret: "elasticsearch-api-key" + apiKeySecretKey: "api-key" + +# Create a secret for the API key (example) +# kubectl create secret generic elasticsearch-api-key --from-literal=api-key="your-api-key-here" \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-opensearch.yaml b/helm-chart/stac-fastapi/values-opensearch.yaml new file mode 100644 index 000000000..b9ecde069 --- /dev/null +++ b/helm-chart/stac-fastapi/values-opensearch.yaml @@ -0,0 +1,164 @@ +# Example values for deploying STAC FastAPI with OpenSearch backend +# Override the default values.yaml with these settings + +# Set backend to opensearch +backend: opensearch + +# STAC FastAPI application configuration +app: + replicaCount: 2 + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + # Service configuration + service: + type: LoadBalancer + port: 80 + targetPort: 8080 + + # Ingress configuration + ingress: + enabled: true + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + cert-manager.io/cluster-issuer: "letsencrypt-prod" + hosts: + - host: stac-api-os.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: stac-fastapi-os-tls + hosts: + - stac-api-os.example.com + + # Resource limits + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + + # Autoscaling + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + + # Environment variables + env: + STAC_FASTAPI_TITLE: "Production STAC API with OpenSearch" + STAC_FASTAPI_DESCRIPTION: "High-performance STAC API for geospatial data discovery with OpenSearch" + ENVIRONMENT: "production" + WEB_CONCURRENCY: "4" + ENABLE_DIRECT_RESPONSE: "true" # Enable for maximum performance + DATABASE_REFRESH: "false" # Better performance for bulk operations + ENABLE_DATETIME_INDEX_FILTERING: "true" # Enable for large datasets + DATETIME_INDEX_MAX_SIZE_GB: "50" + STAC_FASTAPI_RATE_LIMIT: "1000/minute" + +# Disable Elasticsearch since we're using OpenSearch +elasticsearch: + enabled: false + +# OpenSearch configuration +opensearch: + enabled: true + + # Cluster configuration + clusterName: "stac-opensearch-prod" + replicas: 3 + + # OpenSearch roles for each node + roles: + - master + - ingest + - data + - remote_cluster_client + + # Resource configuration for production + resources: + requests: + cpu: "1000m" + memory: "4Gi" + limits: + cpu: "2000m" + memory: "4Gi" + + # JVM heap size (should be 50% of memory limit) + opensearchJavaOpts: "-Xmx2g -Xms2g" + + # Production-ready OpenSearch configuration + config: + opensearch.yml: | + cluster.name: stac-opensearch-prod + network.host: 0.0.0.0 + discovery.seed_hosts: ["stac-opensearch-prod-cluster-master-headless"] + cluster.initial_cluster_manager_nodes: ["stac-opensearch-prod-cluster-master-0", "stac-opensearch-prod-cluster-master-1", "stac-opensearch-prod-cluster-master-2"] + plugins.security.disabled: true + action.destructive_requires_name: true + indices.memory.index_buffer_size: 20% + thread_pool.write.queue_size: 1000 + thread_pool.search.queue_size: 1000 + cluster.routing.allocation.disk.threshold_enabled: true + cluster.routing.allocation.disk.watermark.low: 85% + cluster.routing.allocation.disk.watermark.high: 90% + cluster.routing.allocation.disk.watermark.flood_stage: 95% + + # Persistent storage + persistence: + enabled: true + storageClass: "fast-ssd" # Use high-performance storage class + accessModes: + - ReadWriteOnce + size: 100Gi + annotations: {} + +# External database disabled since we're using bundled OpenSearch +externalDatabase: + enabled: false + +# Monitoring configuration +monitoring: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: + monitoring: "prometheus" + +# Pod disruption budget for high availability +podDisruptionBudget: + enabled: true + minAvailable: 1 + +# Network policies for security +networkPolicy: + enabled: true + ingress: + - from: + - namespaceSelector: + matchLabels: + name: nginx-ingress + ports: + - protocol: TCP + port: 8080 + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: opensearch + ports: + - protocol: TCP + port: 9200 \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml new file mode 100644 index 000000000..b520d701d --- /dev/null +++ b/helm-chart/stac-fastapi/values.yaml @@ -0,0 +1,359 @@ +# Default values for stac-fastapi. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Backend configuration - choose 'elasticsearch' or 'opensearch' +backend: elasticsearch + +# Global configuration +global: + imageRegistry: "" + storageClass: "" + +# STAC FastAPI application configuration +app: + # Number of replicas for the application + replicaCount: 2 + + image: + # Repository for the STAC FastAPI image + repository: ghcr.io/stac-utils/stac-fastapi + # Image pull policy + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion + tag: "" + + # Image pull secrets + imagePullSecrets: [] + + # Override the name of the chart + nameOverride: "" + # Override the full name of the chart + fullnameOverride: "" + + # Service account configuration + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + # Pod annotations + podAnnotations: {} + + # Pod security context + podSecurityContext: {} + # fsGroup: 2000 + + # Security context + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + # Service configuration + service: + type: ClusterIP + port: 80 + targetPort: 8080 + + # Ingress configuration + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: stac-fastapi.local + paths: + - path: / + pathType: Prefix + tls: [] + # - secretName: stac-fastapi-tls + # hosts: + # - stac-fastapi.local + + # Resource limits and requests + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Horizontal Pod Autoscaler + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + + # Node selector + nodeSelector: {} + + # Tolerations + tolerations: [] + + # Affinity + affinity: {} + + # Liveness probe configuration + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + + # Readiness probe configuration + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + + # Environment variables for STAC FastAPI + env: + # Application configuration + STAC_FASTAPI_TITLE: "stac-fastapi" + STAC_FASTAPI_DESCRIPTION: "A STAC FastAPI with configurable backend" + STAC_FASTAPI_VERSION: "6.0.0" + STAC_FASTAPI_LANDING_PAGE_ID: "stac-fastapi" + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + ENVIRONMENT: "production" + WEB_CONCURRENCY: "10" + RELOAD: "false" + + # Database configuration + DATABASE_REFRESH: "false" + ENABLE_TRANSACTIONS_EXTENSIONS: "true" + STAC_ITEM_LIMIT: "10" + ENV_MAX_LIMIT: "10000" + STAC_INDEX_ASSETS: "false" + RAISE_ON_BULK_ERROR: "false" + + # Performance configuration + ENABLE_DIRECT_RESPONSE: "false" + + # Rate limiting + STAC_FASTAPI_RATE_LIMIT: "200/minute" + + # Datetime index filtering + ENABLE_DATETIME_INDEX_FILTERING: "false" + DATETIME_INDEX_MAX_SIZE_GB: "25" + STAC_ITEMS_INDEX_PREFIX: "items_" + + # SSL configuration + ES_USE_SSL: "false" + ES_VERIFY_CERTS: "false" + + # Additional environment variables from secrets + envFromSecret: {} + # ES_API_KEY: "es-api-key-secret" + +# Elasticsearch configuration +elasticsearch: + # Enable/disable Elasticsearch deployment + enabled: true + + # Elasticsearch cluster name + clusterName: "stac-elasticsearch" + + # Node group configuration + nodeGroup: "master" + + # Number of masters eligible nodes + masterService: "stac-elasticsearch-master" + + # Elasticsearch roles + roles: + master: "true" + ingest: "true" + data: "true" + remote_cluster_client: "true" + ml: "false" + + # Number of replicas + replicas: 1 + + # Minimum master nodes + minimumMasterNodes: 1 + + # Elasticsearch configuration + esConfig: + elasticsearch.yml: | + cluster.name: "stac-elasticsearch" + network.host: 0.0.0.0 + discovery.type: single-node + action.destructive_requires_name: false + xpack.security.enabled: false + xpack.security.transport.ssl.enabled: false + xpack.security.http.ssl.enabled: false + + # JVM configuration + esJavaOpts: "-Xmx1g -Xms1g" + + # Resource configuration + resources: + requests: + cpu: "100m" + memory: "2Gi" + limits: + cpu: "1000m" + memory: "2Gi" + + # Volume claim template + volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "" + resources: + requests: + storage: 10Gi + + # Service configuration + service: + type: ClusterIP + nodePort: "" + annotations: {} + httpPortName: http + transportPortName: transport + labels: {} + labelsHeadless: {} + httpPort: 9200 + transportPort: 9300 + +# OpenSearch configuration +opensearch: + # Enable/disable OpenSearch deployment + enabled: false + + # OpenSearch cluster name + clusterName: "stac-opensearch" + + # Node group configuration + nodeGroup: "master" + + # OpenSearch roles + roles: + - master + - ingest + - data + - remote_cluster_client + + # Number of replicas + replicas: 1 + + # Minimum master nodes + majorVersion: "" + + # OpenSearch configuration + config: + opensearch.yml: | + cluster.name: stac-opensearch + network.host: 0.0.0.0 + discovery.type: single-node + plugins.security.disabled: true + action.destructive_requires_name: false + + # JVM configuration + opensearchJavaOpts: "-Xmx1g -Xms1g" + + # Resource configuration + resources: + requests: + cpu: "100m" + memory: "2Gi" + limits: + cpu: "1000m" + memory: "2Gi" + + # Persistence configuration + persistence: + enabled: true + storageClass: "" + accessModes: + - ReadWriteOnce + size: 10Gi + annotations: {} + + # Service configuration + service: + type: ClusterIP + nodePort: "" + annotations: {} + httpPortName: http + transportPortName: transport + labels: {} + httpPort: 9200 + transportPort: 9300 + +# External database configuration (when not using bundled ES/OS) +externalDatabase: + # Whether to use external database instead of bundled one + enabled: false + # Database host + host: "" + # Database port + port: 9200 + # Whether to use SSL + ssl: false + # Whether to verify certificates + verifyCerts: false + # Connection timeout + timeout: 30 + # API key for authentication (reference to secret) + apiKeySecret: "" + # API key field in the secret + apiKeySecretKey: "api-key" + +# Configuration for monitoring and observability +monitoring: + # Enable monitoring + enabled: false + + # Prometheus monitoring + prometheus: + enabled: false + serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + labels: {} + + # Grafana dashboards + grafana: + enabled: false + dashboards: {} + +# Network policies +networkPolicy: + enabled: false + ingress: [] + egress: [] + +# Pod disruption budget +podDisruptionBudget: + enabled: false + minAvailable: 1 + # maxUnavailable: 1 + +# Security policies +securityPolicy: + enabled: false + psp: + enabled: false \ No newline at end of file diff --git a/helm-chart/test-chart.sh b/helm-chart/test-chart.sh new file mode 100755 index 000000000..426e4a998 --- /dev/null +++ b/helm-chart/test-chart.sh @@ -0,0 +1,452 @@ +#!/bin/bash + +# STAC FastAPI Helm Chart Testing Script +# This script helps validate and test Helm chart deployments + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +CHART_PATH="./helm-chart/stac-fastapi" +RELEASE_NAME="stac-fastapi-test" +NAMESPACE="stac-fastapi" +BACKEND=${BACKEND:-"elasticsearch"} + +# Functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Help function +show_help() { + cat << EOF +STAC FastAPI Helm Chart Testing Script + +Usage: $0 [OPTIONS] COMMAND + +Commands: + lint Lint the Helm chart + test Run Helm chart tests + install Install the chart for testing + upgrade Upgrade existing installation + uninstall Uninstall the test deployment + validate Validate deployment health + load-data Load sample data into the API + cleanup Clean up all test resources + +Options: + -b, --backend BACKEND Backend to test (elasticsearch|opensearch) [default: elasticsearch] + -n, --namespace NS Kubernetes namespace [default: stac-fastapi] + -r, --release NAME Helm release name [default: stac-fastapi-test] + -h, --help Show this help message + +Examples: + $0 lint # Lint the chart + $0 -b opensearch install # Install with OpenSearch backend + $0 validate # Check deployment health + $0 load-data # Load test data + $0 cleanup # Clean up everything + +EOF +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -b|--backend) + BACKEND="$2" + shift 2 + ;; + -n|--namespace) + NAMESPACE="$2" + shift 2 + ;; + -r|--release) + RELEASE_NAME="$2" + shift 2 + ;; + -h|--help) + show_help + exit 0 + ;; + lint|test|install|upgrade|uninstall|validate|load-data|cleanup) + COMMAND="$1" + shift + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Validate backend +if [[ "$BACKEND" != "elasticsearch" && "$BACKEND" != "opensearch" ]]; then + log_error "Invalid backend: $BACKEND. Must be 'elasticsearch' or 'opensearch'" + exit 1 +fi + +# Check prerequisites +check_prerequisites() { + log_info "Checking prerequisites..." + + if ! command -v helm &> /dev/null; then + log_error "Helm is not installed" + exit 1 + fi + + if ! command -v kubectl &> /dev/null; then + log_error "kubectl is not installed" + exit 1 + fi + + if ! kubectl cluster-info &> /dev/null; then + log_error "Cannot connect to Kubernetes cluster" + exit 1 + fi + + log_success "Prerequisites check passed" +} + +# Lint the Helm chart +lint_chart() { + log_info "Linting Helm chart..." + + if [[ ! -d "$CHART_PATH" ]]; then + log_error "Chart path not found: $CHART_PATH" + exit 1 + fi + + # Update dependencies + log_info "Updating chart dependencies..." + helm dependency update "$CHART_PATH" + + # Lint the chart + helm lint "$CHART_PATH" + + # Template the chart to check for syntax errors + log_info "Testing chart templates..." + helm template test-release "$CHART_PATH" \ + --set backend="$BACKEND" \ + --set "${BACKEND}.enabled=true" \ + --output-dir /tmp/helm-test || { + log_error "Chart templating failed" + exit 1 + } + + log_success "Chart linting completed successfully" +} + +# Run Helm chart tests +test_chart() { + log_info "Running Helm chart tests..." + + # Dry run installation + log_info "Testing chart installation (dry run)..." + helm install "$RELEASE_NAME" "$CHART_PATH" \ + --namespace "$NAMESPACE" \ + --create-namespace \ + --dry-run \ + --set backend="$BACKEND" \ + --set "${BACKEND}.enabled=true" \ + --set "app.image.tag=latest" + + log_success "Chart tests completed successfully" +} + +# Install the chart +install_chart() { + log_info "Installing STAC FastAPI chart with $BACKEND backend..." + + # Create namespace if it doesn't exist + kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - + + # Update dependencies + helm dependency update "$CHART_PATH" + + # Install the chart + helm install "$RELEASE_NAME" "$CHART_PATH" \ + --namespace "$NAMESPACE" \ + --set backend="$BACKEND" \ + --set "${BACKEND}.enabled=true" \ + --set "app.image.tag=latest" \ + --set "app.service.type=ClusterIP" \ + --wait \ + --timeout=10m + + log_success "Chart installed successfully" + + # Show installation status + helm status "$RELEASE_NAME" -n "$NAMESPACE" +} + +# Upgrade the chart +upgrade_chart() { + log_info "Upgrading STAC FastAPI chart..." + + helm dependency update "$CHART_PATH" + + helm upgrade "$RELEASE_NAME" "$CHART_PATH" \ + --namespace "$NAMESPACE" \ + --set backend="$BACKEND" \ + --set "${BACKEND}.enabled=true" \ + --set "app.image.tag=latest" \ + --wait \ + --timeout=10m + + log_success "Chart upgraded successfully" +} + +# Uninstall the chart +uninstall_chart() { + log_info "Uninstalling STAC FastAPI chart..." + + helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" || true + + log_success "Chart uninstalled" +} + +# Validate deployment +validate_deployment() { + log_info "Validating deployment health..." + + # Check if deployment exists + if ! kubectl get deployment "$RELEASE_NAME" -n "$NAMESPACE" &> /dev/null; then + log_error "Deployment not found: $RELEASE_NAME" + exit 1 + fi + + # Check pod status + log_info "Checking pod status..." + kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/name=stac-fastapi" + + # Wait for pods to be ready + log_info "Waiting for pods to be ready..." + kubectl wait --for=condition=ready pod \ + -l "app.kubernetes.io/name=stac-fastapi" \ + -n "$NAMESPACE" \ + --timeout=300s + + # Check service + log_info "Checking service..." + kubectl get service "$RELEASE_NAME" -n "$NAMESPACE" + + # Test API endpoint + log_info "Testing API endpoint..." + kubectl port-forward -n "$NAMESPACE" service/"$RELEASE_NAME" 8080:80 & + PORT_FORWARD_PID=$! + + # Wait for port-forward to be ready + sleep 5 + + # Test the API + if curl -s -f http://localhost:8080/ > /dev/null; then + log_success "API is responding" + + # Test specific endpoints + log_info "Testing API endpoints..." + + # Root endpoint + curl -s http://localhost:8080/ | jq -r '.title // "No title"' 2>/dev/null || echo "Root endpoint accessible" + + # Collections endpoint + if curl -s -f http://localhost:8080/collections > /dev/null; then + log_success "Collections endpoint accessible" + else + log_warning "Collections endpoint not accessible" + fi + + # Search endpoint + if curl -s -f -X POST http://localhost:8080/search -H "Content-Type: application/json" -d '{}' > /dev/null; then + log_success "Search endpoint accessible" + else + log_warning "Search endpoint not accessible" + fi + + else + log_error "API is not responding" + fi + + # Clean up port-forward + kill $PORT_FORWARD_PID 2>/dev/null || true + + # Check database connectivity + log_info "Checking database connectivity..." + DB_POD=$(kubectl get pods -n "$NAMESPACE" -l "app=${RELEASE_NAME}-${BACKEND}-master" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "") + + if [[ -n "$DB_POD" ]]; then + if kubectl exec -n "$NAMESPACE" "$DB_POD" -- curl -s -f http://localhost:9200/_health > /dev/null 2>&1; then + log_success "Database is healthy" + else + log_warning "Database health check failed" + fi + else + log_warning "Database pod not found (might be using external database)" + fi + + log_success "Deployment validation completed" +} + +# Load sample data +load_sample_data() { + log_info "Loading sample data..." + + # Port forward to the service + kubectl port-forward -n "$NAMESPACE" service/"$RELEASE_NAME" 8080:80 & + PORT_FORWARD_PID=$! + + # Wait for port-forward + sleep 5 + + # Create a simple collection + log_info "Creating test collection..." + curl -X POST http://localhost:8080/collections \ + -H "Content-Type: application/json" \ + -d '{ + "id": "test-collection", + "title": "Test Collection", + "description": "A test collection for validation", + "extent": { + "spatial": { + "bbox": [[-180, -90, 180, 90]] + }, + "temporal": { + "interval": [["2020-01-01T00:00:00Z", "2024-12-31T23:59:59Z"]] + } + }, + "license": "public-domain" + }' || log_warning "Failed to create collection (might already exist)" + + # Create a test item + log_info "Creating test item..." + curl -X POST http://localhost:8080/collections/test-collection/items \ + -H "Content-Type: application/json" \ + -d '{ + "id": "test-item-001", + "type": "Feature", + "stac_version": "1.0.0", + "collection": "test-collection", + "geometry": { + "type": "Polygon", + "coordinates": [[[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]] + }, + "bbox": [-1, -1, 1, 1], + "properties": { + "datetime": "2023-06-15T12:00:00Z" + }, + "assets": { + "thumbnail": { + "href": "https://example.com/thumbnail.jpg", + "type": "image/jpeg", + "title": "Thumbnail" + } + } + }' || log_warning "Failed to create item" + + # Test search + log_info "Testing search functionality..." + SEARCH_RESULT=$(curl -s -X POST http://localhost:8080/search \ + -H "Content-Type: application/json" \ + -d '{"collections": ["test-collection"], "limit": 1}') + + ITEM_COUNT=$(echo "$SEARCH_RESULT" | jq -r '.features | length' 2>/dev/null || echo "0") + + if [[ "$ITEM_COUNT" -gt 0 ]]; then + log_success "Sample data loaded and searchable (found $ITEM_COUNT items)" + else + log_warning "Sample data might not be immediately searchable (indexing delay)" + fi + + # Clean up port-forward + kill $PORT_FORWARD_PID 2>/dev/null || true + + log_success "Sample data loading completed" +} + +# Clean up all resources +cleanup() { + log_info "Cleaning up all test resources..." + + # Uninstall Helm release + helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" --ignore-not-found + + # Delete namespace (this will delete all resources) + kubectl delete namespace "$NAMESPACE" --ignore-not-found + + # Clean up any remaining persistent volumes + kubectl get pv | grep "$NAMESPACE" | awk '{print $1}' | xargs -r kubectl delete pv + + log_success "Cleanup completed" +} + +# Main script logic +main() { + case "${COMMAND:-}" in + lint) + check_prerequisites + lint_chart + ;; + test) + check_prerequisites + lint_chart + test_chart + ;; + install) + check_prerequisites + install_chart + ;; + upgrade) + check_prerequisites + upgrade_chart + ;; + uninstall) + uninstall_chart + ;; + validate) + check_prerequisites + validate_deployment + ;; + load-data) + check_prerequisites + load_sample_data + ;; + cleanup) + cleanup + ;; + "") + log_error "No command specified" + show_help + exit 1 + ;; + *) + log_error "Unknown command: $COMMAND" + show_help + exit 1 + ;; + esac +} + +# Trap to ensure cleanup on script exit +trap 'kill $PORT_FORWARD_PID 2>/dev/null || true' EXIT + +# Run main function +main \ No newline at end of file From 55eac51104178d11ff526aa4f6cdc8d07990b546 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Tue, 23 Sep 2025 13:52:16 +0930 Subject: [PATCH 02/12] Fix dependency versions and add gitignore --- .gitignore | 11 +++++++++++ helm-chart/stac-fastapi/Chart.yaml | 6 +++--- helm-chart/stac-fastapi/values.yaml | 9 ++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e66d06727..6490d54d5 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,14 @@ venv /docs/src/api/* .DS_Store + +# Helm +*.tgz +charts/*/charts/ +charts/*/requirements.lock +charts/*/Chart.lock +helm-chart/stac-fastapi/charts/ +helm-chart/stac-fastapi/Chart.lock +helm-chart/stac-fastapi/*.tgz +helm-chart/test-results/ +helm-chart/tmp/ diff --git a/helm-chart/stac-fastapi/Chart.yaml b/helm-chart/stac-fastapi/Chart.yaml index b3e96f1b7..a197cdd47 100644 --- a/helm-chart/stac-fastapi/Chart.yaml +++ b/helm-chart/stac-fastapi/Chart.yaml @@ -3,7 +3,7 @@ name: stac-fastapi description: A Helm chart for STAC FastAPI with Elasticsearch and OpenSearch backends type: application version: 0.1.0 -appVersion: "6.0.0" +appVersion: "6.3.0" keywords: - stac @@ -23,10 +23,10 @@ maintainers: dependencies: - name: elasticsearch - version: 19.19.4 + version: 8.5.1 repository: https://helm.elastic.co condition: elasticsearch.enabled - name: opensearch - version: 2.18.0 + version: 3.2.1 repository: https://opensearch-project.github.io/helm-charts/ condition: opensearch.enabled \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index b520d701d..7b0fcd907 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -182,11 +182,10 @@ elasticsearch: # Elasticsearch roles roles: - master: "true" - ingest: "true" - data: "true" - remote_cluster_client: "true" - ml: "false" + - master + - ingest + - data + - remote_cluster_client # Number of replicas replicas: 1 From 6b865bb491a1e1e1086d6c5b830f7cc8be749f8f Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Thu, 25 Sep 2025 10:22:54 +0930 Subject: [PATCH 03/12] Clean up some auto generated content --- .github/actions/setup-helm-repos/action.yml | 12 + .github/ct-config.yaml | 11 - .github/ct-lint.yaml | 43 ++ .github/ct.yaml | 35 + .github/workflows/helm-chart-test.yml | 202 ++++++ .github/workflows/helm-chart.yml | 292 -------- helm-chart/DEPLOYMENT.md | 635 ------------------ helm-chart/stac-fastapi/README.md | 39 +- .../stac-fastapi/templates/_helpers.tpl | 22 + helm-chart/stac-fastapi/values-dev.yaml | 124 ++++ .../stac-fastapi/values-minimal-storage.yaml | 102 +++ .../stac-fastapi/values-no-persistence.yaml | 128 ++++ helm-chart/stac-fastapi/values.yaml | 5 +- helm-chart/test-chart.sh | 346 +++++++++- 14 files changed, 1047 insertions(+), 949 deletions(-) create mode 100644 .github/actions/setup-helm-repos/action.yml delete mode 100644 .github/ct-config.yaml create mode 100644 .github/ct-lint.yaml create mode 100644 .github/ct.yaml create mode 100644 .github/workflows/helm-chart-test.yml delete mode 100644 .github/workflows/helm-chart.yml delete mode 100644 helm-chart/DEPLOYMENT.md create mode 100644 helm-chart/stac-fastapi/values-dev.yaml create mode 100644 helm-chart/stac-fastapi/values-minimal-storage.yaml create mode 100644 helm-chart/stac-fastapi/values-no-persistence.yaml diff --git a/.github/actions/setup-helm-repos/action.yml b/.github/actions/setup-helm-repos/action.yml new file mode 100644 index 000000000..0ea44f837 --- /dev/null +++ b/.github/actions/setup-helm-repos/action.yml @@ -0,0 +1,12 @@ +name: 'Setup Helm Repositories' +description: 'Add required Helm repositories for STAC FastAPI' + +runs: + using: 'composite' + steps: + - name: Add Helm repositories + shell: bash + run: | + helm repo add elasticsearch https://helm.elastic.co + helm repo add opensearch https://opensearch-project.github.io/helm-charts/ + helm repo update \ No newline at end of file diff --git a/.github/ct-config.yaml b/.github/ct-config.yaml deleted file mode 100644 index 7a0653bd4..000000000 --- a/.github/ct-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Configuration for chart-testing -chart-dirs: - - helm-chart - -chart-repos: - - elastic=https://helm.elastic.co - - opensearch=https://opensearch-project.github.io/helm-charts/ - -helm-extra-args: --timeout 10m - -validate-maintainers: false \ No newline at end of file diff --git a/.github/ct-lint.yaml b/.github/ct-lint.yaml new file mode 100644 index 000000000..befb4dc7e --- /dev/null +++ b/.github/ct-lint.yaml @@ -0,0 +1,43 @@ +# Chart Testing Lint Configuration +# Additional linting rules for chart-testing + +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 1 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + min-spaces-from-content: 1 + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + max: 2 + max-start: 0 + max-end: 1 + hyphens: + max-spaces-after: 1 + indentation: + spaces: 2 + indent-sequences: true + key-duplicates: enable + line-length: + max: 120 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: true + new-line-at-end-of-file: enable + octal-values: disable + quoted-strings: disable + trailing-spaces: enable + truthy: + allowed-values: ['true', 'false'] + check-keys: false \ No newline at end of file diff --git a/.github/ct.yaml b/.github/ct.yaml new file mode 100644 index 000000000..ce1b5b7ee --- /dev/null +++ b/.github/ct.yaml @@ -0,0 +1,35 @@ +# Chart Testing Configuration +# This configuration is used by the chart-testing (ct) tool + +# Chart directories relative to repository root +chart-dirs: + - helm-chart + +# Target branch for pull request testing +target-branch: main + +# Chart repositories to add +chart-repos: + - name: elasticsearch + url: https://helm.elastic.co + - name: opensearch + url: https://opensearch-project.github.io/helm-charts/ + +# Helm version to use +helm-version: v3.13.0 + +# Additional configurations +validate-chart-schema: true +validate-maintainers: false +check-version-increment: true +process-skipped-charts: false + +# Test configuration +upgrade: false +skip-missing-values: true + +# Linting configuration +lint-conf: .github/ct-lint.yaml + +# Additional Helm lint settings +helm-lint-extra-args: --set app.image.tag=latest \ No newline at end of file diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml new file mode 100644 index 000000000..218baa228 --- /dev/null +++ b/.github/workflows/helm-chart-test.yml @@ -0,0 +1,202 @@ +name: Helm Chart Tests + +on: + push: + branches: [ main, develop, feat/* ] + paths: + - 'helm-chart/**' + - '.github/workflows/helm-chart-test.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'helm-chart/**' + - '.github/workflows/helm-chart-test.yml' + +env: + HELM_VERSION: v3.13.0 + KUBECTL_VERSION: v1.28.0 + KIND_VERSION: v0.20.0 + +jobs: + lint: + name: Lint Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Setup Helm repositories + uses: ./.github/actions/setup-helm-repos + + - name: Lint Helm chart + run: | + cd helm-chart/stac-fastapi + helm dependency update + helm lint . + + test-matrix: + name: Test Chart + runs-on: ubuntu-latest + needs: lint + strategy: + matrix: + backend: [elasticsearch, opensearch] + kubernetes-version: [v1.27.3, v1.28.0] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBECTL_VERSION }} + + - name: Set up Kind + uses: helm/kind-action@v1.8.0 + with: + version: ${{ env.KIND_VERSION }} + node_image: kindest/node:${{ matrix.kubernetes-version }} + cluster_name: stac-fastapi-test + + - name: Setup Helm repositories + uses: ./.github/actions/setup-helm-repos + + - name: Run matrix tests + env: + BACKEND: ${{ matrix.backend }} + MATRIX_MODE: true + run: | + chmod +x ./helm-chart/test-chart.sh + ./helm-chart/test-chart.sh -m -b ${{ matrix.backend }} ci + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-report-${{ matrix.backend }}-k8s-${{ matrix.kubernetes-version }} + path: test-report-*.json + + integration-test: + name: Integration Tests + runs-on: ubuntu-latest + needs: test-matrix + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBECTL_VERSION }} + + - name: Set up Kind + uses: helm/kind-action@v1.8.0 + with: + version: ${{ env.KIND_VERSION }} + cluster_name: stac-fastapi-integration + + - name: Setup Helm repositories + uses: ./.github/actions/setup-helm-repos + + - name: Run full integration tests + run: | + chmod +x ./helm-chart/test-chart.sh + ./helm-chart/test-chart.sh test-all + + - name: Test upgrade scenarios + run: | + # Test elasticsearch to opensearch migration scenario + ./helm-chart/test-chart.sh -b elasticsearch install + ./helm-chart/test-chart.sh validate + ./helm-chart/test-chart.sh cleanup + + # Test opensearch deployment + ./helm-chart/test-chart.sh -b opensearch install + ./helm-chart/test-chart.sh validate + ./helm-chart/test-chart.sh cleanup + + security-scan: + name: Security Scan + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Setup Helm repositories + uses: ./.github/actions/setup-helm-repos + + - name: Run Checkov security scan + uses: bridgecrewio/checkov-action@master + with: + directory: helm-chart/ + framework: kubernetes + output_format: sarif + output_file_path: reports/results.sarif + + - name: Upload Checkov results + if: always() + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: reports/results.sarif + + chart-testing: + name: Chart Testing (ct) + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.6.1 + + - name: Setup Helm repositories + uses: ./.github/actions/setup-helm-repos + + - name: Run chart-testing (list) + run: ct list --config .github/ct.yaml + + - name: Run chart-testing (lint) + run: ct lint --config .github/ct.yaml + + - name: Set up Kind cluster + uses: helm/kind-action@v1.8.0 + with: + version: ${{ env.KIND_VERSION }} + + - name: Run chart-testing (install) + run: ct install --config .github/ct.yaml \ No newline at end of file diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml deleted file mode 100644 index 6f014ba14..000000000 --- a/.github/workflows/helm-chart.yml +++ /dev/null @@ -1,292 +0,0 @@ -name: Helm Chart CI - -on: - push: - paths: - - 'helm-chart/**' - - '.github/workflows/helm-chart.yml' - pull_request: - paths: - - 'helm-chart/**' - - '.github/workflows/helm-chart.yml' - -env: - HELM_VERSION: v3.13.0 - KUBECTL_VERSION: v1.28.0 - -jobs: - lint-and-test: - runs-on: ubuntu-latest - strategy: - matrix: - backend: [elasticsearch, opensearch] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: ${{ env.HELM_VERSION }} - - - name: Set up kubectl - uses: azure/setup-kubectl@v3 - with: - version: ${{ env.KUBECTL_VERSION }} - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Set up chart-testing - uses: helm/chart-testing-action@v2.6.1 - - - name: Add Helm repositories - run: | - helm repo add elastic https://helm.elastic.co - helm repo add opensearch https://opensearch-project.github.io/helm-charts/ - helm repo update - - - name: Lint Helm chart - run: | - cd helm-chart/stac-fastapi - helm dependency update - helm lint . - - - name: Template Helm chart - run: | - cd helm-chart/stac-fastapi - helm template test-release . \ - --set backend=${{ matrix.backend }} \ - --set ${{ matrix.backend }}.enabled=true \ - --set app.image.tag=latest \ - --output-dir /tmp/helm-test-${{ matrix.backend }} - - - name: Validate templated manifests - run: | - # Check that all required resources are created - ls -la /tmp/helm-test-${{ matrix.backend }}/stac-fastapi/templates/ - - # Validate YAML syntax - find /tmp/helm-test-${{ matrix.backend }} -name "*.yaml" -exec kubectl apply --dry-run=client -f {} \; - - - name: Run chart-testing (lint) - run: | - ct lint --config .github/ct-config.yaml --charts helm-chart/stac-fastapi - - integration-test: - runs-on: ubuntu-latest - needs: lint-and-test - strategy: - matrix: - backend: [elasticsearch, opensearch] - k8s-version: ['1.26.6', '1.27.3', '1.28.0'] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: ${{ env.HELM_VERSION }} - - - name: Set up kubectl - uses: azure/setup-kubectl@v3 - with: - version: ${{ env.KUBECTL_VERSION }} - - - name: Create kind cluster - uses: helm/kind-action@v1.8.0 - with: - node_image: kindest/node:v${{ matrix.k8s-version }} - cluster_name: kind - config: | - kind: Cluster - apiVersion: kind.x-k8s.io/v1alpha4 - nodes: - - role: control-plane - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - extraPortMappings: - - containerPort: 80 - hostPort: 80 - protocol: TCP - - containerPort: 443 - hostPort: 443 - protocol: TCP - - - name: Add Helm repositories - run: | - helm repo add elastic https://helm.elastic.co - helm repo add opensearch https://opensearch-project.github.io/helm-charts/ - helm repo update - - - name: Install STAC FastAPI chart - run: | - cd helm-chart/stac-fastapi - helm dependency update - - # Install with specific backend - helm install stac-fastapi-test . \ - --namespace stac-fastapi \ - --create-namespace \ - --set backend=${{ matrix.backend }} \ - --set ${{ matrix.backend }}.enabled=true \ - --set app.image.tag=latest \ - --set app.service.type=ClusterIP \ - --wait \ - --timeout=10m - - - name: Wait for deployment - run: | - kubectl wait --for=condition=ready pod \ - -l "app.kubernetes.io/name=stac-fastapi" \ - -n stac-fastapi \ - --timeout=300s - - - name: Check deployment status - run: | - kubectl get all -n stac-fastapi - helm status stac-fastapi-test -n stac-fastapi - - - name: Test API endpoints - run: | - # Port forward to the service - kubectl port-forward -n stac-fastapi service/stac-fastapi-test 8080:80 & - sleep 10 - - # Test root endpoint - curl -f http://localhost:8080/ || exit 1 - - # Test collections endpoint - curl -f http://localhost:8080/collections || exit 1 - - # Test search endpoint - curl -f -X POST http://localhost:8080/search \ - -H "Content-Type: application/json" \ - -d '{}' || exit 1 - - echo "All API endpoints are working!" - - - name: Test database connectivity - run: | - # Check if database is responding - DB_SERVICE=$(kubectl get svc -n stac-fastapi -l "app=stac-fastapi-test-${{ matrix.backend }}-master" -o jsonpath="{.items[0].metadata.name}" || echo "") - - if [[ -n "$DB_SERVICE" ]]; then - kubectl port-forward -n stac-fastapi service/$DB_SERVICE 9200:9200 & - sleep 5 - curl -f http://localhost:9200/_health || echo "Database health check failed" - else - echo "Database service not found or using external database" - fi - - - name: Load test data - run: | - # Port forward to the API service - kubectl port-forward -n stac-fastapi service/stac-fastapi-test 8080:80 & - sleep 5 - - # Create test collection - curl -X POST http://localhost:8080/collections \ - -H "Content-Type: application/json" \ - -d '{ - "id": "test-collection", - "title": "Test Collection", - "description": "A test collection", - "extent": { - "spatial": {"bbox": [[-180, -90, 180, 90]]}, - "temporal": {"interval": [["2020-01-01T00:00:00Z", "2024-12-31T23:59:59Z"]]} - }, - "license": "public-domain" - }' || echo "Collection creation failed" - - # Create test item - curl -X POST http://localhost:8080/collections/test-collection/items \ - -H "Content-Type: application/json" \ - -d '{ - "id": "test-item", - "type": "Feature", - "stac_version": "1.0.0", - "collection": "test-collection", - "geometry": { - "type": "Polygon", - "coordinates": [[[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]] - }, - "bbox": [-1, -1, 1, 1], - "properties": {"datetime": "2023-06-15T12:00:00Z"}, - "assets": { - "thumbnail": { - "href": "https://example.com/thumbnail.jpg", - "type": "image/jpeg" - } - } - }' || echo "Item creation failed" - - # Test search - sleep 5 # Allow time for indexing - RESULT=$(curl -s -X POST http://localhost:8080/search \ - -H "Content-Type: application/json" \ - -d '{"collections": ["test-collection"]}') - - echo "Search result: $RESULT" - - - name: Cleanup - if: always() - run: | - helm uninstall stac-fastapi-test -n stac-fastapi || true - kubectl delete namespace stac-fastapi || true - - security-scan: - runs-on: ubuntu-latest - needs: lint-and-test - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: ${{ env.HELM_VERSION }} - - - name: Add Helm repositories - run: | - helm repo add elastic https://helm.elastic.co - helm repo add opensearch https://opensearch-project.github.io/helm-charts/ - helm repo update - - - name: Template chart for security scan - run: | - cd helm-chart/stac-fastapi - helm dependency update - helm template security-scan . \ - --set backend=elasticsearch \ - --set elasticsearch.enabled=true \ - --output-dir /tmp/security-scan - - - name: Run Checkov security scan - uses: bridgecrewio/checkov-action@master - with: - directory: /tmp/security-scan - framework: kubernetes - soft_fail: true - output_format: sarif - output_file_path: checkov-results.sarif - - - name: Upload Checkov results - uses: github/codeql-action/upload-sarif@v2 - if: always() - with: - sarif_file: checkov-results.sarif \ No newline at end of file diff --git a/helm-chart/DEPLOYMENT.md b/helm-chart/DEPLOYMENT.md deleted file mode 100644 index 27ed93648..000000000 --- a/helm-chart/DEPLOYMENT.md +++ /dev/null @@ -1,635 +0,0 @@ -# STAC FastAPI Kubernetes Deployment Guide - -This guide provides comprehensive instructions for deploying STAC FastAPI on Kubernetes using the provided Helm chart. - -## Table of Contents - -1. [Overview](#overview) -2. [Prerequisites](#prerequisites) -3. [Quick Start](#quick-start) -4. [Deployment Options](#deployment-options) -5. [Configuration](#configuration) -6. [Production Deployment](#production-deployment) -7. [Monitoring and Observability](#monitoring-and-observability) -8. [Troubleshooting](#troubleshooting) -9. [Maintenance](#maintenance) - -## Overview - -The STAC FastAPI Helm chart supports multiple deployment configurations: - -- **Elasticsearch Backend**: Deploy with bundled or external Elasticsearch -- **OpenSearch Backend**: Deploy with bundled or external OpenSearch -- **High Availability**: Multi-replica deployments with proper load balancing -- **Auto-scaling**: Horizontal Pod Autoscaler based on CPU/memory metrics -- **Security**: Network policies, RBAC, and pod security contexts -- **Monitoring**: Prometheus metrics and Grafana dashboards - -## Prerequisites - -### Required - -- Kubernetes cluster (v1.16+) -- Helm 3.0+ -- kubectl configured to access your cluster -- Sufficient cluster resources (see resource requirements below) - -### Optional - -- NGINX Ingress Controller (for ingress) -- cert-manager (for TLS certificates) -- Prometheus Operator (for monitoring) -- Persistent storage class (for database persistence) - -### Resource Requirements - -**Minimum (Development)**: -- 2 CPU cores -- 4 GB RAM -- 20 GB storage - -**Recommended (Production)**: -- 8 CPU cores -- 16 GB RAM -- 200 GB SSD storage - -## Quick Start - -### 1. Clone and Prepare - -```bash -git clone https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch.git -cd stac-fastapi-elasticsearch-opensearch/helm-chart -``` - -### 2. Update Dependencies - -```bash -helm dependency update stac-fastapi -``` - -### 3. Install with Elasticsearch - -```bash -helm install my-stac-api stac-fastapi \ - --set backend=elasticsearch \ - --set elasticsearch.enabled=true \ - --set opensearch.enabled=false \ - --create-namespace \ - --namespace stac-fastapi -``` - -### 4. Install with OpenSearch - -```bash -helm install my-stac-api stac-fastapi \ - --set backend=opensearch \ - --set elasticsearch.enabled=false \ - --set opensearch.enabled=true \ - --create-namespace \ - --namespace stac-fastapi -``` - -### 5. Access the API - -```bash -# Port forward to access locally -kubectl port-forward -n stac-fastapi service/my-stac-api 8080:80 - -# Test the API -curl http://localhost:8080/ -curl http://localhost:8080/collections -``` - -## Deployment Options - -### 1. Development Deployment - -**Use case**: Local development, testing, minimal resources - -```bash -helm install dev-stac stac-fastapi \ - --set backend=elasticsearch \ - --set app.replicaCount=1 \ - --set elasticsearch.replicas=1 \ - --set elasticsearch.minimumMasterNodes=1 \ - --set elasticsearch.resources.requests.memory=1Gi \ - --set elasticsearch.volumeClaimTemplate.resources.requests.storage=10Gi -``` - -### 2. Production Deployment with Elasticsearch - -**Use case**: High-availability production environment - -```bash -helm install prod-stac stac-fastapi \ - --values stac-fastapi/values-elasticsearch.yaml \ - --set app.ingress.enabled=true \ - --set app.ingress.hosts[0].host=stac-api.yourdomain.com \ - --set elasticsearch.volumeClaimTemplate.storageClassName=fast-ssd -``` - -### 3. Production Deployment with OpenSearch - -**Use case**: Production environment with OpenSearch preference - -```bash -helm install prod-stac stac-fastapi \ - --values stac-fastapi/values-opensearch.yaml \ - --set app.ingress.enabled=true \ - --set app.ingress.hosts[0].host=stac-api.yourdomain.com \ - --set opensearch.persistence.storageClass=fast-ssd -``` - -### 4. External Database Deployment - -**Use case**: Connect to existing Elasticsearch/OpenSearch cluster - -```bash -# First create secret for API key -kubectl create secret generic elasticsearch-credentials \ - --from-literal=api-key="your-api-key-here" - -# Deploy -helm install external-stac stac-fastapi \ - --values stac-fastapi/values-external.yaml \ - --set externalDatabase.host=elasticsearch.example.com \ - --set externalDatabase.port=9200 \ - --set externalDatabase.ssl=true -``` - -### 5. Multi-Environment Deployment - -**Use case**: Deploy multiple environments (dev, staging, prod) - -```bash -# Development -helm install dev-stac stac-fastapi \ - --namespace stac-dev \ - --create-namespace \ - --set backend=elasticsearch \ - --set app.env.ENVIRONMENT=development - -# Staging -helm install staging-stac stac-fastapi \ - --namespace stac-staging \ - --create-namespace \ - --values stac-fastapi/values-elasticsearch.yaml \ - --set app.env.ENVIRONMENT=staging - -# Production -helm install prod-stac stac-fastapi \ - --namespace stac-production \ - --create-namespace \ - --values stac-fastapi/values-elasticsearch.yaml \ - --set app.env.ENVIRONMENT=production -``` - -## Configuration - -### Application Configuration - -```yaml -app: - env: - # API Configuration - STAC_FASTAPI_TITLE: "My STAC API" - STAC_FASTAPI_DESCRIPTION: "A production STAC API" - STAC_FASTAPI_VERSION: "6.0.0" - - # Performance Tuning - WEB_CONCURRENCY: "8" # Number of worker processes - ENABLE_DIRECT_RESPONSE: "true" # Maximum performance mode - DATABASE_REFRESH: "false" # Better bulk performance - - # Large Dataset Optimization - ENABLE_DATETIME_INDEX_FILTERING: "true" # Temporal partitioning - DATETIME_INDEX_MAX_SIZE_GB: "50" # Index size limit - - # Rate Limiting - STAC_FASTAPI_RATE_LIMIT: "1000/minute" # API rate limit - - # Feature Toggles - ENABLE_TRANSACTIONS_EXTENSIONS: "true" # Enable POST operations - STAC_INDEX_ASSETS: "true" # Index asset metadata -``` - -### Database Configuration - -#### Elasticsearch Production Settings - -```yaml -elasticsearch: - replicas: 3 - minimumMasterNodes: 2 - - esConfig: - elasticsearch.yml: | - # Cluster settings - cluster.name: "stac-elasticsearch-prod" - action.destructive_requires_name: true - - # Performance tuning - indices.memory.index_buffer_size: 20% - thread_pool.write.queue_size: 1000 - thread_pool.search.queue_size: 1000 - - # Disk usage thresholds - cluster.routing.allocation.disk.threshold_enabled: true - cluster.routing.allocation.disk.watermark.low: 85% - cluster.routing.allocation.disk.watermark.high: 90% - cluster.routing.allocation.disk.watermark.flood_stage: 95% - - resources: - requests: - cpu: "2000m" - memory: "8Gi" - limits: - cpu: "4000m" - memory: "8Gi" - - esJavaOpts: "-Xmx4g -Xms4g" -``` - -#### OpenSearch Production Settings - -```yaml -opensearch: - replicas: 3 - - config: - opensearch.yml: | - # Cluster settings - cluster.name: stac-opensearch-prod - action.destructive_requires_name: true - - # Performance tuning - indices.memory.index_buffer_size: 20% - thread_pool.write.queue_size: 1000 - thread_pool.search.queue_size: 1000 - - # Disk usage thresholds - cluster.routing.allocation.disk.threshold_enabled: true - cluster.routing.allocation.disk.watermark.low: 85% - cluster.routing.allocation.disk.watermark.high: 90% - cluster.routing.allocation.disk.watermark.flood_stage: 95% - - resources: - requests: - cpu: "2000m" - memory: "8Gi" - limits: - cpu: "4000m" - memory: "8Gi" - - opensearchJavaOpts: "-Xmx4g -Xms4g" -``` - -### Ingress Configuration - -```yaml -app: - ingress: - enabled: true - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - nginx.ingress.kubernetes.io/proxy-read-timeout: "300" - cert-manager.io/cluster-issuer: "letsencrypt-prod" - - hosts: - - host: stac-api.yourdomain.com - paths: - - path: / - pathType: Prefix - - tls: - - secretName: stac-api-tls - hosts: - - stac-api.yourdomain.com -``` - -## Production Deployment - -### 1. Pre-deployment Checklist - -- [ ] Kubernetes cluster properly sized -- [ ] Storage classes configured -- [ ] Ingress controller installed -- [ ] DNS records configured -- [ ] SSL certificates ready -- [ ] Monitoring stack deployed -- [ ] Backup strategy defined - -### 2. Production Values File - -Create a production-specific values file: - -```yaml -# values-production.yaml -backend: elasticsearch - -app: - replicaCount: 3 - - image: - tag: "v6.0.0" # Use specific version - pullPolicy: IfNotPresent - - resources: - requests: - cpu: "1000m" - memory: "2Gi" - limits: - cpu: "2000m" - memory: "4Gi" - - autoscaling: - enabled: true - minReplicas: 3 - maxReplicas: 20 - targetCPUUtilizationPercentage: 70 - targetMemoryUtilizationPercentage: 80 - - env: - ENVIRONMENT: "production" - WEB_CONCURRENCY: "8" - ENABLE_DIRECT_RESPONSE: "true" - DATABASE_REFRESH: "false" - ENABLE_DATETIME_INDEX_FILTERING: "true" - DATETIME_INDEX_MAX_SIZE_GB: "100" - STAC_FASTAPI_RATE_LIMIT: "2000/minute" - -elasticsearch: - enabled: true - replicas: 5 - minimumMasterNodes: 3 - - resources: - requests: - cpu: "2000m" - memory: "8Gi" - limits: - cpu: "4000m" - memory: "8Gi" - - volumeClaimTemplate: - storageClassName: "fast-ssd" - resources: - requests: - storage: 500Gi - -podDisruptionBudget: - enabled: true - minAvailable: 2 - -monitoring: - enabled: true - prometheus: - enabled: true - serviceMonitor: - enabled: true - -networkPolicy: - enabled: true -``` - -### 3. Deploy to Production - -```bash -# Deploy -helm install stac-prod stac-fastapi \ - --namespace stac-production \ - --create-namespace \ - --values values-production.yaml \ - --wait \ - --timeout=15m - -# Verify deployment -kubectl get all -n stac-production -helm status stac-prod -n stac-production -``` - -### 4. Post-deployment Verification - -```bash -# Test API endpoints -curl https://stac-api.yourdomain.com/ -curl https://stac-api.yourdomain.com/collections -curl -X POST https://stac-api.yourdomain.com/search \ - -H "Content-Type: application/json" \ - -d '{}' - -# Check database health -kubectl port-forward -n stac-production \ - service/stac-prod-elasticsearch-master 9200:9200 & -curl http://localhost:9200/_cluster/health -``` - -## Monitoring and Observability - -### 1. Enable Prometheus Monitoring - -```yaml -monitoring: - enabled: true - prometheus: - enabled: true - serviceMonitor: - enabled: true - interval: 30s - scrapeTimeout: 10s -``` - -### 2. Grafana Dashboards - -Create custom dashboards for: -- API request rates and latency -- Database performance metrics -- Resource utilization -- Error rates and patterns - -### 3. Alerting Rules - -Example Prometheus alerting rules: - -```yaml -# stac-fastapi-alerts.yaml -groups: - - name: stac-fastapi - rules: - - alert: STACAPIHighErrorRate - expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1 - for: 5m - labels: - severity: warning - annotations: - summary: "High error rate detected" - - - alert: STACAPIHighLatency - expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2 - for: 5m - labels: - severity: warning - annotations: - summary: "High latency detected" -``` - -### 4. Log Aggregation - -Configure log shipping to your preferred logging solution: - -```yaml -app: - podAnnotations: - fluentd.io/include: "true" - fluentd.io/exclude: "false" -``` - -## Troubleshooting - -### Common Issues - -#### 1. Pod Startup Issues - -```bash -# Check pod status -kubectl get pods -n stac-fastapi - -# View pod logs -kubectl logs -f deployment/my-stac-api -n stac-fastapi - -# Describe pod for events -kubectl describe pod -n stac-fastapi -``` - -#### 2. Database Connectivity Issues - -```bash -# Check database service -kubectl get svc -n stac-fastapi - -# Test database connectivity -kubectl exec -it deployment/my-stac-api -n stac-fastapi -- \ - curl http://elasticsearch:9200/_health - -# Check database logs -kubectl logs -f statefulset/my-stac-api-elasticsearch-master -n stac-fastapi -``` - -#### 3. Performance Issues - -```bash -# Check resource usage -kubectl top pods -n stac-fastapi -kubectl top nodes - -# Check HPA status -kubectl get hpa -n stac-fastapi - -# View detailed metrics -kubectl describe hpa my-stac-api -n stac-fastapi -``` - -#### 4. Ingress Issues - -```bash -# Check ingress status -kubectl get ingress -n stac-fastapi - -# Check ingress controller logs -kubectl logs -f deployment/nginx-controller -n nginx-ingress - -# Test DNS resolution -nslookup stac-api.yourdomain.com -``` - -### Debugging Tools - -Use the provided test script for comprehensive debugging: - -```bash -# Validate deployment -./test-chart.sh validate - -# Load test data -./test-chart.sh load-data - -# Full cleanup -./test-chart.sh cleanup -``` - -## Maintenance - -### 1. Upgrades - -```bash -# Update chart dependencies -helm dependency update stac-fastapi - -# Upgrade deployment -helm upgrade stac-prod stac-fastapi \ - --namespace stac-production \ - --values values-production.yaml \ - --wait - -# Rollback if needed -helm rollback stac-prod 1 -n stac-production -``` - -### 2. Scaling - -```bash -# Manual scaling -kubectl scale deployment stac-prod \ - --replicas=5 -n stac-production - -# Update HPA limits -helm upgrade stac-prod stac-fastapi \ - --set app.autoscaling.maxReplicas=30 \ - --reuse-values -``` - -### 3. Backup and Restore - -#### Database Snapshots - -```bash -# Create snapshot -kubectl exec -it stac-prod-elasticsearch-master-0 -n stac-production -- \ - curl -X PUT "localhost:9200/_snapshot/backup/snapshot_$(date +%Y%m%d)" \ - -H 'Content-Type: application/json' \ - -d '{"indices": "*", "ignore_unavailable": true}' - -# List snapshots -kubectl exec -it stac-prod-elasticsearch-master-0 -n stac-production -- \ - curl "localhost:9200/_snapshot/backup/_all" -``` - -#### Configuration Backup - -```bash -# Backup Helm values -helm get values stac-prod -n stac-production > backup-values.yaml - -# Backup Kubernetes resources -kubectl get all -n stac-production -o yaml > backup-resources.yaml -``` - -### 4. Security Updates - -```bash -# Update to latest images -helm upgrade stac-prod stac-fastapi \ - --set app.image.tag=latest \ - --set elasticsearch.imageTag=8.11.0 \ - --reuse-values - -# Apply security patches -kubectl patch deployment stac-prod \ - -n stac-production \ - -p '{"spec":{"template":{"metadata":{"annotations":{"kubectl.kubernetes.io/restartedAt":"'$(date +%Y-%m-%dT%H:%M:%S%z)'"}}}}}' -``` - -This deployment guide provides comprehensive instructions for deploying and maintaining STAC FastAPI in production Kubernetes environments. Adapt the configurations based on your specific requirements and infrastructure constraints. \ No newline at end of file diff --git a/helm-chart/stac-fastapi/README.md b/helm-chart/stac-fastapi/README.md index 08589837e..2fc76365b 100644 --- a/helm-chart/stac-fastapi/README.md +++ b/helm-chart/stac-fastapi/README.md @@ -62,19 +62,44 @@ helm install my-stac-api ./helm-chart/stac-fastapi \ ## Configuration -### Backend Selection +## Backend Selection -The chart supports two backends: +The chart supports both Elasticsearch and OpenSearch backends, but only deploys **one backend at a time** based on the `backend` configuration: -- `elasticsearch`: Uses Elasticsearch as the backend database -- `opensearch`: Uses OpenSearch as the backend database - -Set the backend using: +### Elasticsearch Backend +```yaml +backend: elasticsearch +elasticsearch: + enabled: true +opensearch: + enabled: false +``` +### OpenSearch Backend ```yaml -backend: elasticsearch # or opensearch +backend: opensearch +elasticsearch: + enabled: false +opensearch: + enabled: true ``` +### How It Works + +1. **Chart Dependencies**: Both Elasticsearch and OpenSearch charts are listed as dependencies with conditions +2. **Conditional Deployment**: Only the backend specified by `backend` value is enabled +3. **Resource Isolation**: When deploying with elasticsearch, no OpenSearch resources are created (and vice versa) +4. **Automatic Configuration**: The application automatically connects to the correct backend service + +### Values Files + +Use the provided values files for easy backend selection: + +- **Elasticsearch**: `helm install stac-fastapi ./stac-fastapi -f values-elasticsearch.yaml` +- **OpenSearch**: `helm install stac-fastapi ./stac-fastapi -f values-opensearch.yaml` + +This ensures efficient resource usage and prevents conflicts between backends. + ### Application Configuration Key application settings: diff --git a/helm-chart/stac-fastapi/templates/_helpers.tpl b/helm-chart/stac-fastapi/templates/_helpers.tpl index 87c3ad307..becd5eaa0 100644 --- a/helm-chart/stac-fastapi/templates/_helpers.tpl +++ b/helm-chart/stac-fastapi/templates/_helpers.tpl @@ -146,4 +146,26 @@ Create environment variables for the application name: {{ $secretName }} key: {{ $key }} {{- end }} +{{- end }} + +{{/* +Determine if Elasticsearch should be enabled based on backend selection +*/}} +{{- define "stac-fastapi.elasticsearch.enabled" -}} +{{- if eq .Values.backend "elasticsearch" }} +{{- true }} +{{- else }} +{{- false }} +{{- end }} +{{- end }} + +{{/* +Determine if OpenSearch should be enabled based on backend selection +*/}} +{{- define "stac-fastapi.opensearch.enabled" -}} +{{- if eq .Values.backend "opensearch" }} +{{- true }} +{{- else }} +{{- false }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-dev.yaml b/helm-chart/stac-fastapi/values-dev.yaml new file mode 100644 index 000000000..08d49afaa --- /dev/null +++ b/helm-chart/stac-fastapi/values-dev.yaml @@ -0,0 +1,124 @@ +# Development values for STAC FastAPI +# This configuration is optimized for local development and testing +# It disables persistent storage to avoid PVC issues in development clusters + +# Set backend to elasticsearch by default +backend: elasticsearch + +# STAC FastAPI application configuration +app: + replicaCount: 1 # Single replica for development + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + # Service configuration + service: + type: ClusterIP + port: 80 + targetPort: 8080 + + # No ingress for development + ingress: + enabled: false + + # Minimal resources for development + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "500m" + memory: "1Gi" + + # Disable autoscaling + autoscaling: + enabled: false + + # Development environment variables + env: + STAC_FASTAPI_TITLE: "Development STAC API" + STAC_FASTAPI_DESCRIPTION: "STAC API for development and testing" + ENVIRONMENT: "development" + WEB_CONCURRENCY: "2" + RELOAD: "true" + DATABASE_REFRESH: "true" # Allow fresh starts in development + ENABLE_TRANSACTIONS_EXTENSIONS: "true" + ENABLE_DIRECT_RESPONSE: "false" # Easier debugging + STAC_FASTAPI_RATE_LIMIT: "1000/minute" + +# Elasticsearch configuration for development +elasticsearch: + enabled: true + + # Single node cluster for development + clusterName: "stac-elasticsearch-dev" + replicas: 1 + minimumMasterNodes: 1 + + # Minimal resources + resources: + requests: + cpu: "100m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "2Gi" + + # Small heap size + esJavaOpts: "-Xmx512m -Xms512m" + + # Development Elasticsearch configuration + esConfig: + elasticsearch.yml: | + cluster.name: "stac-elasticsearch-dev" + network.host: 0.0.0.0 + discovery.type: single-node + action.destructive_requires_name: false + xpack.security.enabled: false + xpack.security.transport.ssl.enabled: false + xpack.security.http.ssl.enabled: false + # Development-friendly settings + cluster.routing.allocation.disk.threshold_enabled: false + indices.memory.index_buffer_size: 10% + + # DISABLE PERSISTENT STORAGE FOR DEVELOPMENT + # This prevents PVC issues in development clusters + volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + storageClassName: "" # Use default storage class if available + resources: + requests: + storage: 1Gi # Minimal storage + + # Alternative: Use emptyDir instead of PVC (uncomment to use) + # persistence: + # enabled: false + # extraVolumes: + # - name: elasticsearch-data + # emptyDir: {} + # extraVolumeMounts: + # - name: elasticsearch-data + # mountPath: /usr/share/elasticsearch/data + +# Disable OpenSearch for Elasticsearch backend +opensearch: + enabled: false + +# External database disabled since we're using bundled Elasticsearch +externalDatabase: + enabled: false + +# Disable monitoring for development +monitoring: + enabled: false + +# Disable pod disruption budget +podDisruptionBudget: + enabled: false + +# Disable network policies for development +networkPolicy: + enabled: false \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-minimal-storage.yaml b/helm-chart/stac-fastapi/values-minimal-storage.yaml new file mode 100644 index 000000000..312792cc9 --- /dev/null +++ b/helm-chart/stac-fastapi/values-minimal-storage.yaml @@ -0,0 +1,102 @@ +# Minimal storage values for STAC FastAPI +# This configuration uses minimal storage requirements for development clusters +# It will still create PVCs but with very small storage requests + +# Set backend to elasticsearch by default +backend: elasticsearch + +# STAC FastAPI application configuration +app: + replicaCount: 1 + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + service: + type: ClusterIP + port: 80 + targetPort: 8080 + + ingress: + enabled: false + + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "500m" + memory: "1Gi" + + autoscaling: + enabled: false + + env: + STAC_FASTAPI_TITLE: "Test STAC API (Minimal Storage)" + STAC_FASTAPI_DESCRIPTION: "STAC API for testing with minimal storage" + ENVIRONMENT: "test" + WEB_CONCURRENCY: "2" + RELOAD: "true" + DATABASE_REFRESH: "true" + ENABLE_TRANSACTIONS_EXTENSIONS: "true" + ENABLE_DIRECT_RESPONSE: "false" + STAC_FASTAPI_RATE_LIMIT: "1000/minute" + +# Elasticsearch with minimal storage requirements +elasticsearch: + enabled: true + + clusterName: "stac-elasticsearch-test" + replicas: 1 + # IMPORTANT: Set minimumMasterNodes to 0 for single-node with discovery.type: single-node + minimumMasterNodes: 0 + + resources: + requests: + cpu: "100m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "2Gi" + + esJavaOpts: "-Xmx512m -Xms512m" + + esConfig: + elasticsearch.yml: | + cluster.name: "stac-elasticsearch-test" + network.host: 0.0.0.0 + discovery.type: single-node + action.destructive_requires_name: false + xpack.security.enabled: false + xpack.security.transport.ssl.enabled: false + xpack.security.http.ssl.enabled: false + cluster.routing.allocation.disk.threshold_enabled: false + indices.memory.index_buffer_size: 10% + path.data: /usr/share/elasticsearch/data + + # MINIMAL STORAGE - This will create small PVCs + # Most development clusters should be able to handle 1Gi PVCs + volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + # Use default storage class (remove storageClassName line to use default) + # storageClassName: "" + +opensearch: + enabled: false + +externalDatabase: + enabled: false + +monitoring: + enabled: false + +podDisruptionBudget: + enabled: false + +networkPolicy: + enabled: false \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-no-persistence.yaml b/helm-chart/stac-fastapi/values-no-persistence.yaml new file mode 100644 index 000000000..a53b37262 --- /dev/null +++ b/helm-chart/stac-fastapi/values-no-persistence.yaml @@ -0,0 +1,128 @@ +# No-persistence values for STAC FastAPI +# This configuration completely disables persistent storage +# Data will be lost when pods restart - USE ONLY FOR TESTING + +# Set backend to elasticsearch by default +backend: elasticsearch + +# STAC FastAPI application configuration +app: + replicaCount: 1 + + image: + repository: ghcr.io/stac-utils/stac-fastapi + tag: "latest" + pullPolicy: IfNotPresent + + service: + type: ClusterIP + port: 80 + targetPort: 8080 + + ingress: + enabled: false + + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "500m" + memory: "1Gi" + + autoscaling: + enabled: false + + env: + STAC_FASTAPI_TITLE: "Test STAC API (No Persistence)" + STAC_FASTAPI_DESCRIPTION: "STAC API for testing - data not persisted" + ENVIRONMENT: "test" + WEB_CONCURRENCY: "2" + RELOAD: "true" + DATABASE_REFRESH: "true" + ENABLE_TRANSACTIONS_EXTENSIONS: "true" + ENABLE_DIRECT_RESPONSE: "false" + STAC_FASTAPI_RATE_LIMIT: "1000/minute" + +# Elasticsearch with NO persistent storage +elasticsearch: + enabled: true + + clusterName: "stac-elasticsearch-test" + replicas: 1 + minimumMasterNodes: 1 + + resources: + requests: + cpu: "100m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "2Gi" + + esJavaOpts: "-Xmx512m -Xms512m" + + esConfig: + elasticsearch.yml: | + cluster.name: "stac-elasticsearch-test" + network.host: 0.0.0.0 + discovery.type: single-node + action.destructive_requires_name: false + xpack.security.enabled: false + xpack.security.transport.ssl.enabled: false + xpack.security.http.ssl.enabled: false + cluster.routing.allocation.disk.threshold_enabled: false + indices.memory.index_buffer_size: 10% + + # COMPLETELY DISABLE PERSISTENT STORAGE + # The official chart requires volumeClaimTemplate but we can make it very small + # and override with emptyDir in extraVolumes + volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + # Use emptyDir storage class to prevent actual PVC creation + storageClassName: "" + + # Override the data volume with emptyDir + extraVolumes: + - name: elasticsearch-data-override + emptyDir: + sizeLimit: 2Gi + + extraVolumeMounts: + - name: elasticsearch-data-override + mountPath: /usr/share/elasticsearch/data-override + + # Alternative approach: Use extraInitContainers to symlink data directory + extraInitContainers: + - name: setup-data-dir + image: "busybox:1.35" + command: + - /bin/sh + - -c + - | + # Remove the PVC mounted directory and create symlink to emptyDir + rm -rf /usr/share/elasticsearch/data/* + ln -sfn /usr/share/elasticsearch/data-override/* /usr/share/elasticsearch/data/ || true + volumeMounts: + - name: stac-elasticsearch-master + mountPath: /usr/share/elasticsearch/data + - name: elasticsearch-data-override + mountPath: /usr/share/elasticsearch/data-override + +opensearch: + enabled: false + +externalDatabase: + enabled: false + +monitoring: + enabled: false + +podDisruptionBudget: + enabled: false + +networkPolicy: + enabled: false \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index 7b0fcd907..131cc829b 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -167,6 +167,7 @@ app: # ES_API_KEY: "es-api-key-secret" # Elasticsearch configuration +# Note: This will be automatically set based on 'backend' value elasticsearch: # Enable/disable Elasticsearch deployment enabled: true @@ -219,10 +220,10 @@ elasticsearch: # Volume claim template volumeClaimTemplate: accessModes: [ "ReadWriteOnce" ] - storageClassName: "" + # storageClassName: "" # Leave empty to use cluster default resources: requests: - storage: 10Gi + storage: 1Gi # Service configuration service: diff --git a/helm-chart/test-chart.sh b/helm-chart/test-chart.sh index 426e4a998..37ebeeb4d 100755 --- a/helm-chart/test-chart.sh +++ b/helm-chart/test-chart.sh @@ -17,6 +17,7 @@ CHART_PATH="./helm-chart/stac-fastapi" RELEASE_NAME="stac-fastapi-test" NAMESPACE="stac-fastapi" BACKEND=${BACKEND:-"elasticsearch"} +MATRIX_MODE=${MATRIX_MODE:-false} # Functions log_info() { @@ -45,22 +46,29 @@ Usage: $0 [OPTIONS] COMMAND Commands: lint Lint the Helm chart test Run Helm chart tests + test-all Test both elasticsearch and opensearch backends + test-matrix Run GitHub workflow matrix testing install Install the chart for testing upgrade Upgrade existing installation uninstall Uninstall the test deployment validate Validate deployment health load-data Load sample data into the API cleanup Clean up all test resources + ci Run CI pipeline (lint + test + validate) Options: -b, --backend BACKEND Backend to test (elasticsearch|opensearch) [default: elasticsearch] -n, --namespace NS Kubernetes namespace [default: stac-fastapi] -r, --release NAME Helm release name [default: stac-fastapi-test] + -m, --matrix Run matrix testing for CI -h, --help Show this help message Examples: $0 lint # Lint the chart $0 -b opensearch install # Install with OpenSearch backend + $0 test-all # Test both backends + $0 test-matrix # Run GitHub workflow matrix tests + $0 ci # Run full CI pipeline $0 validate # Check deployment health $0 load-data # Load test data $0 cleanup # Clean up everything @@ -83,11 +91,15 @@ while [[ $# -gt 0 ]]; do RELEASE_NAME="$2" shift 2 ;; + -m|--matrix) + MATRIX_MODE=true + shift + ;; -h|--help) show_help exit 0 ;; - lint|test|install|upgrade|uninstall|validate|load-data|cleanup) + lint|test|test-all|test-matrix|install|upgrade|uninstall|validate|load-data|cleanup|ci) COMMAND="$1" shift ;; @@ -183,9 +195,27 @@ install_chart() { # Update dependencies helm dependency update "$CHART_PATH" - # Install the chart + # Select appropriate values file for backend + local values_file="" + case $BACKEND in + elasticsearch) + values_file="values-minimal-storage.yaml" + ;; + opensearch) + values_file="values-opensearch.yaml" + ;; + *) + log_error "Unknown backend: $BACKEND" + return 1 + ;; + esac + + log_info "Using values file: $values_file" + + # Install the chart with appropriate values helm install "$RELEASE_NAME" "$CHART_PATH" \ --namespace "$NAMESPACE" \ + --values "$CHART_PATH/$values_file" \ --set backend="$BACKEND" \ --set "${BACKEND}.enabled=true" \ --set "app.image.tag=latest" \ @@ -205,8 +235,24 @@ upgrade_chart() { helm dependency update "$CHART_PATH" + # Select appropriate values file for backend + local values_file="" + case $BACKEND in + elasticsearch) + values_file="values-minimal-storage.yaml" + ;; + opensearch) + values_file="values-opensearch.yaml" + ;; + *) + log_error "Unknown backend: $BACKEND" + return 1 + ;; + esac + helm upgrade "$RELEASE_NAME" "$CHART_PATH" \ --namespace "$NAMESPACE" \ + --values "$CHART_PATH/$values_file" \ --set backend="$BACKEND" \ --set "${BACKEND}.enabled=true" \ --set "app.image.tag=latest" \ @@ -382,6 +428,290 @@ load_sample_data() { log_success "Sample data loading completed" } +# Test both backends comprehensively +test_all_backends() { + log_info "Testing all backends (elasticsearch and opensearch)..." + + local original_backend="$BACKEND" + local failed_backends=() + + for backend in elasticsearch opensearch; do + log_info "================== Testing $backend backend ==================" + + BACKEND="$backend" + RELEASE_NAME="stac-fastapi-test-$backend" + NAMESPACE="stac-fastapi-$backend" + + log_info "Running tests for $backend backend..." + + # Run comprehensive test + if run_backend_test "$backend"; then + log_success "$backend backend tests passed" + else + log_error "$backend backend tests failed" + failed_backends+=("$backend") + fi + + # Cleanup between tests + cleanup + sleep 5 + done + + # Restore original values + BACKEND="$original_backend" + RELEASE_NAME="stac-fastapi-test" + NAMESPACE="stac-fastapi" + + # Report results + if [[ ${#failed_backends[@]} -eq 0 ]]; then + log_success "All backend tests passed successfully!" + else + log_error "The following backends failed: ${failed_backends[*]}" + exit 1 + fi +} + +# Run GitHub workflow matrix testing +test_matrix() { + log_info "Running GitHub workflow matrix testing..." + + if [[ "$MATRIX_MODE" == "true" ]]; then + log_info "Matrix mode enabled - testing single backend: $BACKEND" + run_backend_test "$BACKEND" + else + log_info "Running full matrix test locally..." + test_all_backends + fi +} + +# Run tests for a specific backend +run_backend_test() { + local backend="$1" + local test_failed=false + + log_info "Starting comprehensive test for $backend backend..." + + # Set backend-specific values + BACKEND="$backend" + + # Step 1: Lint + log_info "Step 1: Linting chart for $backend..." + if ! lint_chart; then + log_error "Linting failed for $backend" + return 1 + fi + + # Step 2: Template test + log_info "Step 2: Testing chart templates for $backend..." + if ! test_chart; then + log_error "Template test failed for $backend" + return 1 + fi + + # Step 3: Install and validate + log_info "Step 3: Installing chart for $backend..." + if ! install_chart; then + log_error "Installation failed for $backend" + return 1 + fi + + # Step 4: Validate deployment + log_info "Step 4: Validating deployment for $backend..." + if ! validate_deployment; then + log_error "Deployment validation failed for $backend" + test_failed=true + fi + + # Step 5: Load and test data + log_info "Step 5: Testing data operations for $backend..." + if ! load_sample_data; then + log_error "Data operations failed for $backend" + test_failed=true + fi + + # Step 6: Test backend-specific functionality + log_info "Step 6: Testing $backend-specific functionality..." + if ! test_backend_specifics "$backend"; then + log_error "Backend-specific tests failed for $backend" + test_failed=true + fi + + if [[ "$test_failed" == "true" ]]; then + log_error "Some tests failed for $backend backend" + return 1 + else + log_success "All tests passed for $backend backend" + return 0 + fi +} + +# Test backend-specific functionality +test_backend_specifics() { + local backend="$1" + + log_info "Testing $backend-specific functionality..." + + # Port forward to the service + kubectl port-forward -n "$NAMESPACE" service/"$RELEASE_NAME" 8080:80 & + local pf_pid=$! + + # Wait for port-forward + sleep 5 + + case "$backend" in + elasticsearch) + # Test Elasticsearch-specific endpoints + log_info "Testing Elasticsearch-specific features..." + + # Check if using correct image + local image=$(kubectl get deployment "$RELEASE_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}') + if [[ "$image" == *"-es:"* ]]; then + log_success "Using correct Elasticsearch image: $image" + else + log_warning "Image might not be Elasticsearch-specific: $image" + fi + + # Test backend environment variable + local backend_env=$(kubectl get deployment "$RELEASE_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="BACKEND")].value}') + if [[ "$backend_env" == "elasticsearch" ]]; then + log_success "Backend environment correctly set to elasticsearch" + else + log_error "Backend environment incorrect: $backend_env" + kill $pf_pid 2>/dev/null || true + return 1 + fi + ;; + + opensearch) + # Test OpenSearch-specific endpoints + log_info "Testing OpenSearch-specific features..." + + # Check if using correct image + local image=$(kubectl get deployment "$RELEASE_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}') + if [[ "$image" == *"-os:"* ]]; then + log_success "Using correct OpenSearch image: $image" + else + log_warning "Image might not be OpenSearch-specific: $image" + fi + + # Test backend environment variable + local backend_env=$(kubectl get deployment "$RELEASE_NAME" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="BACKEND")].value}') + if [[ "$backend_env" == "opensearch" ]]; then + log_success "Backend environment correctly set to opensearch" + else + log_error "Backend environment incorrect: $backend_env" + kill $pf_pid 2>/dev/null || true + return 1 + fi + ;; + esac + + # Test that only the correct backend is deployed + log_info "Verifying only $backend backend is deployed..." + + local es_pods=$(kubectl get pods -n "$NAMESPACE" -l "app=stac-elasticsearch-master" --no-headers 2>/dev/null | wc -l) + local os_pods=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/name=opensearch" --no-headers 2>/dev/null | wc -l) + + case "$backend" in + elasticsearch) + if [[ "$es_pods" -gt 0 && "$os_pods" -eq 0 ]]; then + log_success "Only Elasticsearch backend is deployed" + else + log_error "Wrong backends deployed - ES pods: $es_pods, OS pods: $os_pods" + kill $pf_pid 2>/dev/null || true + return 1 + fi + ;; + opensearch) + if [[ "$os_pods" -gt 0 && "$es_pods" -eq 0 ]]; then + log_success "Only OpenSearch backend is deployed" + else + log_error "Wrong backends deployed - ES pods: $es_pods, OS pods: $os_pods" + kill $pf_pid 2>/dev/null || true + return 1 + fi + ;; + esac + + # Clean up port-forward + kill $pf_pid 2>/dev/null || true + + log_success "$backend-specific tests completed successfully" + return 0 +} + +# Run CI pipeline +run_ci_pipeline() { + log_info "Running CI pipeline..." + + if [[ "$MATRIX_MODE" == "true" ]]; then + log_info "CI Matrix Mode: Testing $BACKEND backend" + + # Set CI-specific configuration + export HELM_EXPERIMENTAL_OCI=1 + + # Run the backend test + if run_backend_test "$BACKEND"; then + log_success "CI pipeline passed for $BACKEND backend" + + # Generate test report + generate_test_report "$BACKEND" + else + log_error "CI pipeline failed for $BACKEND backend" + exit 1 + fi + else + log_info "CI Full Mode: Testing all backends" + test_all_backends + + # Generate combined test report + generate_test_report "all" + fi +} + +# Generate test report +generate_test_report() { + local backend="$1" + local report_file="test-report-$backend-$(date +%Y%m%d-%H%M%S).json" + + log_info "Generating test report: $report_file" + + # Get cluster info + local k8s_version=$(kubectl version --short 2>/dev/null | grep "Server Version" | cut -d: -f2 | xargs) + local helm_version=$(helm version --short 2>/dev/null) + + # Create test report + cat > "$report_file" << EOF +{ + "test_run": { + "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "backend": "$backend", + "kubernetes_version": "$k8s_version", + "helm_version": "$helm_version", + "chart_version": "$(helm show chart $CHART_PATH | grep version | head -1 | cut -d: -f2 | xargs)", + "status": "passed" + }, + "tests_performed": [ + "chart_linting", + "template_validation", + "deployment_installation", + "health_validation", + "data_operations", + "backend_specific_tests" + ], + "resources_tested": { + "deployment": true, + "service": true, + "configmap": true, + "secrets": true, + "backend_statefulset": true + } +} +EOF + + log_success "Test report generated: $report_file" +} + # Clean up all resources cleanup() { log_info "Cleaning up all test resources..." @@ -410,6 +740,18 @@ main() { lint_chart test_chart ;; + test-all) + check_prerequisites + test_all_backends + ;; + test-matrix) + check_prerequisites + test_matrix + ;; + ci) + check_prerequisites + run_ci_pipeline + ;; install) check_prerequisites install_chart From 3a7c86d6560567fb863aae5e3b78ee74fb0f182e Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Wed, 8 Oct 2025 14:13:25 +1030 Subject: [PATCH 04/12] Allow opensearch deployement, resolving app to storage cluster connection issues still --- helm-chart/stac-fastapi/README.md | 1 - .../stac-fastapi/templates/_helpers.tpl | 40 +++++++++++++++++++ helm-chart/stac-fastapi/templates/hpa.yaml | 15 ++++++- .../templates/poddisruptionbudget.yaml | 3 +- .../stac-fastapi/values-elasticsearch.yaml | 2 +- .../stac-fastapi/values-opensearch.yaml | 3 +- helm-chart/stac-fastapi/values.yaml | 17 ++++---- 7 files changed, 69 insertions(+), 12 deletions(-) diff --git a/helm-chart/stac-fastapi/README.md b/helm-chart/stac-fastapi/README.md index 2fc76365b..b4afb72d8 100644 --- a/helm-chart/stac-fastapi/README.md +++ b/helm-chart/stac-fastapi/README.md @@ -169,7 +169,6 @@ opensearch: persistence: enabled: true size: 100Gi - storageClass: "fast-ssd" ``` #### External Database diff --git a/helm-chart/stac-fastapi/templates/_helpers.tpl b/helm-chart/stac-fastapi/templates/_helpers.tpl index becd5eaa0..040fe6c1b 100644 --- a/helm-chart/stac-fastapi/templates/_helpers.tpl +++ b/helm-chart/stac-fastapi/templates/_helpers.tpl @@ -69,13 +69,29 @@ Create the database host based on backend selection {{- .Values.externalDatabase.host }} {{- else if eq .Values.backend "elasticsearch" }} {{- if .Values.elasticsearch.enabled }} +{{- if .Values.elasticsearch.masterService }} +{{- .Values.elasticsearch.masterService }} +{{- else if .Values.elasticsearch.fullnameOverride }} +{{- printf "%s-master" .Values.elasticsearch.fullnameOverride }} +{{- else if .Values.elasticsearch.clusterName }} +{{- printf "%s-master" .Values.elasticsearch.clusterName }} +{{- else }} {{- printf "%s-%s" .Release.Name "elasticsearch-master" }} +{{- end }} {{- else }} {{- fail "Elasticsearch is not enabled but backend is set to elasticsearch" }} {{- end }} {{- else if eq .Values.backend "opensearch" }} {{- if .Values.opensearch.enabled }} +{{- if .Values.opensearch.masterService }} +{{- .Values.opensearch.masterService }} +{{- else if .Values.opensearch.fullnameOverride }} +{{- printf "%s-master" .Values.opensearch.fullnameOverride }} +{{- else if .Values.opensearch.clusterName }} +{{- printf "%s-master" .Values.opensearch.clusterName }} +{{- else }} {{- printf "%s-%s" .Release.Name "opensearch-cluster-master" }} +{{- end }} {{- else }} {{- fail "OpenSearch is not enabled but backend is set to opensearch" }} {{- end }} @@ -168,4 +184,28 @@ Determine if OpenSearch should be enabled based on backend selection {{- else }} {{- false }} {{- end }} +{{- end }} + +{{/* +Determine PodDisruptionBudget apiVersion based on cluster capabilities +*/}} +{{- define "stac-fastapi.pdb.apiVersion" -}} +{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.GitVersion }} +policy/v1 +{{- else }} +policy/v1beta1 +{{- end }} +{{- end }} + +{{/* +Determine HorizontalPodAutoscaler apiVersion based on cluster capabilities +*/}} +{{- define "stac-fastapi.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +autoscaling/v2 +{{- else if .Capabilities.APIVersions.Has "autoscaling/v2beta2" }} +autoscaling/v2beta2 +{{- else }} +autoscaling/v2beta1 +{{- end }} {{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/hpa.yaml b/helm-chart/stac-fastapi/templates/hpa.yaml index 5e00c217e..eb233e12d 100644 --- a/helm-chart/stac-fastapi/templates/hpa.yaml +++ b/helm-chart/stac-fastapi/templates/hpa.yaml @@ -1,5 +1,6 @@ {{- if .Values.app.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +{{- $hpaApiVersion := (include "stac-fastapi.hpa.apiVersion" . | trim) }} +apiVersion: {{ $hpaApiVersion }} kind: HorizontalPodAutoscaler metadata: name: {{ include "stac-fastapi.fullname" . }} @@ -17,12 +18,24 @@ spec: - type: Resource resource: name: cpu + {{- if or (eq $hpaApiVersion "autoscaling/v2") (eq $hpaApiVersion "autoscaling/v2beta2") }} + target: + type: Utilization + averageUtilization: {{ .Values.app.autoscaling.targetCPUUtilizationPercentage }} + {{- else }} targetAverageUtilization: {{ .Values.app.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} {{- end }} {{- if .Values.app.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory + {{- if or (eq $hpaApiVersion "autoscaling/v2") (eq $hpaApiVersion "autoscaling/v2beta2") }} + target: + type: Utilization + averageUtilization: {{ .Values.app.autoscaling.targetMemoryUtilizationPercentage }} + {{- else }} targetAverageUtilization: {{ .Values.app.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml b/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml index d27ada320..b8f510362 100644 --- a/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml +++ b/helm-chart/stac-fastapi/templates/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- if .Values.podDisruptionBudget.enabled }} -apiVersion: policy/v1beta1 +{{- $pdbApiVersion := (include "stac-fastapi.pdb.apiVersion" . | trim) }} +apiVersion: {{ $pdbApiVersion }} kind: PodDisruptionBudget metadata: name: {{ include "stac-fastapi.fullname" . }} diff --git a/helm-chart/stac-fastapi/values-elasticsearch.yaml b/helm-chart/stac-fastapi/values-elasticsearch.yaml index bd914c514..c001a8eff 100644 --- a/helm-chart/stac-fastapi/values-elasticsearch.yaml +++ b/helm-chart/stac-fastapi/values-elasticsearch.yaml @@ -104,7 +104,7 @@ elasticsearch: # Persistent storage volumeClaimTemplate: accessModes: ["ReadWriteOnce"] - storageClassName: "fast-ssd" # Use high-performance storage class + # storageClassName: "fast-ssd" # Use high-performance storage class resources: requests: storage: 100Gi diff --git a/helm-chart/stac-fastapi/values-opensearch.yaml b/helm-chart/stac-fastapi/values-opensearch.yaml index b9ecde069..4007f3be9 100644 --- a/helm-chart/stac-fastapi/values-opensearch.yaml +++ b/helm-chart/stac-fastapi/values-opensearch.yaml @@ -75,6 +75,7 @@ opensearch: # Cluster configuration clusterName: "stac-opensearch-prod" + masterService: "opensearch-cluster-master" replicas: 3 # OpenSearch roles for each node @@ -116,7 +117,7 @@ opensearch: # Persistent storage persistence: enabled: true - storageClass: "fast-ssd" # Use high-performance storage class + # storageClass: "fast-ssd" # Use high-performance storage class accessModes: - ReadWriteOnce size: 100Gi diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index 131cc829b..42d8a620f 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -81,13 +81,13 @@ app: # - stac-fastapi.local # Resource limits and requests - resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi # Horizontal Pod Autoscaler autoscaling: @@ -245,6 +245,9 @@ opensearch: # OpenSearch cluster name clusterName: "stac-opensearch" + # Service name used to expose the master nodes + masterService: "opensearch-cluster-master" + # Node group configuration nodeGroup: "master" From 42ade28a64b5e7312fa97e53733b713e0d3a69e9 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Wed, 8 Oct 2025 15:06:09 +1030 Subject: [PATCH 05/12] Added FQDN support to experiment with resolution for db cluster --- helm-chart/stac-fastapi/README.md | 38 +++++- .../stac-fastapi/templates/_helpers.tpl | 104 +++++++++----- .../stac-fastapi/templates/deployment.yaml | 33 +++++ .../stac-fastapi/values-elasticsearch.yaml | 5 + helm-chart/stac-fastapi/values-external.yaml | 52 ------- .../stac-fastapi/values-minimal-storage.yaml | 102 -------------- .../stac-fastapi/values-no-persistence.yaml | 128 ------------------ .../stac-fastapi/values-opensearch.yaml | 12 ++ helm-chart/stac-fastapi/values.yaml | 25 ++++ helm-chart/test-chart.sh | 4 +- 10 files changed, 186 insertions(+), 317 deletions(-) delete mode 100644 helm-chart/stac-fastapi/values-external.yaml delete mode 100644 helm-chart/stac-fastapi/values-minimal-storage.yaml delete mode 100644 helm-chart/stac-fastapi/values-no-persistence.yaml diff --git a/helm-chart/stac-fastapi/README.md b/helm-chart/stac-fastapi/README.md index b4afb72d8..c1026b10d 100644 --- a/helm-chart/stac-fastapi/README.md +++ b/helm-chart/stac-fastapi/README.md @@ -62,11 +62,23 @@ helm install my-stac-api ./helm-chart/stac-fastapi \ ## Configuration +### Global Options + +```yaml +global: + imageRegistry: "" + storageClass: "" + clusterDomain: "cluster.local" +``` + +The chart builds fully qualified service endpoints for bundled databases using the Kubernetes cluster domain. Adjust `clusterDomain` if your cluster doesn't use the default `cluster.local` suffix. + ## Backend Selection The chart supports both Elasticsearch and OpenSearch backends, but only deploys **one backend at a time** based on the `backend` configuration: ### Elasticsearch Backend + ```yaml backend: elasticsearch elasticsearch: @@ -76,6 +88,7 @@ opensearch: ``` ### OpenSearch Backend + ```yaml backend: opensearch elasticsearch: @@ -113,6 +126,11 @@ app: tag: "latest" pullPolicy: IfNotPresent + waitForDatabase: + enabled: true + intervalSeconds: 2 + maxAttempts: 120 + env: STAC_FASTAPI_TITLE: "STAC API" STAC_FASTAPI_DESCRIPTION: "A STAC FastAPI implementation" @@ -124,8 +142,24 @@ app: STAC_FASTAPI_RATE_LIMIT: "200/minute" ``` +The optional `waitForDatabase` block adds a lightweight init container that blocks STAC FastAPI startup until the backing Elasticsearch/OpenSearch service is reachable—mirroring the docker-compose `wait-for-it` helper. Disable it by setting `app.waitForDatabase.enabled=false` if you prefer the application to start immediately and rely on internal retries instead. + ### Database Configuration +#### Application Credentials + +If your Elasticsearch or OpenSearch cluster requires authentication, provide credentials to the application with the `app.databaseAuth` block. You can reference an existing secret or supply literal values: + +```yaml +app: + databaseAuth: + existingSecret: "stac-opensearch-admin" # Optional. When set, keys are read from this secret. + usernameKey: "username" # Secret key that stores the username (defaults to "username"). + passwordKey: "password" # Secret key that stores the password (defaults to "password"). + # username: "admin" # Optional literal username when not using a secret. + # password: "changeme" # Optional literal password when not using a secret. +``` + #### Bundled Elasticsearch ```yaml @@ -394,7 +428,7 @@ kubectl exec -it deployment/my-stac-api -- curl http://elasticsearch:9200/_healt kubectl port-forward service/my-stac-api 8080:80 ``` -Then visit http://localhost:8080 +Then visit ## Configuration Reference @@ -423,4 +457,4 @@ Contributions are welcome! Please ensure any changes maintain compatibility with ## License -This chart is released under the same license as the STAC FastAPI project. \ No newline at end of file +This chart is released under the same license as the STAC FastAPI project. diff --git a/helm-chart/stac-fastapi/templates/_helpers.tpl b/helm-chart/stac-fastapi/templates/_helpers.tpl index 040fe6c1b..40e5efd69 100644 --- a/helm-chart/stac-fastapi/templates/_helpers.tpl +++ b/helm-chart/stac-fastapi/templates/_helpers.tpl @@ -62,41 +62,53 @@ Create the name of the service account to use {{- end }} {{/* -Create the database host based on backend selection +Create the bundled database service name based on backend selection */}} -{{- define "stac-fastapi.databaseHost" -}} -{{- if .Values.externalDatabase.enabled }} -{{- .Values.externalDatabase.host }} -{{- else if eq .Values.backend "elasticsearch" }} -{{- if .Values.elasticsearch.enabled }} -{{- if .Values.elasticsearch.masterService }} -{{- .Values.elasticsearch.masterService }} -{{- else if .Values.elasticsearch.fullnameOverride }} -{{- printf "%s-master" .Values.elasticsearch.fullnameOverride }} -{{- else if .Values.elasticsearch.clusterName }} -{{- printf "%s-master" .Values.elasticsearch.clusterName }} -{{- else }} -{{- printf "%s-%s" .Release.Name "elasticsearch-master" }} -{{- end }} -{{- else }} -{{- fail "Elasticsearch is not enabled but backend is set to elasticsearch" }} -{{- end }} +{{- define "stac-fastapi.databaseServiceName" -}} +{{- if eq .Values.backend "elasticsearch" }} + {{- if .Values.elasticsearch.enabled }} + {{- if .Values.elasticsearch.masterService }} + {{- .Values.elasticsearch.masterService }} + {{- else if .Values.elasticsearch.fullnameOverride }} + {{- printf "%s-master" .Values.elasticsearch.fullnameOverride }} + {{- else if .Values.elasticsearch.clusterName }} + {{- printf "%s-master" .Values.elasticsearch.clusterName }} + {{- else }} + {{- printf "%s-%s" .Release.Name "elasticsearch-master" }} + {{- end }} + {{- else }} + {{- fail "Elasticsearch is not enabled but backend is set to elasticsearch" }} + {{- end }} {{- else if eq .Values.backend "opensearch" }} -{{- if .Values.opensearch.enabled }} -{{- if .Values.opensearch.masterService }} -{{- .Values.opensearch.masterService }} -{{- else if .Values.opensearch.fullnameOverride }} -{{- printf "%s-master" .Values.opensearch.fullnameOverride }} -{{- else if .Values.opensearch.clusterName }} -{{- printf "%s-master" .Values.opensearch.clusterName }} + {{- if .Values.opensearch.enabled }} + {{- if .Values.opensearch.masterService }} + {{- .Values.opensearch.masterService }} + {{- else if .Values.opensearch.fullnameOverride }} + {{- printf "%s-master" .Values.opensearch.fullnameOverride }} + {{- else if .Values.opensearch.clusterName }} + {{- printf "%s-master" .Values.opensearch.clusterName }} + {{- else }} + {{- printf "%s-%s" .Release.Name "opensearch-cluster-master" }} + {{- end }} + {{- else }} + {{- fail "OpenSearch is not enabled but backend is set to opensearch" }} + {{- end }} {{- else }} -{{- printf "%s-%s" .Release.Name "opensearch-cluster-master" }} + {{- fail "Invalid backend specified. Must be 'elasticsearch' or 'opensearch'" }} {{- end }} -{{- else }} -{{- fail "OpenSearch is not enabled but backend is set to opensearch" }} {{- end }} + +{{/* +Create the database host based on backend selection +*/}} +{{- define "stac-fastapi.databaseHost" -}} +{{- if .Values.externalDatabase.enabled }} + {{- .Values.externalDatabase.host }} {{- else }} -{{- fail "Invalid backend specified. Must be 'elasticsearch' or 'opensearch'" }} + {{- $service := include "stac-fastapi.databaseServiceName" . | trim }} + {{- $namespace := .Release.Namespace | default "default" }} + {{- $clusterDomain := default "cluster.local" .Values.global.clusterDomain }} + {{- printf "%s.%s.svc.%s" $service $namespace $clusterDomain }} {{- end }} {{- end }} @@ -130,12 +142,42 @@ Create the image repository with tag Create environment variables for the application */}} {{- define "stac-fastapi.environment" -}} +{{- $env := default (dict) .Values.app.env -}} +{{- $envFromSecret := default (dict) .Values.app.envFromSecret -}} +{{- $dbAuth := default (dict) .Values.app.databaseAuth -}} - name: BACKEND value: {{ .Values.backend | quote }} - name: ES_HOST value: {{ include "stac-fastapi.databaseHost" . | quote }} - name: ES_PORT value: {{ include "stac-fastapi.databasePort" . | quote }} +{{- if $dbAuth.existingSecret }} +{{- $usernameKey := default "username" $dbAuth.usernameKey }} +{{- $passwordKey := default "password" $dbAuth.passwordKey }} +{{- if not (hasKey $env "ES_USER") }} +- name: ES_USER + valueFrom: + secretKeyRef: + name: {{ $dbAuth.existingSecret }} + key: {{ $usernameKey }} +{{- end }} +{{- if not (hasKey $env "ES_PASS") }} +- name: ES_PASS + valueFrom: + secretKeyRef: + name: {{ $dbAuth.existingSecret }} + key: {{ $passwordKey }} +{{- end }} +{{- else }} +{{- if and (not (hasKey $env "ES_USER")) $dbAuth.username }} +- name: ES_USER + value: {{ $dbAuth.username | quote }} +{{- end }} +{{- if and (not (hasKey $env "ES_PASS")) $dbAuth.password }} +- name: ES_PASS + value: {{ $dbAuth.password | quote }} +{{- end }} +{{- end }} {{- if .Values.externalDatabase.enabled }} - name: ES_USE_SSL value: {{ .Values.externalDatabase.ssl | quote }} @@ -151,11 +193,11 @@ Create environment variables for the application key: {{ .Values.externalDatabase.apiKeySecretKey }} {{- end }} {{- end }} -{{- range $key, $value := .Values.app.env }} +{{- range $key, $value := $env }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} -{{- range $key, $secretName := .Values.app.envFromSecret }} +{{- range $key, $secretName := $envFromSecret }} - name: {{ $key }} valueFrom: secretKeyRef: diff --git a/helm-chart/stac-fastapi/templates/deployment.yaml b/helm-chart/stac-fastapi/templates/deployment.yaml index 4d94b8dee..c246cc891 100644 --- a/helm-chart/stac-fastapi/templates/deployment.yaml +++ b/helm-chart/stac-fastapi/templates/deployment.yaml @@ -20,6 +20,39 @@ spec: labels: {{- include "stac-fastapi.selectorLabels" . | nindent 8 }} spec: + {{- if .Values.app.waitForDatabase.enabled }} + initContainers: + - name: wait-for-database + image: {{ .Values.app.waitForDatabase.image | default "busybox:1.36.1" }} + imagePullPolicy: {{ .Values.app.waitForDatabase.pullPolicy | default "IfNotPresent" }} + command: + - sh + - -c + - | + set -e + HOST="{{ include "stac-fastapi.databaseHost" . | trim }}" + PORT="{{ include "stac-fastapi.databasePort" . | trim }}" + ATTEMPTS={{ .Values.app.waitForDatabase.maxAttempts | default 120 }} + INTERVAL={{ .Values.app.waitForDatabase.intervalSeconds | default 2 }} + COUNT=0 + echo "Waiting for ${HOST}:${PORT} before starting STAC FastAPI..." + while true; do + if nc -z "$HOST" "$PORT"; then + echo "${HOST}:${PORT} is reachable." + exit 0 + fi + COUNT=$((COUNT + 1)) + if [ "$COUNT" -ge "$ATTEMPTS" ]; then + echo "Timed out waiting for ${HOST}:${PORT} after ${COUNT} attempts." >&2 + exit 1 + fi + sleep "$INTERVAL" + done + {{- with .Values.app.waitForDatabase.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} {{- with .Values.app.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm-chart/stac-fastapi/values-elasticsearch.yaml b/helm-chart/stac-fastapi/values-elasticsearch.yaml index c001a8eff..6cae6bd6e 100644 --- a/helm-chart/stac-fastapi/values-elasticsearch.yaml +++ b/helm-chart/stac-fastapi/values-elasticsearch.yaml @@ -19,6 +19,11 @@ app: port: 80 targetPort: 8080 + waitForDatabase: + enabled: true + intervalSeconds: 2 + maxAttempts: 180 + # Ingress configuration ingress: enabled: true diff --git a/helm-chart/stac-fastapi/values-external.yaml b/helm-chart/stac-fastapi/values-external.yaml deleted file mode 100644 index 8a5780907..000000000 --- a/helm-chart/stac-fastapi/values-external.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Example values for using external Elasticsearch/OpenSearch -# Use this when you have an existing database cluster - -# Choose your backend -backend: elasticsearch # or opensearch - -# STAC FastAPI application configuration -app: - replicaCount: 3 - - image: - repository: ghcr.io/stac-utils/stac-fastapi - tag: "latest" - pullPolicy: IfNotPresent - - # Resource configuration - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 250m - memory: 512Mi - - # Environment variables - env: - STAC_FASTAPI_TITLE: "STAC API with External Database" - STAC_FASTAPI_DESCRIPTION: "STAC API connected to external Elasticsearch/OpenSearch cluster" - ENVIRONMENT: "production" - WEB_CONCURRENCY: "4" - -# Disable bundled databases -elasticsearch: - enabled: false - -opensearch: - enabled: false - -# External database configuration -externalDatabase: - enabled: true - host: "elasticsearch.database.svc.cluster.local" # Your external database host - port: 9200 - ssl: true - verifyCerts: true - timeout: 30 - # Reference to secret containing API key - apiKeySecret: "elasticsearch-api-key" - apiKeySecretKey: "api-key" - -# Create a secret for the API key (example) -# kubectl create secret generic elasticsearch-api-key --from-literal=api-key="your-api-key-here" \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-minimal-storage.yaml b/helm-chart/stac-fastapi/values-minimal-storage.yaml deleted file mode 100644 index 312792cc9..000000000 --- a/helm-chart/stac-fastapi/values-minimal-storage.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# Minimal storage values for STAC FastAPI -# This configuration uses minimal storage requirements for development clusters -# It will still create PVCs but with very small storage requests - -# Set backend to elasticsearch by default -backend: elasticsearch - -# STAC FastAPI application configuration -app: - replicaCount: 1 - - image: - repository: ghcr.io/stac-utils/stac-fastapi - tag: "latest" - pullPolicy: IfNotPresent - - service: - type: ClusterIP - port: 80 - targetPort: 8080 - - ingress: - enabled: false - - resources: - requests: - cpu: "100m" - memory: "512Mi" - limits: - cpu: "500m" - memory: "1Gi" - - autoscaling: - enabled: false - - env: - STAC_FASTAPI_TITLE: "Test STAC API (Minimal Storage)" - STAC_FASTAPI_DESCRIPTION: "STAC API for testing with minimal storage" - ENVIRONMENT: "test" - WEB_CONCURRENCY: "2" - RELOAD: "true" - DATABASE_REFRESH: "true" - ENABLE_TRANSACTIONS_EXTENSIONS: "true" - ENABLE_DIRECT_RESPONSE: "false" - STAC_FASTAPI_RATE_LIMIT: "1000/minute" - -# Elasticsearch with minimal storage requirements -elasticsearch: - enabled: true - - clusterName: "stac-elasticsearch-test" - replicas: 1 - # IMPORTANT: Set minimumMasterNodes to 0 for single-node with discovery.type: single-node - minimumMasterNodes: 0 - - resources: - requests: - cpu: "100m" - memory: "1Gi" - limits: - cpu: "1000m" - memory: "2Gi" - - esJavaOpts: "-Xmx512m -Xms512m" - - esConfig: - elasticsearch.yml: | - cluster.name: "stac-elasticsearch-test" - network.host: 0.0.0.0 - discovery.type: single-node - action.destructive_requires_name: false - xpack.security.enabled: false - xpack.security.transport.ssl.enabled: false - xpack.security.http.ssl.enabled: false - cluster.routing.allocation.disk.threshold_enabled: false - indices.memory.index_buffer_size: 10% - path.data: /usr/share/elasticsearch/data - - # MINIMAL STORAGE - This will create small PVCs - # Most development clusters should be able to handle 1Gi PVCs - volumeClaimTemplate: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 1Gi - # Use default storage class (remove storageClassName line to use default) - # storageClassName: "" - -opensearch: - enabled: false - -externalDatabase: - enabled: false - -monitoring: - enabled: false - -podDisruptionBudget: - enabled: false - -networkPolicy: - enabled: false \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-no-persistence.yaml b/helm-chart/stac-fastapi/values-no-persistence.yaml deleted file mode 100644 index a53b37262..000000000 --- a/helm-chart/stac-fastapi/values-no-persistence.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# No-persistence values for STAC FastAPI -# This configuration completely disables persistent storage -# Data will be lost when pods restart - USE ONLY FOR TESTING - -# Set backend to elasticsearch by default -backend: elasticsearch - -# STAC FastAPI application configuration -app: - replicaCount: 1 - - image: - repository: ghcr.io/stac-utils/stac-fastapi - tag: "latest" - pullPolicy: IfNotPresent - - service: - type: ClusterIP - port: 80 - targetPort: 8080 - - ingress: - enabled: false - - resources: - requests: - cpu: "100m" - memory: "512Mi" - limits: - cpu: "500m" - memory: "1Gi" - - autoscaling: - enabled: false - - env: - STAC_FASTAPI_TITLE: "Test STAC API (No Persistence)" - STAC_FASTAPI_DESCRIPTION: "STAC API for testing - data not persisted" - ENVIRONMENT: "test" - WEB_CONCURRENCY: "2" - RELOAD: "true" - DATABASE_REFRESH: "true" - ENABLE_TRANSACTIONS_EXTENSIONS: "true" - ENABLE_DIRECT_RESPONSE: "false" - STAC_FASTAPI_RATE_LIMIT: "1000/minute" - -# Elasticsearch with NO persistent storage -elasticsearch: - enabled: true - - clusterName: "stac-elasticsearch-test" - replicas: 1 - minimumMasterNodes: 1 - - resources: - requests: - cpu: "100m" - memory: "1Gi" - limits: - cpu: "1000m" - memory: "2Gi" - - esJavaOpts: "-Xmx512m -Xms512m" - - esConfig: - elasticsearch.yml: | - cluster.name: "stac-elasticsearch-test" - network.host: 0.0.0.0 - discovery.type: single-node - action.destructive_requires_name: false - xpack.security.enabled: false - xpack.security.transport.ssl.enabled: false - xpack.security.http.ssl.enabled: false - cluster.routing.allocation.disk.threshold_enabled: false - indices.memory.index_buffer_size: 10% - - # COMPLETELY DISABLE PERSISTENT STORAGE - # The official chart requires volumeClaimTemplate but we can make it very small - # and override with emptyDir in extraVolumes - volumeClaimTemplate: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 1Gi - # Use emptyDir storage class to prevent actual PVC creation - storageClassName: "" - - # Override the data volume with emptyDir - extraVolumes: - - name: elasticsearch-data-override - emptyDir: - sizeLimit: 2Gi - - extraVolumeMounts: - - name: elasticsearch-data-override - mountPath: /usr/share/elasticsearch/data-override - - # Alternative approach: Use extraInitContainers to symlink data directory - extraInitContainers: - - name: setup-data-dir - image: "busybox:1.35" - command: - - /bin/sh - - -c - - | - # Remove the PVC mounted directory and create symlink to emptyDir - rm -rf /usr/share/elasticsearch/data/* - ln -sfn /usr/share/elasticsearch/data-override/* /usr/share/elasticsearch/data/ || true - volumeMounts: - - name: stac-elasticsearch-master - mountPath: /usr/share/elasticsearch/data - - name: elasticsearch-data-override - mountPath: /usr/share/elasticsearch/data-override - -opensearch: - enabled: false - -externalDatabase: - enabled: false - -monitoring: - enabled: false - -podDisruptionBudget: - enabled: false - -networkPolicy: - enabled: false \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-opensearch.yaml b/helm-chart/stac-fastapi/values-opensearch.yaml index 4007f3be9..8f240a25f 100644 --- a/helm-chart/stac-fastapi/values-opensearch.yaml +++ b/helm-chart/stac-fastapi/values-opensearch.yaml @@ -19,6 +19,11 @@ app: port: 80 targetPort: 8080 + waitForDatabase: + enabled: true + intervalSeconds: 2 + maxAttempts: 180 + # Ingress configuration ingress: enabled: true @@ -65,6 +70,13 @@ app: DATETIME_INDEX_MAX_SIZE_GB: "50" STAC_FASTAPI_RATE_LIMIT: "1000/minute" + # Optional: pull connection credentials from a secret managed by the + # OpenSearch chart (or provide plain values via username/password). + databaseAuth: + existingSecret: "" + usernameKey: "username" + passwordKey: "password" + # Disable Elasticsearch since we're using OpenSearch elasticsearch: enabled: false diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index 42d8a620f..04e033bc4 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -9,6 +9,7 @@ backend: elasticsearch global: imageRegistry: "" storageClass: "" + clusterDomain: "cluster.local" # STAC FastAPI application configuration app: @@ -63,6 +64,15 @@ app: port: 80 targetPort: 8080 + # Wait for bundled database before starting the app (similar to docker-compose setup) + waitForDatabase: + enabled: true + image: busybox:1.36.1 + pullPolicy: IfNotPresent + intervalSeconds: 2 + maxAttempts: 120 + resources: {} + # Ingress configuration ingress: enabled: false @@ -166,6 +176,21 @@ app: envFromSecret: {} # ES_API_KEY: "es-api-key-secret" + # Database authentication configuration for bundled backends + databaseAuth: + # Plain username to connect to Elasticsearch/OpenSearch. Leave empty to skip. + username: "" + # Plain password to connect to Elasticsearch/OpenSearch. Leave empty to skip. + password: "" + # Reference an existing secret containing credentials. When set, the + # username/password keys below will be looked up from this secret instead of + # using the plain values above. + existingSecret: "" + # Secret key that stores the username. Defaults to "username" when empty. + usernameKey: "username" + # Secret key that stores the password. Defaults to "password" when empty. + passwordKey: "password" + # Elasticsearch configuration # Note: This will be automatically set based on 'backend' value elasticsearch: diff --git a/helm-chart/test-chart.sh b/helm-chart/test-chart.sh index 37ebeeb4d..2494cabda 100755 --- a/helm-chart/test-chart.sh +++ b/helm-chart/test-chart.sh @@ -199,7 +199,7 @@ install_chart() { local values_file="" case $BACKEND in elasticsearch) - values_file="values-minimal-storage.yaml" + values_file="values-elasticsearch.yaml" ;; opensearch) values_file="values-opensearch.yaml" @@ -239,7 +239,7 @@ upgrade_chart() { local values_file="" case $BACKEND in elasticsearch) - values_file="values-minimal-storage.yaml" + values_file="values-elasticsearch.yaml" ;; opensearch) values_file="values-opensearch.yaml" From 0a82ef50069568044ae7e8887874cda8d84743a6 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Wed, 8 Oct 2025 18:38:14 +1030 Subject: [PATCH 06/12] Made network policy more permissive --- helm-chart/stac-fastapi/README.md | 12 +++++++ .../stac-fastapi/templates/networkpolicy.yaml | 33 +++++++++++++++---- .../stac-fastapi/values-elasticsearch.yaml | 2 ++ .../stac-fastapi/values-opensearch.yaml | 2 ++ helm-chart/stac-fastapi/values.yaml | 2 ++ 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/helm-chart/stac-fastapi/README.md b/helm-chart/stac-fastapi/README.md index c1026b10d..70b0e78d0 100644 --- a/helm-chart/stac-fastapi/README.md +++ b/helm-chart/stac-fastapi/README.md @@ -308,6 +308,8 @@ Enable network policies for additional security: ```yaml networkPolicy: enabled: true + allowNamespaceCommunication: true # allow pods in the release namespace to talk to each other + allowDNS: true # keep kube-dns reachable for service discovery ingress: - from: - namespaceSelector: @@ -316,6 +318,14 @@ networkPolicy: ports: - protocol: TCP port: 8080 + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: opensearch + ports: + - protocol: TCP + port: 9200 ``` ### Pod Security Context @@ -447,6 +457,8 @@ Then visit | `externalDatabase.enabled` | Use external database | `false` | | `monitoring.enabled` | Enable monitoring | `false` | | `networkPolicy.enabled` | Enable network policies | `false` | +| `networkPolicy.allowNamespaceCommunication` | Allow ingress/egress within the release namespace | `true` | +| `networkPolicy.allowDNS` | Allow egress to kube-dns for service discovery | `true` | | `podDisruptionBudget.enabled` | Enable pod disruption budget | `false` | For a complete list of configuration options, see the `values.yaml` file. diff --git a/helm-chart/stac-fastapi/templates/networkpolicy.yaml b/helm-chart/stac-fastapi/templates/networkpolicy.yaml index 10f9bd9c7..d83c19ba1 100644 --- a/helm-chart/stac-fastapi/templates/networkpolicy.yaml +++ b/helm-chart/stac-fastapi/templates/networkpolicy.yaml @@ -5,23 +5,44 @@ metadata: name: {{ include "stac-fastapi.fullname" . }} labels: {{- include "stac-fastapi.labels" . | nindent 4 }} +{{- $ingressRules := list }} +{{- if .Values.networkPolicy.allowNamespaceCommunication }} + {{- $ingressRules = append $ingressRules (dict "from" (list (dict "podSelector" (dict)))) }} +{{- end }} +{{- range $_, $rule := .Values.networkPolicy.ingress }} + {{- $ingressRules = append $ingressRules $rule }} +{{- end }} +{{- $egressRules := list }} +{{- if .Values.networkPolicy.allowNamespaceCommunication }} + {{- $egressRules = append $egressRules (dict "to" (list (dict "podSelector" (dict)))) }} +{{- end }} +{{- if .Values.networkPolicy.allowDNS }} + {{- $dnsTo := dict "namespaceSelector" (dict "matchLabels" (dict "kubernetes.io/metadata.name" "kube-system")) "podSelector" (dict "matchLabels" (dict "k8s-app" "kube-dns")) }} + {{- $dnsPorts := list (dict "protocol" "UDP" "port" 53) (dict "protocol" "TCP" "port" 53) }} + {{- $egressRules = append $egressRules (dict "to" (list $dnsTo) "ports" $dnsPorts) }} +{{- end }} +{{- range $_, $rule := .Values.networkPolicy.egress }} + {{- $egressRules = append $egressRules $rule }} +{{- end }} +{{- $hasIngress := gt (len $ingressRules) 0 }} +{{- $hasEgress := gt (len $egressRules) 0 }} spec: podSelector: matchLabels: {{- include "stac-fastapi.selectorLabels" . | nindent 6 }} policyTypes: - {{- if .Values.networkPolicy.ingress }} + {{- if $hasIngress }} - Ingress {{- end }} - {{- if .Values.networkPolicy.egress }} + {{- if $hasEgress }} - Egress {{- end }} - {{- if .Values.networkPolicy.ingress }} + {{- if $hasIngress }} ingress: - {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- toYaml $ingressRules | nindent 4 }} {{- end }} - {{- if .Values.networkPolicy.egress }} + {{- if $hasEgress }} egress: - {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- toYaml $egressRules | nindent 4 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-elasticsearch.yaml b/helm-chart/stac-fastapi/values-elasticsearch.yaml index 6cae6bd6e..aedc80636 100644 --- a/helm-chart/stac-fastapi/values-elasticsearch.yaml +++ b/helm-chart/stac-fastapi/values-elasticsearch.yaml @@ -142,6 +142,8 @@ podDisruptionBudget: # Network policies for security networkPolicy: enabled: true + allowNamespaceCommunication: true + allowDNS: true ingress: - from: - namespaceSelector: diff --git a/helm-chart/stac-fastapi/values-opensearch.yaml b/helm-chart/stac-fastapi/values-opensearch.yaml index 8f240a25f..3dae8c02b 100644 --- a/helm-chart/stac-fastapi/values-opensearch.yaml +++ b/helm-chart/stac-fastapi/values-opensearch.yaml @@ -159,6 +159,8 @@ podDisruptionBudget: # Network policies for security networkPolicy: enabled: true + allowNamespaceCommunication: true + allowDNS: true ingress: - from: - namespaceSelector: diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index 04e033bc4..16994cc3e 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -371,6 +371,8 @@ monitoring: # Network policies networkPolicy: enabled: false + allowNamespaceCommunication: true + allowDNS: true ingress: [] egress: [] From 69d19358b9358beab9ec7284833510adaf7d8a7b Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Thu, 9 Oct 2025 13:56:43 +1030 Subject: [PATCH 07/12] Create secrets in helm --- helm-chart/stac-fastapi/README.md | 23 ++++++++ .../stac-fastapi/templates/_helpers.tpl | 55 ++++++++++++++++++- .../stac-fastapi/values-opensearch.yaml | 30 +++++++++- helm-chart/stac-fastapi/values.yaml | 10 ++++ 4 files changed, 114 insertions(+), 4 deletions(-) diff --git a/helm-chart/stac-fastapi/README.md b/helm-chart/stac-fastapi/README.md index 70b0e78d0..8694f7027 100644 --- a/helm-chart/stac-fastapi/README.md +++ b/helm-chart/stac-fastapi/README.md @@ -328,6 +328,27 @@ networkPolicy: port: 9200 ``` +### OpenSearch Admin Credentials + +When deploying with the OpenSearch backend you can instruct the chart to generate +an initial admin password and store it in a Kubernetes secret. Enable this by +setting `opensearchSecurity.generateAdminPassword=true` (already enabled in +`values-opensearch.yaml`). The chart will create a secret named +`-stac-fastapi-opensearch-admin` by default and automatically wires it to +the STAC FastAPI deployment through environment variables. + +Retrieve the generated credentials with: + +```bash +kubectl get secret -stac-fastapi-opensearch-admin \ + -o jsonpath='{.data.username}' | base64 --decode +kubectl get secret -stac-fastapi-opensearch-admin \ + -o jsonpath='{.data.password}' | base64 --decode +``` + +You can provide your own secret name, username key, or password key through the +`opensearchSecurity` values block if you already manage credentials externally. + ### Pod Security Context Configure security contexts: @@ -455,6 +476,8 @@ Then visit | `elasticsearch.enabled` | Deploy Elasticsearch | `true` | | `opensearch.enabled` | Deploy OpenSearch | `false` | | `externalDatabase.enabled` | Use external database | `false` | +| `opensearchSecurity.generateAdminPassword` | Generate random OpenSearch admin password | `false` | +| `opensearchSecurity.secretName` | Override name of generated OpenSearch admin secret | `""` | | `monitoring.enabled` | Enable monitoring | `false` | | `networkPolicy.enabled` | Enable network policies | `false` | | `networkPolicy.allowNamespaceCommunication` | Allow ingress/egress within the release namespace | `true` | diff --git a/helm-chart/stac-fastapi/templates/_helpers.tpl b/helm-chart/stac-fastapi/templates/_helpers.tpl index 40e5efd69..cc5daae3c 100644 --- a/helm-chart/stac-fastapi/templates/_helpers.tpl +++ b/helm-chart/stac-fastapi/templates/_helpers.tpl @@ -61,6 +61,46 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +Name for the autogenerated OpenSearch admin secret. +*/}} +{{- define "stac-fastapi.opensearchAdminSecretName" -}} +{{- if .Values.opensearchSecurity.secretName }} +{{- .Values.opensearchSecurity.secretName }} +{{- else }} +{{- printf "%s-opensearch-admin" (include "stac-fastapi.fullname" .) }} +{{- end }} +{{- end }} + +{{/* +Mutate values to ensure OpenSearch initial admin credentials are wired when +password generation is enabled. +*/}} +{{- define "stac-fastapi.configureOpensearchSecurity" -}} +{{- if and (eq .Values.backend "opensearch") (.Values.opensearch.enabled) (.Values.opensearchSecurity.generateAdminPassword | default false) }} + {{- $secretName := include "stac-fastapi.opensearchAdminSecretName" . -}} + {{- $passwordKey := default "password" .Values.opensearchSecurity.passwordKey -}} + {{- $usernameKey := default "username" .Values.opensearchSecurity.usernameKey -}} + {{- $flags := dict "hasPassword" false "hasUsername" false -}} + {{- range $env := (default (list) .Values.opensearch.extraEnvs) }} + {{- if eq (get $env "name") "OPENSEARCH_INITIAL_ADMIN_PASSWORD" }} + {{- $_ := set $flags "hasPassword" true -}} + {{- end }} + {{- if eq (get $env "name") "OPENSEARCH_INITIAL_ADMIN_USERNAME" }} + {{- $_ := set $flags "hasUsername" true -}} + {{- end }} + {{- end }} + {{- if not (get $flags "hasPassword") }} + {{- $passwordEnv := dict "name" "OPENSEARCH_INITIAL_ADMIN_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" $secretName "key" $passwordKey)) -}} + {{- $_ := set .Values.opensearch "extraEnvs" (append (default (list) .Values.opensearch.extraEnvs) $passwordEnv) -}} + {{- end }} + {{- if not (get $flags "hasUsername") }} + {{- $usernameEnv := dict "name" "OPENSEARCH_INITIAL_ADMIN_USERNAME" "valueFrom" (dict "secretKeyRef" (dict "name" $secretName "key" $usernameKey)) -}} + {{- $_ := set .Values.opensearch "extraEnvs" (append (default (list) .Values.opensearch.extraEnvs) $usernameEnv) -}} + {{- end }} +{{- end }} +{{- end }} + {{/* Create the bundled database service name based on backend selection */}} @@ -144,7 +184,20 @@ Create environment variables for the application {{- define "stac-fastapi.environment" -}} {{- $env := default (dict) .Values.app.env -}} {{- $envFromSecret := default (dict) .Values.app.envFromSecret -}} -{{- $dbAuth := default (dict) .Values.app.databaseAuth -}} +{{- $dbAuth := deepCopy (default (dict) .Values.app.databaseAuth) -}} +{{- $opensearchSecurity := default (dict) .Values.opensearchSecurity -}} +{{- if and (eq .Values.backend "opensearch") (get $opensearchSecurity "generateAdminPassword") }} + {{- $secretName := include "stac-fastapi.opensearchAdminSecretName" . -}} + {{- if or (not (hasKey $dbAuth "existingSecret")) (not $dbAuth.existingSecret) }} + {{- $_ := set $dbAuth "existingSecret" $secretName -}} + {{- end }} + {{- if not (hasKey $dbAuth "usernameKey") }} + {{- $_ := set $dbAuth "usernameKey" (default "username" (get $opensearchSecurity "usernameKey")) -}} + {{- end }} + {{- if not (hasKey $dbAuth "passwordKey") }} + {{- $_ := set $dbAuth "passwordKey" (default "password" (get $opensearchSecurity "passwordKey")) -}} + {{- end }} +{{- end }} - name: BACKEND value: {{ .Values.backend | quote }} - name: ES_HOST diff --git a/helm-chart/stac-fastapi/values-opensearch.yaml b/helm-chart/stac-fastapi/values-opensearch.yaml index 3dae8c02b..2f0c9413c 100644 --- a/helm-chart/stac-fastapi/values-opensearch.yaml +++ b/helm-chart/stac-fastapi/values-opensearch.yaml @@ -77,6 +77,14 @@ app: usernameKey: "username" passwordKey: "password" +# Enable automatic OpenSearch admin password generation +opensearchSecurity: + generateAdminPassword: true + # Use a deterministic name so the generated secret can be referenced below + secretName: stac-opensearch-admin + usernameKey: username + passwordKey: password + # Disable Elasticsearch since we're using OpenSearch elasticsearch: enabled: false @@ -114,9 +122,8 @@ opensearch: opensearch.yml: | cluster.name: stac-opensearch-prod network.host: 0.0.0.0 - discovery.seed_hosts: ["stac-opensearch-prod-cluster-master-headless"] - cluster.initial_cluster_manager_nodes: ["stac-opensearch-prod-cluster-master-0", "stac-opensearch-prod-cluster-master-1", "stac-opensearch-prod-cluster-master-2"] - plugins.security.disabled: true + discovery.seed_hosts: ["opensearch-cluster-master-headless"] + cluster.initial_cluster_manager_nodes: ["stac-opensearch-prod-master-0", "stac-opensearch-prod-master-1", "stac-opensearch-prod-master-2"] action.destructive_requires_name: true indices.memory.index_buffer_size: 20% thread_pool.write.queue_size: 1000 @@ -126,6 +133,23 @@ opensearch: cluster.routing.allocation.disk.watermark.high: 90% cluster.routing.allocation.disk.watermark.flood_stage: 95% + # Extra environment variables + extraEnvs: + - name: "DISABLE_INSTALL_DEMO_CONFIG" + value: "false" + - name: "DISABLE_SECURITY_PLUGIN" + value: "false" + - name: "OPENSEARCH_INITIAL_ADMIN_USERNAME" + valueFrom: + secretKeyRef: + name: stac-opensearch-admin + key: username + - name: "OPENSEARCH_INITIAL_ADMIN_PASSWORD" + valueFrom: + secretKeyRef: + name: stac-opensearch-admin + key: password + # Persistent storage persistence: enabled: true diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index 16994cc3e..57fbaf7dc 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -191,6 +191,16 @@ app: # Secret key that stores the password. Defaults to "password" when empty. passwordKey: "password" +# OpenSearch security helper +opensearchSecurity: + generateAdminPassword: false + username: "admin" + usernameKey: "username" + passwordKey: "password" + passwordLength: 32 + secretName: "" + annotations: {} + # Elasticsearch configuration # Note: This will be automatically set based on 'backend' value elasticsearch: From 98268d9873b0789c9df74e36273d701b110992f9 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Thu, 9 Oct 2025 14:04:11 +1030 Subject: [PATCH 08/12] Allow SSL --- .../templates/opensearch-admin-secret.yaml | 30 +++++++++++++++++++ .../templates/opensearch-security-config.yaml | 11 +++++++ .../stac-fastapi/values-opensearch.yaml | 2 ++ 3 files changed, 43 insertions(+) create mode 100644 helm-chart/stac-fastapi/templates/opensearch-admin-secret.yaml create mode 100644 helm-chart/stac-fastapi/templates/opensearch-security-config.yaml diff --git a/helm-chart/stac-fastapi/templates/opensearch-admin-secret.yaml b/helm-chart/stac-fastapi/templates/opensearch-admin-secret.yaml new file mode 100644 index 000000000..e7abc0082 --- /dev/null +++ b/helm-chart/stac-fastapi/templates/opensearch-admin-secret.yaml @@ -0,0 +1,30 @@ +{{- if and (eq .Values.backend "opensearch") (.Values.opensearchSecurity.generateAdminPassword | default false) }} +{{- include "stac-fastapi.configureOpensearchSecurity" . -}} +{{- $secretName := include "stac-fastapi.opensearchAdminSecretName" . -}} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName -}} +{{- $usernameKey := default "username" .Values.opensearchSecurity.usernameKey -}} +{{- $passwordKey := default "password" .Values.opensearchSecurity.passwordKey -}} +{{- $username := default "admin" .Values.opensearchSecurity.username -}} +{{- $passwordLength := int (default 32 .Values.opensearchSecurity.passwordLength) -}} +{{- $passwordB64 := "" -}} +{{- if and $existing (hasKey $existing.data $passwordKey) }} + {{- $passwordB64 = index $existing.data $passwordKey -}} +{{- else }} + {{- $passwordB64 = randAlphaNum $passwordLength | b64enc -}} +{{- end }} +{{- $usernameB64 := b64enc $username -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + labels: + {{- include "stac-fastapi.labels" . | nindent 4 }} + {{- with .Values.opensearchSecurity.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{ $usernameKey }}: {{ $usernameB64 }} + {{ $passwordKey }}: {{ $passwordB64 }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/templates/opensearch-security-config.yaml b/helm-chart/stac-fastapi/templates/opensearch-security-config.yaml new file mode 100644 index 000000000..2ea17d26c --- /dev/null +++ b/helm-chart/stac-fastapi/templates/opensearch-security-config.yaml @@ -0,0 +1,11 @@ +{{- /* + This template doesn't create resources; it mutates values so the + OpenSearch subchart receives the generated admin credentials. +*/ -}} +{{- include "stac-fastapi.configureOpensearchSecurity" . -}} +{{- if (and (eq .Values.backend "opensearch") (.Values.opensearch.enabled) (.Values.opensearchSecurity.generateAdminPassword | default false)) }} +# debug-extraEnvs-count: {{ len (default (list) .Values.opensearch.extraEnvs) }} +{{- range $env := (default (list) .Values.opensearch.extraEnvs) }} +# debug-extraEnv: {{ toYaml $env | replace "\n" " " | trim }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-opensearch.yaml b/helm-chart/stac-fastapi/values-opensearch.yaml index 2f0c9413c..942933bbf 100644 --- a/helm-chart/stac-fastapi/values-opensearch.yaml +++ b/helm-chart/stac-fastapi/values-opensearch.yaml @@ -69,6 +69,8 @@ app: ENABLE_DATETIME_INDEX_FILTERING: "true" # Enable for large datasets DATETIME_INDEX_MAX_SIZE_GB: "50" STAC_FASTAPI_RATE_LIMIT: "1000/minute" + ES_USE_SSL: "true" + ES_VERIFY_CERTS: "false" # Optional: pull connection credentials from a secret managed by the # OpenSearch chart (or provide plain values via username/password). From 617767e489ddc057f565a56d884da2a92ebc09c5 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Thu, 9 Oct 2025 16:27:42 +1030 Subject: [PATCH 09/12] Set up SSL to make ES version work as well --- helm-chart/stac-fastapi/values-dev.yaml | 124 ------------------ .../stac-fastapi/values-elasticsearch.yaml | 8 ++ 2 files changed, 8 insertions(+), 124 deletions(-) delete mode 100644 helm-chart/stac-fastapi/values-dev.yaml diff --git a/helm-chart/stac-fastapi/values-dev.yaml b/helm-chart/stac-fastapi/values-dev.yaml deleted file mode 100644 index 08d49afaa..000000000 --- a/helm-chart/stac-fastapi/values-dev.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Development values for STAC FastAPI -# This configuration is optimized for local development and testing -# It disables persistent storage to avoid PVC issues in development clusters - -# Set backend to elasticsearch by default -backend: elasticsearch - -# STAC FastAPI application configuration -app: - replicaCount: 1 # Single replica for development - - image: - repository: ghcr.io/stac-utils/stac-fastapi - tag: "latest" - pullPolicy: IfNotPresent - - # Service configuration - service: - type: ClusterIP - port: 80 - targetPort: 8080 - - # No ingress for development - ingress: - enabled: false - - # Minimal resources for development - resources: - requests: - cpu: "100m" - memory: "512Mi" - limits: - cpu: "500m" - memory: "1Gi" - - # Disable autoscaling - autoscaling: - enabled: false - - # Development environment variables - env: - STAC_FASTAPI_TITLE: "Development STAC API" - STAC_FASTAPI_DESCRIPTION: "STAC API for development and testing" - ENVIRONMENT: "development" - WEB_CONCURRENCY: "2" - RELOAD: "true" - DATABASE_REFRESH: "true" # Allow fresh starts in development - ENABLE_TRANSACTIONS_EXTENSIONS: "true" - ENABLE_DIRECT_RESPONSE: "false" # Easier debugging - STAC_FASTAPI_RATE_LIMIT: "1000/minute" - -# Elasticsearch configuration for development -elasticsearch: - enabled: true - - # Single node cluster for development - clusterName: "stac-elasticsearch-dev" - replicas: 1 - minimumMasterNodes: 1 - - # Minimal resources - resources: - requests: - cpu: "100m" - memory: "1Gi" - limits: - cpu: "1000m" - memory: "2Gi" - - # Small heap size - esJavaOpts: "-Xmx512m -Xms512m" - - # Development Elasticsearch configuration - esConfig: - elasticsearch.yml: | - cluster.name: "stac-elasticsearch-dev" - network.host: 0.0.0.0 - discovery.type: single-node - action.destructive_requires_name: false - xpack.security.enabled: false - xpack.security.transport.ssl.enabled: false - xpack.security.http.ssl.enabled: false - # Development-friendly settings - cluster.routing.allocation.disk.threshold_enabled: false - indices.memory.index_buffer_size: 10% - - # DISABLE PERSISTENT STORAGE FOR DEVELOPMENT - # This prevents PVC issues in development clusters - volumeClaimTemplate: - accessModes: ["ReadWriteOnce"] - storageClassName: "" # Use default storage class if available - resources: - requests: - storage: 1Gi # Minimal storage - - # Alternative: Use emptyDir instead of PVC (uncomment to use) - # persistence: - # enabled: false - # extraVolumes: - # - name: elasticsearch-data - # emptyDir: {} - # extraVolumeMounts: - # - name: elasticsearch-data - # mountPath: /usr/share/elasticsearch/data - -# Disable OpenSearch for Elasticsearch backend -opensearch: - enabled: false - -# External database disabled since we're using bundled Elasticsearch -externalDatabase: - enabled: false - -# Disable monitoring for development -monitoring: - enabled: false - -# Disable pod disruption budget -podDisruptionBudget: - enabled: false - -# Disable network policies for development -networkPolicy: - enabled: false \ No newline at end of file diff --git a/helm-chart/stac-fastapi/values-elasticsearch.yaml b/helm-chart/stac-fastapi/values-elasticsearch.yaml index aedc80636..cd2f5e897 100644 --- a/helm-chart/stac-fastapi/values-elasticsearch.yaml +++ b/helm-chart/stac-fastapi/values-elasticsearch.yaml @@ -69,6 +69,13 @@ app: ENABLE_DATETIME_INDEX_FILTERING: "true" # Enable for large datasets DATETIME_INDEX_MAX_SIZE_GB: "50" STAC_FASTAPI_RATE_LIMIT: "1000/minute" + ES_USE_SSL: "true" + ES_VERIFY_CERTS: "false" + + databaseAuth: + existingSecret: "stac-elasticsearch-prod-master-credentials" + usernameKey: "username" + passwordKey: "password" # Elasticsearch configuration elasticsearch: @@ -105,6 +112,7 @@ elasticsearch: indices.memory.index_buffer_size: 20% thread_pool.write.queue_size: 1000 thread_pool.search.queue_size: 1000 + xpack.security.http.ssl.client_authentication: optional # Persistent storage volumeClaimTemplate: From 21f8af4245690b62b345d90a19e114647f1b1393 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Thu, 9 Oct 2025 16:31:04 +1030 Subject: [PATCH 10/12] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0f028bc9..3f60a9488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - CloudFerro logo to sponsors and supporters list [#485](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/485) - Latest news section to README [#485](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/485) +- Added Helm chart for ES or OS in-cluster deployment [#455](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/455) ### Changed From 58b098bd097d2304d0a1def425b37f09c0a60d77 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Thu, 9 Oct 2025 16:45:21 +1030 Subject: [PATCH 11/12] Enhance pre-commit to cover helm chart --- .github/ct.yaml | 6 +-- .pre-commit-config.yaml | 18 ++++++++- helm-chart/stac-fastapi/Chart.yaml | 2 +- helm-chart/stac-fastapi/values.yaml | 62 ++++++++++++++--------------- 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/.github/ct.yaml b/.github/ct.yaml index ce1b5b7ee..c49b55b29 100644 --- a/.github/ct.yaml +++ b/.github/ct.yaml @@ -10,10 +10,8 @@ target-branch: main # Chart repositories to add chart-repos: - - name: elasticsearch - url: https://helm.elastic.co - - name: opensearch - url: https://opensearch-project.github.io/helm-charts/ + - elasticsearch=https://helm.elastic.co + - opensearch=https://opensearch-project.github.io/helm-charts/ # Helm version to use helm-version: v3.13.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e867050b4..65cd67583 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,4 +41,20 @@ repos: #args: [ # Don't require docstrings for tests #'--match=(?!test|alembic|scripts).*\.py', - #] \ No newline at end of file + #] + - repo: local + hooks: + - id: helm-chart-lint + name: Helm chart lint + language: docker_image + entry: quay.io/helmpack/chart-testing:v3.7.1 + args: + - bash + - -c + - >- + export HELM_CONFIG_HOME=/tmp/helm && + export HELM_DATA_HOME=/tmp/helm && + export HELM_CACHE_HOME=/tmp/helm && + ct lint --config /src/.github/ct.yaml + pass_filenames: false + files: ^helm-chart/ \ No newline at end of file diff --git a/helm-chart/stac-fastapi/Chart.yaml b/helm-chart/stac-fastapi/Chart.yaml index a197cdd47..c635dfa29 100644 --- a/helm-chart/stac-fastapi/Chart.yaml +++ b/helm-chart/stac-fastapi/Chart.yaml @@ -29,4 +29,4 @@ dependencies: - name: opensearch version: 3.2.1 repository: https://opensearch-project.github.io/helm-charts/ - condition: opensearch.enabled \ No newline at end of file + condition: opensearch.enabled diff --git a/helm-chart/stac-fastapi/values.yaml b/helm-chart/stac-fastapi/values.yaml index 57fbaf7dc..9744caf36 100644 --- a/helm-chart/stac-fastapi/values.yaml +++ b/helm-chart/stac-fastapi/values.yaml @@ -148,7 +148,7 @@ app: ENVIRONMENT: "production" WEB_CONCURRENCY: "10" RELOAD: "false" - + # Database configuration DATABASE_REFRESH: "false" ENABLE_TRANSACTIONS_EXTENSIONS: "true" @@ -156,18 +156,18 @@ app: ENV_MAX_LIMIT: "10000" STAC_INDEX_ASSETS: "false" RAISE_ON_BULK_ERROR: "false" - + # Performance configuration ENABLE_DIRECT_RESPONSE: "false" - + # Rate limiting STAC_FASTAPI_RATE_LIMIT: "200/minute" - + # Datetime index filtering ENABLE_DATETIME_INDEX_FILTERING: "false" DATETIME_INDEX_MAX_SIZE_GB: "25" STAC_ITEMS_INDEX_PREFIX: "items_" - + # SSL configuration ES_USE_SSL: "false" ES_VERIFY_CERTS: "false" @@ -206,29 +206,29 @@ opensearchSecurity: elasticsearch: # Enable/disable Elasticsearch deployment enabled: true - + # Elasticsearch cluster name clusterName: "stac-elasticsearch" - + # Node group configuration nodeGroup: "master" - + # Number of masters eligible nodes masterService: "stac-elasticsearch-master" - + # Elasticsearch roles roles: - master - ingest - data - remote_cluster_client - + # Number of replicas replicas: 1 - + # Minimum master nodes minimumMasterNodes: 1 - + # Elasticsearch configuration esConfig: elasticsearch.yml: | @@ -239,10 +239,10 @@ elasticsearch: xpack.security.enabled: false xpack.security.transport.ssl.enabled: false xpack.security.http.ssl.enabled: false - + # JVM configuration esJavaOpts: "-Xmx1g -Xms1g" - + # Resource configuration resources: requests: @@ -251,15 +251,15 @@ elasticsearch: limits: cpu: "1000m" memory: "2Gi" - + # Volume claim template volumeClaimTemplate: - accessModes: [ "ReadWriteOnce" ] + accessModes: ["ReadWriteOnce"] # storageClassName: "" # Leave empty to use cluster default resources: requests: storage: 1Gi - + # Service configuration service: type: ClusterIP @@ -276,29 +276,29 @@ elasticsearch: opensearch: # Enable/disable OpenSearch deployment enabled: false - + # OpenSearch cluster name clusterName: "stac-opensearch" - + # Service name used to expose the master nodes masterService: "opensearch-cluster-master" - + # Node group configuration nodeGroup: "master" - + # OpenSearch roles roles: - master - ingest - data - remote_cluster_client - + # Number of replicas replicas: 1 - + # Minimum master nodes majorVersion: "" - + # OpenSearch configuration config: opensearch.yml: | @@ -307,10 +307,10 @@ opensearch: discovery.type: single-node plugins.security.disabled: true action.destructive_requires_name: false - + # JVM configuration opensearchJavaOpts: "-Xmx1g -Xms1g" - + # Resource configuration resources: requests: @@ -319,7 +319,7 @@ opensearch: limits: cpu: "1000m" memory: "2Gi" - + # Persistence configuration persistence: enabled: true @@ -328,7 +328,7 @@ opensearch: - ReadWriteOnce size: 10Gi annotations: {} - + # Service configuration service: type: ClusterIP @@ -363,7 +363,7 @@ externalDatabase: monitoring: # Enable monitoring enabled: false - + # Prometheus monitoring prometheus: enabled: false @@ -372,7 +372,7 @@ monitoring: interval: 30s scrapeTimeout: 10s labels: {} - + # Grafana dashboards grafana: enabled: false @@ -396,4 +396,4 @@ podDisruptionBudget: securityPolicy: enabled: false psp: - enabled: false \ No newline at end of file + enabled: false From 422ffd0534f48614998f64cb315334b2399e92e7 Mon Sep 17 00:00:00 2001 From: "Tisham (whatnick) Dhar" Date: Tue, 14 Oct 2025 10:29:30 +1030 Subject: [PATCH 12/12] Fixup some testing process --- .github/workflows/helm-chart-test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 218baa228..53142457a 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -81,7 +81,7 @@ jobs: - name: Upload test reports if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-report-${{ matrix.backend }}-k8s-${{ matrix.kubernetes-version }} path: test-report-*.json @@ -186,10 +186,7 @@ jobs: - name: Setup Helm repositories uses: ./.github/actions/setup-helm-repos - - - name: Run chart-testing (list) - run: ct list --config .github/ct.yaml - + - name: Run chart-testing (lint) run: ct lint --config .github/ct.yaml