From 0063113824bda6855ba8ef662761be933ea69f94 Mon Sep 17 00:00:00 2001 From: Nico Schieder Date: Thu, 30 Jul 2026 13:59:55 +0200 Subject: [PATCH 1/2] feat: add Firestore storage backend with integration tests Implement a pluggable storage backend using Google Cloud Firestore with transaction-based resource versioning, snapshot listeners for real-time event broadcasting, and native duplicate detection via Create(). Adds Makefile targets (firestore-start, firestore-stop, test-firestore) and a README documenting limitations and comparison with other backends. Co-Authored-By: Claude Opus 4.6 --- orlop/Makefile | 49 +- orlop/go.mod | 45 +- orlop/go.sum | 107 +-- .../pkg/apiserver/storage/firestore/README.md | 79 +++ .../storage/firestore/broadcaster.go | 316 +++++++++ .../apiserver/storage/firestore/factory.go | 74 ++ .../pkg/apiserver/storage/firestore/store.go | 632 ++++++++++++++++++ .../apiserver/storage/firestore/store_test.go | 573 ++++++++++++++++ 8 files changed, 1814 insertions(+), 61 deletions(-) create mode 100644 orlop/pkg/apiserver/storage/firestore/README.md create mode 100644 orlop/pkg/apiserver/storage/firestore/broadcaster.go create mode 100644 orlop/pkg/apiserver/storage/firestore/factory.go create mode 100644 orlop/pkg/apiserver/storage/firestore/store.go create mode 100644 orlop/pkg/apiserver/storage/firestore/store_test.go diff --git a/orlop/Makefile b/orlop/Makefile index 9e9de846..d27ab4e5 100644 --- a/orlop/Makefile +++ b/orlop/Makefile @@ -70,6 +70,38 @@ postgres-logs: postgres-psql: @docker exec -it $(POSTGRES_CONTAINER_NAME) psql -U $(POSTGRES_USER) -d $(POSTGRES_DB) +# Firestore emulator configuration +FIRESTORE_EMULATOR_PORT ?= 8090 +FIRESTORE_EMULATOR_HOST ?= localhost:$(FIRESTORE_EMULATOR_PORT) +FIRESTORE_EMULATOR_PID_FILE ?= /tmp/orlop-firestore-emulator.pid + +.PHONY: firestore-start +firestore-start: + @echo "Starting Firestore emulator..." + @gcloud emulators firestore start --host-port=$(FIRESTORE_EMULATOR_HOST) > /dev/null 2>&1 & \ + echo $$! > $(FIRESTORE_EMULATOR_PID_FILE); \ + for i in $$(seq 1 30); do \ + nc -z localhost $(FIRESTORE_EMULATOR_PORT) 2>/dev/null && break || sleep 0.5; \ + done + @echo "Firestore emulator is ready at $(FIRESTORE_EMULATOR_HOST)" + +.PHONY: firestore-stop +firestore-stop: + @echo "Stopping Firestore emulator..." + @if [ -f $(FIRESTORE_EMULATOR_PID_FILE) ]; then \ + kill $$(cat $(FIRESTORE_EMULATOR_PID_FILE)) 2>/dev/null || true; \ + rm -f $(FIRESTORE_EMULATOR_PID_FILE); \ + fi + @echo "Firestore emulator stopped" + +.PHONY: test-firestore +test-firestore: firestore-start + @echo "Running Firestore integration tests..." + @FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) go test -v -count=1 ./pkg/apiserver/storage/firestore/... ; \ + EXIT_CODE=$$?; \ + $(MAKE) firestore-stop; \ + exit $$EXIT_CODE + .PHONY: test test: @echo "Running all tests..." @@ -89,14 +121,19 @@ test-postgres: postgres-start @echo "Done!" .PHONY: test-all -test-all: postgres-start - @echo "Running all tests with PostgreSQL..." - @POSTGRES_TEST_URL=$(POSTGRES_TEST_URL) go test -v ./... - @echo "Done!" +test-all: postgres-start firestore-start + @echo "Running all tests with PostgreSQL and Firestore..." + @POSTGRES_TEST_URL=$(POSTGRES_TEST_URL) FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) go test -v -count=1 ./... ; \ + EXIT_CODE=$$?; \ + $(MAKE) firestore-stop; \ + exit $$EXIT_CODE .PHONY: test-coverage -test-coverage: postgres-start +test-coverage: postgres-start firestore-start @echo "Running tests with coverage..." - @POSTGRES_TEST_URL=$(POSTGRES_TEST_URL) go test -v -coverprofile=coverage.out ./pkg/... + @POSTGRES_TEST_URL=$(POSTGRES_TEST_URL) FIRESTORE_EMULATOR_HOST=$(FIRESTORE_EMULATOR_HOST) go test -v -count=1 -coverprofile=coverage.out ./pkg/... ; \ + EXIT_CODE=$$?; \ + $(MAKE) firestore-stop; \ + exit $$EXIT_CODE @go tool cover -html=coverage.out -o coverage.html @echo "Coverage report generated: coverage.html" diff --git a/orlop/go.mod b/orlop/go.mod index a3d1fb58..9fe61289 100644 --- a/orlop/go.mod +++ b/orlop/go.mod @@ -3,6 +3,7 @@ module github.com/openshift-online/gecko/orlop go 1.26.0 require ( + cloud.google.com/go/firestore v1.24.0 github.com/evanphx/json-patch/v5 v5.9.11 github.com/go-chi/chi/v5 v5.0.10 github.com/go-chi/cors v1.2.1 @@ -11,6 +12,8 @@ require ( github.com/google/uuid v1.6.0 github.com/lib/pq v1.12.3 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 + google.golang.org/api v0.287.1 + google.golang.org/grpc v1.82.0 k8s.io/apiextensions-apiserver v0.36.0 k8s.io/apimachinery v0.36.2 k8s.io/apiserver v0.36.0 @@ -23,6 +26,11 @@ require ( require ( cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -55,6 +63,9 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/cel-go v0.26.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect @@ -76,33 +87,33 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.50.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.53.0 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/term v0.42.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.44.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/grpc v1.79.3 // indirect + golang.org/x/tools v0.45.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/orlop/go.sum b/orlop/go.sum index 53c4548c..a1691b72 100644 --- a/orlop/go.sum +++ b/orlop/go.sum @@ -1,5 +1,17 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/firestore v1.24.0 h1:x0Z3hrgjYgo2wI9whuBRQcNc2hYwzZDQy/7pkUXbXcs= +cloud.google.com/go/firestore v1.24.0/go.mod h1:5aojyjN4olKUnBZDCRWwM+NsdrrCX3t1qfyERZGOonM= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= @@ -14,6 +26,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= @@ -27,6 +41,11 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -101,8 +120,14 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= @@ -145,6 +170,8 @@ github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -205,24 +232,24 @@ go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -238,46 +265,46 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -286,14 +313,18 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/orlop/pkg/apiserver/storage/firestore/README.md b/orlop/pkg/apiserver/storage/firestore/README.md new file mode 100644 index 00000000..b827bc78 --- /dev/null +++ b/orlop/pkg/apiserver/storage/firestore/README.md @@ -0,0 +1,79 @@ +# Firestore Storage Backend + +A pluggable storage backend for the platform API server using Google Cloud Firestore. + +## Usage + +```go +import "github.com/openshift-online/gecko/orlop/pkg/apiserver/storage/firestore" + +factory := firestore.NewStorageFactory(firestore.StorageFactoryConfig{ + ProjectID: "my-project", +}) +``` + +## Data Model + +| Collection | Document ID | Purpose | +|---|---|---| +| `resources_{type}` | `{namespace}_{name}` | Resource documents (JSON data, labels, resource version) | +| `eventlog_{type}` | Zero-padded resource version | Event log for watch/subscribe replay | +| `counters` | `rv_{type}` | Monotonic resource version counter per type | + +## Limitations + +### Resource Version Counter Bottleneck + +Firestore has no atomic increment-and-return operation. Resource versioning uses a Firestore transaction (read-increment-write) on a single counter document per resource type. Firestore enforces a **~1 sustained write/second soft limit per document**, which makes the counter a bottleneck under load. + +- At **< 10 writes/sec** per resource type: works reliably with occasional transaction retries. +- At **10–50 writes/sec**: transaction contention causes retry storms and increased latency. +- At **> 50 writes/sec**: not viable without architectural changes. + +Distributed counter shards could increase throughput but would break the total ordering required by the Kubernetes watch protocol. + +### Composite Index Requirements + +Firestore requires composite indexes for queries combining multiple fields. Since Kubernetes labels are arbitrary key-value pairs, server-side label filtering is impractical for arbitrary selectors. Label selectors, shard selectors, and field filters are evaluated **client-side** after fetching documents. + +### Snapshot Listener Reconnection + +Firestore snapshot listeners (used for event broadcasting) can return `iterator.Done` unexpectedly due to network issues. The broadcaster includes reconnection logic with a 1-second backoff, but there may be brief gaps in real-time event delivery during reconnections. + +### Document ID Constraints + +Firestore document IDs cannot contain `/`. Resource names and namespaces are joined with `_` instead of `/` or `\x00`. If resource names or namespaces contain underscores, there is a theoretical (but unlikely in practice) collision risk. + +## Comparison with Other Backends + +| | In-Memory | PostgreSQL | Bigtable | Firestore | +|---|---|---|---|---| +| **Resource versioning** | `atomic.Int64` | `pg_current_xact_id()` | `ReadModifyWriteRow` (atomic increment + return) | Transaction (read-increment-write, ~1 write/sec/doc limit) | +| **Event broadcasting** | In-process ring buffer | `LISTEN/NOTIFY` | Change streams (low-level gRPC) | Snapshot listeners (high-level API) | +| **Duplicate detection** | Map key check | `INSERT ... ON CONFLICT` | `CheckAndMutateRow` (conditional mutation) | `DocumentRef.Create()` (native `AlreadyExists`) | +| **List pagination** | Client-side slice | SQL `LIMIT`/`OFFSET` with tuple comparison | Manual key range (`NewRange`) | Native `OrderBy` + `StartAfter` + `Limit` | +| **Label/field filtering** | Client-side | Server-side (JSONB operators) | Client-side | Client-side (composite index constraints) | +| **Consistency** | Immediate | Serializable transactions | Row-level strong consistency | Strongly consistent | +| **Persistence** | None | Durable | Durable | Durable | +| **Scale ceiling** | Single process | Single instance (with replicas) | Petabyte-scale, linear scaling | 10K writes/sec per database | +| **Operational complexity** | None | Manage DB instances | Provision clusters/nodes | Fully serverless | +| **Cost at small scale** | Free | DB instance cost | Minimum node cost | Pay-per-operation | +| **Emulator** | N/A | Docker container | `go install .../bigtable/cmd/emulator` | `gcloud emulators firestore start` (requires Java) | + +### When to Use Each + +- **In-Memory**: Development, testing, single-instance deployments where persistence is unnecessary. +- **PostgreSQL**: Production deployments needing server-side query capabilities (label filtering in SQL), strong transactional guarantees, and moderate scale. +- **Bigtable**: High-throughput production workloads (> 50 writes/sec per resource type). Best resource versioning performance due to atomic increment-and-return. +- **Firestore**: Small-to-moderate deployments (< 10 writes/sec per resource type) where serverless operations, zero capacity planning, and pay-per-use pricing are priorities. + +## Running Tests + +```bash +# Start the Firestore emulator and run tests +make test-firestore + +# Or manually +gcloud emulators firestore start --host-port=localhost:8090 & +FIRESTORE_EMULATOR_HOST=localhost:8090 go test -v -count=1 ./pkg/apiserver/storage/firestore/... +``` diff --git a/orlop/pkg/apiserver/storage/firestore/broadcaster.go b/orlop/pkg/apiserver/storage/firestore/broadcaster.go new file mode 100644 index 00000000..44b047fa --- /dev/null +++ b/orlop/pkg/apiserver/storage/firestore/broadcaster.go @@ -0,0 +1,316 @@ +package firestore + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "sync" + "time" + + "cloud.google.com/go/firestore" + "github.com/openshift-online/gecko/orlop/pkg/apiserver/storage" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type FirestoreBroadcaster struct { + client *firestore.Client + eventLogColl string + ctx context.Context + cancel context.CancelFunc + scheme *runtime.Scheme + gvk schema.GroupVersionKind + + mu sync.RWMutex + subscribers map[int]chan storage.ResourceEvent + nextID int + closed bool +} + +type FirestoreBroadcasterConfig struct { + Client *firestore.Client + EventLogColl string + Scheme *runtime.Scheme + GVK schema.GroupVersionKind +} + +func NewFirestoreBroadcaster(ctx context.Context, config FirestoreBroadcasterConfig) (*FirestoreBroadcaster, error) { + if config.Client == nil { + return nil, fmt.Errorf("firestore client is required") + } + + eventLogColl := config.EventLogColl + if eventLogColl == "" { + eventLogColl = "event_log" + } + + bCtx, cancel := context.WithCancel(ctx) + + b := &FirestoreBroadcaster{ + client: config.Client, + eventLogColl: eventLogColl, + ctx: bCtx, + cancel: cancel, + scheme: config.Scheme, + gvk: config.GVK, + subscribers: make(map[int]chan storage.ResourceEvent), + } + + go b.watchSnapshots() + + return b, nil +} + +func (b *FirestoreBroadcaster) watchSnapshots() { + for { + select { + case <-b.ctx.Done(): + return + default: + } + + err := b.readSnapshots() + if err != nil { + if b.ctx.Err() != nil { + return + } + time.Sleep(time.Second) + } + } +} + +func (b *FirestoreBroadcaster) readSnapshots() error { + q := b.client.Collection(b.eventLogColl).OrderBy("rv", firestore.Asc) + iter := q.Snapshots(b.ctx) + defer iter.Stop() + + for { + snap, err := iter.Next() + if err != nil { + return fmt.Errorf("snapshot iterator error: %w", err) + } + + for _, change := range snap.Changes { + if change.Kind != firestore.DocumentAdded { + continue + } + + event, err := b.docToEvent(change.Doc) + if err != nil { + continue + } + + b.broadcastToSubscribers(event) + } + } +} + +func (b *FirestoreBroadcaster) docToEvent(snap *firestore.DocumentSnapshot) (storage.ResourceEvent, error) { + data := snap.Data() + + eventType, _ := data["type"].(string) + rv := toInt64(data["rv"]) + contextFilter, _ := data["contextFilter"].(string) + + rawData, ok := data["data"] + if !ok { + return storage.ResourceEvent{}, fmt.Errorf("no data field in event log document") + } + + obj, err := b.reconstructObject(rawData) + if err != nil { + return storage.ResourceEvent{}, fmt.Errorf("failed to reconstruct object: %w", err) + } + obj.SetResourceVersion(strconv.FormatInt(rv, 10)) + + return storage.ResourceEvent{ + Type: storage.EventType(eventType), + ResourceVersion: strconv.FormatInt(rv, 10), + Object: obj, + ContextFilterValue: contextFilter, + }, nil +} + +func (b *FirestoreBroadcaster) reconstructObject(rawData any) (client.Object, error) { + dataMap, ok := rawData.(map[string]any) + if !ok { + return nil, fmt.Errorf("data is not a map") + } + + jsonBytes, err := json.Marshal(dataMap) + if err != nil { + return nil, err + } + + if b.scheme != nil && !b.gvk.Empty() { + rObj, err := b.scheme.New(b.gvk) + if err == nil { + if err := json.Unmarshal(jsonBytes, rObj); err == nil { + rObj.GetObjectKind().SetGroupVersionKind(b.gvk) + if clientObj, ok := rObj.(client.Object); ok { + return clientObj, nil + } + } + } + } + + unstruct := &unstructured.Unstructured{} + if err := json.Unmarshal(jsonBytes, &unstruct.Object); err != nil { + return nil, err + } + if !b.gvk.Empty() { + unstruct.SetGroupVersionKind(b.gvk) + } + return unstruct, nil +} + +func (b *FirestoreBroadcaster) broadcastToSubscribers(event storage.ResourceEvent) { + b.mu.RLock() + defer b.mu.RUnlock() + + for _, ch := range b.subscribers { + select { + case ch <- event: + default: + } + } +} + +func (b *FirestoreBroadcaster) Broadcast(event storage.ResourceEvent) { + b.mu.RLock() + if b.closed { + b.mu.RUnlock() + return + } + b.mu.RUnlock() + + rv, _ := strconv.ParseInt(event.ResourceVersion, 10, 64) + docID := padResourceVersion(rv) + + objectData, err := marshalData(event.Object) + if err != nil { + return + } + + doc := map[string]any{ + "type": string(event.Type), + "rv": rv, + "data": objectData, + "contextFilter": event.ContextFilterValue, + "timestamp": time.Now(), + } + + _, _ = b.client.Collection(b.eventLogColl).Doc(docID).Set(b.ctx, doc) +} + +func (b *FirestoreBroadcaster) Subscribe(sinceResourceVersion string) (<-chan storage.ResourceEvent, func(), error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.closed { + return nil, nil, fmt.Errorf("broadcaster is closed") + } + + id := b.nextID + b.nextID++ + + ch := make(chan storage.ResourceEvent, 100) + b.subscribers[id] = ch + + if sinceResourceVersion != "" { + go b.sendHistoricalEvents(ch, sinceResourceVersion) + } + + stopFunc := func() { + b.unsubscribe(id) + } + + return ch, stopFunc, nil +} + +func (b *FirestoreBroadcaster) sendHistoricalEvents(ch chan storage.ResourceEvent, sinceResourceVersion string) { + rv, err := strconv.ParseInt(sinceResourceVersion, 10, 64) + if err != nil { + return + } + + docs, err := b.client.Collection(b.eventLogColl). + Where("rv", ">", rv). + OrderBy("rv", firestore.Asc). + Limit(1000). + Documents(b.ctx). + GetAll() + if err != nil { + return + } + + for _, snap := range docs { + event, parseErr := b.docToEvent(snap) + if parseErr != nil { + continue + } + + select { + case ch <- event: + default: + return + } + } +} + +func (b *FirestoreBroadcaster) unsubscribe(id int) { + b.mu.Lock() + defer b.mu.Unlock() + + if ch, exists := b.subscribers[id]; exists { + close(ch) + delete(b.subscribers, id) + } +} + +func (b *FirestoreBroadcaster) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + + if b.closed { + return nil + } + + b.closed = true + b.cancel() + + for id, ch := range b.subscribers { + close(ch) + delete(b.subscribers, id) + } + + return nil +} + +func (b *FirestoreBroadcaster) PruneOldEvents(ctx context.Context, olderThan time.Duration) error { + cutoff := time.Now().Add(-olderThan) + + docs, err := b.client.Collection(b.eventLogColl). + Where("timestamp", "<", cutoff). + Documents(ctx). + GetAll() + if err != nil { + return fmt.Errorf("failed to query events for pruning: %w", err) + } + + for _, snap := range docs { + if _, err := snap.Ref.Delete(ctx); err != nil { + return fmt.Errorf("failed to delete event %s: %w", snap.Ref.ID, err) + } + } + + return nil +} + +var ( + _ storage.EventBroadcaster = (*FirestoreBroadcaster)(nil) + _ storage.EventPruner = (*FirestoreBroadcaster)(nil) +) diff --git a/orlop/pkg/apiserver/storage/firestore/factory.go b/orlop/pkg/apiserver/storage/firestore/factory.go new file mode 100644 index 00000000..6c0f0588 --- /dev/null +++ b/orlop/pkg/apiserver/storage/firestore/factory.go @@ -0,0 +1,74 @@ +package firestore + +import ( + "context" + "fmt" + "strings" + + "cloud.google.com/go/firestore" + "github.com/openshift-online/gecko/orlop/pkg/apiserver/storage" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type StorageFactoryConfig struct { + ProjectID string + Client *firestore.Client + CollectionPrefix string + Context context.Context +} + +func NewStorageFactory(config StorageFactoryConfig) func(string, *runtime.Scheme, schema.GroupVersionKind) (storage.ResourceStore, error) { + return func(resourceType string, scheme *runtime.Scheme, gvk schema.GroupVersionKind) (storage.ResourceStore, error) { + ctx := config.Context + if ctx == nil { + ctx = context.Background() + } + + fsClient := config.Client + var err error + + if fsClient == nil { + fsClient, err = firestore.NewClient(ctx, config.ProjectID) + if err != nil { + return nil, fmt.Errorf("failed to create firestore client: %w", err) + } + } + + prefix := config.CollectionPrefix + safeType := sanitizeCollectionName(resourceType) + resourcesColl := prefix + "resources_" + safeType + eventLogColl := prefix + "eventlog_" + safeType + countersColl := prefix + "counters" + + broadcaster, err := NewFirestoreBroadcaster(ctx, FirestoreBroadcasterConfig{ + Client: fsClient, + EventLogColl: eventLogColl, + Scheme: scheme, + GVK: gvk, + }) + if err != nil { + return nil, fmt.Errorf("failed to create broadcaster: %w", err) + } + + store, err := NewFirestoreStore(ctx, FirestoreStoreConfig{ + Client: fsClient, + ResourceType: resourceType, + Scheme: scheme, + GVK: gvk, + Broadcaster: broadcaster, + ResourcesColl: resourcesColl, + CountersColl: countersColl, + }) + if err != nil { + broadcaster.Close() + return nil, fmt.Errorf("failed to create store: %w", err) + } + + return store, nil + } +} + +func sanitizeCollectionName(name string) string { + return strings.ReplaceAll(name, ".", "-") +} diff --git a/orlop/pkg/apiserver/storage/firestore/store.go b/orlop/pkg/apiserver/storage/firestore/store.go new file mode 100644 index 00000000..c2cb186e --- /dev/null +++ b/orlop/pkg/apiserver/storage/firestore/store.go @@ -0,0 +1,632 @@ +package firestore + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "cloud.google.com/go/firestore" + "github.com/openshift-online/gecko/orlop/pkg/apiserver/storage" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type FirestoreStore struct { + client *firestore.Client + resourceType string + scheme *runtime.Scheme + gvk schema.GroupVersionKind + broadcaster storage.EventBroadcaster + resourcesColl string + countersColl string + counterDocID string + contextFilterKey any +} + +type FirestoreStoreConfig struct { + Client *firestore.Client + ResourceType string + Scheme *runtime.Scheme + GVK schema.GroupVersionKind + Broadcaster storage.EventBroadcaster + ResourcesColl string + CountersColl string + ContextFilterKey any +} + +func NewFirestoreStore(_ context.Context, config FirestoreStoreConfig) (*FirestoreStore, error) { + if config.Client == nil { + return nil, fmt.Errorf("firestore client is required") + } + if config.ResourceType == "" { + return nil, fmt.Errorf("resource type is required") + } + if config.Scheme == nil { + return nil, fmt.Errorf("scheme is required") + } + + resourcesColl := config.ResourcesColl + if resourcesColl == "" { + resourcesColl = "resources_" + config.ResourceType + } + countersColl := config.CountersColl + if countersColl == "" { + countersColl = "counters" + } + + return &FirestoreStore{ + client: config.Client, + resourceType: config.ResourceType, + scheme: config.Scheme, + gvk: config.GVK, + broadcaster: config.Broadcaster, + resourcesColl: resourcesColl, + countersColl: countersColl, + counterDocID: "rv_" + config.ResourceType, + contextFilterKey: config.ContextFilterKey, + }, nil +} + +func (s *FirestoreStore) nextResourceVersion(ctx context.Context) (int64, error) { + counterRef := s.client.Collection(s.countersColl).Doc(s.counterDocID) + var rv int64 + + err := s.client.RunTransaction(ctx, func(ctx context.Context, tx *firestore.Transaction) error { + snap, err := tx.Get(counterRef) + if err != nil { + if status.Code(err) == codes.NotFound { + rv = 1 + return tx.Create(counterRef, map[string]any{"value": int64(1)}) + } + return err + } + current, err := snap.DataAt("value") + if err != nil { + return fmt.Errorf("failed to read counter value: %w", err) + } + rv = current.(int64) + 1 + return tx.Set(counterRef, map[string]any{"value": rv}) + }) + if err != nil { + return 0, fmt.Errorf("failed to increment resource version: %w", err) + } + return rv, nil +} + +func (s *FirestoreStore) contextFilterValue(ctx context.Context) (string, error) { + if s.contextFilterKey == nil { + return "", nil + } + v := ctx.Value(s.contextFilterKey) + if v == nil { + return "", fmt.Errorf("context filter key %v not found in context", s.contextFilterKey) + } + str, ok := v.(string) + if !ok { + return "", fmt.Errorf("context filter value must be a string, got %T", v) + } + return str, nil +} + +func (s *FirestoreStore) Create(ctx context.Context, obj client.Object) error { + filterValue, err := s.contextFilterValue(ctx) + if err != nil { + return err + } + + namespace := obj.GetNamespace() + name := obj.GetName() + useGenerateName := name == "" && obj.GetGenerateName() != "" + + maxAttempts := 1 + if useGenerateName { + maxAttempts = 5 + } + + for attempt := range maxAttempts { + if useGenerateName { + name = storage.GenerateName(obj.GetGenerateName()) + obj.SetName(name) + } + + creationTime := obj.GetCreationTimestamp() + if creationTime.IsZero() { + obj.SetCreationTimestamp(metav1.NewTime(time.Now())) + } + + rv, err := s.nextResourceVersion(ctx) + if err != nil { + return err + } + + data, err := marshalData(obj) + if err != nil { + return err + } + + docID := buildDocID(filterValue, namespace, name) + docRef := s.client.Collection(s.resourcesColl).Doc(docID) + + doc := map[string]any{ + "data": data, + "rv": rv, + "labels": obj.GetLabels(), + "namespace": namespace, + "name": name, + "contextFilter": filterValue, + "createdAt": time.Now(), + } + + _, err = docRef.Create(ctx, doc) + if err != nil { + if status.Code(err) == codes.AlreadyExists { + if useGenerateName && attempt < maxAttempts-1 { + continue + } + return errors.NewAlreadyExists( + schema.GroupResource{Resource: s.resourceType}, + name, + ) + } + return fmt.Errorf("failed to create document: %w", err) + } + + obj.SetResourceVersion(strconv.FormatInt(rv, 10)) + + if s.broadcaster != nil { + s.broadcaster.Broadcast(storage.ResourceEvent{ + Type: storage.EventAdded, + Object: obj.DeepCopyObject().(client.Object), + ResourceVersion: strconv.FormatInt(rv, 10), + ContextFilterValue: filterValue, + }) + } + + return nil + } + + return fmt.Errorf("failed to generate unique name after retries") +} + +func (s *FirestoreStore) Get(ctx context.Context, namespace, name string) (client.Object, error) { + filterValue, err := s.contextFilterValue(ctx) + if err != nil { + return nil, err + } + + docID := buildDocID(filterValue, namespace, name) + snap, err := s.client.Collection(s.resourcesColl).Doc(docID).Get(ctx) + if err != nil { + if status.Code(err) == codes.NotFound { + return nil, errors.NewNotFound( + schema.GroupResource{Resource: s.resourceType}, + name, + ) + } + return nil, fmt.Errorf("failed to get document: %w", err) + } + + return snapToObject(snap) +} + +func (s *FirestoreStore) List(ctx context.Context, opts storage.ListOptions) (client.ObjectList, error) { + filterValue, err := s.contextFilterValue(ctx) + if err != nil { + return nil, err + } + + var labelSelector labels.Selector + if opts.LabelSelector != "" { + labelSelector, err = labels.Parse(opts.LabelSelector) + if err != nil { + return nil, err + } + } + + var continueToken *storage.ContinueToken + if opts.Continue != "" { + continueToken, err = storage.DecodeContinueToken(opts.Continue) + if err != nil { + return nil, fmt.Errorf("invalid continue token: %w", err) + } + } + + q := s.client.Collection(s.resourcesColl).Query + + if filterValue != "" { + q = q.Where("contextFilter", "==", filterValue) + } + + if opts.Namespace != "" { + q = q.Where("namespace", "==", opts.Namespace) + } + + q = q.OrderBy("namespace", firestore.Asc).OrderBy("name", firestore.Asc) + + if continueToken != nil { + q = q.StartAfter(continueToken.Namespace, continueToken.Name) + } + + if opts.Limit > 0 { + q = q.Limit(int(opts.Limit) + 1) + } + + docs, err := q.Documents(ctx).GetAll() + if err != nil { + return nil, fmt.Errorf("failed to list documents: %w", err) + } + + var items []unstructured.Unstructured + var maxRV int64 + limit := opts.Limit + hasMore := false + + for _, snap := range docs { + obj, rv, parseErr := parseResourceSnap(snap) + if parseErr != nil { + continue + } + + if labelSelector != nil && !labelSelector.Matches(labels.Set(obj.GetLabels())) { + continue + } + + if opts.ShardSelector != nil { + matches, matchErr := storage.MatchesShard(obj, opts.ShardSelector) + if matchErr != nil || !matches { + continue + } + } + + if len(opts.FieldFilters) > 0 && !matchesFieldFilters(obj, opts.FieldFilters) { + continue + } + + if limit > 0 && int64(len(items)) >= limit { + hasMore = true + break + } + + items = append(items, *obj) + if rv > maxRV { + maxRV = rv + } + } + + listGVK := s.gvk.GroupVersion().WithKind(s.gvk.Kind + "List") + listObj, err := s.scheme.New(listGVK) + if err != nil { + return nil, fmt.Errorf("failed to create list object: %w", err) + } + + list := listObj.(*unstructured.UnstructuredList) + list.SetResourceVersion(strconv.FormatInt(maxRV, 10)) + list.Items = items + + if hasMore && len(items) > 0 { + listMeta, err := meta.ListAccessor(list) + if err == nil { + lastItem := &items[len(items)-1] + token := &storage.ContinueToken{ + Namespace: lastItem.GetNamespace(), + Name: lastItem.GetName(), + ResourceVersion: strconv.FormatInt(maxRV, 10), + } + continueStr, encErr := storage.EncodeContinueToken(token) + if encErr == nil { + listMeta.SetContinue(continueStr) + } + } + } + + return list, nil +} + +func (s *FirestoreStore) Update(ctx context.Context, obj client.Object) error { + filterValue, err := s.contextFilterValue(ctx) + if err != nil { + return err + } + + namespace := obj.GetNamespace() + name := obj.GetName() + + _, err = s.Get(ctx, namespace, name) + if err != nil { + return err + } + + rv, err := s.nextResourceVersion(ctx) + if err != nil { + return err + } + + data, err := marshalData(obj) + if err != nil { + return err + } + + docID := buildDocID(filterValue, namespace, name) + docRef := s.client.Collection(s.resourcesColl).Doc(docID) + + _, err = docRef.Set(ctx, map[string]any{ + "data": data, + "rv": rv, + "labels": obj.GetLabels(), + "namespace": namespace, + "name": name, + "contextFilter": filterValue, + "createdAt": time.Now(), + }) + if err != nil { + return fmt.Errorf("failed to update document: %w", err) + } + + obj.SetResourceVersion(strconv.FormatInt(rv, 10)) + + if s.broadcaster != nil { + s.broadcaster.Broadcast(storage.ResourceEvent{ + Type: storage.EventModified, + Object: obj.DeepCopyObject().(client.Object), + ResourceVersion: strconv.FormatInt(rv, 10), + ContextFilterValue: filterValue, + }) + } + + return nil +} + +func (s *FirestoreStore) Delete(ctx context.Context, namespace, name string) error { + filterValue, err := s.contextFilterValue(ctx) + if err != nil { + return err + } + + obj, err := s.Get(ctx, namespace, name) + if err != nil { + return err + } + + rv, err := s.nextResourceVersion(ctx) + if err != nil { + return err + } + + docID := buildDocID(filterValue, namespace, name) + docRef := s.client.Collection(s.resourcesColl).Doc(docID) + + _, err = docRef.Delete(ctx) + if err != nil { + return fmt.Errorf("failed to delete document: %w", err) + } + + if s.broadcaster != nil { + s.broadcaster.Broadcast(storage.ResourceEvent{ + Type: storage.EventDeleted, + Object: obj, + ResourceVersion: strconv.FormatInt(rv, 10), + ContextFilterValue: filterValue, + }) + } + + return nil +} + +func (s *FirestoreStore) Watch(ctx context.Context, opts storage.ListOptions, resourceVersion string) (<-chan storage.ResourceEvent, func(), error) { + filterValue, err := s.contextFilterValue(ctx) + if err != nil { + return nil, nil, err + } + + if s.broadcaster == nil { + return nil, nil, fmt.Errorf("broadcaster not configured") + } + + eventCh, stopSubscription, err := s.broadcaster.Subscribe(resourceVersion) + if err != nil { + return nil, nil, err + } + + outCh := make(chan storage.ResourceEvent, 100) + stopCh := make(chan struct{}) + + go func() { + defer close(outCh) + defer stopSubscription() + + var labelSelector labels.Selector + if opts.LabelSelector != "" { + var parseErr error + labelSelector, parseErr = labels.Parse(opts.LabelSelector) + if parseErr != nil { + labelSelector = nil + } + } + + for { + select { + case <-stopCh: + return + case event, ok := <-eventCh: + if !ok { + return + } + + if s.contextFilterKey != nil && event.ContextFilterValue != filterValue { + continue + } + + clientObj, ok := event.Object.(client.Object) + if !ok { + continue + } + + if opts.Namespace != "" && clientObj.GetNamespace() != opts.Namespace { + continue + } + + if labelSelector != nil && !labelSelector.Matches(labels.Set(clientObj.GetLabels())) { + continue + } + + if opts.ShardSelector != nil { + matches, matchErr := storage.MatchesShard(clientObj, opts.ShardSelector) + if matchErr != nil || !matches { + continue + } + } + + if len(opts.FieldFilters) > 0 && !matchesFieldFilters(clientObj, opts.FieldFilters) { + continue + } + + select { + case outCh <- event: + case <-stopCh: + return + } + } + } + }() + + stopFunc := func() { + close(stopCh) + } + + return outCh, stopFunc, nil +} + +// --- helpers --- + +func buildDocID(filterValue, namespace, name string) string { + if filterValue != "" { + return filterValue + "_" + namespace + "_" + name + } + return namespace + "_" + name +} + +func marshalData(obj client.Object) (map[string]any, error) { + rv := obj.GetResourceVersion() + obj.SetResourceVersion("") + data, err := json.Marshal(obj) + obj.SetResourceVersion(rv) + if err != nil { + return nil, fmt.Errorf("failed to marshal object: %w", err) + } + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("failed to unmarshal to map: %w", err) + } + return m, nil +} + +func unmarshalData(data map[string]any, rv int64) (*unstructured.Unstructured, error) { + obj := &unstructured.Unstructured{Object: data} + obj.SetResourceVersion(strconv.FormatInt(rv, 10)) + return obj, nil +} + +func snapToObject(snap *firestore.DocumentSnapshot) (client.Object, error) { + docData := snap.Data() + rawData, ok := docData["data"] + if !ok { + return nil, fmt.Errorf("no data field in document") + } + + dataMap, ok := rawData.(map[string]any) + if !ok { + return nil, fmt.Errorf("data field is not a map") + } + + var rv int64 + if rvVal, ok := docData["rv"]; ok { + rv = toInt64(rvVal) + } + + return unmarshalData(dataMap, rv) +} + +func parseResourceSnap(snap *firestore.DocumentSnapshot) (*unstructured.Unstructured, int64, error) { + docData := snap.Data() + rawData, ok := docData["data"] + if !ok { + return nil, 0, fmt.Errorf("no data field in document") + } + + dataMap, ok := rawData.(map[string]any) + if !ok { + return nil, 0, fmt.Errorf("data field is not a map") + } + + var rv int64 + if rvVal, ok := docData["rv"]; ok { + rv = toInt64(rvVal) + } + + obj, err := unmarshalData(dataMap, rv) + if err != nil { + return nil, 0, err + } + return obj, rv, nil +} + +func toInt64(v any) int64 { + switch val := v.(type) { + case int64: + return val + case float64: + return int64(val) + case int: + return int64(val) + default: + return 0 + } +} + +func padResourceVersion(rv int64) string { + return fmt.Sprintf("%020d", rv) +} + +func matchesFieldFilters(obj client.Object, filters map[string]string) bool { + data, err := json.Marshal(obj) + if err != nil { + return false + } + var objMap map[string]any + if err := json.Unmarshal(data, &objMap); err != nil { + return false + } + for path, expected := range filters { + if fieldValueFromMap(objMap, path) != expected { + return false + } + } + return true +} + +func fieldValueFromMap(m map[string]any, path string) string { + parts := strings.Split(path, ".") + current := any(m) + for _, part := range parts { + cm, ok := current.(map[string]any) + if !ok { + return "" + } + current = cm[part] + } + s, _ := current.(string) + return s +} + +var _ storage.ResourceStore = (*FirestoreStore)(nil) diff --git a/orlop/pkg/apiserver/storage/firestore/store_test.go b/orlop/pkg/apiserver/storage/firestore/store_test.go new file mode 100644 index 00000000..e99625a6 --- /dev/null +++ b/orlop/pkg/apiserver/storage/firestore/store_test.go @@ -0,0 +1,573 @@ +package firestore + +import ( + "context" + "fmt" + "os" + "strconv" + "testing" + "time" + + "cloud.google.com/go/firestore" + "github.com/openshift-online/gecko/orlop/pkg/apiserver/storage" + "google.golang.org/api/iterator" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type objectOption func(*unstructured.Unstructured) + +func newTestObject(opts ...objectOption) *unstructured.Unstructured { + obj := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "test.example.com/v1", + "kind": "TestObject", + "metadata": map[string]any{ + "name": "test", + "namespace": "default", + }, + }, + } + for _, opt := range opts { + opt(obj) + } + return obj +} + +func withName(name string) objectOption { + return func(obj *unstructured.Unstructured) { obj.SetName(name) } +} + +func withNamespace(namespace string) objectOption { + return func(obj *unstructured.Unstructured) { obj.SetNamespace(namespace) } +} + +func withLabels(labels map[string]string) objectOption { + return func(obj *unstructured.Unstructured) { obj.SetLabels(labels) } +} + +func withSpec(spec map[string]any) objectOption { + return func(obj *unstructured.Unstructured) { obj.Object["spec"] = spec } +} + +func withGenerateName(generateName string) objectOption { + return func(obj *unstructured.Unstructured) { + obj.SetGenerateName(generateName) + obj.SetName("") + } +} + +func testScheme() (*runtime.Scheme, schema.GroupVersionKind) { + scheme := runtime.NewScheme() + gv := schema.GroupVersion{Group: "test.example.com", Version: "v1"} + scheme.AddKnownTypeWithName(gv.WithKind("TestObject"), &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(gv.WithKind("TestObjectList"), &unstructured.UnstructuredList{}) + gvk := schema.GroupVersionKind{Group: "test.example.com", Version: "v1", Kind: "TestObject"} + return scheme, gvk +} + +func setupTestStore(t *testing.T) (*FirestoreStore, func()) { + t.Helper() + + emulatorHost := os.Getenv("FIRESTORE_EMULATOR_HOST") + if emulatorHost == "" { + t.Skip("FIRESTORE_EMULATOR_HOST not set, skipping firestore integration test") + } + + ctx := context.Background() + project := "test-project" + + fsClient, err := firestore.NewClient(ctx, project) + if err != nil { + t.Fatalf("Failed to create firestore client: %v", err) + } + + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + resourcesColl := "resources_test_" + suffix + countersColl := "counters_test_" + suffix + + scheme, gvk := testScheme() + + store, err := NewFirestoreStore(ctx, FirestoreStoreConfig{ + Client: fsClient, + ResourceType: "testobjects", + Scheme: scheme, + GVK: gvk, + ResourcesColl: resourcesColl, + CountersColl: countersColl, + }) + if err != nil { + fsClient.Close() + t.Fatalf("Failed to create store: %v", err) + } + + cleanup := func() { + deleteCollection(ctx, fsClient, resourcesColl) + deleteCollection(ctx, fsClient, countersColl) + fsClient.Close() + } + + return store, cleanup +} + +func deleteCollection(ctx context.Context, client *firestore.Client, collName string) { + iter := client.Collection(collName).Documents(ctx) + for { + doc, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + break + } + doc.Ref.Delete(ctx) + } +} + +func TestFirestoreStore_Create(t *testing.T) { + t.Run("creates new object with resourceVersion", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("test-obj"), withNamespace("default")) + + err := store.Create(context.Background(), obj) + if err != nil { + t.Fatalf("Create() failed: %v", err) + } + + retrieved, err := store.Get(context.Background(), "default", "test-obj") + if err != nil { + t.Fatalf("Get() failed: %v", err) + } + + if retrieved.GetResourceVersion() == "" { + t.Error("Created object missing resourceVersion") + } + if _, err := strconv.ParseInt(retrieved.GetResourceVersion(), 10, 64); err != nil { + t.Errorf("resourceVersion is not a valid integer: %s", retrieved.GetResourceVersion()) + } + }) + + t.Run("returns error for duplicate", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("duplicate"), withNamespace("default")) + + store.Create(context.Background(), obj) + err := store.Create(context.Background(), obj) + + if err == nil { + t.Error("Expected error for duplicate object, got nil") + } + }) + + t.Run("creates in different namespaces", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj1 := newTestObject(withName("obj"), withNamespace("ns1")) + obj2 := newTestObject(withName("obj"), withNamespace("ns2")) + + if err := store.Create(context.Background(), obj1); err != nil { + t.Errorf("Create in ns1 failed: %v", err) + } + if err := store.Create(context.Background(), obj2); err != nil { + t.Errorf("Create in ns2 failed: %v", err) + } + + if _, err := store.Get(context.Background(), "ns1", "obj"); err != nil { + t.Error("Object in ns1 not found") + } + if _, err := store.Get(context.Background(), "ns2", "obj"); err != nil { + t.Error("Object in ns2 not found") + } + }) + + t.Run("generateName produces a unique name", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withGenerateName("gen-"), withNamespace("default")) + + err := store.Create(context.Background(), obj) + if err != nil { + t.Fatalf("Create() with generateName failed: %v", err) + } + + name := obj.GetName() + if name == "" { + t.Fatal("Name was not set after Create with generateName") + } + if len(name) < len("gen-")+5 { + t.Errorf("Generated name too short: %q", name) + } + + retrieved, err := store.Get(context.Background(), "default", name) + if err != nil { + t.Fatalf("Get() by generated name failed: %v", err) + } + if retrieved.GetName() != name { + t.Errorf("Retrieved name %q != generated name %q", retrieved.GetName(), name) + } + }) + + t.Run("generateName creates distinct names", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + seen := make(map[string]bool) + + for range 20 { + obj := newTestObject(withGenerateName("multi-"), withNamespace("default")) + if err := store.Create(context.Background(), obj); err != nil { + t.Fatalf("Create() failed: %v", err) + } + name := obj.GetName() + if seen[name] { + t.Fatalf("Duplicate generated name: %q", name) + } + seen[name] = true + } + + listObj, _ := store.List(context.Background(), storage.ListOptions{Namespace: "default"}) + list := listObj.(*unstructured.UnstructuredList) + if len(list.Items) != 20 { + t.Errorf("Expected 20 objects, got %d", len(list.Items)) + } + }) + + t.Run("sets creation timestamp", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("test"), withNamespace("default")) + if err := store.Create(context.Background(), obj); err != nil { + t.Fatalf("Create() failed: %v", err) + } + + retrieved, err := store.Get(context.Background(), "default", "test") + if err != nil { + t.Fatalf("Get() failed: %v", err) + } + creationTime := retrieved.GetCreationTimestamp() + if creationTime.IsZero() { + t.Error("Creation timestamp not set") + } + }) +} + +func TestFirestoreStore_Get(t *testing.T) { + t.Run("gets existing object", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("test"), withNamespace("default")) + store.Create(context.Background(), obj) + + retrieved, err := store.Get(context.Background(), "default", "test") + if err != nil { + t.Fatalf("Get() failed: %v", err) + } + if retrieved.GetName() != "test" { + t.Errorf("Got wrong object: %s", retrieved.GetName()) + } + }) + + t.Run("returns error for non-existent object", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + _, err := store.Get(context.Background(), "default", "missing") + if err == nil { + t.Error("Expected error for missing object, got nil") + } + }) + + t.Run("returns error for wrong namespace", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("test"), withNamespace("default")) + store.Create(context.Background(), obj) + + _, err := store.Get(context.Background(), "kube-system", "test") + if err == nil { + t.Error("Expected error for wrong namespace, got nil") + } + }) +} + +func TestFirestoreStore_List(t *testing.T) { + t.Run("lists objects in namespace", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + store.Create(context.Background(), newTestObject(withName("obj1"), withNamespace("default"))) + store.Create(context.Background(), newTestObject(withName("obj2"), withNamespace("default"))) + store.Create(context.Background(), newTestObject(withName("obj3"), withNamespace("kube-system"))) + + listObj, err := store.List(context.Background(), storage.ListOptions{Namespace: "default"}) + if err != nil { + t.Fatalf("List() failed: %v", err) + } + + list := listObj.(*unstructured.UnstructuredList) + if len(list.Items) != 2 { + t.Errorf("Expected 2 objects, got %d", len(list.Items)) + } + }) + + t.Run("lists all namespaces", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + store.Create(context.Background(), newTestObject(withName("obj1"), withNamespace("default"))) + store.Create(context.Background(), newTestObject(withName("obj2"), withNamespace("kube-system"))) + store.Create(context.Background(), newTestObject(withName("obj3"), withNamespace("kube-public"))) + + listObj, err := store.List(context.Background(), storage.ListOptions{}) + if err != nil { + t.Fatalf("List() failed: %v", err) + } + + list := listObj.(*unstructured.UnstructuredList) + if len(list.Items) != 3 { + t.Errorf("Expected 3 objects, got %d", len(list.Items)) + } + }) + + t.Run("returns empty list for empty store", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + listObj, err := store.List(context.Background(), storage.ListOptions{Namespace: "default"}) + if err != nil { + t.Fatalf("List() failed: %v", err) + } + + list := listObj.(*unstructured.UnstructuredList) + if len(list.Items) != 0 { + t.Errorf("Expected empty list, got %d objects", len(list.Items)) + } + }) + + t.Run("sets resourceVersion on list", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + store.Create(context.Background(), newTestObject(withName("obj1"), withNamespace("default"))) + store.Create(context.Background(), newTestObject(withName("obj2"), withNamespace("default"))) + + listObj, _ := store.List(context.Background(), storage.ListOptions{Namespace: "default"}) + list := listObj.(*unstructured.UnstructuredList) + + if list.GetResourceVersion() == "" { + t.Error("List resourceVersion not set") + } + }) + + t.Run("filters by label selector", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + store.Create(context.Background(), newTestObject(withName("a"), withNamespace("default"), withLabels(map[string]string{"env": "prod"}))) + store.Create(context.Background(), newTestObject(withName("b"), withNamespace("default"), withLabels(map[string]string{"env": "dev"}))) + store.Create(context.Background(), newTestObject(withName("c"), withNamespace("default"), withLabels(map[string]string{"env": "prod"}))) + + opts := storage.ListOptions{Namespace: "default"} + opts.LabelSelector = "env=prod" + filteredObj, err := store.List(context.Background(), opts) + if err != nil { + t.Fatalf("List() with label selector failed: %v", err) + } + + filtered := filteredObj.(*unstructured.UnstructuredList) + if len(filtered.Items) != 2 { + t.Errorf("Expected 2 objects with env=prod, got %d", len(filtered.Items)) + } + }) + + t.Run("pagination with limit and continue", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + for i := range 5 { + store.Create(context.Background(), newTestObject( + withName(fmt.Sprintf("obj-%02d", i)), + withNamespace("default"), + )) + } + + opts := storage.ListOptions{Namespace: "default"} + opts.Limit = 2 + page1, err := store.List(context.Background(), opts) + if err != nil { + t.Fatalf("List page 1 failed: %v", err) + } + + list1 := page1.(*unstructured.UnstructuredList) + if len(list1.Items) != 2 { + t.Fatalf("Expected 2 items in page 1, got %d", len(list1.Items)) + } + + continueToken := list1.GetContinue() + if continueToken == "" { + t.Fatal("Expected continue token for page 1") + } + + opts2 := storage.ListOptions{Namespace: "default"} + opts2.Limit = 2 + opts2.Continue = continueToken + page2, err := store.List(context.Background(), opts2) + if err != nil { + t.Fatalf("List page 2 failed: %v", err) + } + + list2 := page2.(*unstructured.UnstructuredList) + if len(list2.Items) != 2 { + t.Fatalf("Expected 2 items in page 2, got %d", len(list2.Items)) + } + + for _, item1 := range list1.Items { + for _, item2 := range list2.Items { + if item1.GetName() == item2.GetName() { + t.Errorf("Page 2 contains item from page 1: %s", item1.GetName()) + } + } + } + }) +} + +func TestFirestoreStore_Update(t *testing.T) { + t.Run("updates existing object", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject( + withName("test"), + withNamespace("default"), + withSpec(map[string]any{"field": "original"}), + ) + store.Create(context.Background(), obj) + + retrieved, _ := store.Get(context.Background(), "default", "test") + updated := retrieved.DeepCopyObject().(client.Object) + u := updated.(*unstructured.Unstructured) + u.Object["spec"] = map[string]any{"field": "updated"} + + err := store.Update(context.Background(), updated) + if err != nil { + t.Fatalf("Update() failed: %v", err) + } + + final, _ := store.Get(context.Background(), "default", "test") + finalU := final.(*unstructured.Unstructured) + spec := finalU.Object["spec"].(map[string]any) + if spec["field"] != "updated" { + t.Errorf("Update did not persist: got %v", spec["field"]) + } + }) + + t.Run("increments resourceVersion", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("test"), withNamespace("default")) + store.Create(context.Background(), obj) + + retrieved, _ := store.Get(context.Background(), "default", "test") + initialRV := retrieved.GetResourceVersion() + + store.Update(context.Background(), retrieved) + + updated, _ := store.Get(context.Background(), "default", "test") + if updated.GetResourceVersion() == initialRV { + t.Error("ResourceVersion not incremented after update") + } + }) + + t.Run("returns error for non-existent object", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("missing"), withNamespace("default")) + + err := store.Update(context.Background(), obj) + if err == nil { + t.Error("Expected error for missing object, got nil") + } + }) +} + +func TestFirestoreStore_Delete(t *testing.T) { + t.Run("deletes existing object", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + obj := newTestObject(withName("test"), withNamespace("default")) + store.Create(context.Background(), obj) + + err := store.Delete(context.Background(), "default", "test") + if err != nil { + t.Fatalf("Delete() failed: %v", err) + } + + _, err = store.Get(context.Background(), "default", "test") + if err == nil { + t.Error("Object still exists after delete") + } + }) + + t.Run("returns error for non-existent object", func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + err := store.Delete(context.Background(), "default", "missing") + if err == nil { + t.Error("Expected error for missing object, got nil") + } + }) +} + +func TestFirestoreStore_ResourceVersionIncrement(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + parseRV := func(obj client.Object) int64 { + rv, err := strconv.ParseInt(obj.GetResourceVersion(), 10, 64) + if err != nil { + t.Fatalf("invalid resourceVersion %q: %v", obj.GetResourceVersion(), err) + } + return rv + } + + obj1 := newTestObject(withName("obj1"), withNamespace("default")) + store.Create(context.Background(), obj1) + + retrieved1, _ := store.Get(context.Background(), "default", "obj1") + rv1 := parseRV(retrieved1) + if rv1 <= 0 { + t.Errorf("After first create, rv = %d, want > 0", rv1) + } + + obj2 := newTestObject(withName("obj2"), withNamespace("default")) + store.Create(context.Background(), obj2) + + retrieved2, _ := store.Get(context.Background(), "default", "obj2") + rv2 := parseRV(retrieved2) + if rv2 <= rv1 { + t.Errorf("After second create, rv = %d, want > %d", rv2, rv1) + } + + store.Update(context.Background(), retrieved1) + + retrievedUpdated, _ := store.Get(context.Background(), "default", "obj1") + rv3 := parseRV(retrievedUpdated) + if rv3 <= rv2 { + t.Errorf("After update, rv = %d, want > %d", rv3, rv2) + } +} From 3428268d26ed174e2d28e235568a6c46e16bc4b2 Mon Sep 17 00:00:00 2001 From: Nico Schieder Date: Thu, 30 Jul 2026 14:02:20 +0200 Subject: [PATCH 2/2] Run go mod tidy --- controllers/go.mod | 14 +++++++------- controllers/go.sum | 28 ++++++++++++++-------------- platform-api/go.mod | 18 +++++++++--------- platform-api/go.sum | 44 ++++++++++++++++++++++---------------------- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/controllers/go.mod b/controllers/go.mod index 98388df4..ec9845ea 100644 --- a/controllers/go.mod +++ b/controllers/go.mod @@ -11,7 +11,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel/trace v1.44.0 - google.golang.org/api v0.274.0 + google.golang.org/api v0.287.1 k8s.io/apimachinery v0.36.3 k8s.io/client-go v0.36.0 open-cluster-management.io/api v1.3.0 @@ -21,7 +21,7 @@ require ( ) require ( - cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.7.0 // indirect @@ -56,8 +56,8 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect - github.com/googleapis/gax-go/v2 v2.21.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -73,7 +73,7 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect @@ -91,8 +91,8 @@ require ( golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/controllers/go.sum b/controllers/go.sum index 83f0a8d2..4b92d862 100644 --- a/controllers/go.sum +++ b/controllers/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= -cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= @@ -105,10 +105,10 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= -github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= -github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -173,8 +173,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= @@ -221,14 +221,14 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA= -google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/platform-api/go.mod b/platform-api/go.mod index 9f3fab90..14cc04cb 100644 --- a/platform-api/go.mod +++ b/platform-api/go.mod @@ -72,14 +72,14 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect @@ -96,9 +96,9 @@ require ( golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.45.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/grpc v1.79.3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/platform-api/go.sum b/platform-api/go.sum index 81221ca2..2b21db5c 100644 --- a/platform-api/go.sum +++ b/platform-api/go.sum @@ -205,24 +205,24 @@ go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -286,14 +286,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=