diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index e1ad0359..27398f46 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -8,7 +8,7 @@ on: branches: ['main'] env: - GO_VERSION: '1.23' + GO_VERSION: '1.25' jobs: test: @@ -23,6 +23,13 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true + - name: Check formatting + run: | + if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then + echo "Code is not formatted properly:" + gofmt -s -l . + exit 1 + fi - name: Install Clang run: sudo apt-get install -y clang @@ -41,13 +48,6 @@ jobs: # - name: Run vet # run: go vet ./... - - name: Check formatting - run: | - if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then - echo "Code is not formatted properly:" - gofmt -s -l . - exit 1 - fi build: needs: test runs-on: ubuntu-22.04 diff --git a/cmd/verifier/main.go b/cmd/verifier/main.go index 4c36605d..81511321 100644 --- a/cmd/verifier/main.go +++ b/cmd/verifier/main.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/hibiken/asynq" @@ -84,19 +85,29 @@ func main() { // Initialize metrics based on configuration var httpMetrics *internalMetrics.HTTPMetrics + var appStoreCollector *internalMetrics.AppStoreCollector if cfg.Metrics.Enabled { logger.Info("Metrics enabled, setting up Prometheus metrics") - // Start metrics HTTP server with HTTP metrics - services := []string{internalMetrics.ServiceHTTP} + services := []string{internalMetrics.ServiceHTTP, internalMetrics.ServiceAppStore} _ = internalMetrics.StartMetricsServer(internalMetrics.Config{ Enabled: true, Host: cfg.Metrics.Host, Port: cfg.Metrics.Port, + Token: cfg.Metrics.Token, }, services, logger) - // Create HTTP metrics implementation httpMetrics = internalMetrics.NewHTTPMetrics() + + appStoreMetrics := internalMetrics.NewAppStoreMetrics() + appStoreCollector = internalMetrics.NewAppStoreCollector(db, appStoreMetrics, logger, 30*time.Second) + appStoreCollector.Start() + + defer func() { + if appStoreCollector != nil { + appStoreCollector.Stop() + } + }() } else { logger.Info("Verifier metrics disabled") } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 4c15ef9a..b6b86ba1 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -134,12 +134,12 @@ func main() { if cfg.Metrics.Enabled { logger.Info("Metrics enabled, setting up Prometheus metrics") - // Start metrics HTTP server with worker metrics services := []string{internalMetrics.ServiceWorker} _ = internalMetrics.StartMetricsServer(internalMetrics.Config{ Enabled: true, Host: cfg.Metrics.Host, Port: cfg.Metrics.Port, + Token: cfg.Metrics.Token, }, services, logger) // Create worker metrics instance diff --git a/config/config.go b/config/config.go index 6d63c679..564c19a5 100644 --- a/config/config.go +++ b/config/config.go @@ -56,6 +56,7 @@ type MetricsConfig struct { Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"` Host string `mapstructure:"host" json:"host,omitempty"` Port int `mapstructure:"port" json:"port,omitempty"` + Token string `mapstructure:"token" json:"token,omitempty"` } type PluginAssetsConfig struct { diff --git a/grafana-dashboard-appstore.json b/grafana-dashboard-appstore.json new file mode 100644 index 00000000..c00b26d5 --- /dev/null +++ b/grafana-dashboard-appstore.json @@ -0,0 +1,438 @@ +{ + "dashboard": { + "id": null, + "title": "Vultisig App Store Metrics", + "tags": ["vultisig", "appstore", "business"], + "style": "dark", + "timezone": "", + "editable": true, + "graphTooltip": 0, + "panels": [ + { + "id": 1, + "title": "Installations by Plugin", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "verifier_appstore_installations_total", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + }, + "renameByName": { + "plugin_id": "Plugin ID", + "Value": "Installations" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Value", + "desc": true + } + ] + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "title": "Policies by Plugin", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "verifier_appstore_policies_total", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + }, + "renameByName": { + "plugin_id": "Plugin ID", + "Value": "Active Policies" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Value", + "desc": true + } + ] + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + } + }, + { + "id": 3, + "title": "Fees Earned by Plugin", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "verifier_appstore_fees_earned_total", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + }, + "renameByName": { + "plugin_id": "Plugin ID", + "Value": "Total Fees (USDC)" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Value", + "desc": true + } + ] + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyUSD", + "decimals": 0 + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + } + }, + { + "id": 4, + "title": "Total Installations Over Time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "verifier_appstore_installations_grand_total", + "legendFormat": "Total Installations", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + } + }, + { + "id": 5, + "title": "Total Policies Over Time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "verifier_appstore_policies_grand_total", + "legendFormat": "Total Active Policies", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + } + }, + { + "id": 6, + "title": "Total Fees Earned Over Time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "verifier_appstore_fees_grand_total", + "legendFormat": "Total Fees (USDC)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "currencyUSD", + "decimals": 0 + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + } + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "refresh": "1m", + "schemaVersion": 36, + "version": 0 + } +} \ No newline at end of file diff --git a/internal/metrics/appstore.go b/internal/metrics/appstore.go new file mode 100644 index 00000000..0be2aa8a --- /dev/null +++ b/internal/metrics/appstore.go @@ -0,0 +1,123 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + appstoreInstallationsTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "installations_total", + Help: "Current number of installations per plugin", + }, + []string{"plugin_id"}, + ) + + appstorePoliciesTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "policies_total", + Help: "Current number of active policies per plugin", + }, + []string{"plugin_id"}, + ) + + appstoreFeesEarnedTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "fees_earned_total", + Help: "Total fees earned per plugin in USDC", + }, + []string{"plugin_id"}, + ) + + appstoreInstallationsGrandTotal = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "installations_grand_total", + Help: "Total installations across all plugins", + }, + ) + + appstorePoliciesGrandTotal = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "policies_grand_total", + Help: "Total active policies across all plugins", + }, + ) + + appstoreFeesGrandTotal = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "fees_grand_total", + Help: "Total fees earned across all plugins in USDC", + }, + ) + + appstoreCollectorLastUpdateTimestamp = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "verifier", + Subsystem: "appstore", + Name: "collector_last_update_timestamp", + Help: "Unix timestamp of last successful metrics update", + }, + ) +) + +type AppStoreMetrics struct{} + +func NewAppStoreMetrics() *AppStoreMetrics { + return &AppStoreMetrics{} +} + +func (a *AppStoreMetrics) UpdateInstallations(data map[string]int64) { + appstoreInstallationsTotal.Reset() + + var grandTotal int64 + for pluginID, count := range data { + appstoreInstallationsTotal.WithLabelValues(pluginID).Set(float64(count)) + grandTotal += count + } + + appstoreInstallationsGrandTotal.Set(float64(grandTotal)) +} + +func (a *AppStoreMetrics) UpdatePolicies(data map[string]int64) { + appstorePoliciesTotal.Reset() + + var grandTotal int64 + for pluginID, count := range data { + appstorePoliciesTotal.WithLabelValues(pluginID).Set(float64(count)) + grandTotal += count + } + + appstorePoliciesGrandTotal.Set(float64(grandTotal)) +} + +const usdcDecimals = 1e6 + +func (a *AppStoreMetrics) UpdateFees(data map[string]int64) { + appstoreFeesEarnedTotal.Reset() + + var grandTotal int64 + for pluginID, total := range data { + appstoreFeesEarnedTotal.WithLabelValues(pluginID).Set(float64(total) / usdcDecimals) + grandTotal += total + } + + appstoreFeesGrandTotal.Set(float64(grandTotal) / usdcDecimals) +} + +func (a *AppStoreMetrics) UpdateTimestamp() { + appstoreCollectorLastUpdateTimestamp.Set(float64(time.Now().Unix())) +} diff --git a/internal/metrics/appstore_collector.go b/internal/metrics/appstore_collector.go new file mode 100644 index 00000000..b8ddd3dd --- /dev/null +++ b/internal/metrics/appstore_collector.go @@ -0,0 +1,94 @@ +package metrics + +import ( + "context" + "time" + + "github.com/sirupsen/logrus" +) + +type DatabaseQuerier interface { + GetInstallationsByPlugin(ctx context.Context) (map[string]int64, error) + GetPoliciesByPlugin(ctx context.Context) (map[string]int64, error) + GetFeesByPlugin(ctx context.Context) (map[string]int64, error) +} + +type AppStoreCollector struct { + db DatabaseQuerier + metrics *AppStoreMetrics + logger *logrus.Logger + interval time.Duration + stopCh chan struct{} + doneCh chan struct{} +} + +func NewAppStoreCollector( + db DatabaseQuerier, + metrics *AppStoreMetrics, + logger *logrus.Logger, + interval time.Duration, +) *AppStoreCollector { + return &AppStoreCollector{ + db: db, + metrics: metrics, + logger: logger, + interval: interval, + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + } +} + +func (c *AppStoreCollector) Start() { + c.logger.Info("Starting App Store metrics collector") + + go func() { + defer close(c.doneCh) + + ticker := time.NewTicker(c.interval) + defer ticker.Stop() + + c.collect() + + for { + select { + case <-ticker.C: + c.collect() + case <-c.stopCh: + c.logger.Info("Stopping App Store metrics collector") + return + } + } + }() +} + +func (c *AppStoreCollector) Stop() { + close(c.stopCh) + <-c.doneCh +} + +func (c *AppStoreCollector) collect() { + ctx := context.Background() + + installations, err := c.db.GetInstallationsByPlugin(ctx) + if err != nil { + c.logger.Errorf("Failed to collect installations: %v", err) + } else { + c.metrics.UpdateInstallations(installations) + } + + policies, err := c.db.GetPoliciesByPlugin(ctx) + if err != nil { + c.logger.Errorf("Failed to collect policies: %v", err) + } else { + c.metrics.UpdatePolicies(policies) + } + + fees, err := c.db.GetFeesByPlugin(ctx) + if err != nil { + c.logger.Errorf("Failed to collect fees: %v", err) + } else { + c.metrics.UpdateFees(fees) + } + + c.metrics.UpdateTimestamp() +} diff --git a/internal/metrics/registry.go b/internal/metrics/registry.go index bd14e5eb..7039723a 100644 --- a/internal/metrics/registry.go +++ b/internal/metrics/registry.go @@ -15,6 +15,7 @@ const ( ServiceVault = "vault" ServiceWorker = "worker" ServiceHTTP = "http" + ServiceAppStore = "appstore" ) // RegisterMetrics registers metrics for the specified services with a custom registry @@ -32,6 +33,8 @@ func RegisterMetrics(services []string, registry *prometheus.Registry, logger *l registerWorkerMetrics(registry, logger) case ServiceHTTP: registerHTTPMetrics(registry, logger) + case ServiceAppStore: + registerAppStoreMetrics(registry, logger) default: logger.Warnf("Unknown service type for metrics registration: %s", service) } @@ -77,8 +80,18 @@ func registerWorkerMetrics(registry *prometheus.Registry, logger *logrus.Logger) // registerHTTPMetrics registers HTTP-related metrics func registerHTTPMetrics(registry *prometheus.Registry, logger *logrus.Logger) { - // Register each HTTP metric individually with defensive pattern registerIfNotExists(httpRequestsTotal, "http_requests_total", registry, logger) registerIfNotExists(httpRequestDuration, "http_request_duration", registry, logger) registerIfNotExists(httpActiveRequests, "http_active_requests", registry, logger) } + +// registerAppStoreMetrics registers App Store business metrics +func registerAppStoreMetrics(registry *prometheus.Registry, logger *logrus.Logger) { + registerIfNotExists(appstoreInstallationsTotal, "appstore_installations_total", registry, logger) + registerIfNotExists(appstorePoliciesTotal, "appstore_policies_total", registry, logger) + registerIfNotExists(appstoreFeesEarnedTotal, "appstore_fees_earned_total", registry, logger) + registerIfNotExists(appstoreInstallationsGrandTotal, "appstore_installations_grand_total", registry, logger) + registerIfNotExists(appstorePoliciesGrandTotal, "appstore_policies_grand_total", registry, logger) + registerIfNotExists(appstoreFeesGrandTotal, "appstore_fees_grand_total", registry, logger) + registerIfNotExists(appstoreCollectorLastUpdateTimestamp, "appstore_collector_last_update_timestamp", registry, logger) +} diff --git a/internal/metrics/server.go b/internal/metrics/server.go index dfb223af..be9410b1 100644 --- a/internal/metrics/server.go +++ b/internal/metrics/server.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -16,6 +17,7 @@ type Config struct { Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"` Host string `mapstructure:"host" json:"host,omitempty"` Port int `mapstructure:"port" json:"port,omitempty"` + Token string `mapstructure:"token" json:"token,omitempty"` } // DefaultConfig returns default metrics configuration @@ -33,32 +35,51 @@ type Server struct { logger *logrus.Logger } +// bearerAuthMiddleware wraps an http.Handler with Bearer token authentication +func bearerAuthMiddleware(handler http.Handler, token string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + providedToken := strings.TrimPrefix(authHeader, "Bearer ") + if providedToken != token { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + handler.ServeHTTP(w, r) + }) +} + // NewServer creates a new metrics server with a custom registry -func NewServer(host string, port int, logger *logrus.Logger, registry *prometheus.Registry) *Server { +func NewServer(cfg Config, logger *logrus.Logger, registry *prometheus.Registry) *Server { mux := http.NewServeMux() - // Register the Prometheus metrics handler with custom registry + var metricsHandler http.Handler if registry != nil { - mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) + metricsHandler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) } else { - mux.Handle("/metrics", promhttp.Handler()) + metricsHandler = promhttp.Handler() } - // Add a health check endpoint + if cfg.Token != "" { + metricsHandler = bearerAuthMiddleware(metricsHandler, cfg.Token) + logger.Info("Metrics endpoint authentication enabled") + } + + mux.Handle("/metrics", metricsHandler) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) }) - addr := fmt.Sprintf("%s:%d", host, port) - if host == "" { - addr = fmt.Sprintf(":%d", port) + addr := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) + if cfg.Host == "" { + addr = fmt.Sprintf(":%d", cfg.Port) } - if port <= 0 { - if host == "" { + if cfg.Port <= 0 { + if cfg.Host == "" { addr = ":8088" } else { - addr = fmt.Sprintf("%s:8088", host) + addr = fmt.Sprintf("%s:8088", cfg.Host) } } @@ -99,23 +120,10 @@ func StartMetricsServer(cfg Config, services []string, logger *logrus.Logger) *S return nil } - // Create registry and register metrics for specified services registry := prometheus.NewRegistry() RegisterMetrics(services, registry, logger) - server := NewServer(cfg.Host, cfg.Port, logger, registry) - server.Start() - return server -} - -// StartMetricsServerWithRegistry starts a metrics server with a pre-configured registry -func StartMetricsServerWithRegistry(cfg Config, registry *prometheus.Registry, logger *logrus.Logger) *Server { - if !cfg.Enabled { - logger.Info("Metrics server disabled") - return nil - } - - server := NewServer(cfg.Host, cfg.Port, logger, registry) + server := NewServer(cfg, logger, registry) server.Start() return server } diff --git a/internal/storage/postgres/metrics.go b/internal/storage/postgres/metrics.go new file mode 100644 index 00000000..6599d476 --- /dev/null +++ b/internal/storage/postgres/metrics.go @@ -0,0 +1,103 @@ +package postgres + +import ( + "context" + "fmt" +) + +func (p *PostgresBackend) GetInstallationsByPlugin(ctx context.Context) (map[string]int64, error) { + query := ` + SELECT plugin_id, COUNT(*) as count + FROM plugin_installations + GROUP BY plugin_id + ` + + rows, err := p.pool.Query(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query installations: %w", err) + } + defer rows.Close() + + result := make(map[string]int64) + for rows.Next() { + var pluginID string + var count int64 + err = rows.Scan(&pluginID, &count) + if err != nil { + return nil, fmt.Errorf("failed to scan installation row: %w", err) + } + result[pluginID] = count + } + + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating installations: %w", err) + } + + return result, nil +} + +func (p *PostgresBackend) GetPoliciesByPlugin(ctx context.Context) (map[string]int64, error) { + query := ` + SELECT plugin_id, COUNT(*) as count + FROM plugin_policies + WHERE is_active = true + GROUP BY plugin_id + ` + + rows, err := p.pool.Query(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query policies: %w", err) + } + defer rows.Close() + + result := make(map[string]int64) + for rows.Next() { + var pluginID string + var count int64 + err = rows.Scan(&pluginID, &count) + if err != nil { + return nil, fmt.Errorf("failed to scan policy row: %w", err) + } + result[pluginID] = count + } + + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating policies: %w", err) + } + + return result, nil +} + +func (p *PostgresBackend) GetFeesByPlugin(ctx context.Context) (map[string]int64, error) { + query := ` + SELECT f.plugin_id, COALESCE(SUM(f.amount), 0) as total + FROM fees f + JOIN fee_batch_members fbm ON f.id = fbm.fee_id + JOIN fee_batches fb ON fbm.batch_id = fb.id + WHERE fb.status = 'COMPLETED' + GROUP BY f.plugin_id + ` + + rows, err := p.pool.Query(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query fees: %w", err) + } + defer rows.Close() + + result := make(map[string]int64) + for rows.Next() { + var pluginID string + var total int64 + err = rows.Scan(&pluginID, &total) + if err != nil { + return nil, fmt.Errorf("failed to scan fee row: %w", err) + } + result[pluginID] = total + } + + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating fees: %w", err) + } + + return result, nil +} diff --git a/plugin/progress/service.go b/plugin/progress/service.go new file mode 100644 index 00000000..269f3449 --- /dev/null +++ b/plugin/progress/service.go @@ -0,0 +1,17 @@ +package progress + +import ( + "context" + + "github.com/google/uuid" +) + +type Progress struct { + Kind string `json:"kind"` + Value uint32 `json:"value"` +} + +type Service interface { + GetProgress(ctx context.Context, policyID uuid.UUID) (*Progress, error) + GetProgressBatch(ctx context.Context, policyIDs []uuid.UUID) (map[uuid.UUID]*Progress, error) +} diff --git a/plugin/progress/service_nil.go b/plugin/progress/service_nil.go new file mode 100644 index 00000000..20fa5411 --- /dev/null +++ b/plugin/progress/service_nil.go @@ -0,0 +1,22 @@ +package progress + +import ( + "context" + + "github.com/google/uuid" +) + +// NilService implements Service for plugins where progress tracking is not required +type NilService struct{} + +func NewNilService() *NilService { + return &NilService{} +} + +func (s *NilService) GetProgress(_ context.Context, _ uuid.UUID) (*Progress, error) { + return nil, nil +} + +func (s *NilService) GetProgressBatch(_ context.Context, _ []uuid.UUID) (map[uuid.UUID]*Progress, error) { + return nil, nil +} diff --git a/product-metrics.json b/product-metrics.json new file mode 100644 index 00000000..0a5ec0da --- /dev/null +++ b/product-metrics.json @@ -0,0 +1,319 @@ +{ + "dashboard": { + "id": null, + "title": "App Store Product Metrics", + "tags": ["vultisig", "verifier", "product"], + "style": "dark", + "timezone": "", + "panels": [ + { + "id": 1, + "title": "Total Installations", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_installations_grand_total", + "legendFormat": "Installations" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "green", "value": null}]}, + "unit": "short" + } + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto" + }, + "gridPos": {"h": 4, "w": 8, "x": 0, "y": 0} + }, + { + "id": 2, + "title": "Total Active Policies", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_policies_grand_total", + "legendFormat": "Policies" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "blue", "value": null}]}, + "unit": "short" + } + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto" + }, + "gridPos": {"h": 4, "w": 8, "x": 8, "y": 0} + }, + { + "id": 3, + "title": "Total Fees Earned (USDC)", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_fees_grand_total", + "legendFormat": "USDC" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "orange", "value": null}]}, + "unit": "currencyUSD", + "decimals": 2 + } + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto" + }, + "gridPos": {"h": 4, "w": 8, "x": 16, "y": 0} + }, + { + "id": 4, + "title": "Installations by Plugin", + "type": "piechart", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_installations_total", + "legendFormat": "{{plugin_id}}" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"} + } + }, + "options": { + "legend": {"displayMode": "table", "placement": "right", "values": ["value", "percent"]}, + "pieType": "pie", + "tooltip": {"mode": "single"} + }, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 4} + }, + { + "id": 5, + "title": "Active Policies by Plugin", + "type": "piechart", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_policies_total", + "legendFormat": "{{plugin_id}}" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"} + } + }, + "options": { + "legend": {"displayMode": "table", "placement": "right", "values": ["value", "percent"]}, + "pieType": "pie", + "tooltip": {"mode": "single"} + }, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 4} + }, + { + "id": 6, + "title": "Fees Earned by Plugin (USDC)", + "type": "piechart", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_fees_earned_total", + "legendFormat": "{{plugin_id}}" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "unit": "currencyUSD", + "decimals": 2 + } + }, + "options": { + "legend": {"displayMode": "table", "placement": "right", "values": ["value", "percent"]}, + "pieType": "pie", + "tooltip": {"mode": "single"} + }, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 4} + }, + { + "id": 7, + "title": "Installations by Plugin (Table)", + "type": "table", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_installations_total", + "legendFormat": "{{plugin_id}}", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "green", "value": null}]} + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "plugin_id"}, + "properties": [{"id": "displayName", "value": "Plugin"}] + }, + { + "matcher": {"id": "byName", "options": "Value"}, + "properties": [{"id": "displayName", "value": "Installations"}] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{"displayName": "Installations", "desc": true}] + }, + "transformations": [ + {"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "instance": true, "job": true}}} + ], + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 12} + }, + { + "id": 8, + "title": "Active Policies by Plugin (Table)", + "type": "table", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_policies_total", + "legendFormat": "{{plugin_id}}", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "blue", "value": null}]} + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "plugin_id"}, + "properties": [{"id": "displayName", "value": "Plugin"}] + }, + { + "matcher": {"id": "byName", "options": "Value"}, + "properties": [{"id": "displayName", "value": "Policies"}] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{"displayName": "Policies", "desc": true}] + }, + "transformations": [ + {"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "instance": true, "job": true}}} + ], + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 12} + }, + { + "id": 9, + "title": "Fees by Plugin (Table)", + "type": "table", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_fees_earned_total", + "legendFormat": "{{plugin_id}}", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "orange", "value": null}]}, + "unit": "currencyUSD", + "decimals": 2 + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "plugin_id"}, + "properties": [{"id": "displayName", "value": "Plugin"}] + }, + { + "matcher": {"id": "byName", "options": "Value"}, + "properties": [{"id": "displayName", "value": "Fees (USDC)"}] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{"displayName": "Fees (USDC)", "desc": true}] + }, + "transformations": [ + {"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "instance": true, "job": true}}} + ], + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 12} + }, + { + "id": 10, + "title": "Metrics Collector Last Update", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "verifier_appstore_collector_last_update_timestamp * 1000", + "legendFormat": "Last Update" + } + ], + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": {"steps": [{"color": "green", "value": null}]}, + "unit": "dateTimeFromNow" + } + }, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto" + }, + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 20} + } + ], + "time": {"from": "now-24h", "to": "now"}, + "timepicker": {"refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h"]}, + "templating": {"list": []}, + "annotations": {"list": []}, + "refresh": "30s", + "schemaVersion": 27, + "version": 0, + "links": [] + } +} diff --git a/verifier.example.json b/verifier.example.json index a5d35f29..db9c4299 100644 --- a/verifier.example.json +++ b/verifier.example.json @@ -38,6 +38,7 @@ "metrics": { "enabled": true, "host": "0.0.0.0", - "port": 8088 + "port": 8088, + "token": "your-secret-token" } } diff --git a/worker.example.json b/worker.example.json index d0e60490..855e2dd0 100644 --- a/worker.example.json +++ b/worker.example.json @@ -29,6 +29,7 @@ "metrics": { "enabled": true, "host": "0.0.0.0", - "port": 8088 + "port": 8088, + "token": "your-secret-token" } }