nginx-metrics: expose status codes metrics

Combine connections metrics from stub_status and create status codes
metrics. Expose everything under /metrics endpoint of eth-rpc-proxy.
Remove the prometheus nginx metrics exporter since it is replaced.

Author: markoburcul

.gitignore

@@ -0,0 +1 @@
+secrets

docker-compose-local.yml

@@ -75,18 +75,6 @@ services:
     volumes:
       - './secrets:/config:ro'
 
-  rpc-proxy-metrics:
-    image: 'nginx/nginx-prometheus-exporter:1.4'
-    container_name: 'rpc-proxy-metrics'
-    restart: 'always'
-    command: 
-      - '--nginx.scrape-uri=http://rpc-proxy:8080/stub_status'
-      - '--web.listen-address=:8090'
-    ports:
-      - '8090:8090'
-    networks:
-      - 'rpc-network'
-
   keydb:
     image: 'eqalpha/keydb:latest'
     container_name: 'keydb'

docker-compose.yml

@@ -75,19 +75,6 @@ services:
       retries: 3
       start_period: '10s'
 
-  rpc-proxy-metrics:
-    image: 'nginx/nginx-prometheus-exporter:1.4'
-    container_name: 'rpc-proxy-metrics'
-    restart: 'always'
-    command: 
-      - '--nginx.scrape-uri=http://rpc-proxy:8080/stub_status'
-      - '--web.listen-address=:8090'
-    ports:
-      - '8090:8090'
-    networks:
-      - 'rpc-network'
-
-
   cache-service:
     build:
       context: './go-proxy-cache'

nginx-proxy/nginx.conf

@@ -1,10 +1,8 @@
 worker_processes auto;
 pid /var/run/nginx.pid;
-
 events {
     worker_connections 1024;
 }
-
 # Health-checker URL to get the latest provider list
 env CONFIG_HEALTH_CHECKER_URL;
 # custom local DNS servers to use for resolving the provider list
@@ -17,25 +15,41 @@ env GO_AUTH_SERVICE_URL;
 env AUTH_CONFIG_FILE;
 # Go Cache Service Socket (Unix socket)
 env GO_CACHE_SOCKET;
-
 http {
     resolver 1.1.1.1 8.8.8.8 valid=300s ipv6=off;
     resolver_timeout 5s;
     lua_package_path "/usr/local/openresty/nginx/lua/?.lua;/usr/local/openresty/lualib/?.lua;;";
-
-        lua_shared_dict providers 10m;
+    
+    lua_shared_dict providers 10m;
     lua_shared_dict jwt_tokens 10m;  # Shared memory for JWT token validation
+    lua_shared_dict stats 10m;       # Shared memory for metrics
+    
     client_body_buffer_size 10M;
     client_max_body_size 10M;
-
     access_log /dev/stdout;
     error_log /dev/stderr info;
 
     # Schedule periodic reload of the provider list
     init_worker_by_lua_file lua/init_worker.lua;
-
+    
     server {
         listen 8080;
+        
+        # Track status codes for all requests
+        log_by_lua_block {
+            local stats = ngx.shared.stats
+            local status = tonumber(ngx.var.status)
+            
+            if status >= 200 and status < 300 then
+                stats:incr("requests_2xx", 1, 0)
+            elseif status >= 300 and status < 400 then
+                stats:incr("requests_3xx", 1, 0)
+            elseif status >= 400 and status < 500 then
+                stats:incr("requests_4xx", 1, 0)
+            elseif status >= 500 then
+                stats:incr("requests_5xx", 1, 0)
+            end
+        }
 
         # Auth endpoints - no authentication required, handled by Go service
         location /auth/ {
@@ -70,10 +84,76 @@ http {
             proxy_set_header Authorization $http_authorization;
         }
 
-        # Health check endpoint
+        # Connections metrics endpoint - Nginx exposes by default
         location /stub_status {
+            internal;
             stub_status;
         }
+        
+        # Prometheus metrics endpoint (combines stub_status + status codes)
+        location /metrics {
+            access_log off;
+            
+            content_by_lua_block {
+                local stats = ngx.shared.stats
+                
+                -- Read stub_status
+                local res = ngx.location.capture("/stub_status")
+                
+                if res.status == 200 then
+                    -- Parse stub_status output
+                    local active = res.body:match("Active connections:%s*(%d+)")
+                    local accepts, handled, requests = res.body:match("%s*(%d+)%s+(%d+)%s+(%d+)")
+                    local reading, writing, waiting = res.body:match("Reading:%s*(%d+)%s+Writing:%s*(%d+)%s+Waiting:%s*(%d+)")
+                    
+                    -- Output metrics in Prometheus format (matching nginx-exporter format)
+                    ngx.say("# HELP nginx_connections_accepted Accepted client connections")
+                    ngx.say("# TYPE nginx_connections_accepted counter")
+                    ngx.say("nginx_connections_accepted ", accepts or 0)
+                    
+                    ngx.say("# HELP nginx_connections_active Active client connections")
+                    ngx.say("# TYPE nginx_connections_active gauge")
+                    ngx.say("nginx_connections_active ", active or 0)
+                    
+                    ngx.say("# HELP nginx_connections_handled Handled client connections")
+                    ngx.say("# TYPE nginx_connections_handled counter")
+                    ngx.say("nginx_connections_handled ", handled or 0)
+                    
+                    ngx.say("# HELP nginx_connections_reading Connections where NGINX is reading the request header")
+                    ngx.say("# TYPE nginx_connections_reading gauge")
+                    ngx.say("nginx_connections_reading ", reading or 0)
+                    
+                    ngx.say("# HELP nginx_connections_waiting Idle client connections")
+                    ngx.say("# TYPE nginx_connections_waiting gauge")
+                    ngx.say("nginx_connections_waiting ", waiting or 0)
+                    
+                    ngx.say("# HELP nginx_connections_writing Connections where NGINX is writing the response back to the client")
+                    ngx.say("# TYPE nginx_connections_writing gauge")
+                    ngx.say("nginx_connections_writing ", writing or 0)
+                    
+                    ngx.say("# HELP nginx_http_requests_total Total http requests")
+                    ngx.say("# TYPE nginx_http_requests_total counter")
+                    ngx.say("nginx_http_requests_total ", requests or 0)
+                end
+                
+                -- Status code metrics
+                ngx.say("# HELP nginx_http_requests_by_status HTTP requests by status code class")
+                ngx.say("# TYPE nginx_http_requests_by_status counter")
+                ngx.say(string.format('nginx_http_requests_by_status{status="2xx"} %d', 
+                    stats:get("requests_2xx") or 0))
+                ngx.say(string.format('nginx_http_requests_by_status{status="3xx"} %d', 
+                    stats:get("requests_3xx") or 0))
+                ngx.say(string.format('nginx_http_requests_by_status{status="4xx"} %d', 
+                    stats:get("requests_4xx") or 0))
+                ngx.say(string.format('nginx_http_requests_by_status{status="5xx"} %d', 
+                    stats:get("requests_5xx") or 0))
+                
+                -- Optional: Add a "scrape success" metric like nginx_up
+                ngx.say("# HELP nginx_up Status of the last metric scrape")
+                ngx.say("# TYPE nginx_up gauge")
+                ngx.say("nginx_up 1")
+            }
+        }
 
         # Cache metrics endpoints
         # Cache metrics endpoint - proxy to go-proxy-cache service

prometheus.yml

@@ -11,7 +11,9 @@ scrape_configs:
 
   - job_name: 'rpc-proxy-metrics'
     static_configs:
-      - targets: ['rpc-proxy-metrics:8090']
+      - targets: ['rpc-proxy-metrics:8080']
+    scrape_interval: 5s
+    metrics_path: '/metrics'
 
   - job_name: 'auth-service'
     static_configs: