diff --git a/engine/docker/template/grafana-dashboards/gasoline.json b/engine/docker/template/grafana-dashboards/gasoline.json index f2d7b8ee58..d7a8b8de86 100644 --- a/engine/docker/template/grafana-dashboards/gasoline.json +++ b/engine/docker/template/grafana-dashboards/gasoline.json @@ -1188,11 +1188,187 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 0, "y": 51 }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, "id": 23, "interval": "15s", "options": { @@ -1279,7 +1455,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 60 }, "id": 24, "interval": "15s", @@ -1359,12 +1535,10 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 30, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1373,6 +1547,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1382,14 +1559,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1402,7 +1578,7 @@ } ] }, - "unit": "s" + "unit": "none" }, "overrides": [] }, @@ -1410,23 +1586,20 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 68 }, - "id": 13, + "id": 36, + "interval": "15s", "options": { "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true + "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -1438,14 +1611,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", - "instant": false, - "legendFormat": "{{worker_id}}", + "expr": "sum by (sub_workflow_name) (rate(rivet_gasoline_workflow_dispatched_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{signal_name}}", "range": true, "refId": "A" } ], - "title": "Last Pull Workflows Duration", + "title": "Workflows Dispatched/s", "type": "timeseries" }, { @@ -1464,12 +1637,10 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 30, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1478,6 +1649,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1487,14 +1661,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1507,7 +1680,7 @@ } ] }, - "unit": "s" + "unit": "none" }, "overrides": [] }, @@ -1515,200 +1688,21 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 68 }, - "id": 14, + "id": 35, + "interval": "15s", "options": { "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true + "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", - "instant": false, - "legendFormat": "{{worker_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "" - } - }, - "pluginVersion": "11.6.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", - "format": "heatmap", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Core Usage", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 26, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "percentunit" + "sort": "none" } }, "pluginVersion": "11.6.7", @@ -1719,15 +1713,15 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "expr": "sum by (sub_workflow_name) (rate(rivet_gasoline_workflow_dispatched_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=\"\"} [$__rate_interval]))", "format": "heatmap", - "legendFormat": "{{le}}", + "legendFormat": "{{sub_workflow_name}}", "range": true, "refId": "A" } ], - "title": "Load Shedding Ratio", - "type": "heatmap" + "title": "Workflows Dispatched/s (Not From Workflow)", + "type": "timeseries" }, { "datasource": { @@ -2603,7 +2597,9 @@ }, { "current": { - "text": "All", + "text": [ + "All" + ], "value": [ "$__all" ] @@ -2630,9 +2626,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -2659,12 +2653,12 @@ ] }, "time": { - "from": "now-5m", + "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Gasoline", "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 8 + "version": 9 } \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/pegboard.json b/engine/docker/template/grafana-dashboards/pegboard.json index 52bd7422d4..8ed083805d 100644 --- a/engine/docker/template/grafana-dashboards/pegboard.json +++ b/engine/docker/template/grafana-dashboards/pegboard.json @@ -196,9 +196,9 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter) (rivet_pegboard_actor_pending_allocation{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "expr": "sum by (rivet_datacenter) (\n max by(rivet_project, rivet_datacenter) (\n rivet_pegboard_actor_pending_allocation{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}\n * on(k8s_pod_name) group_left(rivet_project, rivet_datacenter)\n (\n (rivet_gasoline_worker_last_metrics_publish{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} == bool\n on(rivet_project, rivet_datacenter) group_left()\n max by (rivet_project, rivet_datacenter) (rivet_gasoline_worker_last_metrics_publish{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}))\n )\n )\n)", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -381,9 +381,9 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter) (rivet_pegboard_serverless_desired_slots{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "expr": "sum by (rivet_datacenter) (\n max by(rivet_project, rivet_datacenter) (\n rivet_pegboard_serverless_desired_slots{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}\n * on(k8s_pod_name) group_left(rivet_project, rivet_datacenter)\n (\n (rivet_gasoline_worker_last_metrics_publish{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} == bool\n on(rivet_project, rivet_datacenter) group_left()\n max by (rivet_project, rivet_datacenter) (rivet_gasoline_worker_last_metrics_publish{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}))\n )\n )\n)", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -487,7 +487,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rivet_pegboard_serverless_outbound_req_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -591,7 +591,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rate(rivet_pegboard_serverless_outbound_req_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval]))", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -695,7 +695,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rivet_pegboard_runner_connection_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -799,7 +799,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rate(rivet_pegboard_runner_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval]))", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -903,7 +903,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rivet_pegboard_event_multiplexer_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -1007,7 +1007,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rate(rivet_pegboard_ingested_events_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval]))", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -1192,7 +1192,7 @@ "editorMode": "code", "expr": "sum by (rivet_datacenter) (rivet_pegboard_runner_version_upgrade_drain_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", "instant": false, - "legendFormat": "{{workflow_name}}", + "legendFormat": "{{rivet_datacenter}}", "range": true, "refId": "A" } @@ -1209,10 +1209,10 @@ { "current": { "text": [ - "prod" + "staging" ], "value": [ - "prod" + "staging" ] }, "datasource": { @@ -1270,5 +1270,5 @@ "timezone": "browser", "title": "Pegboard", "uid": "afa77odsjjk74d", - "version": 1 + "version": 2 } \ No newline at end of file diff --git a/engine/docker/universal/Dockerfile b/engine/docker/universal/Dockerfile index ae4a67641f..1732694647 100644 --- a/engine/docker/universal/Dockerfile +++ b/engine/docker/universal/Dockerfile @@ -72,6 +72,7 @@ RUN \ fi && \ # cargo install --locked tokio-console && \ mkdir /app/dist/ && \ + echo "Copying binary" && \ cp target/$CARGO_BUILD_MODE/rivet-engine /app/dist/ # MARK: Engine (full, base) diff --git a/engine/packages/gasoline/src/db/kv/mod.rs b/engine/packages/gasoline/src/db/kv/mod.rs index 552a46d25d..9571ff6051 100644 --- a/engine/packages/gasoline/src/db/kv/mod.rs +++ b/engine/packages/gasoline/src/db/kv/mod.rs @@ -1174,7 +1174,7 @@ impl Database for DatabaseKv { 0 }; - let active_worker_count = active_worker_ids.len() as u64; + let active_worker_count = active_worker_ids.len().max(1) as u64; // Collect name and deadline ts for each wf id let mut dedup_workflows = HashMap::::new(); diff --git a/engine/packages/util/build.rs b/engine/packages/util/build.rs index a14f5b34f0..1722b3418c 100644 --- a/engine/packages/util/build.rs +++ b/engine/packages/util/build.rs @@ -9,6 +9,8 @@ fn main() -> Result<()> { .add_instructions(&vergen_gitcl::GitclBuilder::all_git()?)? .emit()?; + println!("cargo:rerun-if-env-changed=OVERRIDE_GIT_SHA"); + if let Ok(git_sha) = std::env::var("OVERRIDE_GIT_SHA") { println!("cargo:rustc-env=VERGEN_GIT_SHA={git_sha}"); }