From f43d09ea42a6508875e218bb5bbfad41f9e91b6c Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 15 Oct 2025 20:55:57 +0200 Subject: [PATCH 1/2] Prometheus: Refresh usage guide based on Docker Compose --- docs/howto/index.md | 6 +- docs/integrate/prometheus/compose.yaml | 46 +++++ .../cratedb-prometheus-adapter.yaml | 22 +++ docs/integrate/prometheus/ddl.sql | 11 ++ docs/integrate/prometheus/index.md | 20 +-- docs/integrate/prometheus/prometheus.yml | 26 +++ docs/integrate/prometheus/usage.md | 161 +++--------------- 7 files changed, 133 insertions(+), 159 deletions(-) create mode 100644 docs/integrate/prometheus/compose.yaml create mode 100644 docs/integrate/prometheus/cratedb-prometheus-adapter.yaml create mode 100644 docs/integrate/prometheus/ddl.sql create mode 100644 docs/integrate/prometheus/prometheus.yml diff --git a/docs/howto/index.md b/docs/howto/index.md index 3f7c5c71..44c58f19 100644 --- a/docs/howto/index.md +++ b/docs/howto/index.md @@ -11,6 +11,7 @@ Instructions how to get tasks done with CrateDB. :::{rubric} 2025 ::: +- {ref}`prometheus-usage` - {ref}`influxdb-usage` - {ref}`amqp-usage` - {ref}`mqtt-usage` @@ -62,11 +63,6 @@ Instructions how to get tasks done with CrateDB. - {ref}`terraform-usage` - {ref}`trino-usage` -:::{rubric} 2022 -::: - -- {ref}`prometheus-usage` - :::{note} You can expect the more recent documents to be more up-to-date than previous diff --git a/docs/integrate/prometheus/compose.yaml b/docs/integrate/prometheus/compose.yaml new file mode 100644 index 00000000..f098e834 --- /dev/null +++ b/docs/integrate/prometheus/compose.yaml @@ -0,0 +1,46 @@ +# Service composition file for Docker Compose or Podman Compose +# https://cratedb.com/docs/guide/integrate/prometheus/ + +services: + + cratedb: + image: "docker.io/crate:latest" + command: > + crate -Cdiscovery.type=single-node + ports: + - "4200:4200" + - "5432:5432" + volumes: + - cratedb-data:/data + restart: unless-stopped + + cratedb-ddl: + image: docker.io/crate/crate:latest + command: sh -c "crash --hosts http://cratedb:4200/ -c 'SELECT 1'; crash --hosts http://cratedb:4200/ < /var/ddl.sql" + volumes: + - ./ddl.sql:/var/ddl.sql + depends_on: + - cratedb + + prometheus: + image: "docker.io/prom/prometheus:latest" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + restart: unless-stopped + + cratedb-prometheus-adapter: + image: ghcr.io/crate/cratedb-prometheus-adapter:0.5.8 + command: -config.file /etc/cratedb-prometheus-adapter.yaml + ports: + - "9268:9268/tcp" + volumes: + - ./cratedb-prometheus-adapter.yaml:/etc/cratedb-prometheus-adapter.yaml + depends_on: + - cratedb-ddl + - prometheus + restart: unless-stopped + +volumes: + cratedb-data: diff --git a/docs/integrate/prometheus/cratedb-prometheus-adapter.yaml b/docs/integrate/prometheus/cratedb-prometheus-adapter.yaml new file mode 100644 index 00000000..d688d461 --- /dev/null +++ b/docs/integrate/prometheus/cratedb-prometheus-adapter.yaml @@ -0,0 +1,22 @@ +# https://cratedb.com/docs/guide/integrate/prometheus/ + +cratedb_endpoints: +- host: "cratedb" # Host to connect to (default: "localhost"). + port: 5432 # Port to connect to (default: 5432). + user: "crate" # Username to use (default: "crate") + password: "" # Password to use (default: ""). + schema: "testdrive" # Schema to use (default: ""). + max_connections: 0 # The maximum number of concurrent connections (default: runtime.NumCPU()). + # It will get forwarded to pgx's `pool_max_conns`, and determines + # the maximum number of connections in the connection pool for + # both connection pools (read and write). + read_pool_size_max: 0 # Configure the maximum pool size for read operations individually. + # (default: runtime.NumCPU()) + write_pool_size_max: 0 # Configure the maximum pool size for write operations individually. + # (default: runtime.NumCPU()) + connect_timeout: 10 # TCP connect timeout (seconds) (default: 10). + # It has the same meaning as libpq's `connect_timeout`. + read_timeout: 5 # Query context timeout for read queries (seconds) (default: 5). + write_timeout: 5 # Query context timeout for write queries (seconds) (default: 5). + enable_tls: false # Whether to connect using TLS (default: false). + allow_insecure_tls: false # Whether to allow insecure / invalid TLS certificates (default: false). diff --git a/docs/integrate/prometheus/ddl.sql b/docs/integrate/prometheus/ddl.sql new file mode 100644 index 00000000..2cf7e61b --- /dev/null +++ b/docs/integrate/prometheus/ddl.sql @@ -0,0 +1,11 @@ +-- https://cratedb.com/docs/guide/integrate/prometheus/ + +CREATE TABLE IF NOT EXISTS "testdrive"."metrics" ( + "timestamp" TIMESTAMP, + "labels_hash" STRING, + "labels" OBJECT(DYNAMIC), + "value" DOUBLE, + "valueRaw" LONG, + "day__generated" TIMESTAMP GENERATED ALWAYS AS date_trunc('day', "timestamp"), + PRIMARY KEY ("timestamp", "labels_hash", "day__generated") +) PARTITIONED BY ("day__generated"); diff --git a/docs/integrate/prometheus/index.md b/docs/integrate/prometheus/index.md index 64c4c696..5df98929 100644 --- a/docs/integrate/prometheus/index.md +++ b/docs/integrate/prometheus/index.md @@ -7,8 +7,8 @@ ```{div} .clearfix ``` - -## About +:::{rubric} About +::: [Prometheus] is an open-source systems monitoring and alerting toolkit for collecting metrics data from applications and infrastructures. @@ -37,8 +37,6 @@ Prometheus's main features are: :::: -## Learn - :::{rubric} Remote Endpoints and Storage ::: @@ -57,15 +55,10 @@ massively scale-out Prometheus. ::::{grid} -:::{grid-item-card} Blog: Getting started -:link: https://cratedb.com/blog/getting-started-prometheus-cratedb-long-term-storage -:link-type: url -Getting started with Prometheus and CrateDB for long-term storage. -::: - :::{grid-item-card} Store Prometheus long-term metrics into CrateDB :link: prometheus-usage :link-type: ref +:columns: 6 Set up CrateDB as a long-term metrics store for Prometheus using Docker Compose. ::: @@ -99,13 +92,6 @@ usage guide. :::: -:::{rubric} Repositories -::: - -[CrateDB] • -[Prometheus] • -[CrateDB Prometheus Adapter] - ```{seealso} [CrateDB and Prometheus] diff --git a/docs/integrate/prometheus/prometheus.yml b/docs/integrate/prometheus/prometheus.yml new file mode 100644 index 00000000..3fc7f50f --- /dev/null +++ b/docs/integrate/prometheus/prometheus.yml @@ -0,0 +1,26 @@ +# https://cratedb.com/docs/guide/integrate/prometheus/ + +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any time-series scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['prometheus:9090'] + +remote_write: + - url: http://cratedb-prometheus-adapter:9268/write +remote_read: + - url: http://cratedb-prometheus-adapter:9268/read diff --git a/docs/integrate/prometheus/usage.md b/docs/integrate/prometheus/usage.md index 74099e0c..edca47f5 100644 --- a/docs/integrate/prometheus/usage.md +++ b/docs/integrate/prometheus/usage.md @@ -2,149 +2,39 @@ # Store Prometheus long-term metrics into CrateDB This usage guide shows how to use Docker Compose to run the services -CrateDB, Prometheus, and the CrateDB Prometheus Adapter. - -## Motivation - -[Prometheus] is a monitoring system that collects metrics from applications and -infrastructure. It focuses on ingesting large volumes of concise time‑series -events—timestamped points with key‑value labels. - -This data helps you track a system’s state and trajectory. Many Prometheus users -want to retain it long term. - -[CrateDB] helps here. With the [CrateDB Prometheus Adapter], you can store metrics -in CrateDB and use its fast ingestion and query performance to scale Prometheus. - -## Set up Docker Compose - -Run CrateDB, Prometheus, and the CrateDB Prometheus Adapter as [Docker containers]. -Use [Docker Compose] to centralize container management so you can build and run -all containers with one command and define their connections in a YAML file. - -Install Docker by following the [Docker installation guide]. - -Create a directory on your machine to host the configuration files. -Create three YAML files with your preferred editor and save them with the `.yml` extension. - -### Create `docker-compose.yml` - -Create `docker-compose.yml` to configure the three containers. -Define services for CrateDB, Prometheus, and the adapter. Mount `config.yml` -and `prometheus.yml` into the adapter and Prometheus containers, respectively. -You will create these files in the following steps. -```yaml -services: - cratedb: - image: "docker.io/crate:latest" - command: > - crate -Cdiscovery.type=single-node - ports: - - "4200:4200" - - "5432:5432" - volumes: - - cratedb-data:/data - restart: unless-stopped - prometheus: - image: "docker.io/prom/prometheus:latest" - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - restart: unless-stopped - cratedb-prometheus-adapter: - image: "ghcr.io/crate/cratedb-prometheus-adapter:latest" - volumes: - - ./config.yml:/etc/cratedb-prometheus-adapter/config.yml - ports: - - "9268:9268" - depends_on: - - cratedb - - prometheus - restart: unless-stopped -volumes: - cratedb-data: -``` - -### Create `prometheus.yml` - -Next, create `prometheus.yml` following the [Prometheus documentation]. -This file defines the services that Prometheus scrapes. - -To keep it simple, monitor Prometheus itself. - -To forward samples to CrateDB, set `remote_write` and `remote_read` to the adapter URL (see the [CrateDB Prometheus Adapter setup](https://github.com/crate/cratedb-prometheus-adapter)). +CrateDB, Prometheus, and the [CrateDB Prometheus Adapter]. -Because the adapter runs in Docker, use the adapter service name from `docker-compose.yml` (`cratedb-prometheus-adapter`) instead of `localhost` in the URLs. +## Prerequisites -Use the following `prometheus.yml`: -```yaml -global: - scrape_interval: 15s # By default, scrape targets every 15 seconds. +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. - # Attach these labels to any time series or alerts when communicating with - # external systems (federation, remote storage, Alertmanager). - external_labels: - monitor: 'codelab-monitor' +### Files -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. -scrape_configs: - # The job name is added as a label `job=` to any time-series scraped from this config. - - job_name: 'prometheus' +First, download and save all required files to your machine. +- {download}`compose.yaml` +- {download}`cratedb-prometheus-adapter.yaml` +- {download}`ddl.sql` +- {download}`prometheus.yml` - # Override the global default and scrape targets from this job every 5 seconds. - scrape_interval: 5s +### Services - static_configs: - - targets: ['prometheus:9090'] +Start services using Docker Compose or Podman Compose. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. -remote_write: - - url: http://cratedb-prometheus-adapter:9268/write -remote_read: - - url: http://cratedb-prometheus-adapter:9268/read -``` -### Create `config.yml` - -Finally, create `config.yml` following the [CrateDB Prometheus Adapter]. -This file defines the CrateDB endpoints that the adapter writes to. - -Set the host to `cratedb` (the service name in `docker-compose.yml`) instead of `localhost`. Keep the remaining variables at their defaults. -```yaml -cratedb_endpoints: -- host: "cratedb" # Host to connect to (default: "localhost") - port: 5432 # Port to connect to (default: 5432). - user: "crate" # Username to use (default: "crate") - password: "" # Password to use (default: ""). - schema: "" # Schema to use (default: ""). - connect_timeout: 10 # TCP connect timeout (seconds) (default: 10). - max_connections: 5 # The maximum number of concurrent connections (default: 5). - enable_tls: false # Whether to connect using TLS (default: false). - allow_insecure_tls: false # Whether to allow insecure / invalid TLS certificates (default: false). +```shell +docker compose up ``` -Place `docker-compose.yml`, `config.yml`, and `prometheus.yml` in the same directory on your machine. -## Start the services +## Explore data -Finally, navigate to your CrateDB–Prometheus directory and start Docker Compose: +CrateDB stores the metrics in the designated table, ready for inspection and analysis. ```shell -cd /Users/Path/To/Directory/CrateDB-Prometheus -docker-compose up +docker compose exec cratedb crash -c "SELECT * FROM testdrive.metrics ORDER BY timestamp LIMIT 5;" ``` -After Docker Compose starts, open the CrateDB Admin UI at `http://localhost:4200` -and create a `metrics` table to store metrics gathered by Prometheus. -```sql -CREATE TABLE "metrics" ( - "timestamp" TIMESTAMP, - "labels_hash" STRING, - "labels" OBJECT(DYNAMIC), - "value" DOUBLE, - "valueRaw" LONG, - "day__generated" TIMESTAMP GENERATED ALWAYS AS date_trunc('day', "timestamp"), - PRIMARY KEY ("timestamp", "labels_hash", "day__generated") - ) PARTITIONED BY ("day__generated") -``` +## Screenshots Navigate to `http://localhost:9090` to open the Prometheus UI. Go to **Status** → **Targets**. @@ -156,17 +46,13 @@ Confirm that Prometheus monitors itself. Return to the CrateDB Admin UI and select the `metrics` table you created. -After a few minutes, Prometheus will have gathered hundreds of thousands of data points. +After a few minutes, Prometheus will have gathered a few thousands of data points. ![CrateDB Admin UI showing the populated metrics table](https://us1.discourse-cdn.com/flex020/uploads/crate/original/1X/22e8c7d5a90ec9240a4cb4269774e143759aa92e.jpeg) Use CrateDB’s query engine to analyze and visualize this data with tools -like {ref}`grafana`. - -Explore these related resources: - -* [Visualizing time‑series data with Grafana and CrateDB](https://cratedb.com/blog/visualizing-time-series-data-with-grafana-and-cratedb) -* [Monitoring CrateDB with Prometheus and Grafana](https://cratedb.com/blog/monitoring-cratedb-with-prometheus-and-grafana) +like {ref}`grafana`, see also +[Monitoring a self-managed CrateDB cluster with Prometheus and Grafana]. [CrateDB]: https://cratedb.com/database @@ -174,5 +60,6 @@ Explore these related resources: [Docker Compose]: https://docs.docker.com/compose/ [Docker containers]: https://www.docker.com/resources/what-container [Docker installation guide]: https://docs.docker.com/get-docker/ +[Monitoring a self-managed CrateDB cluster with Prometheus and Grafana]: https://community.cratedb.com/t/monitoring-a-self-managed-cratedb-cluster-with-prometheus-and-grafana/1236 [Prometheus]: https://prometheus.io/docs/introduction/overview/ [Prometheus documentation]: https://prometheus.io/docs/prometheus/latest/getting_started/ From 6310bb63447e0ad94723cd709c11c633af449c5f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 15 Oct 2025 21:14:33 +0200 Subject: [PATCH 2/2] Prometheus: Implement suggestions by CodeRabbit --- docs/integrate/prometheus/compose.yaml | 5 +++++ docs/integrate/prometheus/usage.md | 10 ++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/integrate/prometheus/compose.yaml b/docs/integrate/prometheus/compose.yaml index f098e834..1f451c62 100644 --- a/docs/integrate/prometheus/compose.yaml +++ b/docs/integrate/prometheus/compose.yaml @@ -12,6 +12,11 @@ services: - "5432:5432" volumes: - cratedb-data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 restart: unless-stopped cratedb-ddl: diff --git a/docs/integrate/prometheus/usage.md b/docs/integrate/prometheus/usage.md index edca47f5..ef715c8c 100644 --- a/docs/integrate/prometheus/usage.md +++ b/docs/integrate/prometheus/usage.md @@ -30,6 +30,7 @@ docker compose up ## Explore data CrateDB stores the metrics in the designated table, ready for inspection and analysis. +By default, the included `ddl.sql` creates the table `testdrive.metrics`. ```shell docker compose exec cratedb crash -c "SELECT * FROM testdrive.metrics ORDER BY timestamp LIMIT 5;" ``` @@ -44,22 +45,19 @@ Confirm that Prometheus monitors itself. ![Prometheus target details for the self-scrape job](https://us1.discourse-cdn.com/flex020/uploads/crate/original/1X/57ccb5374b0ab524466de08feefbafde559dac87.png) -Return to the CrateDB Admin UI and select the `metrics` table you created. +Return to the CrateDB Admin UI at `http://localhost:4200` and select the `testdrive.metrics` table. -After a few minutes, Prometheus will have gathered a few thousands of data points. +After a few minutes, Prometheus will have gathered a few thousand data points. ![CrateDB Admin UI showing the populated metrics table](https://us1.discourse-cdn.com/flex020/uploads/crate/original/1X/22e8c7d5a90ec9240a4cb4269774e143759aa92e.jpeg) Use CrateDB’s query engine to analyze and visualize this data with tools -like {ref}`grafana`, see also +like {ref}`grafana`. See also [Monitoring a self-managed CrateDB cluster with Prometheus and Grafana]. [CrateDB]: https://cratedb.com/database [CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter -[Docker Compose]: https://docs.docker.com/compose/ -[Docker containers]: https://www.docker.com/resources/what-container -[Docker installation guide]: https://docs.docker.com/get-docker/ [Monitoring a self-managed CrateDB cluster with Prometheus and Grafana]: https://community.cratedb.com/t/monitoring-a-self-managed-cratedb-cluster-with-prometheus-and-grafana/1236 [Prometheus]: https://prometheus.io/docs/introduction/overview/ [Prometheus documentation]: https://prometheus.io/docs/prometheus/latest/getting_started/