[Feature] Prometheus + Grafana 모니터링 시스템 구축

build.gradle

@@ -54,6 +54,9 @@ dependencies {
 	implementation 'net.logstash.logback:logstash-logback-encoder:8.1'
 	implementation 'com.github.napstr:logback-discord-appender:1.0.0'
 
+	// Metrics
+	implementation 'io.micrometer:micrometer-registry-prometheus'
+
 	// Test
 	testImplementation 'org.springframework.boot:spring-boot-starter-test'
 	testRuntimeOnly 'org.junit.platform:junit-platform-launcher'

docker-compose.monitoring.yml

@@ -0,0 +1,53 @@
+services:
+  prometheus:
+    image: prom/prometheus:v3.5.1
+    container_name: cherrish-prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=15d'
+      - '--web.enable-lifecycle'
+    healthcheck:
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9090/-/healthy"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.6.9
+    container_name: cherrish-grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin}
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - DISCORD_MONITORING_WEBHOOK_URL=${DISCORD_MONITORING_WEBHOOK_URL}
+    volumes:
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - grafana_data:/var/lib/grafana
+    networks:
+      - monitoring
+    depends_on:
+      prometheus:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  prometheus_data:
+  grafana_data:
+
+networks:
+  monitoring:
+    driver: bridge

monitoring/grafana/provisioning/alerting/contactpoints.yml

@@ -0,0 +1,23 @@
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: discord-monitoring
+    receivers:
+      - uid: discord-monitoring
+        type: discord
+        settings:
+          url: ${DISCORD_MONITORING_WEBHOOK_URL}
+          use_discord_username: true
+          avatar_url: "https://grafana.com/static/assets/img/fav32.png"
+          title: '{{ if eq .Status "firing" }}:rotating_light: ALERT{{ else }}:white_check_mark: RESOLVED{{ end }} ({{ len .Alerts }} alerts)'
+          message: |
+            {{ range .Alerts }}
+            {{ if eq $.Status "firing" }}:red_circle:{{ else }}:green_circle:{{ end }} **{{ .Labels.rulename }}**
+            :warning: **Severity:** {{ .Labels.severity | toUpper }}
+            :clipboard: **Summary:** {{ .Annotations.summary }}
+            :memo: **Detail:** {{ .Annotations.description }}
+            :link: **View:** {{ .GeneratorURL }}
+
+            {{ end }}
+        disableResolveMessage: false

monitoring/grafana/provisioning/alerting/policies.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: discord-monitoring
+    group_by:
+      - grafana_folder
+    group_wait: 60s
+    group_interval: 5m
+    repeat_interval: 4h

monitoring/grafana/provisioning/alerting/rules.yml

@@ -0,0 +1,161 @@
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: cherrish-alerts
+    folder: Cherrish
+    interval: 1m
+    rules:
+      - uid: high-error-rate
+        title: High Error Rate
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: sum(rate(http_server_requests_seconds_count{application="cherrish", status=~"5.."}[5m])) or vector(0)
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: A
+          - refId: B
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: sum(rate(http_server_requests_seconds_count{application="cherrish"}[5m])) or vector(1)
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: B
+          - refId: C
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0.05
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                      - C
+                  reducer:
+                    type: last
+              expression: $A / $B
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: C
+              type: math
+        noDataState: NoData
+        execErrState: Error
+        for: 2m
+        annotations:
+          summary: "High error rate detected on Cherrish Server"
+          description: "Error rate is above 5% for the last 2 minutes"
+        labels:
+          severity: critical
+          rulename: "High Error Rate"
+        isPaused: false
+
+      - uid: high-latency
+        title: High Latency (P95)
+        condition: B
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{application="cherrish"}[5m])) by (le))
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: A
+          - refId: B
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 1
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                      - B
+                  reducer:
+                    type: last
+              expression: A
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: B
+              type: threshold
+        noDataState: NoData
+        execErrState: Error
+        for: 5m
+        annotations:
+          summary: "High latency detected on Cherrish Server"
+          description: "P95 response time is above 1 second for the last 5 minutes"
+        labels:
+          severity: warning
+          rulename: "High Latency (P95)"
+        isPaused: false
+
+      - uid: metrics-collection-health
+        title: Metrics Collection Down
+        condition: B
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: prometheus
+            model:
+              expr: up{job="cherrish-server"}
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: A
+          - refId: B
+            relativeTimeRange:
+              from: 300
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 1
+                    type: lt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                      - B
+                  reducer:
+                    type: last
+              expression: A
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: B
+              type: threshold
+        noDataState: NoData
+        execErrState: Error
+        for: 2m
+        annotations:
+          summary: "Cherrish Server metrics collection is down"
+          description: "Prometheus cannot scrape metrics from Cherrish Server. The application may be down or unreachable."
+        labels:
+          severity: critical
+          rulename: "Metrics Collection Down"
+        isPaused: false

monitoring/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: 'Cherrish Dashboards'
+    orgId: 1
+    folder: 'Cherrish'
+    type: file
+    disableDeletion: true
+    updateIntervalSeconds: 120
+    options:
+      path: /etc/grafana/provisioning/dashboards/json