WIP (fleet/mimir) modernize rgw bucket deployment

fleet/lib/kube-prometheus-stack/overlays/dev/values.yaml

@@ -156,6 +156,54 @@ prometheus:
             target_label: network_function
           - target_label: __address__
             replacement: prometheus-snmp-exporter.snmp-exporter:9116
+      - job_name: deliverator-ls
+        scrape_interval: 100ms
+        puppetdb_sd_configs:
+          - url: https://puppetdb.ls.lsst.org:8443
+            basic_auth:
+              username: svc_prometheus
+              password_file: /etc/prometheus/secrets/puppetdb/password
+            query: resources { type = "S3nd::Instance" }
+            refresh_interval: 30s
+            follow_redirects: true
+            include_parameters: true
+            enable_http2: true
+        relabel_configs:
+          - source_labels: [__meta_puppetdb_certname, __meta_puppetdb_parameter_port]
+            separator: ':'
+            target_label: __address__
+          - source_labels: [__meta_puppetdb_certname]
+            target_label: instance
+          - source_labels: [__meta_puppetdb_parameter_port]
+            target_label: port
+          - source_labels: [__meta_puppetdb_title]
+            target_label: service
+          - target_label: site
+            replacement: ls
+      - job_name: deliverator-cp
+        scrape_interval: 100ms
+        puppetdb_sd_configs:
+          - url: https://puppetdb.cp.lsst.org:8443
+            basic_auth:
+              username: svc_prometheus
+              password_file: /etc/prometheus/secrets/puppetdb/password
+            query: resources { type = "S3nd::Instance" }
+            refresh_interval: 30s
+            follow_redirects: true
+            include_parameters: true
+            enable_http2: true
+        relabel_configs:
+          - source_labels: [__meta_puppetdb_certname, __meta_puppetdb_parameter_port]
+            separator: ':'
+            target_label: __address__
+          - source_labels: [__meta_puppetdb_certname]
+            target_label: instance
+          - source_labels: [__meta_puppetdb_parameter_port]
+            target_label: port
+          - source_labels: [__meta_puppetdb_title]
+            target_label: service
+          - target_label: site
+            replacement: cp
 grafana:
   grafana.ini:
     server:

fleet/lib/mimir/fleet.yaml

@@ -1,10 +1,11 @@
 ---
-# XXX automate provisioning of buckets and rgw user
 defaultNamespace: &name mimir
 labels:
   bundle: *name
 namespaceLabels:
   lsst.io/discover: "true"
+kustomize:
+  dir: kustomize/base
 helm:
   chart: &chart mimir-distributed
   releaseName: *chart
@@ -17,24 +18,47 @@ helm:
 dependsOn:
   - selector:
       matchLabels:
-        bundle: mimir-pre
+        bundle: rook-ceph
   - selector:
       matchLabels:
         bundle: prometheus-operator-crds
 targetCustomizations:
-  - name: rke2
+  - name: antu
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - antu
+    kustomize:
+      dir: kustomize/overlays/antu
+  - name: kona
     clusterSelector:
       matchExpressions:
         - key: management.cattle.io/cluster-display-name
           operator: In
           values:
-            - ruka
-            - kueyen
-            - pillan
             - kona
-    helm:
-      valuesFiles:
-        - overlays/rke2/values.yaml
+    kustomize:
+      dir: kustomize/overlays/kona
+  - name: kueyen
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - kueyen
+    kustomize:
+      dir: kustomize/overlays/kueyen
+  - name: ruka
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - ruka
+    kustomize:
+      dir: kustomize/overlays/ruka
   - name: pillan
     clusterSelector:
       matchExpressions:
@@ -45,3 +69,5 @@ targetCustomizations:
     helm:
       valuesFiles:
         - overlays/pillan/values.yaml
+    kustomize:
+      dir: kustomize/overlays/pillan

fleet/lib/mimir/kustomize/base/cephobjectstoreuser-mimir.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: mimir
+spec:
+  store: o11y
+  clusterNamespace: rook-ceph
+  displayName: mimir
+  quotas:
+    maxBuckets: 4

fleet/lib/mimir/kustomize/base/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+resources:
+  - cephobjectstoreuser-mimir.yaml
+  - obc-mimir-alertmanager.yaml
+  - obc-mimir-blocks.yaml
+  - obc-mimir-ruler.yaml
+  - obc-mimir.yaml

fleet/lib/mimir/kustomize/base/obc-mimir-alertmanager.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-alertmanager
+spec:
+  bucketName: mimir-alertmanager
+  storageClassName: o11y
+  additionalConfig:
+    bucketOwner: mimir
+    bucketLifecycle: |
+      {
+        "Rules": [
+          {
+            "ID": "AbortIncompleteMultipartUploads",
+            "Status": "Enabled",
+            "Prefix": "",
+            "AbortIncompleteMultipartUpload": {
+              "DaysAfterInitiation": 1
+            }
+          }
+        ]
+      }

fleet/lib/mimir/kustomize/base/obc-mimir-blocks.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-blocks
+spec:
+  bucketName: mimir-blocks
+  storageClassName: o11y
+  additionalConfig:
+    bucketOwner: mimir
+    bucketLifecycle: |
+      {
+        "Rules": [
+          {
+            "ID": "AbortIncompleteMultipartUploads",
+            "Status": "Enabled",
+            "Prefix": "",
+            "AbortIncompleteMultipartUpload": {
+              "DaysAfterInitiation": 1
+            }
+          }
+        ]
+      }

fleet/lib/mimir/kustomize/base/obc-mimir-ruler.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-ruler
+spec:
+  bucketName: mimir-ruler
+  storageClassName: o11y
+  additionalConfig:
+    bucketOwner: mimir
+    bucketLifecycle: |
+      {
+        "Rules": [
+          {
+            "ID": "AbortIncompleteMultipartUploads",
+            "Status": "Enabled",
+            "Prefix": "",
+            "AbortIncompleteMultipartUpload": {
+              "DaysAfterInitiation": 1
+            }
+          }
+        ]
+      }

fleet/lib/mimir/kustomize/base/obc-mimir.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir
+spec:
+  bucketName: mimir
+  storageClassName: o11y
+  additionalConfig:
+    bucketOwner: mimir
+    bucketLifecycle: |
+      {
+        "Rules": [
+          {
+            "ID": "AbortIncompleteMultipartUploads",
+            "Status": "Enabled",
+            "Prefix": "",
+            "AbortIncompleteMultipartUpload": {
+              "DaysAfterInitiation": 1
+            }
+          }
+        ]
+      }

fleet/lib/mimir/kustomize/overlays/antu/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+resources:
+  - ../../base
+patches:
+  - path: obc-mimir-alertmanager.yaml
+  - path: obc-mimir-blocks.yaml
+  - path: obc-mimir-ruler.yaml
+  - path: obc-mimir.yaml

fleet/lib/mimir/kustomize/overlays/antu/obc-mimir-alertmanager.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-alertmanager
+spec:
+  bucketName: antu-mimir-alertmanager

fleet/lib/mimir/kustomize/overlays/antu/obc-mimir-blocks.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-blocks
+spec:
+  bucketName: antu-mimir-blocks

fleet/lib/mimir/kustomize/overlays/antu/obc-mimir-ruler.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-ruler
+spec:
+  bucketName: antu-mimir-ruler

fleet/lib/mimir/kustomize/overlays/antu/obc-mimir.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir
+spec:
+  bucketName: antu-mimir

fleet/lib/mimir/kustomize/overlays/ayekan/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+resources:
+  - ../../base
+patches:
+  - path: obc-mimir-alertmanager.yaml
+  - path: obc-mimir-blocks.yaml
+  - path: obc-mimir-ruler.yaml
+  - path: obc-mimir.yaml

fleet/lib/mimir/kustomize/overlays/ayekan/obc-mimir-alertmanager.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-alertmanager
+spec:
+  bucketName: ayekan-mimir-alertmanager

fleet/lib/mimir/kustomize/overlays/ayekan/obc-mimir-blocks.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-blocks
+spec:
+  bucketName: ayekan-mimir-blocks

fleet/lib/mimir/kustomize/overlays/ayekan/obc-mimir-ruler.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-ruler
+spec:
+  bucketName: ayekan-mimir-ruler

fleet/lib/mimir/kustomize/overlays/ayekan/obc-mimir.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir
+spec:
+  bucketName: ayekan-mimir

fleet/lib/mimir/kustomize/overlays/kona/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+resources:
+  - ../../base
+patches:
+  - path: obc-mimir-alertmanager.yaml
+  - path: obc-mimir-blocks.yaml
+  - path: obc-mimir-ruler.yaml
+  - path: obc-mimir.yaml

fleet/lib/mimir/kustomize/overlays/kona/obc-mimir-alertmanager.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-alertmanager
+spec:
+  bucketName: kona-mimir-alertmanager

fleet/lib/mimir/kustomize/overlays/kona/obc-mimir-blocks.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-blocks
+spec:
+  bucketName: kona-mimir-blocks

fleet/lib/mimir/kustomize/overlays/kona/obc-mimir-ruler.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: mimir-ruler
+spec:
+  bucketName: kona-mimir-ruler