Add periodic job with volume pool metrics tests

.zuul.yaml

@@ -108,9 +108,41 @@
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: *required_projects
 
+- job:
+    name: functional-periodic-telemetry-with-ceph
+    parent: podified-multinode-hci-deployment-crc-1comp-backends
+    dependencies: ["openstack-k8s-operators-content-provider"]
+    description: |
+      Deploy OpenStack with Telemetry and Ceph. Run metric-verification with volume pool metric tests
+    vars:
+      patch_observabilityclient: true
+      telemetry_verify_metrics_metric_sources_to_test:
+        - ceilometer_compute_agent
+        - ceilometer_central_agent
+        - ksm
+        - node_exporter
+        - podman_exporter
+        - rabbitmq
+        - openstack_network_exporter
+        - kepler
+        - mariadb
+        - volume_pool
+      cifmw_extras:
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/hci_ceph_backends.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-telemetry-with-ceph.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
+    irrelevant-files: *irrelevant_files
+    required-projects: *required_projects
 
 - project:
     name: infrawatch/feature-verification-tests
+    periodic:
+      jobs:
+        - functional-periodic-telemetry-with-ceph
     github-check:
       jobs:
         - feature-verification-tests-noop:
@@ -152,3 +184,9 @@
               - ci/vars-graphing-test.yml
               - ci/run_graphing_test.yml
               - ci/report_result.yml
+        - functional-periodic-telemetry-with-ceph:
+            files:
+              # Run this job for changes to the volume_pool_metrics test changes as this is
+              # the only job, where those jobs get executed.
+              - roles/telemetry_verify_metrics/tasks/verify_ceilometer_volume_pool_metrics.yml
+

roles/telemetry_verify_metrics/README.md

@@ -35,7 +35,7 @@ Tests:
 Role Variables
 --------------
 openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack"
-telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler, openstack_network_exporter
+telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler, openstack\_network\_exporter, volume\_pool
 
 Example Playbook
 ----------------

roles/telemetry_verify_metrics/defaults/main.yml

@@ -12,3 +12,5 @@ telemetry_verify_metrics_metric_sources_to_test:
   - mariadb
   # NOTE: IPMI metrics can only be verified when compute nodes are baremetal
   # - ceilometer_ipmi_agent
+  # NOTE: volume pool metrics can only be verified when a pool with a correct backend is configured
+  # - volume_pool

roles/telemetry_verify_metrics/tasks/main.yml

@@ -108,3 +108,9 @@
   when: '"kepler_vm" in telemetry_verify_metrics_metric_sources_to_test and "ACTIVE" in kepler_vm_status.stdout'
   vars:
     metrics_cmd_prefix: "{{ openstack_cmd }} metric show --disable-rbac -c value -f value"
+
+- name: Verify Ceilometer volume pool metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ceilometer_volume_pool_metrics.yml
+  tags: test
+  when: '"volume_pool" in telemetry_verify_metrics_metric_sources_to_test'

roles/telemetry_verify_metrics/tasks/verify_ceilometer_volume_pool_metrics.yml

@@ -0,0 +1,45 @@
+- name: Verify ceilometer scrapeconfig exists
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1220
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-ceilometer
+
+- name: Verify ceilometer central agent is running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_pod_test_id: RHOSO-1240
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+    common_pod_list:
+      - ceilometer-0
+
+- name: |
+    TEST Use openstack CLI to verify a cinder volume is configured
+    RHOSO-1313
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} volume backend pool list
+  register: result
+  until: result.rc == 0 and result.stdout_lines | length > 0
+  changed_when: false
+
+- name: |
+    TEST Use openstack observabilityclient to verify ceilometer volume pool metrics are stored in prometheus
+    RHOSO-1314
+  vars:
+    volume_pool_metrics:
+      - ceilometer_volume_provider_pool_capacity_allocated
+      - ceilometer_volume_provider_pool_capacity_free
+      - ceilometer_volume_provider_pool_capacity_provisioned
+      - ceilometer_volume_provider_pool_capacity_total
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show {{ item }}
+  register: result
+  delay: 30
+  retries: 10
+  until: result.rc == 0 and item in result.stdout
+  changed_when: false
+  loop: "{{ volume_pool_metrics }}"