Add a custom rag blueprint chart + dependencies

test/e2e/nemo-dependencies/install.yaml

@@ -92,3 +92,19 @@
          Then open http://localhost:8888/ in your browser.
         If prompted for a token, use: {{ jupyter_token }}
     when: install.jupyter == true
+
+- hosts: localhost
+  vars_files:
+    - values.yaml
+  roles:
+    - role: rag
+      vars:
+        namespace: "{{ installation_namespace }}"
+      when: install.rag == true
+  tasks:
+  - name: RAG Details
+    debug:
+      msg:   
+        - "RAG minio Password: {{ minio.password }}"
+        - "RAG minio Username: {{ minio.username }}"
+    when: install.rag == true

test/e2e/nemo-dependencies/rag/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+# defaults file for rag
+namespace: "rag"
+
+# Milvus Helm chart details
+milvus:
+  enabled: true
+  helm_repo_name: "rag-milvus"
+  helm_repo_url: "https://zilliztech.github.io/milvus-helm/"
+  chart_name: "milvus/milvus"
+  chart_version: "4.1.11"
+
+# Minio Helm chart details
+minio:
+  enabled: true
+  helm_release_name: rag-minio
+  helm_oci_registry: oci://registry-1.docker.io/bitnamicharts/minio
+  username: minioadmin
+  password: minioadmin
+

test/e2e/nemo-dependencies/rag/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for rag blueprint

test/e2e/nemo-dependencies/rag/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

test/e2e/nemo-dependencies/rag/tasks/local-path-provisioner.yaml

@@ -0,0 +1,26 @@
+- name: Check if 'local-path-storage' namespace exists
+  shell: kubectl get namespace local-path-storage --no-headers
+  register: ns_check
+  ignore_errors: true
+
+
+- name: Deploy local-path-storage if not already deployed
+  shell: >
+    kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/{{ localPathProvisioner.version }}/deploy/local-path-storage.yaml
+  when: ns_check.rc != 0
+
+- name: Wait for local-path-provisioner deployment to be available
+  command: kubectl rollout status deployment/local-path-provisioner -n local-path-storage --timeout=120s
+  register: rollout_status
+  retries: 5
+  delay: 10
+  until: rollout_status.rc == 0
+  when: ns_check.rc != 0
+
+- name: Set 'local-path' as the default StorageClass
+  shell: |
+    kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+  register: sc_patch_result
+  changed_when: "'patched' in sc_patch_result.stdout"
+  failed_when: sc_patch_result.rc != 0
+  when: localPathProvisioner.default | bool

test/e2e/nemo-dependencies/rag/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+# tasks file for evaluator
+- include_tasks: local-path-provisioner.yaml
+  when: localPathProvisioner.enabled
+- include_tasks: namespace.yaml
+- include_tasks: milvus.yaml
+  when: milvus.enabled
+- include_tasks: minio.yaml
+  when: minio.enabled

test/e2e/nemo-dependencies/rag/tasks/milvus.yaml

@@ -0,0 +1,95 @@
+---
+- name: Get Kube API resources
+  command: kubectl api-resources --verbs=list --namespaced -o name
+  register: api_resources
+
+- name: Check if the current cluster is OpenShift
+  set_fact:
+    is_openshift: "{{ 'routes.route.openshift.io' in api_resources.stdout_lines }}"
+
+
+- name: OpenShift - Prepare RBAC to use anyuid SCC
+  ansible.builtin.template:
+    src: milvus-oc-rbac.yaml.j2
+    dest: milvus-oc-rbac.yaml
+  when: is_openshift 
+
+- name: OpenShift - apply RBAC to use anyuid SCC
+  command: kubectl apply -f milvus-oc-rbac.yaml
+  when: is_openshift 
+
+- name: Add Helm repository for Milvus
+  command: helm repo add {{ milvus.helm_repo_name }} {{ milvus.helm_repo_url }}
+
+- name: Update Helm repositories
+  command: helm repo update
+
+- name: Template values file
+  ansible.builtin.template:
+    src: milvus-values.yaml.j2
+    dest: milvus-values.yaml
+
+- name: OpenShift - configure Milvus to use its dedicated service account
+  blockinfile:
+    path: milvus-values.yaml
+    marker: "# {mark} ANSIBLE MANAGED BLOCK"
+    insertafter: "^(.*)$"
+    block: |
+      serviceAccount:
+        create: false
+        name: milvus
+  when: is_openshift
+
+- name: Install Milvus Helm chart
+  shell: >
+    helm upgrade --install {{ milvus.helm_repo_name }}
+    {{ milvus.chart_name }}
+    --namespace {{ namespace }}
+    --version {{ milvus.chart_version }}
+    --values milvus-values.yaml
+  register: helm_install_result
+  changed_when: "'STATUS: deployed' in helm_install_result.stdout"
+
+- name: Verify Milvus installation
+  command: kubectl get pods -n {{ namespace }}
+  register: pods
+
+- name: Wait for Milvus pod to be ready
+  command: kubectl wait --for=condition=Ready pod -n {{ namespace }} -l app.kubernetes.io/instance={{ milvus.helm_repo_name }} --timeout=300s
+
+- name: Get Milvus pod details
+  shell: |
+    kubectl get pods -n {{ namespace }} -l app.kubernetes.io/instance={{ milvus.helm_repo_name }} -o json
+  register: milvus_pods
+  changed_when: false
+
+- name: Get Milvus pod name
+  shell: |
+    kubectl get pods -n {{ namespace }} -l app.kubernetes.io/instance={{ milvus.helm_repo_name }} -o jsonpath='{.items[0].metadata.name}'
+  register: milvus_pod_name
+  failed_when: milvus_pod_name.stdout == ""
+  changed_when: false
+
+- name: Debug Milvus pod name
+  debug:
+    msg: "Milvus pod name is {{ milvus_pod_name.stdout }}"
+
+- name: Get the IP of the running Milvus pod
+  shell: |
+    kubectl get pod -n {{ namespace }} {{ milvus_pod_name.stdout }} -o jsonpath='{.status.podIP}'
+  register: milvus_pod_ip
+  failed_when: milvus_pod_ip.stdout == ""
+  changed_when: false
+
+- name: Debug Milvus Pod IP
+  debug:
+    msg: "Milvus Pod IP is {{ milvus_pod_ip.stdout }}"
+  when: milvus_pod_ip | length > 0
+
+- name: Check Milvus service is responding
+  command: kubectl run milvus-check --image=busybox --restart=Never --attach --rm=true -- nc -w 10 -zv {{ milvus_pod_ip.stdout }} 19530
+  register: milvus_status
+
+- name: Display Milvus connectivity status
+  debug:
+    msg: "{{ '✅ Milvus is accessible!' if milvus_status.rc == 0 else '❌ Milvus is not reachable!' }}"

test/e2e/nemo-dependencies/rag/tasks/minio.yaml

@@ -0,0 +1,76 @@
+---
+- name: Template values file
+  ansible.builtin.template:
+    src: minio-values.yaml.j2
+    dest: minio-values.yaml
+
+- name: Install minio Helm chart
+  shell: >
+    helm upgrade --install {{ minio.helm_release_name }}
+    {{ minio.helm_oci_registry }}
+    --namespace {{ namespace }}
+    --values minio-values.yaml
+  register: helm_install_result
+  changed_when: "'STATUS: deployed' in helm_install_result.stdout"
+
+- name: Verify MinIO installation
+  command: kubectl get pods -n {{ namespace }}
+  register: pods
+
+- name: Check if mc binary exists in workspace
+  stat:
+    path: "./mc"
+  register: mc_binary
+
+- name: Download mc for Linux x86_64
+  shell: |
+    curl -O https://dl.min.io/client/mc/release/linux-amd64/mc &&
+    chmod +x mc
+  when: not mc_binary.stat.exists and ansible_system == 'Linux' and ansible_architecture == 'x86_64'
+
+- name: Download mc for Linux aarch64
+  shell: |
+    curl -O https://dl.min.io/client/mc/release/linux-arm64/mc &&
+    chmod +x mc
+  when: not mc_binary.stat.exists and ansible_system == 'Linux' and ansible_architecture == 'aarch64'
+
+- name: Download mc for macOS
+  shell: |
+    curl -O https://dl.min.io/client/mc/release/darwin-amd64/mc &&
+    chmod +x mc
+  when: not mc_binary.stat.exists and ansible_system == 'Darwin'
+
+- name: Wait for MinIO pods to be ready
+  shell: |
+    kubectl get pods -n {{ namespace }} \
+      | grep {{ minio.helm_release_name }} \
+      | grep -v console \
+      | awk '{print $1}' \
+      | xargs kubectl get pod -n {{ namespace }} -o json
+  register: minio_pods
+  retries: 30
+  delay: 10
+  until: minio_pods.stdout | from_json | json_query("items[*].status.phase") | unique == ['Running']
+  failed_when: minio_pods.rc != 0
+
+- name: Get the IP of the running MinIO pod
+  set_fact:
+    minio_pod_ip: "{{ (minio_pods.stdout | from_json).status.podIP }}"
+
+- name: Run validation to connect to MinIO
+  ignore_errors: true
+  shell: |
+    echo "Running MinIO validation script"
+    ./mc alias set myminio http://{{ minio_pod_ip }}:9000 {{ minio.username }} {{ minio.password }} --insecure
+    ./mc mb myminio/testbucket --insecure
+  register: minio_validation_output
+
+- name: MinIO Connection status (Success)
+  debug:
+    msg: "Successfully connected and created test bucket on MinIO."
+  when: minio_validation_output.rc == 0
+
+- name: MinIO Connection status (Failed)
+  debug:
+    msg: "Failed to connect and create test bucket on MinIO."
+  when: minio_validation_output.rc != 0

test/e2e/nemo-dependencies/rag/tasks/namespace.yaml

@@ -0,0 +1,10 @@
+- name: Check if the provided namespace exists
+  shell: kubectl get namespace {{ namespace }} --no-headers
+  register: ns_check
+  ignore_errors: true
+
+
+- name: Create the provided namespace if not already exists
+  shell: >
+    kubectl create namespace {{ namespace }}
+  when: ns_check.rc != 0

test/e2e/nemo-dependencies/rag/tasks/uninstall.yaml

@@ -0,0 +1,24 @@
+- name: Check if RAG dependencies are installed
+  shell: helm list -n {{ namespace }} | egrep 'rag-minio|rag-milvus' | awk '{print $1}' | wc -l | tr -d '\n'
+  register: rag_installed
+  ignore_errors: true
+
+- name: Uninstall Minio and Milvus helm charts
+  shell: helm list -n {{ namespace }} | awk '{print $1}' | grep -v NAME | egrep 'rag-minio|rag-milvus' | xargs helm del -n {{ namespace }}
+  ignore_errors: true
+
+- name: Delete RAG PVCs
+  shell: kubectl get pvc -n {{ namespace }} | egrep 'rag-milvus|rag-minio' | awk '{print $1}' | xargs kubectl delete pvc -n {{ namespace }}
+  ignore_errors: true
+
+- name: Delete Milvus SA
+  command: kubectl delete serviceaccount milvus -n {{ namespace }}
+  ignore_errors: true
+
+- name: Delete Milvus role
+  command: kubectl delete role scc-anyuid -n {{ namespace }}
+  ignore_errors: true
+
+- name: Delete Milvus rolebinding
+  command: kubectl delete rolebinding milvus-scc-anyuid-binding -n {{ namespace }}
+  ignore_errors: true

test/e2e/nemo-dependencies/rag/templates/milvus-oc-rbac.yaml.j2

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rag-milvus
+  namespace: {{ namespace }}
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: rag-scc-anyuid
+  namespace: {{ namespace }}
+rules:
+- apiGroups: ['security.openshift.io']
+  resources: ['securitycontextconstraints']
+  verbs: ['use']
+  resourceNames: ['anyuid']
+
+--- 
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rag-milvus-scc-anyuid-binding
+  namespace: {{ namespace }}
+subjects:
+- kind: ServiceAccount
+  name: rag-milvus
+  namespace: {{ namespace }}
+roleRef:
+  kind: Role
+  name: rag-scc-anyuid
+  apiGroup: rbac.authorization.k8s.io