Version 1.3.0

Author: hannahpullen

.github/workflows/helm-chart.yml

@@ -28,15 +28,13 @@ jobs:
       with:
         push: true
         tags: ghcr.io/mathworks-ref-arch/matlab-parallel-server-k8s/mjs-controller-image:${{ github.event.release.tag_name }}
-        context: ./images/controller
+        context: ./controller
 
   release-helm-chart:
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-    env:
-      CHART_DIR: chart/mjs
 
     steps:
       - name: Check out the repo
@@ -46,13 +44,13 @@ jobs:
         uses: azure/setup-helm@v4
 
       - name: Lint chart
-        run: helm lint ${CHART_DIR} --set maxWorkers=4,matlabPVC=test,checkpointPVC=test,logPVC=test,workerLogPVC=test
+        run: helm lint mjs --set maxWorkers=4,matlabPVC=test,checkpointPVC=test,logPVC=test,workerLogPVC=test
 
       - name: Check chart versions
-        run: grep "version. ${{ github.event.release.tag_name }}" ${CHART_DIR}/Chart.yaml && grep "appVersion. ${{ github.event.release.tag_name }}" ${CHART_DIR}/Chart.yaml # Use "." (any character) rather than ":", since ":" breaks YAML parser
+        run: grep "version. ${{ github.event.release.tag_name }}" mjs/Chart.yaml && grep "appVersion. ${{ github.event.release.tag_name }}" mjs/Chart.yaml # Use "." (any character) rather than ":", since ":" breaks YAML parser
 
       - name: Package chart
-        run: helm package ${CHART_DIR} --version ${{ github.event.release.tag_name }} --app-version ${{ github.event.release.tag_name }}
+        run: helm package mjs --version ${{ github.event.release.tag_name }} --app-version ${{ github.event.release.tag_name }}
 
       - name: Login to GitHub Container Registry
         run: echo ${{ secrets.HELM_TOKEN }} | helm registry login ghcr.io/mathworks-ref-arch --username hannahpullen --password-stdin

README.md

@@ -449,6 +449,56 @@ kubectl create secret generic mjs-ldap-secret --namespace mjs --from-file=cert.p
 
 If you use a persistent volume for the job manager pod (`matlabPVC` is set to a non-empty string and `jobManagerUsesPVC` is set to `true` in your `values.yaml` file), you must add your certificate to the Java trust store of the MATLAB Parallel Server installation in your persistent volume. For detailed instructions, see [Add Certificate to Java Trust Store](https://mathworks.com/help/matlab-parallel-server/configure-ldap-server-authentication-for-matlab-job-scheduler.html#mw_fe8d0f90-2854-42b9-9e04-a2f25a295e61) on the MathWorks website.
 
+### Configure Cluster Monitoring Metrics
+
+You can configure MATLAB Job Scheduler to export cluster monitoring metrics.
+This feature is supported for MATLAB Job Scheduler release R2024b or later.
+To export cluster monitoring metrics for MATLAB Job Scheduler releases older than R2024b, set the `jobManagerImageTag` parameter in your Helm values file to `r2024b` to use a newer release for the job manager.
+
+To enable cluster monitoring metrics, set the following values in your `values.yaml` file:
+```yaml
+exportMetrics: true
+metricsPort: 8001
+useSecureMetrics: true
+openMetricsPortOutsideKubernetes: false
+```
+Modify the following values:
+- `metricsPort` — Specify the port for exporting metrics on the HTTP(S) server.
+- `useSecureMetrics` — Set this to true to export metrics over an encrypted HTTPS connection. Set to false to disable encryption and export metrics on an HTTP server.
+- `openMetricsPortOutsideKubernetes` — Set this to true to expose the metrics endpoint outside of the Kubernetes cluster. Set to false if you only want to scrape metrics from another pod inside the Kubernetes cluster.
+
+If you set `useSecureMetrics` to true, by default the Helm chart generates SSL certificates for you.
+
+Optionally, you can provide your own SSL certificates that the job manager uses to encrypt metrics.
+The server SSL certificate must include a Subject Alternative Name (SAN) that corresponds to a DNS name or domain directed at the job manager.
+- If you set `openMetricsPortOutsideKubernetes` to true, use the domain associated with the load balancer addresses generated within your Kubernetes cluster, or configure a static DNS name that routes to your load balancer after installing the MJS Helm Chart.
+- If you set `openMetricsPortOutsideKubernetes` to false, the DNS name of the job manager is `mjs-job-manager.mjs.svc.cluster.local`.
+
+To use your own SSL certificates, create a Kubernetes secret.
+Using the `kubectl` command, specify the paths to the CA certificate used to sign your client certificate, the certificate to use for the server, and the private key to use for the server. For example, use the CA certificate `ca_cert`, server certificate `server_cert`, and server private key `server_key`:
+```
+kubectl create secret generic mjs-metrics-secret --from-file=ca.crt=ca_cert --from-file=jobmanager.crt=server_cert --from-file=jobmanager.key=server_key --namespace mjs
+```
+
+Install the MATLAB Job Scheduler Helm chart.
+
+#### Integrate with Grafana and Prometheus
+
+To integrate your cluster with Grafana® and Prometheus®, follow the instructions in the [Cluster Monitoring Integration for MATLAB Job Scheduler](https://github.com/mathworks/cluster-monitoring-integration-for-matlab-job-scheduler) GitHub repository.
+
+Configure Prometheus to target the metrics endpoint `job-manager-host:metricsPort`, where `metricsPort` is the value you set in your `values.yaml` file.
+- If you set `openMetricsPortOutsideKubernetes` to true, `job-manager-host` is the external IP address or DNS name of your load balancer service. Find this by running `kubectl get services -n mjs mjs-ingress-proxy`.
+- If you set `openMetricsPortOutsideKubernetes` to false, Prometheus must run inside the same Kubernetes cluster as MATLAB Job Scheduler. Set `job-manager-host` to `mjs-job-manager.mjs.svc.cluster.local`.
+
+If you set `useSecureMetrics` to true, configure Prometheus with certificates to authenticate with the metrics server.
+- If you provided your own SSL certificates, use client certificates corresponding to the certificates you used to set up the metrics server.
+- If the Helm chart generated SSL certificates for you, download and use the generated client certificates from the Kubernetes secret `mjs-metrics-client-certs`:
+```
+kubectl get secrets mjs-metrics-client-certs --template="{{.data.ca.crt | base64decode}}" --namespace mjs > ca.crt
+kubectl get secrets mjs-metrics-client-certs --template="{{.data.prometheus.crt | base64decode}}" --namespace mjs > prometheus.crt
+kubectl get secrets mjs-metrics-client-certs --template="{{.data.prometheus.key | base64decode}}" --namespace mjs > prometheus.key
+```
+
 ### Customize Load Balancer
 
 MATLAB Job Scheduler in Kubernetes uses a Kubernetes load balancer service to expose MATLAB Job Scheduler to MATLAB clients running outside of the Kubernetes cluster.

chart/mjs/Chart.yaml

@@ -3,5 +3,5 @@ apiVersion: v2
 name: mjs
 description: A Helm chart for MATLAB (R) Job Scheduler in Kubernetes
 type: application
-version: 1.2.0
-appVersion: 1.2.0
+version: 1.3.0
+appVersion: 1.3.0

chart/mjs/templates/_paths.tpl

@@ -81,3 +81,8 @@ haproxy.cfg
 {{- define "paths.ldapCert" -}}
 /mjs/ldap/cert.pem
 {{- end -}}
+
+# Path to directory containing mounted metrics certificates
+{{- define "paths.metricsCertDir" -}}
+/mjs/metrics/
+{{- end -}}

chart/mjs/templates/controller-configmap.yaml

@@ -33,6 +33,7 @@ data:
       "ControllerLogfile": {{ include "paths.controllerLog" . | quote }},
       "DeploymentName": {{ include "resources.controller" . | quote}},
       "EnableServiceLinks": {{ include "derived.enableServiceLinks" . }},
+      "OpenMetricsPortOutsideKubernetes": {{ .Values.openMetricsPortOutsideKubernetes }},
       "ExtraWorkerEnvironment": {
       {{- $comma := "" }}
       {{- range $key, $value := .Values.extraWorkerEnv }}
@@ -64,6 +65,7 @@ data:
       "LogPVC": {{ .Values.logPVC | quote }},
       "MatlabPVC": {{ .Values.matlabPVC | default "" | quote }},
       "MatlabRoot": {{ include "paths.matlabroot" . | quote }},
+      "MetricsCertDir": {{ include "paths.metricsCertDir" . | quote }},
       "MaxWorkers": {{ .Values.maxWorkers }},
       "MinWorkers": {{ .Values.minWorkers }},
       "MJSDefConfigMap": {{ include "resources.mjsConfigMap" . | quote }},
@@ -99,6 +101,6 @@ data:
       "WorkerPassword": {{ .Values.workerPassword | quote }},
       "WorkersPerPoolProxy": {{ .Values.workersPerPoolProxy }},
       "WorkerUsername": {{ .Values.workerUsername | quote }},
-      "UsePoolProxy": {{ .Values.usePoolProxy | default true }},
-      "UseSecureCommunication": {{ .Values.useSecureCommunication }}
+      "UseSecureCommunication": {{ .Values.useSecureCommunication }},
+      "UseSecureMetrics": {{ and .Values.exportMetrics .Values.useSecureMetrics }}
     }

chart/mjs/templates/ingress-proxy-configmap.yaml

@@ -27,6 +27,11 @@ data:
         default_backend back-mjs-job-manager
     backend back-mjs-job-manager
         server mjs-job-manager mjs-job-manager
+    {{- if and .Values.exportMetrics .Values.openMetricsPortOutsideKubernetes }}
+    frontend front-jobmanager-metrics
+        bind {{ printf "*:%d" (.Values.metricsPort | int) }}
+        default_backend back-mjs-job-manager
+    {{- end }}
 
     # Rules for proxying traffic to the parallel pool proxies.
     # Each parallel pool proxy has a unique port, which should be mapped to the
@@ -40,4 +45,4 @@ data:
     backend back-{{ $poolProxyName }}
         server {{ $poolProxyName }} {{ $poolProxyName }}
     {{ end -}}
-{{ end -}}
+{{ end -}}

chart/mjs/templates/ingress-proxy-service.yaml

@@ -34,4 +34,12 @@ spec:
     port: {{ $poolProxyPort }}
     targetPort: {{ $poolProxyPort }}
   {{ end -}}
-{{ end -}}
+  {{- if and .Values.exportMetrics .Values.openMetricsPortOutsideKubernetes }}
+  # Job manager metrics port
+  - name: {{ printf "metrics-%.0f" .Values.metricsPort }}
+    protocol: TCP
+    appProtocol: TCP
+    port: {{ .Values.metricsPort }}
+    targetPort: {{ .Values.metricsPort }}
+  {{- end }}
+{{ end -}}

chart/mjs/templates/job-manager-service.yaml

@@ -22,4 +22,11 @@ spec:
     appProtocol: TCP
     port: {{ . }}
     targetPort: {{ . }}
+  {{- end }}
+  {{- if .Values.exportMetrics }}
+  - name: {{ printf "metrics-%.0f" .Values.metricsPort }}
+    protocol: TCP
+    appProtocol: TCP
+    port: {{ .Values.metricsPort }}
+    targetPort: {{ .Values.metricsPort }}
   {{- end }}

chart/mjs/templates/mjs-configmap.yaml

@@ -29,6 +29,14 @@ data:
     LDAP_SECURITY_PRINCIPAL_FORMAT={{ .Values.ldapSecurityPrincipalFormat | quote }}
     LDAP_SYNCHRONIZATION_INTERVAL_SECS={{ .Values.ldapSynchronizationIntervalSecs | quote }}
     {{- end }}
+    EXPORT_METRICS={{ .Values.exportMetrics }}
+    METRICS_PORT={{ .Values.metricsPort }}
+    USE_SECURE_METRICS={{ .Values.useSecureMetrics }}
+    {{- if .Values.useSecureMetrics }}
+    METRICS_CA_FILE={{ include "paths.metricsCertDir" . }}/ca.crt
+    METRICS_CERT_FILE={{ include "paths.metricsCertDir" . }}/jobmanager.crt
+    METRICS_KEY_FILE={{ include "paths.metricsCertDir" . }}/jobmanager.key
+    {{- end }}
 
   jobManager.sh: |
     # Script to run on the MATLAB Job Scheduler job manager pod