feat: 📝 Generate Helm docs

.pre-commit-config.yaml

@@ -2,7 +2,9 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
+      - id: check-merge-conflict
       - id: trailing-whitespace
+      - id: check-added-large-files
       - id: end-of-file-fixer
   - repo: local
     hooks:
@@ -12,15 +14,27 @@ repos:
         entry: bash -c 'cd terraform && pre-commit run --all-files'
         require_serial: true
         verbose: true
+      - id: terraform-docs
+        name: Terraform Documentation
+        language: script
+        entry: ./scripts/terraform_docs.sh
+        require_serial: true
+        verbose: true
       - id: pre-commit-python
         name: pre-commit Python
         language: system
         entry: bash -c 'cd airflow && pre-commit run --all-files'
         require_serial: true
         verbose: true
-      - id: terraform-docs
+      - id: pre-commit-helms
+        name: pre-commit Helms
+        language: system
+        entry: bash -c 'cd helms && pre-commit run --all-files'
+        require_serial: true
+        verbose: true
+      - id: helm-docs
         name: Terraform Documentation
         language: script
-        entry: ./scripts/terraform_docs.sh
+        entry: ./scripts/helm_docs.sh
         require_serial: true
         verbose: true

docs/helms/airflow.md

@@ -0,0 +1,112 @@
+# Airflow
+
+![Version: 8.8.0](https://img.shields.io/badge/Version-8.8.0-informational?style=flat-square) ![AppVersion: 2.6.3](https://img.shields.io/badge/AppVersion-2.6.3-informational?style=flat-square)
+
+!!! tip "Description"
+    This chart aims to deploy an Airflow setup inside minikube, so we validate and develop Airflow inside a local K8s cluster.
+
+---
+
+## Setup
+
+Airflow deployment is managed by :simple-argo: **[ArgoCD](https://argo-cd.readthedocs.io)**, therefore any change in the
+Helm or local drift in K8s will be discovered by ArgoCD.
+
+![Airflow Argo deployment](../images/airflow-argocd.png)
+
+The most important properties when defining our Airflow values are:
+
+<div class="grid cards" markdown>
+
+-   :simple-github:{ .lg .middle } __`dags.gitSync.repo`__
+
+    ---
+
+    Github repo containing our Airflow DAGs code.
+    Airflow [**dags-git-sync**](https://airflow.apache.org/docs/helm-chart/stable/manage-dags-files.html#using-git-sync9)
+    sidecars will be fetching new code periodically from it. So all code pushed there will be automatically deployed.
+
+-   :octicons-code-16:{ .lg .middle } __`dags.gitSync.repoSubPath`__
+
+    ---
+
+    Github folder containing our **DAGs code**. This property is mandatory to use in this project since
+    we are defining multiple tools in it (i.e. K8s, terraform, Helms).
+
+</div>
+
+---
+
+## Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| https://airflow-helm.github.io/charts | airflow | 8.8.0 |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| airflow.airflow.config.AIRFLOW__API__AUTH_BACKENDS | string | `"airflow.api.auth.backend.basic_auth"` |  |
+| airflow.airflow.config.AIRFLOW__CORE__CHECK_SLAS | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION | string | `"True"` |  |
+| airflow.airflow.config.AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG | int | `1` |  |
+| airflow.airflow.config.AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL | int | `150` |  |
+| airflow.airflow.config.AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__LOGGING__REMOTE_LOGGING | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__AUTHENTICATE | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__BASE_URL | string | `"https://airflow.data"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX | string | `"True"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__EXPOSE_CONFIG | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT | string | `"True"` |  |
+| airflow.airflow.executor | string | `"CeleryExecutor"` |  |
+| airflow.airflow.image.repository | string | `"apache/airflow"` |  |
+| airflow.airflow.image.tag | string | `"2.8.2-python3.10"` |  |
+| airflow.createUserJob.applyCustomEnv | bool | `false` |  |
+| airflow.createUserJob.useHelmHooks | bool | `false` |  |
+| airflow.dags.gitSync.branch | string | `"main"` |  |
+| airflow.dags.gitSync.enabled | bool | `true` |  |
+| airflow.dags.gitSync.repo | string | `"https://github.com/afranzi/mini-data-platform.git"` |  |
+| airflow.dags.gitSync.repoSubPath | string | `"airflow"` |  |
+| airflow.dags.gitSync.resources.requests.cpu | string | `"50m"` |  |
+| airflow.dags.gitSync.resources.requests.memory | string | `"64Mi"` |  |
+| airflow.dags.gitSync.revision | string | `"HEAD"` |  |
+| airflow.dags.gitSync.syncWait | int | `60` |  |
+| airflow.ingress.apiVersion | string | `"networking.k8s.io/v1"` |  |
+| airflow.ingress.enabled | bool | `true` |  |
+| airflow.ingress.web.host | string | `"airflow.data"` |  |
+| airflow.ingress.web.ingressClassName | string | `"nginx"` |  |
+| airflow.migrateDatabaseJob.applyCustomEnv | bool | `false` |  |
+| airflow.migrateDatabaseJob.jobAnnotations."argocd.argoproj.io/hook" | string | `"Sync"` |  |
+| airflow.migrateDatabaseJob.useHelmHooks | bool | `false` |  |
+| airflow.postgresql.enabled | bool | `true` |  |
+| airflow.redis.cluster.enabled | bool | `false` |  |
+| airflow.redis.cluster.slaveCount | int | `1` |  |
+| airflow.redis.enabled | bool | `true` |  |
+| airflow.redis.image.tag | string | `"7.2.4-debian-12-r9"` |  |
+| airflow.redis.master.persistence.enabled | bool | `false` |  |
+| airflow.redis.master.persistence.size | string | `"2Gi"` |  |
+| airflow.redis.master.persistence.storageClass | string | `""` |  |
+| airflow.redis.master.resources.requests.cpu | string | `"10m"` |  |
+| airflow.redis.master.resources.requests.memory | string | `"32Mi"` |  |
+| airflow.redis.slave.persistence.enabled | bool | `false` |  |
+| airflow.redis.slave.persistence.size | string | `"8Gi"` |  |
+| airflow.redis.slave.persistence.storageClass | string | `""` |  |
+| airflow.redis.slave.resources.requests.cpu | string | `"10m"` |  |
+| airflow.redis.slave.resources.requests.memory | string | `"32Mi"` |  |
+| airflow.scheduler.resources.limits.cpu | string | `"1000m"` |  |
+| airflow.scheduler.resources.limits.memory | string | `"1Gi"` |  |
+| airflow.scheduler.resources.requests.cpu | string | `"1000m"` |  |
+| airflow.scheduler.resources.requests.memory | string | `"512Mi"` |  |
+| airflow.workers.enabled | bool | `true` |  |
+| airflow.workers.logCleanup.enabled | bool | `true` |  |
+| airflow.workers.logCleanup.resources.requests.cpu | string | `"10m"` |  |
+| airflow.workers.logCleanup.resources.requests.memory | string | `"32Mi"` |  |
+| airflow.workers.logCleanup.retentionMinutes | int | `21600` |  |
+| airflow.workers.podDisruptionBudget.apiVersion | string | `nil` |  |
+| airflow.workers.podDisruptionBudget.enabled | bool | `true` |  |
+| airflow.workers.podDisruptionBudget.maxUnavailable | string | `"20%"` |  |
+| airflow.workers.replicas | int | `1` |  |
+| airflow.workers.resources.requests.cpu | string | `"256m"` |  |
+| airflow.workers.resources.requests.memory | string | `"1Gi"` |  |
+

docs/index.md

@@ -6,7 +6,7 @@ data platforms is essential for any organization looking to leverage data-driven
 The mini data platform cluster project that embodies the spirit of innovation and learning in the fields of
 Infrastructure as Code (IaC), Data Engineering, and Operations (Ops).
 
-![Mini-DataPlatformConcept](images/concept.jpg){ align=left }
+![Mini-DataPlatformConcept](images/concept.jpg)
 
 At the heart of this project lies a carefully curated stack comprising Terraform, Minikube, ArgoCD, and Airflow.
 Each tool has been selected for its strength in automating and managing infrastructure, orchestrating deployments, and

helms/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+files: ^helms/
+repos:
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.35.1
+    hooks:
+      - id: yamllint
+        args: [-c=./helms/.yamllint]
+  - repo: https://github.com/Agilicus/pre-commit-hook-k8svalidate.git
+    rev: v0.2.0
+    hooks:
+      - id: k8svalidate
+        files: .yaml$
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.4.0
+    hooks:
+      - id: detect-secrets
+  - repo: https://github.com/norwoodj/helm-docs
+    rev: v1.13.1
+    hooks:
+      - id: helm-docs

helms/.yamllint

@@ -0,0 +1,8 @@
+extends: default
+
+rules:
+  line-length:
+    max: 120
+    level: warning
+  document-start: disable
+  trailing-spaces: disable

helms/airflow/Chart.yaml

@@ -1,11 +1,11 @@
 apiVersion: v2
 name: airflow
-description: Universal Ranking Algorithm for Natural language Archive and Index
+description: This chart aims to deploy an Airflow setup inside minikube, so we validate and develop Airflow inside a local K8s cluster.
 maintainers:
   - name: Albert Franzi
 version: 8.8.0
 appVersion: 2.6.3
 dependencies:
-  - name: airflow # https://artifacthub.io/packages/helm/airflow-helm/airflow
+  - name: airflow  # https://artifacthub.io/packages/helm/airflow-helm/airflow
     version: 8.8.0
-    repository: https://airflow-helm.github.io/charts #https://airflow.apache.org
+    repository: https://airflow-helm.github.io/charts  # https://airflow.apache.org

helms/airflow/README.md

@@ -0,0 +1,112 @@
+# Airflow
+
+![Version: 8.8.0](https://img.shields.io/badge/Version-8.8.0-informational?style=flat-square) ![AppVersion: 2.6.3](https://img.shields.io/badge/AppVersion-2.6.3-informational?style=flat-square)
+
+!!! tip "Description"
+    This chart aims to deploy an Airflow setup inside minikube, so we validate and develop Airflow inside a local K8s cluster.
+
+---
+
+## Setup
+
+Airflow deployment is managed by :simple-argo: **[ArgoCD](https://argo-cd.readthedocs.io)**, therefore any change in the
+Helm or local drift in K8s will be discovered by ArgoCD.
+
+![Airflow Argo deployment](../images/airflow-argocd.png)
+
+The most important properties when defining our Airflow values are:
+
+<div class="grid cards" markdown>
+
+-   :simple-github:{ .lg .middle } __`dags.gitSync.repo`__
+
+    ---
+
+    Github repo containing our Airflow DAGs code.
+    Airflow [**dags-git-sync**](https://airflow.apache.org/docs/helm-chart/stable/manage-dags-files.html#using-git-sync9)
+    sidecars will be fetching new code periodically from it. So all code pushed there will be automatically deployed.
+
+-   :octicons-code-16:{ .lg .middle } __`dags.gitSync.repoSubPath`__
+
+    ---
+
+    Github folder containing our **DAGs code**. This property is mandatory to use in this project since
+    we are defining multiple tools in it (i.e. K8s, terraform, Helms).
+
+</div>
+
+---
+
+## Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| https://airflow-helm.github.io/charts | airflow | 8.8.0 |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| airflow.airflow.config.AIRFLOW__API__AUTH_BACKENDS | string | `"airflow.api.auth.backend.basic_auth"` |  |
+| airflow.airflow.config.AIRFLOW__CORE__CHECK_SLAS | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION | string | `"True"` |  |
+| airflow.airflow.config.AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG | int | `1` |  |
+| airflow.airflow.config.AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL | int | `150` |  |
+| airflow.airflow.config.AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__LOGGING__REMOTE_LOGGING | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__AUTHENTICATE | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__BASE_URL | string | `"https://airflow.data"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX | string | `"True"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__EXPOSE_CONFIG | string | `"False"` |  |
+| airflow.airflow.config.AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT | string | `"True"` |  |
+| airflow.airflow.executor | string | `"CeleryExecutor"` |  |
+| airflow.airflow.image.repository | string | `"apache/airflow"` |  |
+| airflow.airflow.image.tag | string | `"2.8.2-python3.10"` |  |
+| airflow.createUserJob.applyCustomEnv | bool | `false` |  |
+| airflow.createUserJob.useHelmHooks | bool | `false` |  |
+| airflow.dags.gitSync.branch | string | `"main"` |  |
+| airflow.dags.gitSync.enabled | bool | `true` |  |
+| airflow.dags.gitSync.repo | string | `"https://github.com/afranzi/mini-data-platform.git"` |  |
+| airflow.dags.gitSync.repoSubPath | string | `"airflow"` |  |
+| airflow.dags.gitSync.resources.requests.cpu | string | `"50m"` |  |
+| airflow.dags.gitSync.resources.requests.memory | string | `"64Mi"` |  |
+| airflow.dags.gitSync.revision | string | `"HEAD"` |  |
+| airflow.dags.gitSync.syncWait | int | `60` |  |
+| airflow.ingress.apiVersion | string | `"networking.k8s.io/v1"` |  |
+| airflow.ingress.enabled | bool | `true` |  |
+| airflow.ingress.web.host | string | `"airflow.data"` |  |
+| airflow.ingress.web.ingressClassName | string | `"nginx"` |  |
+| airflow.migrateDatabaseJob.applyCustomEnv | bool | `false` |  |
+| airflow.migrateDatabaseJob.jobAnnotations."argocd.argoproj.io/hook" | string | `"Sync"` |  |
+| airflow.migrateDatabaseJob.useHelmHooks | bool | `false` |  |
+| airflow.postgresql.enabled | bool | `true` |  |
+| airflow.redis.cluster.enabled | bool | `false` |  |
+| airflow.redis.cluster.slaveCount | int | `1` |  |
+| airflow.redis.enabled | bool | `true` |  |
+| airflow.redis.image.tag | string | `"7.2.4-debian-12-r9"` |  |
+| airflow.redis.master.persistence.enabled | bool | `false` |  |
+| airflow.redis.master.persistence.size | string | `"2Gi"` |  |
+| airflow.redis.master.persistence.storageClass | string | `""` |  |
+| airflow.redis.master.resources.requests.cpu | string | `"10m"` |  |
+| airflow.redis.master.resources.requests.memory | string | `"32Mi"` |  |
+| airflow.redis.slave.persistence.enabled | bool | `false` |  |
+| airflow.redis.slave.persistence.size | string | `"8Gi"` |  |
+| airflow.redis.slave.persistence.storageClass | string | `""` |  |
+| airflow.redis.slave.resources.requests.cpu | string | `"10m"` |  |
+| airflow.redis.slave.resources.requests.memory | string | `"32Mi"` |  |
+| airflow.scheduler.resources.limits.cpu | string | `"1000m"` |  |
+| airflow.scheduler.resources.limits.memory | string | `"1Gi"` |  |
+| airflow.scheduler.resources.requests.cpu | string | `"1000m"` |  |
+| airflow.scheduler.resources.requests.memory | string | `"512Mi"` |  |
+| airflow.workers.enabled | bool | `true` |  |
+| airflow.workers.logCleanup.enabled | bool | `true` |  |
+| airflow.workers.logCleanup.resources.requests.cpu | string | `"10m"` |  |
+| airflow.workers.logCleanup.resources.requests.memory | string | `"32Mi"` |  |
+| airflow.workers.logCleanup.retentionMinutes | int | `21600` |  |
+| airflow.workers.podDisruptionBudget.apiVersion | string | `nil` |  |
+| airflow.workers.podDisruptionBudget.enabled | bool | `true` |  |
+| airflow.workers.podDisruptionBudget.maxUnavailable | string | `"20%"` |  |
+| airflow.workers.replicas | int | `1` |  |
+| airflow.workers.resources.requests.cpu | string | `"256m"` |  |
+| airflow.workers.resources.requests.memory | string | `"1Gi"` |  |
+

helms/airflow/README.md.gotmpl

@@ -0,0 +1,47 @@
+# Airflow
+
+{{ template "chart.badgesSection" . }}
+
+!!! tip "Description"
+    {{ template "chart.description" . }}
+
+---
+
+## Setup
+
+Airflow deployment is managed by :simple-argo: **[ArgoCD](https://argo-cd.readthedocs.io)**, therefore any change in the
+Helm or local drift in K8s will be discovered by ArgoCD.
+
+![Airflow Argo deployment](../images/airflow-argocd.png)
+
+The most important properties when defining our Airflow values are:
+
+<div class="grid cards" markdown>
+
+-   :simple-github:{ .lg .middle } __`dags.gitSync.repo`__
+
+    ---
+
+    Github repo containing our Airflow DAGs code.
+    Airflow [**dags-git-sync**](https://airflow.apache.org/docs/helm-chart/stable/manage-dags-files.html#using-git-sync9)
+    sidecars will be fetching new code periodically from it. So all code pushed there will be automatically deployed.
+
+-   :octicons-code-16:{ .lg .middle } __`dags.gitSync.repoSubPath`__
+
+    ---
+
+    Github folder containing our **DAGs code**. This property is mandatory to use in this project since
+    we are defining multiple tools in it (i.e. K8s, terraform, Helms).
+
+</div>
+
+---
+
+{{ template "chart.homepageLine" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesSection" . }}
+

scripts/helm_docs.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+docs=($(find helms -type f -path '*/README.md' | sort -r))
+
+for doc in "${docs[@]}"; do
+  folder=$(dirname "${doc}")
+  doc_folder=$(dirname "${folder}")
+  mkdir -p "docs/${doc_folder}"
+  cp "${doc}" "docs/${folder}.md"
+done

scripts/terraform_docs.sh

@@ -4,6 +4,7 @@ docs=($(find terraform -type f -path '*/readme.md' | sort -r))
 
 for doc in "${docs[@]}"; do
   folder=$(dirname "${doc}")
-  mkdir -p "docs/${folder}"
+  doc_folder=$(dirname "${folder}")
+  mkdir -p "docs/${doc_folder}"
   cp "${doc}" "docs/${folder}.md"
 done