Merge pull request #26 from cloudogu/feature/folder_structure_strategy

Feature/folder structure strategy

Author: schnatterer

CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Support different base path in destination gitops repository with `destinationRootPath` #26
+- Support different folder strategies with `folderStructureStrategy` #26
+- Optional `credentialsId` for build images #19
+- Add option for other mainbranches in helm git repositories #19
+
+### Changed
+
+- Bump default cesBuildLib version to 1.62.0 #26
+- Bump default kubectl image to 'lachlanevenson/k8s-kubectl:v1.24.8' #26
+- Bump default helm image to 'ghcr.io/cloudogu/helm:3.11.1-2' #26
+
+### Removed
+
+- Disable kubeval and helm kubeval in default config, because they are deprecated (we will introduce another linting tool later) #26
+
+### Fixed
+- Add namespace to argo helm release #19
+
+## 0.0.1 - 0.1.3
+
+No change log provided. See GitHub release pages for details, e.g.
+https://github.com/cloudogu/gitops-build-lib/releases/tag/0.1.3

Jenkinsfile

@@ -1,5 +1,5 @@
 #!groovy
-@Library('github.com/cloudogu/ces-build-lib@1.45.0')
+@Library('github.com/cloudogu/ces-build-lib@1.62.0')
 import com.cloudogu.ces.cesbuildlib.*
 
 node('docker') {

README.md

@@ -23,7 +23,7 @@ Or if you want to chat with us about gitops in general, visit us [here](https://
   - [GitOps tool](#gitops-tool)
     - [Flux v1](#flux-v1)
     - [ArgoCD](#argocd)
-- [Default Folder Structure](#default-folder-structure)
+- [Default Folder Structure in source repository](#default-folder-structure-in-source-repository)
   - [Plain-k8s](#plain-k8s)
   - [Helm](#helm)
 - [GitOps-Config](#gitops-config)
@@ -35,6 +35,7 @@ Or if you want to chat with us about gitops in general, visit us [here](https://
   - [Helm deployment](#helm-deployment)
     - [Conventions for helm deployment](#conventions-for-helm-deployment)
     - [`helm template` with ArgoCD application](#helm-template-with-argocd-application)
+- [Folder Structure in destination gitops repository](#folder-structure-in-destination-gitops-repository)
 - [Extra Files](#extra-files)
 - [SCM-Provider](#scm-provider)
 - [Validators](#validators)
@@ -101,15 +102,29 @@ def gitopsConfig = [
         repositoryUrl:  'gitops'
     ],
     application: 'spring-petclinic',
-    gitopsTool: 'FLUX' /* or 'ARGO' */
+    gitopsTool: 'FLUX' /* or 'ARGO' */,
+    stages: [
+        staging: [
+            namespace: 'staging',
+            deployDirectly: true
+        ],
+        production: [
+            namespace: 'production',
+            deployDirectly: false
+        ]
+    ],
+    deployments: [
+        plain     : []
+        ]
+    ]
 ]
 
 deployViaGitops(gitopsConfig)
 ```
 
 ### More options
 
-The following is an example of a small and yet complete **gitops-config** for a helm-deployment of an application. 
+The following is an example shows all options of a **gitops-config** for a helm-deployment of an application. 
 This would lead to a deployment of your staging environment by updating the resources of "staging" folder within your 
 gitops-folder in git. For production it will open a PR with the changes.
 
@@ -129,16 +144,19 @@ def gitopsConfig = [
     mainBranch: 'master' /* Default: 'main' */, 
     deployments: [
         sourcePath: 'k8s' /* Default: 'k8s' */,
+        destinationRootPath: '.' /* Default: '.' */,
         /* See docs for helm or plain k8s deployment options */
         helm : [
             repoType : 'HELM',
             credentialsId : 'creds',
+            mainBranch : 'main', /* Default: 'main' */, 
             repoUrl  : <helmChartRepository>,
             chartName: <helmChartName>,
             version  : <helmChartVersion>,
             updateValues  : [[fieldPath: "image.name", newValue: imageName]]
         ]
     ],
+    folderStructureStrategy: 'GLOBAL_ENV', /* or ENV_PER_APP */
     stages: [
         staging: [ 
             namespace: 'my-staging',
@@ -262,7 +280,7 @@ See [Example of ArgoCD application in GitOps Playground](https://github.com/clou
 
 ---
 
-## Default Folder Structure 
+## Default Folder Structure in source repository
 
 A default project structure in your application repo could look like the examples below. Make sure you have your k8s 
 and/or helm resources bundled in a folder. This specific resources folder (here `k8s`) will later be specified by the 
@@ -317,7 +335,7 @@ First of all there are some mandatory properties e.g. the information about your
 * `gitopsTool: 'ARGO'` - Name of the gitops tool. Currently supporting `'FLUX'` (for now only fluxV1) and `'ARGO'`.  
 * and some optional parameters (below are the defaults) for the configuration of the dependency to the ces-build-lib or the default name for the git branch:
    * `cesBuildLibRepo:    'https://github.com/cloudogu/ces-build-lib'`
-   * `cesBuildLibVersion: '1.45.0'`
+   * `cesBuildLibVersion: '1.62.0'`
    * `mainBranch:         'main'`
 
 ---
@@ -326,40 +344,37 @@ First of all there are some mandatory properties e.g. the information about your
 The GitOps-build-lib uses some docker images internally (To run Helm or Kubectl commands and specific Validators inside a docker container).  
 All of these have set default images, but you can change them if you wish to.
 
+```groovy
+def gitopsConfig = [
+    buildImages: [
+        // These are used to run helm and kubectl commands in the core logic
+        helm: 'ghcr.io/cloudogu/helm:3.5.4-1',
+        kubectl: 'lachlanevenson/k8s-kubectl:v1.19.3',
+        // These are used for each specific validator via an imageRef property inside the validators config. See [Validators] for examples.
+        kubeval: 'ghcr.io/cloudogu/helm:3.5.4-1',
+        helmKubeval: 'ghcr.io/cloudogu/helm:3.5.4-1',
+        yamllint: 'cytopia/yamllint:1.25-0.7'
+    ]
+]
+```
+
+Optional - if image is in a private repository, you can pass a `credentialsId` for pulling images.
+
 ```groovy
 def gitopsConfig = [
         buildImages: [
-            // These are used to run helm and kubectl commands in the core logic
-            // 
             helm: [ 
-                image: 'ghcr.io/cloudogu/helm:3.5.4-1'
-                credentialsId: 'myCredentials' (optional - only needed if image is in a private repository. CredentialsId is getting pulled from Jenkins credentials)
+                image: 'ghcr.io/cloudogu/helm:3.11.1-2',
+                credentialsId: 'myCredentials'
                 ],
-            kubectl: [ 
-                image: 'lachlanevenson/k8s-kubectl:v1.19.3'
-                credentialsId: 'myCredentials' (optional - only needed if image is in a private repository. CredentialsId is getting pulled from Jenkins credentials)
-                ],
-            // These are used for each specific validator via an imageRef property inside the validators config. See [Validators] for examples.
-            kubeval: [ 
-                image: 'ghcr.io/cloudogu/helm:3.5.4-1'
-                credentialsId: 'myCredentials' (optional - only needed if image is in a private repository. CredentialsId is getting pulled from Jenkins credentials)
-                ],
-            helmKubeval: [ 
-                image: 'ghcr.io/cloudogu/helm:3.5.4-1'
-                credentialsId: 'myCredentials' (optional - only needed if image is in a private repository. CredentialsId is getting pulled from Jenkins credentials)
-                ],
-            yamllint: [
-                image: 'cytopia/yamllint:1.25-0.7'
-                credentialsId: 'myCredentials' (optional - only needed if image is in a private repository. CredentialsId is getting pulled from Jenkins credentials)
-                ]
+            // ...
         ]
 ]
 ```
 
 ## Stages
 The GitOps-build-lib supports builds on multiple stages. A stage is defined by a name and contains a namespace (used to
-generate the resources) and a deployment-flag. If no stages is passed into the gitops-config by the user, the default
-is set to:
+generate the resources) and a deployment-flag:
 
 ```groovy
 def gitopsConfig = [
@@ -376,7 +391,6 @@ def gitopsConfig = [
 ]
 ```
 
-The defaults above can be overwritten by providing an entry for 'stages' within your config.
 If it is set to deploy directly it will commit and push to your desired `gitops-folder` and therefore triggers a deployment. If it is set to false
 it will create a PR on your `gitops-folder`. **Remember** there are important conventions regarding namespaces and the folder structure (see [namespaces](#namespaces)).
 
@@ -509,6 +523,7 @@ The deployment has to contain the path of your k8s resources within the applicat
 def gitopsConfig = [
         deployments: [
             sourcePath: 'k8s', // path of k8s resources in application repository. Default: 'k8s'
+            destinationRootPath: '.', // Root-Subfolder in the gitops repository, where the following folders for stages and apps shall be created. Default: '.'
             // Either "plain" or "helm" is mandatory
             plain: [], // use plain if you only have, as the name suggests, plain k8s resources
             helm: [] // or if you want to deploy a helm release use `helm`
@@ -601,6 +616,29 @@ We decided to generate plain k8s Resources from Helm applications before we push
 
 ---
 
+## Folder Structure in destination gitops repository
+
+You can customize in which path the final manifests of the application will be created in the gitops repository. For this, you can modify the following parameters:
+```groovy
+def gitopsConfig = [
+    deployments: [
+        destinationRootPath: '.' /* Default: '.' */
+    ],
+    folderStructureStrategy: 'GLOBAL_ENV' /* Default: 'GLOBAL_ENV', or ENV_PER_APP */
+]
+```
+* `destinationRootPath`: Specifies in which subfolder the following folders of `folderStructureStrategy` are created. Defaults to the root of the repository.
+* `folderStructureStrategy`: Possible values: 
+  * `GLOBAL_ENV`: The manifests will be commited into `$DESTINATION_ROOT_PATH/STAGE_NAME/APP_NAME/` in the destination gitops repository
+  * `ENV_PER_APP`: The manifests will be commited into `$DESTINATION_ROOT_PATH/APP_NAME/STAGE_NAME/` in the destination gitops repository
+
+
+Example for **Global Environments** vs **Environment per App** ([Source](https://github.com/cloudogu/gitops-patterns#implementing-release-promotion)):
+
+  ![Global Envs](https://github.com/cloudogu/gitops-talks/blob/1744c1d/images/global-environments.svg)
+  ![Env per app](https://github.com/cloudogu/gitops-talks/blob/1744c1d/images/environment-per-app.svg)
+
+---
 
 ## Extra Files
 

pom.xml

@@ -12,77 +12,59 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <jacoco.version>0.8.5</jacoco.version>
-        <maven.compiler.target>1.8</maven.compiler.target>
-        <maven.compiler.source>1.8</maven.compiler.source>
+        <jacoco.version>0.8.8</jacoco.version>
+        <maven.compiler.target>11</maven.compiler.target>
+        <maven.compiler.source>11</maven.compiler.source>
     </properties>
 
     <dependencies>
-        <dependency>
-            <groupId>com.cloudbees</groupId>
-            <artifactId>groovy-cps</artifactId>
-            <version>1.21</version>
-        </dependency>
-
         <dependency>
             <groupId>org.codehaus.groovy</groupId>
             <artifactId>groovy-all</artifactId>
-            <version>2.4.11</version>
+            <version>2.4.21</version>
+            <!-- Current version can be found out via Jenkins Groovy console: println GroovySystem.version-->
         </dependency>
 
         <dependency>
             <groupId>org.codehaus.groovy</groupId>
             <artifactId>groovy-yaml</artifactId>
-            <version>3.0.7</version>
+            <version>3.0.15</version>
             <scope>test</scope>
         </dependency>
 
 
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <version>5.6.2</version>
+            <version>5.9.2</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-junit-jupiter</artifactId>
-            <version>3.4.6</version>
+            <version>5.1.1</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>
-            <version>3.17.0</version>
+            <version>3.24.2</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>com.lesfurets</groupId>
             <artifactId>jenkins-pipeline-unit</artifactId>
-            <version>1.8</version>
+            <version>1.17</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.spockframework</groupId>
             <artifactId>spock-core</artifactId>
-            <version>1.1-groovy-2.4</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency> <!-- enables mocking of classes (in addition to interfaces) -->
-            <groupId>net.bytebuddy</groupId>
-            <artifactId>byte-buddy</artifactId>
-            <version>1.10.13</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency> <!-- enables mocking of classes without default constructor (together with
-        CGLIB) -->
-            <groupId>org.objenesis</groupId>
-            <artifactId>objenesis</artifactId>
-            <version>2.5.1</version>
+            <version>1.3-groovy-2.4</version>
             <scope>test</scope>
         </dependency>
 
@@ -91,7 +73,7 @@
             <groupId>com.github.cloudogu</groupId>
             <artifactId>ces-build-lib</artifactId>
             <!-- Keep this version in sync with the one used in Jenkinsfile -->
-            <version>1.45.0</version>
+            <version>1.62.0</version>
             <!-- Don't ship this dependency with the app -->
             <optional>true</optional>
             <!-- Don't inherit this dependency! -->
@@ -126,20 +108,20 @@
         <plugins>
             <plugin>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.8.0</version>
+                <version>3.11.0</version>
                 <configuration>
                     <compilerId>groovy-eclipse-compiler</compilerId>
                 </configuration>
                 <dependencies>
                     <dependency>
                         <groupId>org.codehaus.groovy</groupId>
                         <artifactId>groovy-eclipse-compiler</artifactId>
-                        <version>3.3.0-01</version>
+                        <version>3.7.0</version>
                     </dependency>
                     <dependency>
                         <groupId>org.codehaus.groovy</groupId>
                         <artifactId>groovy-eclipse-batch</artifactId>
-                        <version>2.5.6-01</version>
+                        <version>2.5.14-02</version>
                     </dependency>
                 </dependencies>
             </plugin>
@@ -159,7 +141,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-source-plugin</artifactId>
-                <version>3.0.1</version>
+                <version>3.2.1</version>
                 <executions>
                     <execution>
                         <id>attach-sources</id>