feat: Added support for adding providers through mach config directly, without needing plugins

demeyerthom · demeyerthom · commit 5f428d06fff1 · 2025-07-08T15:09:53.000+02:00
diff --git a/.changes/unreleased/Added-20250627-152055.yaml b/.changes/unreleased/Added-20250627-152055.yaml
@@ -0,0 +1,3 @@
+kind: Added
+body: Added support for adding providers through mach config directly, without needing plugins
+time: 2025-06-27T15:20:55.908740714+02:00
diff --git a/go.mod b/go.mod
@@ -52,6 +52,7 @@ require (
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/flosch/pongo2/v5 v5.0.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
diff --git a/go.sum b/go.sum
@@ -745,6 +745,8 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/flosch/pongo2/v5 v5.0.0 h1:ZauMp+iPZzh2aI1QM2UwRb0lXD4BoFcvBuWqefkIuq0=
+github.com/flosch/pongo2/v5 v5.0.0/go.mod h1:6ysKu++8ANFXmc3x6uA6iVaS+PKUoDfdX3yPcv8TIzY=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
diff --git a/internal/cmd/testdata/cases/generate/aws-deployment-type-mixed/expected/main/test-1/component-2/main.tf b/internal/cmd/testdata/cases/generate/aws-deployment-type-mixed/expected/main/test-1/component-2/main.tf
@@ -30,6 +30,10 @@ module "component-2" {
   variables = {
     parent_names = [data.terraform_remote_state.test-1.outputs.component-1.name]
   }
+  component_version = "test"
+  environment       = "test"
+  site              = "test-1"
+  tags              = { "Component" : "component-2", "Site" : "test-1", "Version" : "test" }
 }
 
 output "component-2" {
diff --git a/internal/cmd/testdata/cases/generate/aws-deployment-type-mixed/expected/main/test-1/main.tf b/internal/cmd/testdata/cases/generate/aws-deployment-type-mixed/expected/main/test-1/main.tf
@@ -31,7 +31,11 @@ locals {
 
 # Component: component-1
 module "component-1" {
-  source = "{{ .PWD }}/testdata/modules/application"
+  source            = "{{ .PWD }}/testdata/modules/application"
+  component_version = "test"
+  environment       = "test"
+  site              = "test-1"
+  tags              = { "Component" : "component-1", "Site" : "test-1", "Version" : "test" }
 }
 
 output "component-1" {
diff --git a/internal/cmd/testdata/cases/generate/aws-deployment-type-site/expected/main/test-1/main.tf b/internal/cmd/testdata/cases/generate/aws-deployment-type-site/expected/main/test-1/main.tf
@@ -31,7 +31,11 @@ locals {
 
 # Component: component-1
 module "component-1" {
-  source = "{{ .PWD }}/testdata/modules/application"
+  source            = "{{ .PWD }}/testdata/modules/application"
+  component_version = "test"
+  environment       = "test"
+  site              = "test-1"
+  tags              = { "Component" : "component-1", "Site" : "test-1", "Version" : "test" }
 }
 
 output "component-1" {
@@ -42,7 +46,11 @@ output "component-1" {
 
 # Component: component-2
 module "component-2" {
-  source = "{{ .PWD }}/testdata/modules/application"
+  source            = "{{ .PWD }}/testdata/modules/application"
+  component_version = "test"
+  environment       = "test"
+  site              = "test-1"
+  tags              = { "Component" : "component-2", "Site" : "test-1", "Version" : "test" }
 }
 
 output "component-2" {
diff --git a/internal/cmd/testdata/cases/generate/aws-multisite/expected/main/test-1/component-1/main.tf b/internal/cmd/testdata/cases/generate/aws-multisite/expected/main/test-1/component-1/main.tf
@@ -15,7 +15,11 @@ terraform {
 # Resources
 # Component: component-1
 module "component-1" {
-  source = "{{ .PWD }}/testdata/modules/application"
+  source            = "{{ .PWD }}/testdata/modules/application"
+  component_version = "test"
+  environment       = "test"
+  site              = "test-1"
+  tags              = { "Component" : "component-1", "Site" : "test-1", "Version" : "test" }
 }
 
 output "component-1" {
diff --git a/internal/cmd/testdata/cases/generate/aws-multisite/expected/main/test-2/component-1/main.tf b/internal/cmd/testdata/cases/generate/aws-multisite/expected/main/test-2/component-1/main.tf
@@ -15,7 +15,11 @@ terraform {
 # Resources
 # Component: component-1
 module "component-1" {
-  source = "{{ .PWD }}/testdata/modules/application"
+  source            = "{{ .PWD }}/testdata/modules/application"
+  component_version = "test"
+  environment       = "test"
+  site              = "test-2"
+  tags              = { "Component" : "component-1", "Site" : "test-2", "Version" : "test" }
 }
 
 output "component-1" {
diff --git a/internal/cmd/testdata/modules/application/main.tf b/internal/cmd/testdata/modules/application/main.tf
@@ -16,6 +16,22 @@ variable "variables" {
   })
 }
 
+variable "component_version" {
+  type = string
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "site" {
+  type = string
+}
+
+variable "tags" {
+  type = map(string)
+}
+
 data "http" "example" {
   count = var.variables.fail == true ? 1 : 0
   url   = "fails"
diff --git a/internal/config/global.go b/internal/config/global.go
@@ -5,7 +5,9 @@ import (
 	"github.com/mach-composer/mach-composer-cli/internal/cli"
 	"github.com/mach-composer/mach-composer-cli/internal/config/variable"
 	"github.com/mach-composer/mach-composer-cli/internal/utils"
+	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v3"
+	"slices"
 )
 
 type GlobalConfig struct {
@@ -19,8 +21,9 @@ type GlobalConfig struct {
 }
 
 type TerraformConfig struct {
-	Providers   map[string]string `yaml:"providers"`
-	RemoteState map[string]any    `yaml:"remote_state"`
+	Providers       map[string]string `yaml:"providers"`
+	ProviderConfigs ProviderConfigs   `yaml:"provider_configs"`
+	RemoteState     map[string]any    `yaml:"remote_state"`
 }
 
 func parseGlobalNode(cfg *MachConfig, globalNode *yaml.Node) error {
@@ -56,6 +59,20 @@ func parseGlobalNode(cfg *MachConfig, globalNode *yaml.Node) error {
 		}
 	}
 
+	if cfg.Global.TerraformConfig != nil {
+		pluginNames := cfg.Plugins.AllNames()
+		providerNames, err := cfg.Global.TerraformConfig.ProviderConfigs.Names()
+		if err != nil {
+			return fmt.Errorf("failed to get provider names: %w", err)
+		}
+
+		for _, providerName := range providerNames {
+			if slices.Contains(pluginNames, providerName) {
+				log.Warn().Str("plugin", providerName).Str("name", providerName).Msgf("plugin exists with the same name as a provider: %s, this might cause duplicate providers in the generated Terraform code", providerName)
+			}
+		}
+	}
+
 	if node, ok := nodes["terraform_config"]; ok {
 		children := MapYamlNodes(node.Content)
 
diff --git a/internal/config/provider.go b/internal/config/provider.go
@@ -0,0 +1,18 @@
+package config
+
+type ProviderConfigs []ProviderConfig
+
+func (pc *ProviderConfigs) Names() ([]string, error) {
+	var names []string
+	for _, config := range *pc {
+		names = append(names, config.Name)
+	}
+	return names, nil
+}
+
+type ProviderConfig struct {
+	Name          string         `mapstructure:"name"`
+	Source        string         `mapstructure:"source"`
+	Version       string         `mapstructure:"version"`
+	Configuration map[string]any `mapstructure:"configuration,omitempty"`
+}
diff --git a/internal/config/schemas/schema-1.yaml b/internal/config/schemas/schema-1.yaml
@@ -101,7 +101,6 @@ definitions:
     required:
       - environment
       - terraform_config
-      - cloud
     properties:
       environment:
         type: string
@@ -127,6 +126,11 @@ definitions:
         patternProperties:
           "^[a-zA-Z-]+$":
             type: string
+      provider_configs:
+        description: "A list of provider configurations to generate."
+        type: array
+        items:
+          $ref: "#/definitions/ProviderConfig"
       remote_state:
         allOf:
           - type: object
@@ -145,6 +149,22 @@ definitions:
                   - terraform_cloud
           - $ref: "#/definitions/RemoteState"
 
+  ProviderConfig:
+    type: object
+    description: Provider configuration.
+    required:
+      - name
+    properties:
+      name:
+        type: string
+      source:
+        type: string
+      version:
+        type: string
+      configuration:
+        type: object
+        additionalProperties: true
+
   RemoteState:
     type: object
     properties: { }
diff --git a/internal/generator/component.go b/internal/generator/component.go
@@ -19,6 +19,7 @@ type componentContext struct {
 	ComponentHash       string
 	ComponentVariables  string
 	ComponentSecrets    string
+	ComponentTags       map[string]string
 	SiteName            string
 	Environment         string
 	SourceType          string
@@ -76,18 +77,22 @@ func renderSiteComponent(ctx context.Context, cfg *config.MachConfig, n *graph.S
 }
 
 // renderSiteComponentTerraformConfig uses templates/terraform.tmpl to generate a terraform snippet for each component
-func renderSiteComponentTerraformConfig(cfg *config.MachConfig, n graph.Node) (string, error) {
+func renderSiteComponentTerraformConfig(cfg *config.MachConfig, n *graph.SiteComponent) (string, error) {
 	tpl, err := templates.ReadFile("templates/terraform.tmpl")
 	if err != nil {
 		return "", err
 	}
 
-	site := n.(*graph.SiteComponent).SiteConfig
-	siteComponent := n.(*graph.SiteComponent).SiteComponentConfig
-
 	var providers []string
-	for _, plugin := range cfg.Plugins.Names(siteComponent.Definition.Integrations...) {
-		content, err := plugin.RenderTerraformProviders(site.Identifier)
+
+	genericProviderRequirements, err := renderRequirements(cfg)
+	if err != nil {
+		return "", fmt.Errorf("failed to render generic providers: %w", err)
+	}
+	providers = append(providers, genericProviderRequirements...)
+
+	for _, plugin := range cfg.Plugins.Names(n.SiteComponentConfig.Definition.Integrations...) {
+		content, err := plugin.RenderTerraformProviders(n.SiteConfig.Identifier)
 		if err != nil {
 			return "", fmt.Errorf("plugin %s failed to render providers: %w", plugin.Name, err)
 		}
@@ -98,7 +103,7 @@ func renderSiteComponentTerraformConfig(cfg *config.MachConfig, n graph.Node) (s
 
 	s, ok := cfg.StateRepository.Get(n.Identifier())
 	if !ok {
-		return "", fmt.Errorf("state repository does not have a backend for site %s", site.Identifier)
+		return "", fmt.Errorf("state repository does not have a backend for site %s", n.SiteConfig.Identifier)
 	}
 
 	bc, err := s.Backend()
@@ -113,7 +118,7 @@ func renderSiteComponentTerraformConfig(cfg *config.MachConfig, n graph.Node) (s
 	}{
 		Providers:     providers,
 		BackendConfig: bc,
-		IncludeSOPS:   cfg.Variables.HasEncrypted(site.Identifier),
+		IncludeSOPS:   cfg.Variables.HasEncrypted(n.SiteConfig.Identifier),
 	}
 	return utils.RenderGoTemplate(string(tpl), templateContext)
 }
@@ -125,7 +130,11 @@ func renderSiteComponentResources(cfg *config.MachConfig, n *graph.SiteComponent
 		return "", err
 	}
 
-	var resources []string
+	resources, err := renderProviders(cfg)
+	if err != nil {
+		return "", fmt.Errorf("failed to render generic providers: %w", err)
+	}
+
 	for _, plugin := range cfg.Plugins.Names(n.SiteComponentConfig.Definition.Integrations...) {
 		content, err := plugin.RenderTerraformResources(n.SiteConfig.Identifier)
 		if err != nil {
@@ -158,11 +167,17 @@ func renderComponentModule(_ context.Context, cfg *config.MachConfig, n *graph.S
 		SiteName:         n.SiteConfig.Identifier,
 		Environment:      cfg.Global.Environment,
 		Version:          n.SiteComponentConfig.Definition.Version,
-		SourceType:       string(sourceType),
-		PluginResources:  []string{},
-		PluginVariables:  []string{},
-		PluginDependsOn:  []string{},
-		PluginProviders:  []string{},
+		//TODO: allow setting additional tags via config
+		ComponentTags: map[string]string{
+			"Site":      n.SiteConfig.Identifier,
+			"Component": n.SiteComponentConfig.Name,
+			"Version":   n.SiteComponentConfig.Definition.Version,
+		},
+		SourceType:      string(sourceType),
+		PluginResources: []string{},
+		PluginVariables: []string{},
+		PluginDependsOn: []string{},
+		PluginProviders: []string{},
 	}
 
 	for _, plugin := range cfg.Plugins.Names(n.SiteComponentConfig.Definition.Integrations...) {
diff --git a/internal/generator/providers.go b/internal/generator/providers.go
@@ -0,0 +1,52 @@
+package generator
+
+import (
+	"embed"
+	"fmt"
+	"github.com/mach-composer/mach-composer-cli/internal/config"
+	"github.com/mach-composer/mach-composer-cli/internal/utils"
+)
+
+//go:embed templates/provider.tmpl
+var providerTmpl embed.FS
+
+type RenderProviderConfiguration struct {
+	Name          string
+	Configuration map[string]any `yaml:"configuration,omitempty"`
+}
+
+func renderProviders(cfg *config.MachConfig) ([]string, error) {
+	tpl, err := providerTmpl.ReadFile("templates/provider.tmpl")
+	if err != nil {
+		return nil, err
+	}
+
+	siteConfigs := cfg.Global.TerraformConfig.ProviderConfigs
+
+	var renderedProviders []string
+	for _, provider := range siteConfigs {
+		// Ensure all values in provider.Configuration are not complex types
+		for key, value := range provider.Configuration {
+			if _, ok := value.(map[any]any); ok {
+				return nil, fmt.Errorf("provider configuration for '%s' contains map value for key '%s'", provider.Name, key)
+			}
+			if _, ok := value.([]any); ok {
+				return nil, fmt.Errorf("provider configuration for '%s' contains slice value for key '%s'", provider.Name, key)
+			}
+		}
+
+		data := RenderProviderConfiguration{
+			Name:          provider.Name,
+			Configuration: provider.Configuration,
+		}
+
+		renderedProvider, err := utils.RenderGoTemplate(string(tpl), data)
+		if err != nil {
+			return nil, err
+		}
+
+		renderedProviders = append(renderedProviders, renderedProvider)
+	}
+
+	return renderedProviders, nil
+}
diff --git a/internal/generator/requirements.go b/internal/generator/requirements.go
diff --git a/internal/generator/site.go b/internal/generator/site.go
diff --git a/internal/generator/templates/provider.tmpl b/internal/generator/templates/provider.tmpl
diff --git a/internal/generator/templates/requirement.tmpl b/internal/generator/templates/requirement.tmpl
diff --git a/internal/generator/templates/site_component.tmpl b/internal/generator/templates/site_component.tmpl
diff --git a/internal/plugins/repository.go b/internal/plugins/repository.go
diff --git a/internal/utils/template.go b/internal/utils/template.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+kind: Added`
	`2`	`+body: Added support for adding providers through mach config directly, without needing plugins`
	`3`	`+time: 2025-06-27T15:20:55.908740714+02:00`
Original file line number	Diff line number	Diff line change
`@@ -30,6 +30,10 @@ module "component-2" {`
`30`	`30`	`variables = {`
`31`	`31`	`parent_names = [data.terraform_remote_state.test-1.outputs.component-1.name]`
`32`	`32`	`}`
	`33`	`+ component_version = "test"`
	`34`	`+ environment = "test"`
	`35`	`+ site = "test-1"`
	`36`	`+ tags = { "Component" : "component-2", "Site" : "test-1", "Version" : "test" }`
`33`	`37`	`}`
`34`	`38`
`35`	`39`	`output "component-2" {`