Skip to content

Commit faec607

Browse files
Prashansa-KPrashansa-K
authored
feat: migration utility for converting configuration from 2.8 to 3.4 (#1610)
* feat: added autofixing capability for migrating between 2.8 and 3.4 * feat: added linting rulesets for 2.8 to 3.4 config migrations * style: added clarifying comment * tests: added integration test * style: gofumpted the files * tests: restoring original working directory post tests * fix: addressed PR comments * style: added space where necessary in errors * tests: added other possible scoped plugins
1 parent ef4c758 commit faec607

File tree

12 files changed

+606
-14
lines changed

12 files changed

+606
-14
lines changed

‎cmd/file_convert.go

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,14 @@ can be converted into a 'kong-gateway-3.x' configuration file.`,
114114
Args: validateNoArgs,
115115
RunE: execute,
116116
PreRunE: func(_ *cobra.Command, _ []string) error {
117-
validSourceFormats := []string{string(convert.FormatKongGateway), string(convert.FormatKongGateway2x)}
118-
validDestinationFormats := []string{string(convert.FormatKonnect), string(convert.FormatKongGateway3x)}
117+
validSourceFormats := []string{
118+
string(convert.FormatKongGateway), string(convert.FormatKongGateway2x),
119+
string(convert.FormatKongGatewayVersion28x),
120+
}
121+
validDestinationFormats := []string{
122+
string(convert.FormatKonnect), string(convert.FormatKongGateway3x),
123+
string(convert.FormatKongGatewayVersion34x),
124+
}
119125

120126
err := validateInputFlag("from", convertCmdSourceFormat, validSourceFormats, "")
121127
if err != nil {
@@ -131,8 +137,14 @@ can be converted into a 'kong-gateway-3.x' configuration file.`,
131137
},
132138
}
133139

134-
sourceFormats := []convert.Format{convert.FormatKongGateway, convert.FormatKongGateway2x}
135-
destinationFormats := []convert.Format{convert.FormatKonnect, convert.FormatKongGateway3x}
140+
sourceFormats := []convert.Format{
141+
convert.FormatKongGateway, convert.FormatKongGateway2x,
142+
convert.FormatKongGatewayVersion28x,
143+
}
144+
destinationFormats := []convert.Format{
145+
convert.FormatKonnect, convert.FormatKongGateway3x,
146+
convert.FormatKongGatewayVersion34x,
147+
}
136148
convertCmd.Flags().StringVar(&convertCmdSourceFormat, "from", "",
137149
fmt.Sprintf("format of the source file, allowed formats: %v", sourceFormats))
138150
convertCmd.Flags().StringVar(&convertCmdDestinationFormat, "to", "",

‎convert/convert.go

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"strings"
77

88
"github.com/blang/semver/v4"
9+
"github.com/kong/deck/lint"
910
"github.com/kong/go-database-reconciler/pkg/cprint"
1011
"github.com/kong/go-database-reconciler/pkg/dump"
1112
"github.com/kong/go-database-reconciler/pkg/file"
@@ -27,6 +28,10 @@ const (
2728
FormatKongGateway2x Format = "kong-gateway-2.x"
2829
// FormatKongGateway3x represents the Kong gateway 3.x format.
2930
FormatKongGateway3x Format = "kong-gateway-3.x"
31+
32+
// Adding LTS version strings
33+
FormatKongGatewayVersion28x Format = "2.8"
34+
FormatKongGatewayVersion34x Format = "3.4"
3035
)
3136

3237
// AllFormats contains all available formats.
@@ -45,6 +50,10 @@ func ParseFormat(key string) (Format, error) {
4550
return FormatKongGateway3x, nil
4651
case FormatDistributed:
4752
return FormatDistributed, nil
53+
case FormatKongGatewayVersion28x:
54+
return FormatKongGatewayVersion28x, nil
55+
case FormatKongGatewayVersion34x:
56+
return FormatKongGatewayVersion34x, nil
4857
default:
4958
return "", fmt.Errorf("invalid format: '%v'", key)
5059
}
@@ -79,7 +88,7 @@ func Convert(
7988
if len(inputFilenames) > 1 {
8089
return fmt.Errorf("only one input file can be provided when converting from Kong 2.x to Kong 3.x format")
8190
}
82-
outputContent, err = convertKongGateway2xTo3x(inputContent, inputFilenames[0])
91+
outputContent, err = convertKongGateway2xTo3x(inputContent, inputFilenames[0], true)
8392
if err != nil {
8493
return err
8594
}
@@ -92,6 +101,25 @@ func Convert(
92101
return err
93102
}
94103

104+
case from == FormatKongGatewayVersion28x && to == FormatKongGatewayVersion34x:
105+
if len(inputFilenames) > 1 {
106+
return fmt.Errorf("only one input file can be provided when converting from Kong 2.x to Kong 3.x format")
107+
}
108+
outputContent, err = convertKongGateway28xTo34x(inputContent, inputFilenames[0])
109+
if err != nil {
110+
return err
111+
}
112+
113+
lintErrs, err := lint.Lint(inputFilenames[0], "convert/rulesets/280-to-340/entrypoint.yaml", "error", false)
114+
if err != nil {
115+
return err
116+
}
117+
118+
_, err = lint.GetLintOutput(lintErrs, "plain", "-")
119+
if err != nil {
120+
return err
121+
}
122+
95123
default:
96124
return fmt.Errorf("cannot convert from '%s' to '%s' format", from, to)
97125
}
@@ -100,7 +128,7 @@ func Convert(
100128
return err
101129
}
102130

103-
func convertKongGateway2xTo3x(input *file.Content, filename string) (*file.Content, error) {
131+
func convertKongGateway2xTo3x(input *file.Content, filename string, printFinalWarning bool) (*file.Content, error) {
104132
if input == nil {
105133
return nil, fmt.Errorf("input content is nil")
106134
}
@@ -147,14 +175,18 @@ func convertKongGateway2xTo3x(input *file.Content, filename string) (*file.Conte
147175

148176
cprint.UpdatePrintf(
149177
"From the '%s' config file,\n"+
150-
"the _format_version field has been migrated from '%s' to '%s'.\n"+
151-
"These automatic changes may not be correct or exhaustive enough, please\n"+
152-
"perform a manual audit of the config file.\n\n"+
153-
"For related information, please visit:\n"+
154-
"https://docs.konghq.com/deck/latest/3.0-upgrade\n\n",
178+
"the _format_version field has been migrated from '%s' to '%s'.\n\n",
155179
filename, outputContent.FormatVersion, "3.0")
156180
outputContent.FormatVersion = "3.0"
157181

182+
if printFinalWarning {
183+
cprint.UpdatePrintf(
184+
"\nThese automatic changes may not be correct or exhaustive enough, please\n" +
185+
"perform a manual audit of the config file.\n\n" +
186+
"For related information, please visit:\n" +
187+
"https://docs.konghq.com/deck/latest/3.0-upgrade\n\n")
188+
}
189+
158190
return outputContent, nil
159191
}
160192

@@ -247,3 +279,27 @@ func convertDistributedToKong(
247279
KongVersion: version.String(),
248280
})
249281
}
282+
283+
// convertKongGateway28xTo34x is used to convert a Kong Gateway 2.8.x config
284+
// to a Kong Gateway 3.4.x config. It can be used as a migration utility
285+
// between the two LTS versions. It auto-fixes some configuration. The
286+
// configuration that can't be autofixed is left as is and the user is shown
287+
// warnings/errors about the same.
288+
func convertKongGateway28xTo34x(input *file.Content, filename string) (*file.Content, error) {
289+
preprocessContent, err := convertKongGateway2xTo3x(input, filename, false)
290+
if err != nil {
291+
return nil, err
292+
}
293+
294+
outputContent := preprocessContent.DeepCopy()
295+
296+
updatePlugins(outputContent)
297+
298+
cprint.UpdatePrintf(
299+
"\nThese automatic changes may not be correct or exhaustive enough, please\n" +
300+
"perform a manual audit of the config file.\n\n" +
301+
"For related information, please visit:\n" +
302+
"https://docs.konghq.com/deck/latest/3.0-upgrade\n\n")
303+
304+
return outputContent, nil
305+
}

‎convert/convert_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -561,7 +561,7 @@ func Test_convertAutoFields(t *testing.T) {
561561
},
562562
}
563563

564-
got, err := convertKongGateway2xTo3x(content, "-")
564+
got, err := convertKongGateway2xTo3x(content, "-", true)
565565
require.NoError(t, err)
566566

567567
globalPluginConfig := got.Plugins[0].Config

‎convert/plugin_updates.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,18 @@ package convert
33
import (
44
"fmt"
55

6+
"github.com/kong/go-database-reconciler/pkg/cprint"
67
"github.com/kong/go-database-reconciler/pkg/file"
78
"github.com/kong/go-kong/kong"
89
)
910

1011
const (
1112
rateLimitingAdvancedPluginName = "rate-limiting-advanced"
1213
rlaNamespaceDefaultLength = 32
14+
awsLambdaPluginName = "aws-lambda"
15+
httpLogPluginName = "http-log"
16+
prefunctionPluginName = "pre-function"
17+
postfunctionPluginName = "post-function"
1318
)
1419

1520
func generateAutoFields(content *file.Content) error {
@@ -94,3 +99,94 @@ func autoGenerateNamespaceForRLAPluginConsumerGroups(plugin *kong.ConsumerGroupP
9499
}
95100
return nil
96101
}
102+
103+
func updatePlugins(content *file.Content) {
104+
for idx := range content.Plugins {
105+
plugin := &content.Plugins[idx]
106+
updateLegacyPluginConfig(plugin)
107+
}
108+
109+
for _, service := range content.Services {
110+
for _, plugin := range service.Plugins {
111+
updateLegacyPluginConfig(plugin)
112+
}
113+
114+
for _, route := range service.Routes {
115+
for _, plugin := range route.Plugins {
116+
updateLegacyPluginConfig(plugin)
117+
}
118+
}
119+
}
120+
121+
for _, route := range content.Routes {
122+
for _, plugin := range route.Plugins {
123+
updateLegacyPluginConfig(plugin)
124+
}
125+
}
126+
127+
for _, consumer := range content.Consumers {
128+
for _, plugin := range consumer.Plugins {
129+
updateLegacyPluginConfig(plugin)
130+
}
131+
}
132+
133+
for _, consumerGroup := range content.ConsumerGroups {
134+
for _, plugin := range consumerGroup.Plugins {
135+
updateLegacyPluginConfig(&file.FPlugin{
136+
Plugin: kong.Plugin{
137+
ID: plugin.ID,
138+
Name: plugin.Name,
139+
Config: plugin.Config,
140+
},
141+
})
142+
}
143+
}
144+
}
145+
146+
func updateLegacyPluginConfig(plugin *file.FPlugin) {
147+
if plugin != nil && plugin.Config != nil {
148+
config := plugin.Config.DeepCopy()
149+
150+
var pluginName string
151+
if plugin.Name != nil {
152+
pluginName = *plugin.Name
153+
}
154+
155+
config = updateLegacyFieldToNewField(config, "blacklist", "deny", pluginName)
156+
157+
config = updateLegacyFieldToNewField(config, "whitelist", "allow", pluginName)
158+
159+
if pluginName != "" {
160+
if pluginName == awsLambdaPluginName {
161+
config = removeDeprecatedFields3x(config, "proxy_scheme", pluginName)
162+
}
163+
if pluginName == prefunctionPluginName || pluginName == postfunctionPluginName {
164+
config = updateLegacyFieldToNewField(config, "functions", "access", pluginName)
165+
}
166+
}
167+
168+
plugin.Config = config
169+
}
170+
}
171+
172+
func updateLegacyFieldToNewField(pluginConfig kong.Configuration,
173+
oldField, newField, pluginName string,
174+
) kong.Configuration {
175+
if _, ok := pluginConfig[oldField]; ok {
176+
pluginConfig[newField] = pluginConfig[oldField]
177+
delete(pluginConfig, oldField)
178+
179+
cprint.UpdatePrintf("Automatically converted legacy configuration field \"%s\""+
180+
" to the new field \"%s\" in plugin %s\n",
181+
oldField, newField, pluginName)
182+
}
183+
return pluginConfig
184+
}
185+
186+
func removeDeprecatedFields3x(pluginConfig kong.Configuration, fieldName, pluginName string) kong.Configuration {
187+
if _, ok := pluginConfig[fieldName]; ok {
188+
delete(pluginConfig, fieldName)
189+
cprint.UpdatePrintf("Automatically removed deprecated config field \"%s\" from plugin %s\n", fieldName, pluginName)
190+
}
191+
return pluginConfig
192+
}

‎convert/rulesets/280-to-340/entrypoint.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
extends:
2+
- convert/rulesets/280-to-340/routes.yaml
3+
- convert/rulesets/280-to-340/plugins.yaml

‎convert/rulesets/280-to-340/plugins.yaml

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
rules:
2+
# plugins rules
3+
http-log-plugin-check:
4+
description: >-
5+
The `headers` field now only takes a single string per header name, where it
6+
previously took an array of values.
7+
given:
8+
- $.plugins[?(@.name == 'http-log')].config
9+
- $.services[*].plugins[?(@.name == 'http-log')].config
10+
- $.routes[*].plugins[?(@.name == 'http-log')].config
11+
- $.services[*].routes[*].plugins[?(@.name == 'http-log')].config
12+
- $.consumers[*].plugins[?(@.name == 'http-log')].config
13+
message: >-
14+
In 2.8, each header in the `headers` field could be specified as an array. In
15+
3.4, each header specified can only be specified as a string, and no longer
16+
supports arrays.
17+
severity: error
18+
then:
19+
field: headers
20+
function: schema
21+
functionOptions:
22+
schema:
23+
type: object
24+
additionalProperties:
25+
type: string
26+
statsd-advanced-plugin-check:
27+
description: >-
28+
The StatsD Advanced plugin has been deprecated and will be removed in 4.0. All capabilities
29+
are now available in the StatsD plugin.
30+
given:
31+
- $.plugins[?(@.name == 'statsd-advanced')].enabled
32+
- $.services[*].plugins[?(@.name == 'statsd-advanced')].enabled
33+
- $.routes[*].plugins[?(@.name == 'statsd-advanced')].enabled
34+
- $.services[*].routes[*].plugins[?(@.name == 'statsd-advanced')].enabled
35+
- $.consumers[*].plugins[?(@.name == 'statsd-advanced')].enabled
36+
message: >-
37+
The StatsD Advanced plugin has been deprecated and will be removed in 4.0. All capabilities
38+
are now available in the StatsD plugin.
39+
severity: error
40+
then:
41+
function: pattern
42+
functionOptions:
43+
match: "^true$"
44+
# The following rules are warning-level lint to remind users to review their own configurations,
45+
# which is hard to do through detection.
46+
acme-plugin-check:
47+
description: >-
48+
The `storage_config.vault.auth_method` configuration parameter now defaults to token. For more
49+
details, please visit the official docs: https://docs.konghq.com/hub/kong-inc/acme/changelog/
50+
given:
51+
- $.plugins[?(@.name == 'acme')].config.storage_config.vault
52+
- $.services[*].plugins[?(@.name == 'acme')].config.storage_config.vault
53+
- $.routes[*].plugins[?(@.name == 'acme')].config.storage_config.vault
54+
- $.services[*].routes[*].plugins[?(@.name == 'acme')].config.storage_config.vault
55+
- $.consumers[*].plugins[?(@.name == 'acme')].config.storage_config.vault
56+
message: >-
57+
The `storage_config.vault.auth_method` configuration parameter now defaults to `token`.
58+
For more details, please visit the official docs:
59+
https://docs.konghq.com/hub/kong-inc/acme/changelog/
60+
severity: warn
61+
then:
62+
- field: auth_method
63+
function: truthy
64+
prometheus-plugin-check:
65+
description: >-
66+
High cardinality metrics are now disabled by default. For more details, please visit the
67+
official docs: https://docs.konghq.com/hub/kong-inc/prometheus/changelog/
68+
given:
69+
- $.plugins[?(@.name == 'prometheus')].enabled
70+
- $.services[*].plugins[?(@.name == 'prometheus')].enabled
71+
- $.routes[*].plugins[?(@.name == 'prometheus')].enabled
72+
- $.services[*].routes[*].plugins[?(@.name == 'prometheus')].enabled
73+
- $.plugins[*].plugins[?(@.name == 'prometheus')].enabled
74+
message: >-
75+
High cardinality metrics are now disabled by default. You can selectively enable them per
76+
the documentation here: https://docs.konghq.com/hub/kong-inc/prometheus/#metrics-disabled-by-default
77+
severity: warn
78+
then:
79+
function: pattern
80+
functionOptions:
81+
match: "^true$"
82+
aws-lambda-plugin-check2:
83+
description: >-
84+
The AWS region is now required. You can set it through the plugin configuration with the
85+
`aws_region` field parameter, or with environment variables.
86+
given:
87+
- $.plugins[?(@.name == 'aws-lambda')].config
88+
- $.services[*].plugins[?(@.name == 'aws-lambda')].config
89+
- $.routes[*].plugins[?(@.name == 'aws-lambda')].config
90+
- $.services[*].routes[*].plugins[?(@.name == 'aws-lambda')].config
91+
- $.consumers[*].plugins[?(@.name == 'aws-lambda')].config
92+
message: >-
93+
The AWS region is now required. You can set it through the plugin configuration with the
94+
`aws_region` field parameter, or with environment variables.
95+
severity: warn
96+
then:
97+
- field: aws_region
98+
function: defined

0 commit comments

Comments
 (0)