From 229446dd0340f9a58fe64339c434307373e85c5e Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 10 Sep 2024 17:34:51 +0200 Subject: [PATCH] Create paired upgrade tests (#8158) * Re-use channel.ChannelChain reconciler-test feature in upgrade tests * Fix glob -> global * Remove redundant feature name from test name * Remove Setup/Verify from test name * Use slices.Concat from Golang 1.22 * Include setupEnv and setupCtx in DurableFeature * Remove most pointers * Refactor and use FeatureGroupWithUpgradeTests * Try workaround for issues/8161 * Renames * Expose EnvOpts and make DurableFeature more configurable --- test/e2e-common.sh | 3 + test/rekt/features/channel/features.go | 28 ++- test/upgrade/postdowngrade.go | 27 --- test/upgrade/postupgrade.go | 14 -- test/upgrade/preupgrade.go | 27 --- test/upgrade/smoke.go | 39 ---- test/upgrade/upgrade.go | 272 ++++++++++++++++++++++++- test/upgrade/upgrade_test.go | 50 ++++- 8 files changed, 333 insertions(+), 127 deletions(-) delete mode 100644 test/upgrade/postdowngrade.go delete mode 100644 test/upgrade/preupgrade.go delete mode 100644 test/upgrade/smoke.go diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 73b87d3b634..1c78f80da88 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -187,6 +187,9 @@ function install_knative_eventing() { UNINSTALL_LIST+=( "${EVENTING_RELEASE_YAML}" ) fi + # Workaround for https://github.com/knative/eventing/issues/8161 + kubectl label namespace "${SYSTEM_NAMESPACE}" bindings.knative.dev/exclude=true --overwrite + # Setup config tracing for tracing tests local TMP_CONFIG_TRACING_CONFIG=${TMP_DIR}/${CONFIG_TRACING_CONFIG##*/} sed "s/namespace: ${KNATIVE_DEFAULT_NAMESPACE}/namespace: ${SYSTEM_NAMESPACE}/g" "${CONFIG_TRACING_CONFIG}" > "${TMP_CONFIG_TRACING_CONFIG}" diff --git a/test/rekt/features/channel/features.go b/test/rekt/features/channel/features.go index bc7238b51ef..da8a6e9bf5f 100644 --- a/test/rekt/features/channel/features.go +++ b/test/rekt/features/channel/features.go @@ -47,8 +47,16 @@ import ( func ChannelChain(length int, createSubscriberFn func(ref *duckv1.KReference, uri string) manifest.CfgFn) *feature.Feature { f := feature.NewFeature() - sink := feature.MakeRandomK8sName("sink") - cs := feature.MakeRandomK8sName("containersource") + + sink, channel := ChannelChainSetup(f, length, createSubscriberFn) + + ChannelChainAssert(f, sink, channel) + + return f +} + +func ChannelChainSetup(f *feature.Feature, length int, createSubscriberFn func(ref *duckv1.KReference, uri string) manifest.CfgFn) (sink string, channel string) { + sink = feature.MakeRandomK8sName("sink") var channels []string for i := 0; i < length; i++ { @@ -79,13 +87,19 @@ func ChannelChain(length int, createSubscriberFn func(ref *duckv1.KReference, ur f.Setup("subscription is ready", subscription.IsReady(sub)) } - // attach the first channel to the source - f.Requirement("install containersource", containersource.Install(cs, containersource.WithSink(channel_impl.AsDestinationRef(channels[0])))) - f.Requirement("containersource goes ready", containersource.IsReady(cs)) + return sink, channels[0] +} - f.Assert("chained channels relay events", assert.OnStore(sink).MatchEvent(test.HasType("dev.knative.eventing.samples.heartbeat")).AtLeast(1)) +func ChannelChainAssert(f *feature.Feature, sink, channel string) { + cs := feature.MakeRandomK8sName("containersource") + eventType := feature.MakeRandomK8sName("et") + args := "--eventType=" + eventType + f.Requirement("install containersource", containersource.Install(cs, + containersource.WithSink(channel_impl.AsDestinationRef(channel)), + containersource.WithArgs(args))) + f.Requirement("containersource goes ready", containersource.IsReady(cs)) - return f + f.Assert("chained channels relay events", assert.OnStore(sink).MatchEvent(test.HasType(eventType)).AtLeast(1)) } func DeadLetterSink(createSubscriberFn func(ref *duckv1.KReference, uri string) manifest.CfgFn) *feature.Feature { diff --git a/test/upgrade/postdowngrade.go b/test/upgrade/postdowngrade.go deleted file mode 100644 index aa8572112c5..00000000000 --- a/test/upgrade/postdowngrade.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - pkgupgrade "knative.dev/pkg/test/upgrade" -) - -func PostDowngradeTest() pkgupgrade.Operation { - return pkgupgrade.NewOperation("EventingPostDowngradeTest", func(c pkgupgrade.Context) { - runSmokeTest(c.T) - }) -} diff --git a/test/upgrade/postupgrade.go b/test/upgrade/postupgrade.go index 4009656bf50..f8de52789a5 100644 --- a/test/upgrade/postupgrade.go +++ b/test/upgrade/postupgrade.go @@ -22,12 +22,6 @@ import ( pkgupgrade "knative.dev/pkg/test/upgrade" ) -func SmokePostUpgradeTest() pkgupgrade.Operation { - return pkgupgrade.NewOperation("EventingPostUpgradeTest", func(c pkgupgrade.Context) { - runSmokeTest(c.T) - }) -} - func CRDPostUpgradeTest() pkgupgrade.Operation { return pkgupgrade.NewOperation("PostUpgradeCRDTest", func(c pkgupgrade.Context) { client := testlib.Setup(c.T, true) @@ -35,11 +29,3 @@ func CRDPostUpgradeTest() pkgupgrade.Operation { migrate.ExpectSingleStoredVersion(c.T, client.Apiextensions.CustomResourceDefinitions(), "knative.dev") }) } - -// PostUpgradeTests is an umbrella function for grouping all Eventing post-upgrade tests. -func PostUpgradeTests() []pkgupgrade.Operation { - return []pkgupgrade.Operation{ - SmokePostUpgradeTest(), - CRDPostUpgradeTest(), - } -} diff --git a/test/upgrade/preupgrade.go b/test/upgrade/preupgrade.go deleted file mode 100644 index 1107a303f10..00000000000 --- a/test/upgrade/preupgrade.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - pkgupgrade "knative.dev/pkg/test/upgrade" -) - -func PreUpgradeTest() pkgupgrade.Operation { - return pkgupgrade.NewOperation("EventingPreUpgradeTest", func(c pkgupgrade.Context) { - runSmokeTest(c.T) - }) -} diff --git a/test/upgrade/smoke.go b/test/upgrade/smoke.go deleted file mode 100644 index 0c57cefb705..00000000000 --- a/test/upgrade/smoke.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "testing" - - cloudevents "github.com/cloudevents/sdk-go/v2" - "knative.dev/eventing/test/e2e/helpers" - "knative.dev/eventing/test/lib" -) - -var channelTestRunner lib.ComponentsTestRunner - -func runSmokeTest(t *testing.T) { - helpers.SingleEventForChannelTestHelper( - context.Background(), - t, - cloudevents.EncodingBinary, - helpers.SubscriptionV1, - "", - channelTestRunner, - ) -} diff --git a/test/upgrade/upgrade.go b/test/upgrade/upgrade.go index 999dcdbeaad..6d00a4f6c68 100644 --- a/test/upgrade/upgrade.go +++ b/test/upgrade/upgrade.go @@ -17,23 +17,32 @@ limitations under the License. package upgrade import ( + "context" "log" "os" + "sync" "testing" - "knative.dev/eventing/test" - testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/rekt/features/channel" + "knative.dev/eventing/test/rekt/resources/channel_impl" + "knative.dev/eventing/test/rekt/resources/subscription" + duckv1 "knative.dev/pkg/apis/duck/v1" + "knative.dev/pkg/system" + pkgupgrade "knative.dev/pkg/test/upgrade" "knative.dev/pkg/test/zipkin" + "knative.dev/reconciler-test/pkg/environment" + "knative.dev/reconciler-test/pkg/feature" + "knative.dev/reconciler-test/pkg/k8s" + "knative.dev/reconciler-test/pkg/knative" + "knative.dev/reconciler-test/pkg/manifest" ) +var channelConfigMux = &sync.Mutex{} + // RunMainTest expects flags to be already initialized. // This function needs to be exposed, so that test cases in other repositories can call the upgrade // main tests in eventing. func RunMainTest(m *testing.M) { - channelTestRunner = testlib.ComponentsTestRunner{ - ComponentFeatureMap: testlib.ChannelFeatureMap, - ComponentsToTest: test.EventingFlags.Channels, - } os.Exit(func() int { // Any tests may SetupZipkinTracing, it will only actually be done once. This should be the ONLY // place that cleans it up. If an individual test calls this instead, then it will break other @@ -42,3 +51,254 @@ func RunMainTest(m *testing.M) { return m.Run() }()) } + +// DurableFeature holds the setup and verify phase of a feature. The "setup" phase should set up +// the feature. The "verify" phase should only verify its function. This function should be +// idempotent. Calling this function multiple times should still properly verify the feature +// (e.g. one call after upgrade, one call after downgrade). +type DurableFeature struct { + SetupF *feature.Feature + // EnvOpts should never include environment.Managed or environment.Cleanup as these functions + // break the functionality. + EnvOpts []environment.EnvOpts + setupEnv environment.Environment + setupCtx context.Context + VerifyF *feature.Feature + Global environment.GlobalEnvironment +} + +func (fe *DurableFeature) Setup(label string) pkgupgrade.Operation { + return pkgupgrade.NewOperation(label, func(c pkgupgrade.Context) { + c.T.Parallel() + ctx, env := fe.Global.Environment( + fe.EnvOpts..., + // Not managed - namespace preserved. + ) + fe.setupEnv = env + fe.setupCtx = ctx + env.Test(ctx, c.T, fe.SetupF) + }) +} + +func (fe *DurableFeature) Verify(label string) pkgupgrade.Operation { + return pkgupgrade.NewOperation(label, func(c pkgupgrade.Context) { + c.T.Parallel() + fe.setupEnv.Test(fe.setupCtx, c.T, fe.VerifyF) + }) +} + +func (fe *DurableFeature) VerifyAndTeardown(label string) pkgupgrade.Operation { + return pkgupgrade.NewOperation(label, func(c pkgupgrade.Context) { + c.T.Parallel() + fe.setupEnv.Test(fe.setupCtx, c.T, fe.VerifyF) + // Ensures teardown of resources/namespace. + fe.setupEnv.Finish() + }) +} + +func (fe *DurableFeature) SetupVerifyAndTeardown(label string) pkgupgrade.Operation { + return pkgupgrade.NewOperation(label, func(c pkgupgrade.Context) { + c.T.Parallel() + ctx, env := fe.Global.Environment( + append(fe.EnvOpts, environment.Managed(c.T))..., + ) + env.Test(ctx, c.T, fe.SetupF) + env.Test(ctx, c.T, fe.VerifyF) + }) +} + +type FeatureWithUpgradeTests interface { + PreUpgradeTests() []pkgupgrade.Operation + PostUpgradeTests() []pkgupgrade.Operation + PostDowngradeTests() []pkgupgrade.Operation +} + +// NewFeatureOnlyUpgrade decorates a feature with these actions: +// Pre-upgrade: Setup, Verify +// Post-upgrade: Verify, Teardown +// Post-downgrade: no-op. +func NewFeatureOnlyUpgrade(f *DurableFeature) FeatureWithUpgradeTests { + return featureOnlyUpgrade{ + label: "OnlyUpgrade", + feature: f, + } +} + +type featureOnlyUpgrade struct { + label string + feature *DurableFeature +} + +func (f featureOnlyUpgrade) PreUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + f.feature.Setup(f.label), + } +} + +func (f featureOnlyUpgrade) PostUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + f.feature.VerifyAndTeardown(f.label), + } +} + +func (f featureOnlyUpgrade) PostDowngradeTests() []pkgupgrade.Operation { + // No-op. Teardown was done post-upgrade. + return nil +} + +// NewFeatureUpgradeDowngrade decorates a feature with these actions: +// Pre-upgrade: Setup, Verify. +// Post-upgrade: Verify. +// Post-downgrade: Verify, Teardown. +func NewFeatureUpgradeDowngrade(f *DurableFeature) FeatureWithUpgradeTests { + return featureUpgradeDowngrade{ + label: "BothUpgradeDowngrade", + feature: f, + } +} + +type featureUpgradeDowngrade struct { + label string + feature *DurableFeature +} + +func (f featureUpgradeDowngrade) PreUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + f.feature.Setup(f.label), + } +} + +func (f featureUpgradeDowngrade) PostUpgradeTests() []pkgupgrade.Operation { + // PostUpgrade only asserts existing resources. Teardown will be done post-downgrade. + return []pkgupgrade.Operation{ + f.feature.Verify(f.label), + } +} + +func (f featureUpgradeDowngrade) PostDowngradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + f.feature.VerifyAndTeardown(f.label), + } +} + +// NewFeatureOnlyDowngrade decorates a feature with these actions: +// Pre-upgrade: no-op. +// Post-upgrade: Setup, Verify. +// Post-downgrade: Verify, Teardown. +func NewFeatureOnlyDowngrade(f *DurableFeature) FeatureWithUpgradeTests { + return featureOnlyDowngrade{ + label: "OnlyDowngrade", + feature: f, + } +} + +type featureOnlyDowngrade struct { + label string + feature *DurableFeature +} + +func (f featureOnlyDowngrade) PreUpgradeTests() []pkgupgrade.Operation { + // No-op. Resources will be created post-upgrade. + return nil +} + +func (f featureOnlyDowngrade) PostUpgradeTests() []pkgupgrade.Operation { + // Resources created post-upgrade. + return []pkgupgrade.Operation{ + f.feature.Setup(f.label), + } +} + +func (f featureOnlyDowngrade) PostDowngradeTests() []pkgupgrade.Operation { + // Assert and Teardown is done post-downgrade. + return []pkgupgrade.Operation{ + f.feature.VerifyAndTeardown(f.label), + } +} + +// NewFeatureSmoke decorates a feature with these actions: +// Pre-upgrade: no-op. +// Post-upgrade: Setup, Verify, Teardown. +// Post-downgrade: Setup, Verify, Teardown. +func NewFeatureSmoke(f *DurableFeature) FeatureWithUpgradeTests { + return featureSmoke{ + label: "Smoke", + feature: f, + } +} + +type featureSmoke struct { + label string + feature *DurableFeature +} + +func (f featureSmoke) PreUpgradeTests() []pkgupgrade.Operation { + // No-op. No need to smoke test before upgrade. + return nil +} + +func (f featureSmoke) PostUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + f.feature.SetupVerifyAndTeardown(f.label), + } +} + +func (f featureSmoke) PostDowngradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + f.feature.SetupVerifyAndTeardown(f.label), + } +} + +// FeatureGroupWithUpgradeTests aggregates tests across a group of features. +type FeatureGroupWithUpgradeTests []FeatureWithUpgradeTests + +func (fg FeatureGroupWithUpgradeTests) PreUpgradeTests() []pkgupgrade.Operation { + ops := make([]pkgupgrade.Operation, 0, len(fg)) + for _, ft := range fg { + ops = append(ops, ft.PreUpgradeTests()...) + } + return ops +} + +func (fg FeatureGroupWithUpgradeTests) PostUpgradeTests() []pkgupgrade.Operation { + ops := make([]pkgupgrade.Operation, 0, len(fg)) + for _, ft := range fg { + ops = append(ops, ft.PostUpgradeTests()...) + } + return ops +} + +func (fg FeatureGroupWithUpgradeTests) PostDowngradeTests() []pkgupgrade.Operation { + ops := make([]pkgupgrade.Operation, 0, len(fg)) + for _, ft := range fg { + ops = append(ops, ft.PostDowngradeTests()...) + } + return ops +} + +func InMemoryChannelFeature(glob environment.GlobalEnvironment) *DurableFeature { + // Prevent race conditions on channel_impl.EnvCfg.ChannelGK when running tests in parallel. + channelConfigMux.Lock() + defer channelConfigMux.Unlock() + channel_impl.EnvCfg.ChannelGK = "InMemoryChannel.messaging.knative.dev" + channel_impl.EnvCfg.ChannelV = "v1" + + createSubscriberFn := func(ref *duckv1.KReference, uri string) manifest.CfgFn { + return subscription.WithSubscriber(ref, uri, "") + } + + setupF := feature.NewFeature() + sink, ch := channel.ChannelChainSetup(setupF, 1, createSubscriberFn) + + verifyF := feature.NewFeature() + channel.ChannelChainAssert(verifyF, sink, ch) + + opts := []environment.EnvOpts{ + knative.WithKnativeNamespace(system.Namespace()), + knative.WithLoggingConfig, + knative.WithTracingConfig, + k8s.WithEventListener, + } + + return &DurableFeature{SetupF: setupF, VerifyF: verifyF, Global: glob, EnvOpts: opts} +} diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index 1e2faac4f7d..2c38f02109d 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -21,15 +21,21 @@ package upgrade import ( "flag" + "log" + "slices" "testing" "knative.dev/eventing/test" testlib "knative.dev/eventing/test/lib" "knative.dev/eventing/test/upgrade/installation" "knative.dev/pkg/system" + pkgtest "knative.dev/pkg/test" pkgupgrade "knative.dev/pkg/test/upgrade" + "knative.dev/reconciler-test/pkg/environment" ) +var global environment.GlobalEnvironment + func TestEventingUpgrades(t *testing.T) { labels := []string{ "eventing-controller", @@ -43,15 +49,31 @@ func TestEventingUpgrades(t *testing.T) { canceler := testlib.ExportLogStreamOnError(t, testlib.SystemLogsDir, system.Namespace(), labels...) defer canceler() + g := FeatureGroupWithUpgradeTests{ + // A feature that will run the same test post-upgrade and post-downgrade. + NewFeatureSmoke(InMemoryChannelFeature(global)), + // A feature that will be created pre-upgrade and verified/removed post-upgrade. + NewFeatureOnlyUpgrade(InMemoryChannelFeature(global)), + // A feature that will be created pre-upgrade, verified post-upgrade, verified and removed post-downgrade. + NewFeatureUpgradeDowngrade(InMemoryChannelFeature(global)), + // A feature that will be created post-upgrade, verified and removed post-downgrade. + NewFeatureOnlyDowngrade(InMemoryChannelFeature(global)), + } + suite := pkgupgrade.Suite{ Tests: pkgupgrade.Tests{ - PreUpgrade: []pkgupgrade.Operation{ - PreUpgradeTest(), - }, - PostUpgrade: PostUpgradeTests(), - PostDowngrade: []pkgupgrade.Operation{ - PostDowngradeTest(), - }, + PreUpgrade: slices.Concat( + g.PreUpgradeTests(), + ), + PostUpgrade: slices.Concat( + []pkgupgrade.Operation{ + CRDPostUpgradeTest(), + }, + g.PostUpgradeTests(), + ), + PostDowngrade: slices.Concat( + g.PostDowngradeTests(), + ), Continual: []pkgupgrade.BackgroundOperation{ ContinualTest(), }, @@ -73,6 +95,20 @@ func TestEventingUpgrades(t *testing.T) { func TestMain(m *testing.M) { test.InitializeEventingFlags() + + restConfig, err := pkgtest.Flags.ClientConfig.GetRESTConfig() + if err != nil { + log.Fatal("Error building client config: ", err) + } + + // Getting the rest config explicitly and passing it further will prevent re-initializing the flagset + // in NewStandardGlobalEnvironment(). The upgrade tests use knative.dev/pkg/test which initializes the + // flagset as well. + global = environment.NewStandardGlobalEnvironment(func(cfg environment.Configuration) environment.Configuration { + cfg.Config = restConfig + return cfg + }) + flag.Parse() RunMainTest(m) }