From 7837d29269970088eaa019a2d7e61ecdfb68d985 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 21 Mar 2025 13:31:19 -0700 Subject: [PATCH] Fix upgrade test container version (#11996) * Fix upgrade test container version Signed-off-by: Derek Nola * Ensure CI cleanup on all docker tests Signed-off-by: Derek Nola Fix cleanup Signed-off-by: Derek Nola * Bump skew test timeout Signed-off-by: Derek Nola --------- Signed-off-by: Derek Nola --- scripts/test | 20 +++++++++---------- tests/docker/basics/basics_test.go | 7 ++++--- .../bootstraptoken/bootstraptoken_test.go | 5 +++-- tests/docker/cacerts/cacerts_test.go | 1 + tests/docker/conformance/conformance_test.go | 5 +++-- tests/docker/etcd/etcd_test.go | 5 +++-- tests/docker/lazypull/lazypull_test.go | 5 +++-- tests/docker/skew/skew_test.go | 9 +++++---- tests/docker/test-helpers.go | 5 +++-- tests/docker/upgrade/upgrade_test.go | 7 ++++--- 10 files changed, 39 insertions(+), 30 deletions(-) diff --git a/scripts/test b/scripts/test index cb7283e3a9a..dd9f05b6c5f 100755 --- a/scripts/test +++ b/scripts/test @@ -26,25 +26,25 @@ export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}" # Run all tests on tag events, as we want test failures to block the release if [ "$ARCH" == 'arm' ] || [ "$DRONE_BUILD_EVENT" = 'tag' ]; then - go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE" + go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE" -ci echo "Did go test basics $?" # Extract v1.XX minor version for skew and upgrade tests minor_version=$(echo $VERSION_K8S | cut -d '.' -f1,2) - go test ./tests/docker/cacerts/cacerts_test.go -k3sImage="$K3S_IMAGE" + go test ./tests/docker/cacerts/cacerts_test.go -k3sImage="$K3S_IMAGE" -ci echo "Did go test cacerts $?" - go test ./tests/docker/skew/skew_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version" + go test ./tests/docker/skew/skew_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version" -ci echo "Did go test skew $?" - go test ./tests/docker/bootstraptoken/bootstraptoken_test.go -k3sImage="$K3S_IMAGE" + go test ./tests/docker/bootstraptoken/bootstraptoken_test.go -k3sImage="$K3S_IMAGE" -ci echo "Did go test bootstraptoken $?" - go test ./tests/docker/upgrade/upgrade_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version" + go test ./tests/docker/upgrade/upgrade_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version" -ci echo "Did go test upgrade $?" - go test ./tests/docker/lazypull/lazypull_test.go -k3sImage="$K3S_IMAGE" + go test ./tests/docker/lazypull/lazypull_test.go -k3sImage="$K3S_IMAGE" -ci echo "Did go test lazypull $?" fi @@ -67,9 +67,9 @@ fi # --- if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then - run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -serial -ginkgo.v + run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -serial -ginkgo.v -ci echo "Did go conformance sqlite serial $?" - run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -serial -ginkgo.v + run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -serial -ginkgo.v -ci echo "Did go conformance etcd serial $?" test-run-sonobuoy mysql serial echo "Did test-run-sonobuoy-mysqk serial $?" @@ -87,9 +87,9 @@ if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then E2E_OUTPUT=$artifacts test-run-sonobuoy parallel echo "Did test-run-sonobuoy parallel $?" - run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -ginkgo.v + run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -ginkgo.v -ci echo "Did go conformance sqlite parallel $?" - run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -ginkgo.v + run-go-test ./tests/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -ginkgo.v -ci echo "Did go test conformance etcd parallel $?" test-run-sonobuoy mysql parallel echo "Did test-run-sonobuoy-mysql parallel $?" diff --git a/tests/docker/basics/basics_test.go b/tests/docker/basics/basics_test.go index bf7a80cf565..aa7479cf791 100644 --- a/tests/docker/basics/basics_test.go +++ b/tests/docker/basics/basics_test.go @@ -15,6 +15,7 @@ import ( ) var k3sImage = flag.String("k3sImage", "", "The image used to provision containers") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig func Test_DockerBasic(t *testing.T) { @@ -34,7 +35,7 @@ var _ = Describe("Basic Tests", Ordered, func() { Expect(config.ProvisionAgents(1)).To(Succeed()) Eventually(func() error { return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) - }, "120s", "5s").Should(Succeed()) + }, "180s", "5s").Should(Succeed()) Eventually(func() error { return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames()) }, "40s", "5s").Should(Succeed()) @@ -79,8 +80,8 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } }) diff --git a/tests/docker/bootstraptoken/bootstraptoken_test.go b/tests/docker/bootstraptoken/bootstraptoken_test.go index 2fe5f27038b..4fa969473a6 100644 --- a/tests/docker/bootstraptoken/bootstraptoken_test.go +++ b/tests/docker/bootstraptoken/bootstraptoken_test.go @@ -13,6 +13,7 @@ import ( ) var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig func Test_DockerBootstrapToken(t *testing.T) { @@ -67,7 +68,7 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } }) diff --git a/tests/docker/cacerts/cacerts_test.go b/tests/docker/cacerts/cacerts_test.go index c8714e608b8..2c9ce80c5b0 100644 --- a/tests/docker/cacerts/cacerts_test.go +++ b/tests/docker/cacerts/cacerts_test.go @@ -16,6 +16,7 @@ import ( ) var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig var testID string diff --git a/tests/docker/conformance/conformance_test.go b/tests/docker/conformance/conformance_test.go index 15129394653..6fd26accaf2 100644 --- a/tests/docker/conformance/conformance_test.go +++ b/tests/docker/conformance/conformance_test.go @@ -21,6 +21,7 @@ import ( var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") var db = flag.String("db", "", "The database to use for the tests (sqlite, etcd, mysql, postgres)") var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig func Test_DockerConformance(t *testing.T) { @@ -128,8 +129,8 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } }) diff --git a/tests/docker/etcd/etcd_test.go b/tests/docker/etcd/etcd_test.go index 87b61d2f6b9..8246e70743c 100644 --- a/tests/docker/etcd/etcd_test.go +++ b/tests/docker/etcd/etcd_test.go @@ -13,6 +13,7 @@ import ( ) var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig func Test_DockerEtcd(t *testing.T) { @@ -77,7 +78,7 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } }) diff --git a/tests/docker/lazypull/lazypull_test.go b/tests/docker/lazypull/lazypull_test.go index 0ad92a534fe..4ddf4137521 100644 --- a/tests/docker/lazypull/lazypull_test.go +++ b/tests/docker/lazypull/lazypull_test.go @@ -14,6 +14,7 @@ import ( ) var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig func Test_DockerLazyPull(t *testing.T) { @@ -76,8 +77,8 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } }) diff --git a/tests/docker/skew/skew_test.go b/tests/docker/skew/skew_test.go index 6d3e4035430..da949e03f98 100644 --- a/tests/docker/skew/skew_test.go +++ b/tests/docker/skew/skew_test.go @@ -18,6 +18,7 @@ import ( // the current commit build of K3s defined by var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s") var channel = flag.String("channel", "latest", "The release channel to test") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig func Test_DockerSkew(t *testing.T) { @@ -56,7 +57,7 @@ var _ = Describe("Skew Tests", Ordered, func() { Expect(config.ProvisionAgents(1)).To(Succeed()) Eventually(func() error { return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) - }, "120s", "5s").Should(Succeed()) + }, "180s", "5s").Should(Succeed()) }) It("should match respective versions", func() { for _, server := range config.Servers { @@ -101,7 +102,7 @@ var _ = Describe("Skew Tests", Ordered, func() { Expect(config.ProvisionServers(3)).To(Succeed()) Eventually(func() error { return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) - }, "120s", "5s").Should(Succeed()) + }, "180s", "5s").Should(Succeed()) Eventually(func(g Gomega) { g.Expect(tests.ParseNodes(config.KubeconfigFile)).To(HaveLen(3)) g.Expect(tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())).To(Succeed()) @@ -139,7 +140,7 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } }) diff --git a/tests/docker/test-helpers.go b/tests/docker/test-helpers.go index 80a030e392c..18c8f94a82d 100644 --- a/tests/docker/test-helpers.go +++ b/tests/docker/test-helpers.go @@ -390,6 +390,7 @@ func (config *TestConfig) RemoveNode(nodeName string) error { if _, err := RunCommand(cmd); err != nil { return fmt.Errorf("failed to remove node %s: %v", nodeName, err) } + fmt.Println("Stopped and removed ", nodeName) return nil } @@ -428,6 +429,7 @@ func (config *TestConfig) Cleanup() error { errs = append(errs, err) } } + config.Servers = nil // Stop and remove all agents for _, agent := range config.Agents { @@ -435,6 +437,7 @@ func (config *TestConfig) Cleanup() error { errs = append(errs, err) } } + config.Agents = nil // Stop DB if it was started if config.DBType == "mysql" || config.DBType == "postgres" { @@ -456,8 +459,6 @@ func (config *TestConfig) Cleanup() error { if config.TestDir != "" { return os.RemoveAll(config.TestDir) } - config.Agents = nil - config.Servers = nil return nil } diff --git a/tests/docker/upgrade/upgrade_test.go b/tests/docker/upgrade/upgrade_test.go index 34cdc3f5b42..94d6074ffa4 100644 --- a/tests/docker/upgrade/upgrade_test.go +++ b/tests/docker/upgrade/upgrade_test.go @@ -20,6 +20,7 @@ import ( // the current commit build of K3s defined by var k3sImage = flag.String("k3sImage", "", "The current commit build of K3s") var channel = flag.String("channel", "latest", "The release channel to test") +var ci = flag.Bool("ci", false, "running on CI, forced cleanup") var config *tester.TestConfig var numServers = 1 @@ -128,7 +129,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() { cVersion = strings.Replace(cVersion, "-amd64", "", 1) cVersion = strings.Replace(cVersion, "-arm64", "", 1) cVersion = strings.Replace(cVersion, "-arm", "", 1) - cVersion = strings.Replace(cVersion, "-", "+", 1) + cVersion = strings.Replace(cVersion, "-k3s", "+k3s", 1) Expect(out).To(ContainSubstring(cVersion)) } }) @@ -151,7 +152,7 @@ var _ = AfterSuite(func() { AddReportEntry("describe", docker.DescribeNodesAndPods(config)) AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...))) } - if config != nil && !failed { - config.Cleanup() + if config != nil && (*ci || !failed) { + Expect(config.Cleanup()).To(Succeed()) } })