From 7c3f21e58155acc9fa1bf7dfb1c2c570fedff199 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 28 Oct 2021 12:35:28 -0700 Subject: [PATCH] K3s Integration test fixes (#4341) * Move tests into sub folders * Updated documentation * Prevent infinite loop is user has not made k3s Signed-off-by: dereknola --- pkg/etcd/etcd_int_test.go | 2 +- scripts/build-tests-sonobuoy | 18 ++++++--- tests/TESTING.md | 10 ++--- .../dualstack_int_test.go} | 22 +++++++---- .../localstorage_int_test.go | 17 +++++---- tests/util/cmd.go | 37 ++++++++++++++----- 6 files changed, 68 insertions(+), 38 deletions(-) rename tests/integration/{dual_stack_int_test.go => dualstack/dualstack_int_test.go} (61%) rename tests/integration/{ => localstorage}/localstorage_int_test.go (83%) diff --git a/pkg/etcd/etcd_int_test.go b/pkg/etcd/etcd_int_test.go index 3d7f17a0343..4ed298fa5f9 100644 --- a/pkg/etcd/etcd_int_test.go +++ b/pkg/etcd/etcd_int_test.go @@ -32,7 +32,7 @@ var _ = Describe("etcd snapshots", func() { It("starts up with no problems", func() { Eventually(func() (string, error) { return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + }, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) }) It("saves an etcd snapshot", func() { Expect(testutil.K3sCmd("etcd-snapshot", "save")). diff --git a/scripts/build-tests-sonobuoy b/scripts/build-tests-sonobuoy index 3e0ff4e05a2..430dc8356e6 100755 --- a/scripts/build-tests-sonobuoy +++ b/scripts/build-tests-sonobuoy @@ -8,16 +8,22 @@ OUTFILE="./dist/artifacts/k3s-int-tests.yaml" # Compile all integration tests and containerize them mkdir -p dist/artifacts -go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-1.test ./tests/integration/... -run Integration +# Integration tests under /pkg PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |sort -u) -INDEX=1 for i in $PKG_TO_TEST; do - echo $i - go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$INDEX.test $i -run Integration - INDEX=$(expr $INDEX + 1) + name=$(echo "${i##*/}") + echo $name + go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration +done + +# Integration tests under /tests +PKG_TO_TEST=$(find ./tests/integration -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |sort -u) +for i in $PKG_TO_TEST; do + name=$(echo "${i##*/}") + echo $name + go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration done -go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$INDEX.test ./tests/integration/... -run Integration docker build -f ./tests/integration/Dockerfile.test -t $REPO . docker save $REPO -o ./dist/artifacts/$REPO.tar diff --git a/tests/TESTING.md b/tests/TESTING.md index 169e7149294..983d58c7462 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -60,25 +60,25 @@ To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions. Integration tests can be placed in two areas: 1. Next to the go package they intend to test. -2. In `tests/integration/` for package agnostic testing. +2. In `tests/integration/` for package agnostic testing. Package specific integration tests should use the `_test` package. Package agnostic integration tests should use the `integration` package. All integration test files should be named: `_int_test.go` All integration test functions should be named: `Test_Integration`. See the [etcd snapshot test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_int_test.go) as a package specific example. -See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage_int_test.go) as a package agnostic example. +See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage/localstorage_int_test.go) as a package agnostic example. ### Running Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs. ```bash -go test ./pkg/... ./tests/... -run Integration +go test ./pkg/... ./tests/integration/... -run Integration ``` Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly. ```bash -go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/... -run Integration +go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/integration/... -run Integration ``` Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster. @@ -111,7 +111,7 @@ See the [upgrade cluster test](https://github.com/k3s-io/k3s/blob/master/tests/e Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. ```bash -go test ./tests/... -run E2E +go test ./tests/e2e... -run E2E ``` ## Contributing New Or Updated Tests diff --git a/tests/integration/dual_stack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go similarity index 61% rename from tests/integration/dual_stack_int_test.go rename to tests/integration/dualstack/dualstack_int_test.go index 44607707a0d..0c26eef99ba 100644 --- a/tests/integration/dual_stack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -10,11 +10,17 @@ import ( testutil "github.com/rancher/k3s/tests/util" ) -var dualStackServerArgs = []string{"--cluster-init", "--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56", "--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112"} +var dualStackServer *testutil.K3sServer +var dualStackServerArgs = []string{ + "--cluster-init", + "--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56", + "--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112", + "--disable-network-policy", +} var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error - server, err = testutil.K3sStartServer(dualStackServerArgs...) + dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs...) Expect(err).ToNot(HaveOccurred()) } }) @@ -29,21 +35,21 @@ var _ = Describe("dual stack", func() { It("starts up with no problems", func() { Eventually(func() (string, error) { return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "90s", "1s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running")) + }, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running")) }) It("creates pods with two IPs", func() { - podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-nkube-system", "-ojsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}") - Expect(err).NotTo(HaveOccurred()) - result, err := testutil.K3sCmd("kubectl", "exec", podname, "-nkube-system", "--", "ip", "a") - Expect(result).To(ContainSubstring("2001:cafe:42:")) + podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}") Expect(err).NotTo(HaveOccurred()) + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "exec", podname, "-n", "kube-system", "--", "ip", "a") + }, "5s", "1s").Should(ContainSubstring("2001:cafe:42:")) }) }) }) var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(server)).To(Succeed()) + Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed()) } }) diff --git a/tests/integration/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go similarity index 83% rename from tests/integration/localstorage_int_test.go rename to tests/integration/localstorage/localstorage_int_test.go index ae3df128567..c2bdc1b8d44 100644 --- a/tests/integration/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -13,20 +13,21 @@ import ( testutil "github.com/rancher/k3s/tests/util" ) -var server *testutil.K3sServer -var serverArgs = []string{"--cluster-init"} +var localStorageServer *testutil.K3sServer +var localStorageServerArgs = []string{"--cluster-init"} +var testDataDir = "../../testdata/" var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error - server, err = testutil.K3sStartServer(serverArgs...) + localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs...) Expect(err).ToNot(HaveOccurred()) } }) var _ = Describe("local storage", func() { BeforeEach(func() { - if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) { - Skip("Test needs k3s server with: " + strings.Join(serverArgs, " ")) + if testutil.IsExistingServer() && !testutil.ServerArgsPresent(localStorageServerArgs) { + Skip("Test needs k3s server with: " + strings.Join(localStorageServerArgs, " ")) } }) When("a new local storage is created", func() { @@ -36,12 +37,12 @@ var _ = Describe("local storage", func() { }, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) }) It("creates a new pvc", func() { - result, err := testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pvc.yaml") + result, err := testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pvc.yaml") Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created")) Expect(err).NotTo(HaveOccurred()) }) It("creates a new pod", func() { - Expect(testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pod.yaml")). + Expect(testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pod.yaml")). To(ContainSubstring("pod/volume-test created")) }) It("shows storage up in kubectl", func() { @@ -81,7 +82,7 @@ var _ = Describe("local storage", func() { var _ = AfterSuite(func() { if !testutil.IsExistingServer() { - Expect(testutil.K3sKillServer(server)).To(Succeed()) + Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed()) } }) diff --git a/tests/util/cmd.go b/tests/util/cmd.go index 49172e8700a..b3c36e46cae 100644 --- a/tests/util/cmd.go +++ b/tests/util/cmd.go @@ -15,6 +15,14 @@ import ( // Compile-time variable var existingServer = "False" +const lockFile = "/var/lock/k3s-test.lock" + +type K3sServer struct { + cmd *exec.Cmd + scanner *bufio.Scanner + lock int +} + func findK3sExecutable() string { // if running on an existing cluster, it maybe installed via k3s.service // or run manually from dist/artifacts/k3s @@ -25,7 +33,8 @@ func findK3sExecutable() string { } } k3sBin := "dist/artifacts/k3s" - for { + i := 0 + for ; i < 20; i++ { _, err := os.Stat(k3sBin) if err != nil { k3sBin = "../" + k3sBin @@ -33,6 +42,9 @@ func findK3sExecutable() string { } break } + if i == 20 { + logrus.Fatal("Unable to find k3s executable") + } return k3sBin } @@ -112,22 +124,21 @@ func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool { return false } -type K3sServer struct { - cmd *exec.Cmd - scanner *bufio.Scanner - lock int -} - // K3sStartServer acquires an exclusive lock on a temporary file, then launches a k3s cluster // with the provided arguments. Subsequent/parallel calls to this function will block until // the original lock is cleared using K3sKillServer -func K3sStartServer(cmdArgs ...string) (*K3sServer, error) { +func K3sStartServer(inputArgs ...string) (*K3sServer, error) { logrus.Info("waiting to get server lock") - k3sLock, err := flock.Acquire("/var/lock/k3s-test.lock") + k3sLock, err := flock.Acquire(lockFile) if err != nil { return nil, err } + var cmdArgs []string + for _, arg := range inputArgs { + cmdArgs = append(cmdArgs, strings.Fields(arg)...) + } + k3sBin := findK3sExecutable() var cmd *exec.Cmd if IsRoot() { @@ -157,5 +168,11 @@ func K3sKillServer(server *K3sServer) error { return err } } - return flock.Release(server.lock) + if err := flock.Release(server.lock); err != nil { + return err + } + if !flock.CheckLock(lockFile) { + return os.Remove(lockFile) + } + return nil }