diff --git a/test/e2e_node/builder/build.go b/test/e2e_node/builder/build.go index cfe8688aa95..b700ae38a56 100644 --- a/test/e2e_node/builder/build.go +++ b/test/e2e_node/builder/build.go @@ -31,51 +31,37 @@ var k8sBinDir = CommandLine.String("k8s-bin-dir", "", "Directory containing k8s var useDockerizedBuild = CommandLine.Bool("use-dockerized-build", false, "Use dockerized build for test artifacts") var targetBuildArch = CommandLine.String("target-build-arch", "linux/amd64", "Target architecture for the test artifacts for dockerized build") -var buildCGOTargets = []string{ +// buildTargets is what the `test/e2e_node/runners/remote` builds via `make WHAT=` +// when invoked via "make test-e2e-node". In this mode, separate binaries are +// used for each command. +var buildTargets = []string{ "cmd/kubelet", -} - -var buildNoCGOTargets = []string{ "test/e2e_node/e2e_node.test", "github.com/onsi/ginkgo/v2/ginkgo", "cluster/gce/gci/mounter", "test/e2e_node/plugins/gcp-credential-provider", } -// BuildGo builds k8s binaries. +// BuildGo builds some default k8s binaries. func BuildGo() error { - if err := BuildTargets(true); err != nil { - return fmt.Errorf("unable to build cgo targets : %w", err) - } - if err := BuildTargets(false); err != nil { - return fmt.Errorf("unable to build non-cgo targets : %w", err) - } - return nil + return BuildTargets(buildTargets...) } -// BuildGo builds k8s binaries. -func BuildTargets(cgo bool) error { - klog.Infof("Building k8s binaries...") +// BuildTargets builds the specified k8s binaries (= WHAT targets). +func BuildTargets(targets ...string) error { k8sRoot, err := utils.GetK8sRootDir() if err != nil { return fmt.Errorf("failed to locate kubernetes root directory %v", err) } - targets := buildCGOTargets - if !cgo { - targets = buildNoCGOTargets - } + arch := GetTargetBuildArch() + klog.Infof("Building k8s binaries %v in %q for %s...", targets, k8sRoot, arch) what := strings.Join(targets, " ") cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", what)) - if cgo { - cmd.Args = append(cmd.Args, "CGO_ENABLED=1") - } else { - cmd.Args = append(cmd.Args, "CGO_ENABLED=0") - } if IsDockerizedBuild() { klog.Infof("Building dockerized k8s binaries targets %s for architecture %s", targets, GetTargetBuildArch()) // Multi-architecture build is only supported in dockerized build - cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", what), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", GetTargetBuildArch())) + cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", what), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", arch)) // Ensure we run this command in k8s root directory for dockerized build cmd.Dir = k8sRoot } diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index b6ac3d71072..74c250db772 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -26,6 +26,8 @@ import ( "encoding/json" "flag" "fmt" + "path/filepath" + "strings" "os" "os/exec" @@ -48,6 +50,9 @@ import ( e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" "k8s.io/kubernetes/test/e2e_node/criproxy" + "k8s.io/kubernetes/test/e2e_node/mounter" + gcpcredentialprovider "k8s.io/kubernetes/test/e2e_node/plugins/gcp-credential-provider/pkg" + "k8s.io/kubernetes/test/e2e_node/runner/node" "k8s.io/kubernetes/test/e2e_node/services" e2enodetestingmanifests "k8s.io/kubernetes/test/e2e_node/testing-manifests" system "k8s.io/system-validators/validators" @@ -120,6 +125,37 @@ func init() { } func TestMain(m *testing.M) { + // e2e_node.test can behave like several other commands which are required + // when doing node testing on a remote virtual machine. + // + // e2e_node.test owns pflag.CommandLine and flag.CommandLine. + // Other commands must use separate FlagSets. + pflag.Usage = func() { + fmt.Fprint(pflag.CommandLine.Output(), `Usage when invoked under a different name: + gcp-credential-provider (no flags) - emulate test/e2e_node/plugins/gcp-credential-provider + mounter (no flags) - emulate cluster/gce/gci/mounter + +Usage as e2e_node.test: + e2e_node.test remote - run remote testing, replacing "testgrid2 noop -test=node", see "remote -help" for flags + e2e_node.test - execute Ginkgo test suite, see following flags + +`) + pflag.CommandLine.PrintDefaults() + } + cmdName := filepath.Base(os.Args[0]) + switch { + case strings.HasPrefix(cmdName, "gcp-credential-provider"): + gcpcredentialprovider.Main() + case strings.HasPrefix(cmdName, "mounter"): + mounter.Main() + case len(os.Args) > 1 && os.Args[1] == "remote": + node.Main(os.Args[2:]) + default: + testMain(m) + } +} + +func testMain(m *testing.M) { // Copy go flags in TestMain, to ensure go test flags are registered (no longer available in init() as of go1.13) e2econfig.CopyFlags(e2econfig.Flags, flag.CommandLine) framework.RegisterCommonFlags(flag.CommandLine) diff --git a/test/e2e_node/mounter/mounter.go b/test/e2e_node/mounter/mounter.go new file mode 100644 index 00000000000..9ae364690e6 --- /dev/null +++ b/test/e2e_node/mounter/mounter.go @@ -0,0 +1,97 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mounter is a temporary copy of cluster/gce/gci/mounter for use in the e2e_node +// binary itself. If that approach works out, cluster/gce/gci/mounter will be refactored +// to remove the code duplication, otherwise this will get reverted. +package mounter + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + // Location of the mount file to use + chrootCmd = "chroot" + mountCmd = "mount" + mountBin = "/bin/mount" + rootfs = "rootfs" + nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n" + rpcBindCmd = "/sbin/rpcbind" + defaultRootfs = "/home/kubernetes/containerized_mounter/rootfs" +) + +func Main() { + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n") + return + } + path, _ := filepath.Split(os.Args[0]) + rootfsPath := filepath.Join(path, rootfs) + if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { + rootfsPath = defaultRootfs + } + command := os.Args[1] + switch command { + case mountCmd: + mountErr := mountInChroot(rootfsPath, os.Args[2:]) + if mountErr != nil { + fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr) + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd) + os.Exit(1) + + } +} + +// mountInChroot runs mount within chroot with the passing root directory +func mountInChroot(rootfsPath string, args []string) error { + if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { + return fmt.Errorf("path <%s> does not exist", rootfsPath) + } + args = append([]string{rootfsPath, mountBin}, args...) + output, err := exec.Command(chrootCmd, args...).CombinedOutput() + if err == nil { + return nil + } + + if !strings.EqualFold(string(output), nfsRPCBindErrMsg) { + // Mount failed but not because of RPC bind error + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s", err, chrootCmd, args, string(output)) + } + + // Mount failed because it is NFS V3 and we need to run rpcBind + output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput() + if err != nil { + return fmt.Errorf("mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err) + } + + // Rpcbind is running, try mounting again + output, err = exec.Command(chrootCmd, args...).CombinedOutput() + + if err != nil { + return fmt.Errorf("mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err) + } + + return nil +} diff --git a/test/e2e_node/plugins/gcp-credential-provider/pkg/main.go b/test/e2e_node/plugins/gcp-credential-provider/pkg/main.go new file mode 100644 index 00000000000..ede36b34e3f --- /dev/null +++ b/test/e2e_node/plugins/gcp-credential-provider/pkg/main.go @@ -0,0 +1,179 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredentialsprovider + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "reflect" + "strings" + "time" + + "gopkg.in/go-jose/go-jose.v2/jwt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" +) + +const ( + metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token" + + pluginModeEnvVar = "PLUGIN_MODE" +) + +func Main() { + if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil { + klog.Fatalf("failed to get credentials: %v", err) + } +} + +func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error { + provider := &provider{ + client: &http.Client{ + Timeout: 10 * time.Second, + }, + tokenEndpoint: tokenEndpoint, + } + + data, err := io.ReadAll(r) + if err != nil { + return err + } + + var authRequest credentialproviderv1.CredentialProviderRequest + err = json.Unmarshal(data, &authRequest) + if err != nil { + return err + } + + pluginUsingServiceAccount := os.Getenv(pluginModeEnvVar) == "serviceaccount" + if pluginUsingServiceAccount { + if len(authRequest.ServiceAccountToken) == 0 { + return errors.New("service account token is empty") + } + expectedAnnotations := map[string]string{ + "domain.io/identity-id": "123456", + "domain.io/identity-type": "serviceaccount", + } + if !reflect.DeepEqual(authRequest.ServiceAccountAnnotations, expectedAnnotations) { + return fmt.Errorf("unexpected service account annotations, want: %v, got: %v", expectedAnnotations, authRequest.ServiceAccountAnnotations) + } + // The service account token is not actually used for authentication by this test plugin. + // We extract the claims from the token to validate the audience. + // This is solely for testing assertions and is not an actual security layer. + // Post validation in this block, we proceed with the default flow for fetching credentials. + c, err := getClaims(authRequest.ServiceAccountToken) + if err != nil { + return err + } + // The audience in the token should match the audience configured in tokenAttributes.serviceAccountTokenAudience + // in CredentialProviderConfig. + if len(c.Audience) != 1 || c.Audience[0] != "test-audience" { + return fmt.Errorf("unexpected audience: %v", c.Audience) + } + } else { + if len(authRequest.ServiceAccountToken) > 0 { + return errors.New("service account token is not expected") + } + if len(authRequest.ServiceAccountAnnotations) > 0 { + return errors.New("service account annotations are not expected") + } + } + + auth, err := provider.Provide(authRequest.Image) + if err != nil { + return err + } + + response := &credentialproviderv1.CredentialProviderResponse{ + TypeMeta: metav1.TypeMeta{ + Kind: "CredentialProviderResponse", + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + }, + CacheKeyType: credentialproviderv1.RegistryPluginCacheKeyType, + Auth: auth, + } + + if pluginUsingServiceAccount { + response.CacheKeyType = credentialproviderv1.GlobalPluginCacheKeyType + } + + if err := json.NewEncoder(w).Encode(response); err != nil { + // The error from json.Marshal is intentionally not included so as to not leak credentials into the logs + return errors.New("error marshaling response") + } + + return nil +} + +// getClaims is used to extract claims from the service account token when the plugin is running in service account mode +// This is solely for testing assertions and is not an actual security layer. +// We get claims and validate the audience of the token (audience in the token matches the audience configured +// in tokenAttributes.serviceAccountTokenAudience in CredentialProviderConfig). +func getClaims(tokenData string) (claims, error) { + if strings.HasPrefix(strings.TrimSpace(tokenData), "{") { + return claims{}, errors.New("token is not a JWS") + } + parts := strings.Split(tokenData, ".") + if len(parts) != 3 { + return claims{}, errors.New("token is not a JWS") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return claims{}, fmt.Errorf("error decoding token payload: %w", err) + } + + var c claims + d := json.NewDecoder(strings.NewReader(string(payload))) + d.DisallowUnknownFields() + if err := d.Decode(&c); err != nil { + return claims{}, fmt.Errorf("error decoding token payload: %w", err) + } + + return c, nil +} + +type claims struct { + jwt.Claims + privateClaims +} + +// copied from https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/serviceaccount/claims.go#L51-L67 + +type privateClaims struct { + Kubernetes kubernetes `json:"kubernetes.io,omitempty"` +} + +type kubernetes struct { + Namespace string `json:"namespace,omitempty"` + Svcacct ref `json:"serviceaccount,omitempty"` + Pod *ref `json:"pod,omitempty"` + Secret *ref `json:"secret,omitempty"` + Node *ref `json:"node,omitempty"` + WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"` +} + +type ref struct { + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` +} diff --git a/test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go b/test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go new file mode 100644 index 00000000000..82a550d34f5 --- /dev/null +++ b/test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go @@ -0,0 +1,55 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredentialsprovider + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +type fakeTokenServer struct { + token string +} + +func (f *fakeTokenServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { + fmt.Fprintf(w, `{"access_token": "%s"}`, f.token) +} + +func Test_getCredentials(t *testing.T) { + server := httptest.NewServer(&fakeTokenServer{token: "abc123"}) + defer server.Close() + + in := bytes.NewBuffer([]byte(`{"kind":"CredentialProviderRequest","apiVersion":"credentialprovider.kubelet.k8s.io/v1","image":"gcr.io/foobar"}`)) + out := bytes.NewBuffer(nil) + + err := getCredentials(server.URL, in, out) + if err != nil { + t.Fatalf("unexpected error running getCredentials: %v", err) + } + + expected := `{"kind":"CredentialProviderResponse","apiVersion":"credentialprovider.kubelet.k8s.io/v1","cacheKeyType":"Registry","auth":{"*.gcr.io":{"username":"_token","password":"abc123"},"*.pkg.dev":{"username":"_token","password":"abc123"},"container.cloud.google.com":{"username":"_token","password":"abc123"},"gcr.io":{"username":"_token","password":"abc123"}}} +` + + if out.String() != expected { + t.Logf("actual response: %v", out) + t.Logf("expected response: %v", expected) + t.Errorf("unexpected credential provider response") + } +} diff --git a/test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go b/test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go new file mode 100644 index 00000000000..96d90faebac --- /dev/null +++ b/test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go @@ -0,0 +1,125 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package gcpcredentialsprovider is a temporary copy of the +// gcp-credentials-provider command for use in the e2e_node binary itself. If +// that approach works out, the command will be refactored to remove the code +// duplication, otherwise this will get reverted. +// +// Originally copied from pkg/credentialproviders/gcp +package gcpcredentialsprovider + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" +) + +const ( + maxReadLength = 10 * 1 << 20 // 10MB +) + +var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"} + +// HTTPError wraps a non-StatusOK error code as an error. +type HTTPError struct { + StatusCode int + URL string +} + +var _ error = &HTTPError{} + +// Error implements error +func (h *HTTPError) Error() string { + return fmt.Sprintf("http status code: %d while fetching url %s", + h.StatusCode, h.URL) +} + +// TokenBlob is used to decode the JSON blob containing an access token +// that is returned by GCE metadata. +type TokenBlob struct { + AccessToken string `json:"access_token"` +} + +type provider struct { + client *http.Client + tokenEndpoint string +} + +func (p *provider) Provide(image string) (map[string]credentialproviderv1.AuthConfig, error) { + cfg := map[string]credentialproviderv1.AuthConfig{} + + tokenJSONBlob, err := readURL(p.tokenEndpoint, p.client) + if err != nil { + return cfg, err + } + + var parsedBlob TokenBlob + if err := json.Unmarshal(tokenJSONBlob, &parsedBlob); err != nil { + return cfg, err + } + + authConfig := credentialproviderv1.AuthConfig{ + Username: "_token", + Password: parsedBlob.AccessToken, + } + + // Add our entry for each of the supported container registry URLs + for _, k := range containerRegistryUrls { + cfg[k] = authConfig + } + return cfg, nil +} + +func readURL(url string, client *http.Client) (body []byte, err error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header = http.Header{ + "Metadata-Flavor": []string{"Google"}, + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, &HTTPError{ + StatusCode: resp.StatusCode, + URL: url, + } + } + + limitedReader := &io.LimitedReader{R: resp.Body, N: maxReadLength} + contents, err := io.ReadAll(limitedReader) + if err != nil { + return nil, err + } + + if limitedReader.N <= 0 { + return nil, errors.New("the read limit is reached") + } + + return contents, nil +} diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 9baf478ce32..ab86e4f6172 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -40,17 +40,45 @@ func init() { RegisterTestSuite("default", &NodeE2ERemote{}) } +var ginkgoBin = CommandLine.String("ginkgo-binary", "", "Existing Ginkgo binary to be used on the target instead of building from source") +var kubeletBin = CommandLine.String("kubelet-binary", "", "Existing kubelet binary to be used on the target instead of building from source") +var e2eNodeBin = CommandLine.String("e2e-node-binary", "", "Existing e2e-node.test binary to be used on the target instead of building from source") + // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { - // Build the executables - if err := builder.BuildGo(); err != nil { - return fmt.Errorf("failed to build the dependencies: %w", err) + requiredBins := map[string]struct { + file string + target string + }{ + "ginkgo": {*ginkgoBin, "github.com/onsi/ginkgo/v2/ginkgo"}, + "kubelet": {*kubeletBin, "cmd/kubelet"}, + "e2e_node.test": {*e2eNodeBin, "test/e2e_node/e2e_node.test"}, } - // Make sure we can find the newly built binaries - buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) - if err != nil { - return fmt.Errorf("failed to locate kubernetes build output directory: %w", err) + // Build only targets for which we don't have a binary already. + var targets []string + for _, entry := range requiredBins { + if entry.file == "" { + targets = append(targets, entry.target) + } + } + if len(targets) > 0 { + // Build the missing executables required below. + if err := builder.BuildTargets(targets...); err != nil { + return fmt.Errorf("failed to build the dependencies: %w", err) + } + + // Make sure we can find the newly built binaries + buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) + if err != nil { + return fmt.Errorf("failed to locate kubernetes build output directory: %w", err) + } + for bin, entry := range requiredBins { + if entry.file == "" { + entry.file = filepath.Join(buildOutputDir, bin) + } + requiredBins[bin] = entry + } } rootDir, err := utils.GetK8sRootDir() @@ -59,9 +87,8 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } // Copy binaries - requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"} - for _, bin := range requiredBins { - source := filepath.Join(buildOutputDir, bin) + for bin, entry := range requiredBins { + source := entry.file klog.V(2).Infof("Copying binaries from %s", source) if _, err := os.Stat(source); err != nil { return fmt.Errorf("failed to locate test binary %s: %w", bin, err) @@ -72,6 +99,17 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } } + // When e2e_node.test is invoked through these symlinks, it behaves like these + // separate binaries. + e2eNodeBinary := "e2e_node.test" + for _, alias := range []string{"mounter", "gcp-credential-provider"} { + symlink := filepath.Join(tardir, alias) + klog.V(2).Infof("Creating symlink %s -> %s", symlink, e2eNodeBinary) + if err := os.Symlink(e2eNodeBinary, symlink); err != nil { + return fmt.Errorf("failed to create symlink %q: %w", alias, err) + } + } + // create a symlink of gcp-credential-provider binary to use for testing // service account token for credential providers. // feature-gate: KubeletServiceAccountTokenForCredentialProviders=true diff --git a/test/e2e_node/remote/run_remote_suite.go b/test/e2e_node/remote/run_remote_suite.go index 7ceacd929fe..dcf8312a421 100644 --- a/test/e2e_node/remote/run_remote_suite.go +++ b/test/e2e_node/remote/run_remote_suite.go @@ -84,7 +84,7 @@ func RunRemoteTestSuite(testSuite TestSuite) { // Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags) - fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags) + klog.Infof("Will use ginkgo flags as: %s", allGinkgoFlags) var runner Runner cfg := Config{ diff --git a/test/e2e_node/runner/node/node.go b/test/e2e_node/runner/node/node.go new file mode 100644 index 00000000000..b7585b65ef9 --- /dev/null +++ b/test/e2e_node/runner/node/node.go @@ -0,0 +1,243 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package node implements `e2e_node.test remote `, i.e. +// it gets built into e2e_node.test. This avoids shipping another binary +// in release packages. Only remote execution of the E2E suite is +// supported. +// +// It gets called by `kubetest2 noop -test=node -- ` +// when the flags include any of the "use or build binaries" variants. +// This enables running E2E node tests without the Kubernetes source code. +// +// It provides the command line flags of the kubetest2 node tester (with some +// changes, see below) and maps them to the way how `make test-e2e-node` would +// have invoked test/e2e_node/runner/remote. +// +// It's derived from +// https://github.com/kubernetes-sigs/kubetest2/blob/master/pkg/testers/node/node.go +// (revision 558f16b589d15d031595af7d035330b8e87bcaaf). +package node + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + "k8s.io/klog/v2" + + "k8s.io/kubernetes/test/e2e_node/remote" + _ "k8s.io/kubernetes/test/e2e_node/remote/gce" // Register remote execution via GCE. +) + +// Tester is intentionally as similar as possible to +// https://github.com/kubernetes-sigs/kubetest2/blob/master/pkg/testers/node/node.go +// to make it more obvious that the command line is the same, +// with a few flags added for binary paths. +type Tester struct { + // Flags passed through by node tester. + GCPProject string `desc:"GCP Project to create VMs in."` + GCPZone string `desc:"GCP Zone to create VMs in."` + SkipRegex string `desc:"Regular expression of jobs to skip."` + FocusRegex string `desc:"Regular expression of jobs to focus on."` + TestArgs string `desc:"A space-separated list of arguments to pass to node e2e test."` + LabelFilter string `desc:"Label filter arguments to be passed to ginkgo."` + ImageConfigFile string `desc:"Path to a file containing image configuration."` + Images string `desc:"List of images to use when creating instances separated by commas"` + ImageProject string `desc:"A GCP Project containing an image to use when creating instances"` + InstanceType string `desc:"Machine/Instance type to use on AWS/GCP"` + InstanceMetadata string `desc:"Instance Metadata to use for creating GCE instance"` + UserDataFile string `desc:"User Data to use for creating EC2 instance"` + Provider string `desc:"Cloud Provider to use for node tests. Valid options are ec2 and gce"` + ImageConfigDir string `desc:"Path to image config files."` + Parallelism int `desc:"The number of nodes to run in parallel."` + RuntimeConfig string `desc:"The runtime configuration for the API server. Format: a list of key=value pairs."` + Timeout time.Duration `desc:"How long (in golang duration format) to wait for ginkgo tests to complete."` + DeleteInstances bool `desc:"Where to delete instances after running the test"` + NodeEnv string `desc:"Additional metadata keys to add to a gce instance"` + + // Flags for pre-built binary support. Required, building from source is not supported. + GinkgoBinary string `desc:"Existing Ginkgo binary to be used on the target instead of building from source"` + KubeletBinary string `desc:"Existing kubelet binary to be used on the target instead of building from source"` + E2ENodeBinary string `desc:"Existing e2e-node.test binary to be used on the target instead of building from source"` + + // Flags for values determined by node tester. + SSHUser string `desc:"Used for ssh into the remote node."` + SSHKey string `desc:"Used for ssh into the remote node."` +} + +func NewDefaultTester() *Tester { + return &Tester{ + Parallelism: 8, + DeleteInstances: true, + } +} + +func (t *Tester) Execute(ctx context.Context, args []string) error { + fs := flag.NewFlagSet(`"e2e_node.test remote"`, flag.ExitOnError) + + // Bind all exported fields to flags. This replaces github.com/octago/sflags/gen/gpflag. + value := reflect.ValueOf(t).Elem() + for field := range reflect.TypeOf(*t).Fields() { + if !field.IsExported() { + continue + } + name := fieldToFlagName(field.Name) + desc := field.Tag.Get("desc") + ptr := value.FieldByIndex(field.Index).Addr() + switch ptr := ptr.Interface().(type) { + case *string: + fs.StringVar(ptr, name, *ptr, desc) + case *int: + fs.IntVar(ptr, name, *ptr, desc) + case *bool: + fs.BoolVar(ptr, name, *ptr, desc) + case *time.Duration: + fs.DurationVar(ptr, name, *ptr, desc) + default: + return fmt.Errorf("unsupported config field type %T", ptr) + } + } + + klog.InitFlags(fs) + if err := fs.Parse(args); err != nil { + return fmt.Errorf("parse flags: %v", err) + } + if err := t.validateFlags(); err != nil { + return fmt.Errorf("validate flags: %v", err) + } + + return t.Test() +} + +// fieldToFlagName converts to lower case and inserts hyphens between the boundary +// between upper and lower characters. As a special case, e.g. "GCPProject" becomes +// "gcp-project". +func fieldToFlagName(in string) string { + parts := regexp.MustCompile(`[a-z0-9]+|[A-Z0-9]+`).FindAllString(in, -1) + for i, part := range parts { + if part[0] < 'A' || part[0] > 'Z' { + continue + } + if l := len(part); l > 1 { + // Split before the last upper character. + part = part[:l-1] + "-" + part[l-1:] + } + if i > 0 { + part = "-" + part + } + parts[i] = strings.ToLower(part) + } + return strings.Join(parts, "") +} + +func (t *Tester) validateFlags() error { + if t.GinkgoBinary == "" { + return errors.New("required --ginkgo-binary path missing") + } + if t.KubeletBinary == "" { + return errors.New("required --kubelet-binary path missing") + } + if t.E2ENodeBinary == "" { + return errors.New("required --e2e-node-binary path missing") + } + if t.GCPZone == "" && t.Provider == "gce" { + return errors.New("required --gcp-zone") + } + return nil +} + +// configureRemote configures the "remote" package through it's flags. +// This corresponds to test/e2e_node/runner/remote/run_remote.go +// as invoked by hack/make-rules/test-e2e-node.sh. +func (t *Tester) configureRemote() (finalErr error) { + for name, value := range map[string]string{ + "ssh-env": "gce", // Hard-coded as in https://github.com/kubernetes/kubernetes/blob/34341909b3e9a4854ab5d336b056b934bbbd9f16/hack/make-rules/test-e2e-node.sh#L218 + + // See test/e2e_node/remote/run_remote_suite.go and test/e2e_node/remote/gce/gce_runner.go + // for the definition of these flags or check `go run ./test/e2e_node/runner/remote/ -help`. + "project": t.GCPProject, + "zone": t.GCPZone, + "test_args": t.TestArgs, + "node-env": t.NodeEnv, + "delete-instances": strconv.FormatBool(t.DeleteInstances), + "image-config-file": t.ImageConfigFile, + "image-config-dir": t.ImageConfigDir, + "image-project": t.ImageProject, + "images": t.Images, + "instance-metadata": t.InstanceMetadata, + "instance-type": t.InstanceType, + "test-timeout": t.Timeout.String(), + + "ginkgo-binary": t.GinkgoBinary, + "kubelet-binary": t.KubeletBinary, + "e2e-node-binary": t.E2ENodeBinary, + + "ssh-user": t.SSHUser, + "ssh-key": t.SSHKey, + + // Not used by the remote runner and cannot be set because the flag is only defined in the command: + // https://github.com/kubernetes/kubernetes/blob/a5098cf9a1405b6b6ed6cf9e0e4e49270c5a0996/test/e2e_node/runner/remote/run_remote.go#L37-L41 + // + // We accept the parameter for the sake of consistency and because it might make + // migrating jobs easier. Maybe it will also be supported in the future, if there + // turns out to be a need for it. + // "runtime-config": t.RuntimeConfig, + + // Combining multiple different parameters inside a single parameter is problematic + // because of quoting, but that is what the existing code uses. A better solution + // would be have a `ginkgo-flag` parameter that can be used once for each individual + // Ginkgo parameter. + "ginkgo-flags": fmt.Sprintf("--nodes=%d --label-filter=%q --skip=%q --focus=%q", t.Parallelism, t.LabelFilter, t.SkipRegex, t.FocusRegex), + } { + if err := remote.CommandLine.Set(name, value); err != nil { + return fmt.Errorf("set --%s to %q: %v", name, value, err) + } + } + + return nil +} + +func (t *Tester) Test() error { + if err := t.configureRemote(); err != nil { + return fmt.Errorf("configure remote E2E node execution: %v", err) + } + + suite, err := remote.GetTestSuite("default") + if err != nil { + return fmt.Errorf("error looking up testsuite [%v] - registered test suites [%v]", err, remote.GetTestSuiteKeys()) + } + remote.RunRemoteTestSuite(suite) + return nil +} + +func Main(args []string) { + // No cancellation, there's nothing to clean up when killed. + ctx := context.Background() + + t := NewDefaultTester() + if err := t.Execute(ctx, args); err != nil { + fmt.Fprintf(os.Stderr, "\"%s remote\" failed: %v\n", os.Args[0], err) + os.Exit(1) + } +}