From 071c858417cccd3dfe3b28a559e0abedd213c109 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 May 2026 17:04:35 +0200 Subject: [PATCH 1/4] e2e_node: invoke make once for all targets The caller does not need to enable or disable CGO explicitly, the build rules do that automatically: $ make WHAT="cmd/kubelet cluster/gce/gci/mounter" +++ [0515 17:02:56] Building go targets for linux/amd64 k8s.io/kubernetes/cluster/gce/gci/mounter (static) k8s.io/kubernetes/cmd/kubelet (non-static) BuildGo builds the same targets as before. BuildTargets gets changed to accept a list of targets from the caller, which is a more useful package API. --- test/e2e_node/builder/build.go | 36 +++++++++++----------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/test/e2e_node/builder/build.go b/test/e2e_node/builder/build.go index cfe8688aa95..b700ae38a56 100644 --- a/test/e2e_node/builder/build.go +++ b/test/e2e_node/builder/build.go @@ -31,51 +31,37 @@ var k8sBinDir = CommandLine.String("k8s-bin-dir", "", "Directory containing k8s var useDockerizedBuild = CommandLine.Bool("use-dockerized-build", false, "Use dockerized build for test artifacts") var targetBuildArch = CommandLine.String("target-build-arch", "linux/amd64", "Target architecture for the test artifacts for dockerized build") -var buildCGOTargets = []string{ +// buildTargets is what the `test/e2e_node/runners/remote` builds via `make WHAT=` +// when invoked via "make test-e2e-node". In this mode, separate binaries are +// used for each command. +var buildTargets = []string{ "cmd/kubelet", -} - -var buildNoCGOTargets = []string{ "test/e2e_node/e2e_node.test", "github.com/onsi/ginkgo/v2/ginkgo", "cluster/gce/gci/mounter", "test/e2e_node/plugins/gcp-credential-provider", } -// BuildGo builds k8s binaries. +// BuildGo builds some default k8s binaries. func BuildGo() error { - if err := BuildTargets(true); err != nil { - return fmt.Errorf("unable to build cgo targets : %w", err) - } - if err := BuildTargets(false); err != nil { - return fmt.Errorf("unable to build non-cgo targets : %w", err) - } - return nil + return BuildTargets(buildTargets...) } -// BuildGo builds k8s binaries. -func BuildTargets(cgo bool) error { - klog.Infof("Building k8s binaries...") +// BuildTargets builds the specified k8s binaries (= WHAT targets). +func BuildTargets(targets ...string) error { k8sRoot, err := utils.GetK8sRootDir() if err != nil { return fmt.Errorf("failed to locate kubernetes root directory %v", err) } - targets := buildCGOTargets - if !cgo { - targets = buildNoCGOTargets - } + arch := GetTargetBuildArch() + klog.Infof("Building k8s binaries %v in %q for %s...", targets, k8sRoot, arch) what := strings.Join(targets, " ") cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", what)) - if cgo { - cmd.Args = append(cmd.Args, "CGO_ENABLED=1") - } else { - cmd.Args = append(cmd.Args, "CGO_ENABLED=0") - } if IsDockerizedBuild() { klog.Infof("Building dockerized k8s binaries targets %s for architecture %s", targets, GetTargetBuildArch()) // Multi-architecture build is only supported in dockerized build - cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", what), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", GetTargetBuildArch())) + cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", what), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", arch)) // Ensure we run this command in k8s root directory for dockerized build cmd.Dir = k8sRoot } From 6ba4d21765bc9416560cabbafe737c735f913176 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 15 May 2026 19:52:25 +0200 Subject: [PATCH 2/4] e2e_node: multiplex different commands in e2e_node.test The additional commands (mounter, gcp-credentials-provider) are needed for E2E node testing. This change makes e2e_node.test entirely self-contained. Copying the commands' code into separate packages is temporary and only done to avoid touching them while it is still unclear whether this approach will work out. Besides avoiding changes to the build rules, bundling the functionality also has a slight size advantage: the size of e2e_node.test increases by 10KB, whereas the other two separate commands would add 10MB. --- test/e2e_node/e2e_node_suite_test.go | 32 ++++ test/e2e_node/mounter/mounter.go | 97 ++++++++++ .../gcp-credential-provider/pkg/main.go | 179 ++++++++++++++++++ .../gcp-credential-provider/pkg/main_test.go | 55 ++++++ .../gcp-credential-provider/pkg/provider.go | 125 ++++++++++++ test/e2e_node/remote/node_e2e.go | 17 +- 6 files changed, 502 insertions(+), 3 deletions(-) create mode 100644 test/e2e_node/mounter/mounter.go create mode 100644 test/e2e_node/plugins/gcp-credential-provider/pkg/main.go create mode 100644 test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go create mode 100644 test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 085e1c0fd2b..b369857f998 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -26,6 +26,8 @@ import ( "encoding/json" "flag" "fmt" + "path/filepath" + "strings" "os" "os/exec" @@ -48,6 +50,8 @@ import ( e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests" "k8s.io/kubernetes/test/e2e_node/criproxy" + "k8s.io/kubernetes/test/e2e_node/mounter" + gcpcredentialprovider "k8s.io/kubernetes/test/e2e_node/plugins/gcp-credential-provider/pkg" "k8s.io/kubernetes/test/e2e_node/services" e2enodetestingmanifests "k8s.io/kubernetes/test/e2e_node/testing-manifests" system "k8s.io/system-validators/validators" @@ -120,6 +124,34 @@ func init() { } func TestMain(m *testing.M) { + // e2e_node.test can behave like several other commands which are required + // when doing node testing on a remote virtual machine. + // + // e2e_node.test owns pflag.CommandLine and flag.CommandLine. + // Other commands must use separate FlagSets. + pflag.Usage = func() { + fmt.Fprint(pflag.CommandLine.Output(), `Usage when invoked under a different name: + gcp-credential-provider (no flags) - emulate test/e2e_node/plugins/gcp-credential-provider + mounter (no flags) - emulate cluster/gce/gci/mounter + +Usage as e2e_node.test: + e2e_node.test - execute Ginkgo test suite, see following flags + +`) + pflag.CommandLine.PrintDefaults() + } + cmdName := filepath.Base(os.Args[0]) + switch { + case strings.HasPrefix(cmdName, "gcp-credential-provider"): + gcpcredentialprovider.Main() + case strings.HasPrefix(cmdName, "mounter"): + mounter.Main() + default: + testMain(m) + } +} + +func testMain(m *testing.M) { // Copy go flags in TestMain, to ensure go test flags are registered (no longer available in init() as of go1.13) e2econfig.CopyFlags(e2econfig.Flags, flag.CommandLine) framework.RegisterCommonFlags(flag.CommandLine) diff --git a/test/e2e_node/mounter/mounter.go b/test/e2e_node/mounter/mounter.go new file mode 100644 index 00000000000..9ae364690e6 --- /dev/null +++ b/test/e2e_node/mounter/mounter.go @@ -0,0 +1,97 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mounter is a temporary copy of cluster/gce/gci/mounter for use in the e2e_node +// binary itself. If that approach works out, cluster/gce/gci/mounter will be refactored +// to remove the code duplication, otherwise this will get reverted. +package mounter + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + // Location of the mount file to use + chrootCmd = "chroot" + mountCmd = "mount" + mountBin = "/bin/mount" + rootfs = "rootfs" + nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n" + rpcBindCmd = "/sbin/rpcbind" + defaultRootfs = "/home/kubernetes/containerized_mounter/rootfs" +) + +func Main() { + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n") + return + } + path, _ := filepath.Split(os.Args[0]) + rootfsPath := filepath.Join(path, rootfs) + if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { + rootfsPath = defaultRootfs + } + command := os.Args[1] + switch command { + case mountCmd: + mountErr := mountInChroot(rootfsPath, os.Args[2:]) + if mountErr != nil { + fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr) + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd) + os.Exit(1) + + } +} + +// mountInChroot runs mount within chroot with the passing root directory +func mountInChroot(rootfsPath string, args []string) error { + if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { + return fmt.Errorf("path <%s> does not exist", rootfsPath) + } + args = append([]string{rootfsPath, mountBin}, args...) + output, err := exec.Command(chrootCmd, args...).CombinedOutput() + if err == nil { + return nil + } + + if !strings.EqualFold(string(output), nfsRPCBindErrMsg) { + // Mount failed but not because of RPC bind error + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s", err, chrootCmd, args, string(output)) + } + + // Mount failed because it is NFS V3 and we need to run rpcBind + output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput() + if err != nil { + return fmt.Errorf("mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err) + } + + // Rpcbind is running, try mounting again + output, err = exec.Command(chrootCmd, args...).CombinedOutput() + + if err != nil { + return fmt.Errorf("mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err) + } + + return nil +} diff --git a/test/e2e_node/plugins/gcp-credential-provider/pkg/main.go b/test/e2e_node/plugins/gcp-credential-provider/pkg/main.go new file mode 100644 index 00000000000..ede36b34e3f --- /dev/null +++ b/test/e2e_node/plugins/gcp-credential-provider/pkg/main.go @@ -0,0 +1,179 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredentialsprovider + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "reflect" + "strings" + "time" + + "gopkg.in/go-jose/go-jose.v2/jwt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" +) + +const ( + metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token" + + pluginModeEnvVar = "PLUGIN_MODE" +) + +func Main() { + if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil { + klog.Fatalf("failed to get credentials: %v", err) + } +} + +func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error { + provider := &provider{ + client: &http.Client{ + Timeout: 10 * time.Second, + }, + tokenEndpoint: tokenEndpoint, + } + + data, err := io.ReadAll(r) + if err != nil { + return err + } + + var authRequest credentialproviderv1.CredentialProviderRequest + err = json.Unmarshal(data, &authRequest) + if err != nil { + return err + } + + pluginUsingServiceAccount := os.Getenv(pluginModeEnvVar) == "serviceaccount" + if pluginUsingServiceAccount { + if len(authRequest.ServiceAccountToken) == 0 { + return errors.New("service account token is empty") + } + expectedAnnotations := map[string]string{ + "domain.io/identity-id": "123456", + "domain.io/identity-type": "serviceaccount", + } + if !reflect.DeepEqual(authRequest.ServiceAccountAnnotations, expectedAnnotations) { + return fmt.Errorf("unexpected service account annotations, want: %v, got: %v", expectedAnnotations, authRequest.ServiceAccountAnnotations) + } + // The service account token is not actually used for authentication by this test plugin. + // We extract the claims from the token to validate the audience. + // This is solely for testing assertions and is not an actual security layer. + // Post validation in this block, we proceed with the default flow for fetching credentials. + c, err := getClaims(authRequest.ServiceAccountToken) + if err != nil { + return err + } + // The audience in the token should match the audience configured in tokenAttributes.serviceAccountTokenAudience + // in CredentialProviderConfig. + if len(c.Audience) != 1 || c.Audience[0] != "test-audience" { + return fmt.Errorf("unexpected audience: %v", c.Audience) + } + } else { + if len(authRequest.ServiceAccountToken) > 0 { + return errors.New("service account token is not expected") + } + if len(authRequest.ServiceAccountAnnotations) > 0 { + return errors.New("service account annotations are not expected") + } + } + + auth, err := provider.Provide(authRequest.Image) + if err != nil { + return err + } + + response := &credentialproviderv1.CredentialProviderResponse{ + TypeMeta: metav1.TypeMeta{ + Kind: "CredentialProviderResponse", + APIVersion: "credentialprovider.kubelet.k8s.io/v1", + }, + CacheKeyType: credentialproviderv1.RegistryPluginCacheKeyType, + Auth: auth, + } + + if pluginUsingServiceAccount { + response.CacheKeyType = credentialproviderv1.GlobalPluginCacheKeyType + } + + if err := json.NewEncoder(w).Encode(response); err != nil { + // The error from json.Marshal is intentionally not included so as to not leak credentials into the logs + return errors.New("error marshaling response") + } + + return nil +} + +// getClaims is used to extract claims from the service account token when the plugin is running in service account mode +// This is solely for testing assertions and is not an actual security layer. +// We get claims and validate the audience of the token (audience in the token matches the audience configured +// in tokenAttributes.serviceAccountTokenAudience in CredentialProviderConfig). +func getClaims(tokenData string) (claims, error) { + if strings.HasPrefix(strings.TrimSpace(tokenData), "{") { + return claims{}, errors.New("token is not a JWS") + } + parts := strings.Split(tokenData, ".") + if len(parts) != 3 { + return claims{}, errors.New("token is not a JWS") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return claims{}, fmt.Errorf("error decoding token payload: %w", err) + } + + var c claims + d := json.NewDecoder(strings.NewReader(string(payload))) + d.DisallowUnknownFields() + if err := d.Decode(&c); err != nil { + return claims{}, fmt.Errorf("error decoding token payload: %w", err) + } + + return c, nil +} + +type claims struct { + jwt.Claims + privateClaims +} + +// copied from https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/serviceaccount/claims.go#L51-L67 + +type privateClaims struct { + Kubernetes kubernetes `json:"kubernetes.io,omitempty"` +} + +type kubernetes struct { + Namespace string `json:"namespace,omitempty"` + Svcacct ref `json:"serviceaccount,omitempty"` + Pod *ref `json:"pod,omitempty"` + Secret *ref `json:"secret,omitempty"` + Node *ref `json:"node,omitempty"` + WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"` +} + +type ref struct { + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` +} diff --git a/test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go b/test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go new file mode 100644 index 00000000000..82a550d34f5 --- /dev/null +++ b/test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go @@ -0,0 +1,55 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcpcredentialsprovider + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +type fakeTokenServer struct { + token string +} + +func (f *fakeTokenServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { + fmt.Fprintf(w, `{"access_token": "%s"}`, f.token) +} + +func Test_getCredentials(t *testing.T) { + server := httptest.NewServer(&fakeTokenServer{token: "abc123"}) + defer server.Close() + + in := bytes.NewBuffer([]byte(`{"kind":"CredentialProviderRequest","apiVersion":"credentialprovider.kubelet.k8s.io/v1","image":"gcr.io/foobar"}`)) + out := bytes.NewBuffer(nil) + + err := getCredentials(server.URL, in, out) + if err != nil { + t.Fatalf("unexpected error running getCredentials: %v", err) + } + + expected := `{"kind":"CredentialProviderResponse","apiVersion":"credentialprovider.kubelet.k8s.io/v1","cacheKeyType":"Registry","auth":{"*.gcr.io":{"username":"_token","password":"abc123"},"*.pkg.dev":{"username":"_token","password":"abc123"},"container.cloud.google.com":{"username":"_token","password":"abc123"},"gcr.io":{"username":"_token","password":"abc123"}}} +` + + if out.String() != expected { + t.Logf("actual response: %v", out) + t.Logf("expected response: %v", expected) + t.Errorf("unexpected credential provider response") + } +} diff --git a/test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go b/test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go new file mode 100644 index 00000000000..96d90faebac --- /dev/null +++ b/test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go @@ -0,0 +1,125 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package gcpcredentialsprovider is a temporary copy of the +// gcp-credentials-provider command for use in the e2e_node binary itself. If +// that approach works out, the command will be refactored to remove the code +// duplication, otherwise this will get reverted. +// +// Originally copied from pkg/credentialproviders/gcp +package gcpcredentialsprovider + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" +) + +const ( + maxReadLength = 10 * 1 << 20 // 10MB +) + +var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"} + +// HTTPError wraps a non-StatusOK error code as an error. +type HTTPError struct { + StatusCode int + URL string +} + +var _ error = &HTTPError{} + +// Error implements error +func (h *HTTPError) Error() string { + return fmt.Sprintf("http status code: %d while fetching url %s", + h.StatusCode, h.URL) +} + +// TokenBlob is used to decode the JSON blob containing an access token +// that is returned by GCE metadata. +type TokenBlob struct { + AccessToken string `json:"access_token"` +} + +type provider struct { + client *http.Client + tokenEndpoint string +} + +func (p *provider) Provide(image string) (map[string]credentialproviderv1.AuthConfig, error) { + cfg := map[string]credentialproviderv1.AuthConfig{} + + tokenJSONBlob, err := readURL(p.tokenEndpoint, p.client) + if err != nil { + return cfg, err + } + + var parsedBlob TokenBlob + if err := json.Unmarshal(tokenJSONBlob, &parsedBlob); err != nil { + return cfg, err + } + + authConfig := credentialproviderv1.AuthConfig{ + Username: "_token", + Password: parsedBlob.AccessToken, + } + + // Add our entry for each of the supported container registry URLs + for _, k := range containerRegistryUrls { + cfg[k] = authConfig + } + return cfg, nil +} + +func readURL(url string, client *http.Client) (body []byte, err error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header = http.Header{ + "Metadata-Flavor": []string{"Google"}, + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, &HTTPError{ + StatusCode: resp.StatusCode, + URL: url, + } + } + + limitedReader := &io.LimitedReader{R: resp.Body, N: maxReadLength} + contents, err := io.ReadAll(limitedReader) + if err != nil { + return nil, err + } + + if limitedReader.N <= 0 { + return nil, errors.New("the read limit is reached") + } + + return contents, nil +} diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 9baf478ce32..d7ef4170c41 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -42,8 +42,8 @@ func init() { // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { - // Build the executables - if err := builder.BuildGo(); err != nil { + // Build the executables required below. + if err := builder.BuildTargets("cmd/kubelet", "test/e2e_node/e2e_node.test", "github.com/onsi/ginkgo/v2/ginkgo"); err != nil { return fmt.Errorf("failed to build the dependencies: %w", err) } @@ -59,7 +59,7 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } // Copy binaries - requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"} + requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo"} for _, bin := range requiredBins { source := filepath.Join(buildOutputDir, bin) klog.V(2).Infof("Copying binaries from %s", source) @@ -72,6 +72,17 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } } + // When e2e_node.test is invoked through these symlinks, it behaves like these + // separate binaries. + e2eNodeBinary := "e2e_node.test" + for _, alias := range []string{"mounter", "gcp-credential-provider"} { + symlink := filepath.Join(tardir, alias) + klog.V(2).Infof("Creating symlink %s -> %s", symlink, e2eNodeBinary) + if err := os.Symlink(e2eNodeBinary, symlink); err != nil { + return fmt.Errorf("failed to create symlink %q: %w", alias, err) + } + } + // create a symlink of gcp-credential-provider binary to use for testing // service account token for credential providers. // feature-gate: KubeletServiceAccountTokenForCredentialProviders=true From 2d574790a671f5d6d59dee99638bd7785c7211de Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 21 May 2026 12:34:32 +0200 Subject: [PATCH 3/4] e2e_node: fix log output fmt.Printf lacked the trailing newline and is inconsistent with other output, which uses klog. --- test/e2e_node/remote/run_remote_suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_node/remote/run_remote_suite.go b/test/e2e_node/remote/run_remote_suite.go index 7ceacd929fe..dcf8312a421 100644 --- a/test/e2e_node/remote/run_remote_suite.go +++ b/test/e2e_node/remote/run_remote_suite.go @@ -84,7 +84,7 @@ func RunRemoteTestSuite(testSuite TestSuite) { // Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags) - fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags) + klog.Infof("Will use ginkgo flags as: %s", allGinkgoFlags) var runner Runner cfg := Config{ From de2d13b27e692270201f9d245350c46133d14811 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 20 May 2026 12:22:17 +0200 Subject: [PATCH 4/4] e2e_node: support pre-built binaries This is not usable through "make test-e2e-node", which (while feasible) would be a bit pointless because the Kubernetes source could would still be needed for the make rules. Instead, "kubetest2 noop -test=node" gets extended to invoke `e2e_node.test remote` with flags that tell e2e_node.test where to find the binaries and flags that were provided by the caller of kubetest2. --- test/e2e_node/e2e_node_suite_test.go | 4 + test/e2e_node/remote/node_e2e.go | 47 ++++-- test/e2e_node/runner/node/node.go | 243 +++++++++++++++++++++++++++ 3 files changed, 284 insertions(+), 10 deletions(-) create mode 100644 test/e2e_node/runner/node/node.go diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index b369857f998..992c6af0295 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -52,6 +52,7 @@ import ( "k8s.io/kubernetes/test/e2e_node/criproxy" "k8s.io/kubernetes/test/e2e_node/mounter" gcpcredentialprovider "k8s.io/kubernetes/test/e2e_node/plugins/gcp-credential-provider/pkg" + "k8s.io/kubernetes/test/e2e_node/runner/node" "k8s.io/kubernetes/test/e2e_node/services" e2enodetestingmanifests "k8s.io/kubernetes/test/e2e_node/testing-manifests" system "k8s.io/system-validators/validators" @@ -135,6 +136,7 @@ func TestMain(m *testing.M) { mounter (no flags) - emulate cluster/gce/gci/mounter Usage as e2e_node.test: + e2e_node.test remote - run remote testing, replacing "testgrid2 noop -test=node", see "remote -help" for flags e2e_node.test - execute Ginkgo test suite, see following flags `) @@ -146,6 +148,8 @@ Usage as e2e_node.test: gcpcredentialprovider.Main() case strings.HasPrefix(cmdName, "mounter"): mounter.Main() + case len(os.Args) > 1 && os.Args[1] == "remote": + node.Main(os.Args[2:]) default: testMain(m) } diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index d7ef4170c41..ab86e4f6172 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -40,17 +40,45 @@ func init() { RegisterTestSuite("default", &NodeE2ERemote{}) } +var ginkgoBin = CommandLine.String("ginkgo-binary", "", "Existing Ginkgo binary to be used on the target instead of building from source") +var kubeletBin = CommandLine.String("kubelet-binary", "", "Existing kubelet binary to be used on the target instead of building from source") +var e2eNodeBin = CommandLine.String("e2e-node-binary", "", "Existing e2e-node.test binary to be used on the target instead of building from source") + // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { - // Build the executables required below. - if err := builder.BuildTargets("cmd/kubelet", "test/e2e_node/e2e_node.test", "github.com/onsi/ginkgo/v2/ginkgo"); err != nil { - return fmt.Errorf("failed to build the dependencies: %w", err) + requiredBins := map[string]struct { + file string + target string + }{ + "ginkgo": {*ginkgoBin, "github.com/onsi/ginkgo/v2/ginkgo"}, + "kubelet": {*kubeletBin, "cmd/kubelet"}, + "e2e_node.test": {*e2eNodeBin, "test/e2e_node/e2e_node.test"}, } - // Make sure we can find the newly built binaries - buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) - if err != nil { - return fmt.Errorf("failed to locate kubernetes build output directory: %w", err) + // Build only targets for which we don't have a binary already. + var targets []string + for _, entry := range requiredBins { + if entry.file == "" { + targets = append(targets, entry.target) + } + } + if len(targets) > 0 { + // Build the missing executables required below. + if err := builder.BuildTargets(targets...); err != nil { + return fmt.Errorf("failed to build the dependencies: %w", err) + } + + // Make sure we can find the newly built binaries + buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch()) + if err != nil { + return fmt.Errorf("failed to locate kubernetes build output directory: %w", err) + } + for bin, entry := range requiredBins { + if entry.file == "" { + entry.file = filepath.Join(buildOutputDir, bin) + } + requiredBins[bin] = entry + } } rootDir, err := utils.GetK8sRootDir() @@ -59,9 +87,8 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { } // Copy binaries - requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo"} - for _, bin := range requiredBins { - source := filepath.Join(buildOutputDir, bin) + for bin, entry := range requiredBins { + source := entry.file klog.V(2).Infof("Copying binaries from %s", source) if _, err := os.Stat(source); err != nil { return fmt.Errorf("failed to locate test binary %s: %w", bin, err) diff --git a/test/e2e_node/runner/node/node.go b/test/e2e_node/runner/node/node.go new file mode 100644 index 00000000000..b7585b65ef9 --- /dev/null +++ b/test/e2e_node/runner/node/node.go @@ -0,0 +1,243 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package node implements `e2e_node.test remote `, i.e. +// it gets built into e2e_node.test. This avoids shipping another binary +// in release packages. Only remote execution of the E2E suite is +// supported. +// +// It gets called by `kubetest2 noop -test=node -- ` +// when the flags include any of the "use or build binaries" variants. +// This enables running E2E node tests without the Kubernetes source code. +// +// It provides the command line flags of the kubetest2 node tester (with some +// changes, see below) and maps them to the way how `make test-e2e-node` would +// have invoked test/e2e_node/runner/remote. +// +// It's derived from +// https://github.com/kubernetes-sigs/kubetest2/blob/master/pkg/testers/node/node.go +// (revision 558f16b589d15d031595af7d035330b8e87bcaaf). +package node + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + "k8s.io/klog/v2" + + "k8s.io/kubernetes/test/e2e_node/remote" + _ "k8s.io/kubernetes/test/e2e_node/remote/gce" // Register remote execution via GCE. +) + +// Tester is intentionally as similar as possible to +// https://github.com/kubernetes-sigs/kubetest2/blob/master/pkg/testers/node/node.go +// to make it more obvious that the command line is the same, +// with a few flags added for binary paths. +type Tester struct { + // Flags passed through by node tester. + GCPProject string `desc:"GCP Project to create VMs in."` + GCPZone string `desc:"GCP Zone to create VMs in."` + SkipRegex string `desc:"Regular expression of jobs to skip."` + FocusRegex string `desc:"Regular expression of jobs to focus on."` + TestArgs string `desc:"A space-separated list of arguments to pass to node e2e test."` + LabelFilter string `desc:"Label filter arguments to be passed to ginkgo."` + ImageConfigFile string `desc:"Path to a file containing image configuration."` + Images string `desc:"List of images to use when creating instances separated by commas"` + ImageProject string `desc:"A GCP Project containing an image to use when creating instances"` + InstanceType string `desc:"Machine/Instance type to use on AWS/GCP"` + InstanceMetadata string `desc:"Instance Metadata to use for creating GCE instance"` + UserDataFile string `desc:"User Data to use for creating EC2 instance"` + Provider string `desc:"Cloud Provider to use for node tests. Valid options are ec2 and gce"` + ImageConfigDir string `desc:"Path to image config files."` + Parallelism int `desc:"The number of nodes to run in parallel."` + RuntimeConfig string `desc:"The runtime configuration for the API server. Format: a list of key=value pairs."` + Timeout time.Duration `desc:"How long (in golang duration format) to wait for ginkgo tests to complete."` + DeleteInstances bool `desc:"Where to delete instances after running the test"` + NodeEnv string `desc:"Additional metadata keys to add to a gce instance"` + + // Flags for pre-built binary support. Required, building from source is not supported. + GinkgoBinary string `desc:"Existing Ginkgo binary to be used on the target instead of building from source"` + KubeletBinary string `desc:"Existing kubelet binary to be used on the target instead of building from source"` + E2ENodeBinary string `desc:"Existing e2e-node.test binary to be used on the target instead of building from source"` + + // Flags for values determined by node tester. + SSHUser string `desc:"Used for ssh into the remote node."` + SSHKey string `desc:"Used for ssh into the remote node."` +} + +func NewDefaultTester() *Tester { + return &Tester{ + Parallelism: 8, + DeleteInstances: true, + } +} + +func (t *Tester) Execute(ctx context.Context, args []string) error { + fs := flag.NewFlagSet(`"e2e_node.test remote"`, flag.ExitOnError) + + // Bind all exported fields to flags. This replaces github.com/octago/sflags/gen/gpflag. + value := reflect.ValueOf(t).Elem() + for field := range reflect.TypeOf(*t).Fields() { + if !field.IsExported() { + continue + } + name := fieldToFlagName(field.Name) + desc := field.Tag.Get("desc") + ptr := value.FieldByIndex(field.Index).Addr() + switch ptr := ptr.Interface().(type) { + case *string: + fs.StringVar(ptr, name, *ptr, desc) + case *int: + fs.IntVar(ptr, name, *ptr, desc) + case *bool: + fs.BoolVar(ptr, name, *ptr, desc) + case *time.Duration: + fs.DurationVar(ptr, name, *ptr, desc) + default: + return fmt.Errorf("unsupported config field type %T", ptr) + } + } + + klog.InitFlags(fs) + if err := fs.Parse(args); err != nil { + return fmt.Errorf("parse flags: %v", err) + } + if err := t.validateFlags(); err != nil { + return fmt.Errorf("validate flags: %v", err) + } + + return t.Test() +} + +// fieldToFlagName converts to lower case and inserts hyphens between the boundary +// between upper and lower characters. As a special case, e.g. "GCPProject" becomes +// "gcp-project". +func fieldToFlagName(in string) string { + parts := regexp.MustCompile(`[a-z0-9]+|[A-Z0-9]+`).FindAllString(in, -1) + for i, part := range parts { + if part[0] < 'A' || part[0] > 'Z' { + continue + } + if l := len(part); l > 1 { + // Split before the last upper character. + part = part[:l-1] + "-" + part[l-1:] + } + if i > 0 { + part = "-" + part + } + parts[i] = strings.ToLower(part) + } + return strings.Join(parts, "") +} + +func (t *Tester) validateFlags() error { + if t.GinkgoBinary == "" { + return errors.New("required --ginkgo-binary path missing") + } + if t.KubeletBinary == "" { + return errors.New("required --kubelet-binary path missing") + } + if t.E2ENodeBinary == "" { + return errors.New("required --e2e-node-binary path missing") + } + if t.GCPZone == "" && t.Provider == "gce" { + return errors.New("required --gcp-zone") + } + return nil +} + +// configureRemote configures the "remote" package through it's flags. +// This corresponds to test/e2e_node/runner/remote/run_remote.go +// as invoked by hack/make-rules/test-e2e-node.sh. +func (t *Tester) configureRemote() (finalErr error) { + for name, value := range map[string]string{ + "ssh-env": "gce", // Hard-coded as in https://github.com/kubernetes/kubernetes/blob/34341909b3e9a4854ab5d336b056b934bbbd9f16/hack/make-rules/test-e2e-node.sh#L218 + + // See test/e2e_node/remote/run_remote_suite.go and test/e2e_node/remote/gce/gce_runner.go + // for the definition of these flags or check `go run ./test/e2e_node/runner/remote/ -help`. + "project": t.GCPProject, + "zone": t.GCPZone, + "test_args": t.TestArgs, + "node-env": t.NodeEnv, + "delete-instances": strconv.FormatBool(t.DeleteInstances), + "image-config-file": t.ImageConfigFile, + "image-config-dir": t.ImageConfigDir, + "image-project": t.ImageProject, + "images": t.Images, + "instance-metadata": t.InstanceMetadata, + "instance-type": t.InstanceType, + "test-timeout": t.Timeout.String(), + + "ginkgo-binary": t.GinkgoBinary, + "kubelet-binary": t.KubeletBinary, + "e2e-node-binary": t.E2ENodeBinary, + + "ssh-user": t.SSHUser, + "ssh-key": t.SSHKey, + + // Not used by the remote runner and cannot be set because the flag is only defined in the command: + // https://github.com/kubernetes/kubernetes/blob/a5098cf9a1405b6b6ed6cf9e0e4e49270c5a0996/test/e2e_node/runner/remote/run_remote.go#L37-L41 + // + // We accept the parameter for the sake of consistency and because it might make + // migrating jobs easier. Maybe it will also be supported in the future, if there + // turns out to be a need for it. + // "runtime-config": t.RuntimeConfig, + + // Combining multiple different parameters inside a single parameter is problematic + // because of quoting, but that is what the existing code uses. A better solution + // would be have a `ginkgo-flag` parameter that can be used once for each individual + // Ginkgo parameter. + "ginkgo-flags": fmt.Sprintf("--nodes=%d --label-filter=%q --skip=%q --focus=%q", t.Parallelism, t.LabelFilter, t.SkipRegex, t.FocusRegex), + } { + if err := remote.CommandLine.Set(name, value); err != nil { + return fmt.Errorf("set --%s to %q: %v", name, value, err) + } + } + + return nil +} + +func (t *Tester) Test() error { + if err := t.configureRemote(); err != nil { + return fmt.Errorf("configure remote E2E node execution: %v", err) + } + + suite, err := remote.GetTestSuite("default") + if err != nil { + return fmt.Errorf("error looking up testsuite [%v] - registered test suites [%v]", err, remote.GetTestSuiteKeys()) + } + remote.RunRemoteTestSuite(suite) + return nil +} + +func Main(args []string) { + // No cancellation, there's nothing to clean up when killed. + ctx := context.Background() + + t := NewDefaultTester() + if err := t.Execute(ctx, args); err != nil { + fmt.Fprintf(os.Stderr, "\"%s remote\" failed: %v\n", os.Args[0], err) + os.Exit(1) + } +}