Merge pull request #139129 from pohly/e2e-node-update-local

E2E node: enable using release archives for periodic jobs, simplified

test/e2e_node/builder/build.go

@@ -31,51 +31,37 @@ var k8sBinDir = CommandLine.String("k8s-bin-dir", "", "Directory containing k8s
 var useDockerizedBuild = CommandLine.Bool("use-dockerized-build", false, "Use dockerized build for test artifacts")
 var targetBuildArch = CommandLine.String("target-build-arch", "linux/amd64", "Target architecture for the test artifacts for dockerized build")
 
-var buildCGOTargets = []string{
+// buildTargets is what the `test/e2e_node/runners/remote` builds via `make WHAT=`
+// when invoked via "make test-e2e-node". In this mode, separate binaries are
+// used for each command.
+var buildTargets = []string{
 	"cmd/kubelet",
-}
-
-var buildNoCGOTargets = []string{
 	"test/e2e_node/e2e_node.test",
 	"github.com/onsi/ginkgo/v2/ginkgo",
 	"cluster/gce/gci/mounter",
 	"test/e2e_node/plugins/gcp-credential-provider",
 }
 
-// BuildGo builds k8s binaries.
+// BuildGo builds some default k8s binaries.
 func BuildGo() error {
-	if err := BuildTargets(true); err != nil {
-		return fmt.Errorf("unable to build cgo targets : %w", err)
-	}
-	if err := BuildTargets(false); err != nil {
-		return fmt.Errorf("unable to build non-cgo targets : %w", err)
-	}
-	return nil
+	return BuildTargets(buildTargets...)
 }
 
-// BuildGo builds k8s binaries.
-func BuildTargets(cgo bool) error {
-	klog.Infof("Building k8s binaries...")
+// BuildTargets builds the specified k8s binaries (= WHAT targets).
+func BuildTargets(targets ...string) error {
 	k8sRoot, err := utils.GetK8sRootDir()
 	if err != nil {
 		return fmt.Errorf("failed to locate kubernetes root directory %v", err)
 	}
-	targets := buildCGOTargets
-	if !cgo {
-		targets = buildNoCGOTargets
-	}
+	arch := GetTargetBuildArch()
+	klog.Infof("Building k8s binaries %v in %q for %s...", targets, k8sRoot, arch)
 	what := strings.Join(targets, " ")
 	cmd := exec.Command("make", "-C", k8sRoot,
 		fmt.Sprintf("WHAT=%s", what))
-	if cgo {
-		cmd.Args = append(cmd.Args, "CGO_ENABLED=1")
-	} else {
-		cmd.Args = append(cmd.Args, "CGO_ENABLED=0")
-	}
 	if IsDockerizedBuild() {
 		klog.Infof("Building dockerized k8s binaries targets %s for architecture %s", targets, GetTargetBuildArch())
 		// Multi-architecture build is only supported in dockerized build
-		cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", what), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", GetTargetBuildArch()))
+		cmd = exec.Command(filepath.Join(k8sRoot, "build/run.sh"), "make", fmt.Sprintf("WHAT=%s", what), fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", arch))
 		// Ensure we run this command in k8s root directory for dockerized build
 		cmd.Dir = k8sRoot
 	}

test/e2e_node/e2e_node_suite_test.go

@@ -26,6 +26,8 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"path/filepath"
+	"strings"
 
 	"os"
 	"os/exec"
@@ -48,6 +50,9 @@ import (
 	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
 	e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
 	"k8s.io/kubernetes/test/e2e_node/criproxy"
+	"k8s.io/kubernetes/test/e2e_node/mounter"
+	gcpcredentialprovider "k8s.io/kubernetes/test/e2e_node/plugins/gcp-credential-provider/pkg"
+	"k8s.io/kubernetes/test/e2e_node/runner/node"
 	"k8s.io/kubernetes/test/e2e_node/services"
 	e2enodetestingmanifests "k8s.io/kubernetes/test/e2e_node/testing-manifests"
 	system "k8s.io/system-validators/validators"
@@ -120,6 +125,37 @@ func init() {
 }
 
 func TestMain(m *testing.M) {
+	// e2e_node.test can behave like several other commands which are required
+	// when doing node testing on a remote virtual machine.
+	//
+	// e2e_node.test owns pflag.CommandLine and flag.CommandLine.
+	// Other commands must use separate FlagSets.
+	pflag.Usage = func() {
+		fmt.Fprint(pflag.CommandLine.Output(), `Usage when invoked under a different name:
+    gcp-credential-provider (no flags) - emulate test/e2e_node/plugins/gcp-credential-provider
+    mounter (no flags) - emulate cluster/gce/gci/mounter
+
+Usage as e2e_node.test:
+    e2e_node.test remote <remote flags> - run remote testing, replacing "testgrid2 noop -test=node", see "remote -help" for flags
+    e2e_node.test <test flags> - execute Ginkgo test suite, see following flags
+
+`)
+		pflag.CommandLine.PrintDefaults()
+	}
+	cmdName := filepath.Base(os.Args[0])
+	switch {
+	case strings.HasPrefix(cmdName, "gcp-credential-provider"):
+		gcpcredentialprovider.Main()
+	case strings.HasPrefix(cmdName, "mounter"):
+		mounter.Main()
+	case len(os.Args) > 1 && os.Args[1] == "remote":
+		node.Main(os.Args[2:])
+	default:
+		testMain(m)
+	}
+}
+
+func testMain(m *testing.M) {
 	// Copy go flags in TestMain, to ensure go test flags are registered (no longer available in init() as of go1.13)
 	e2econfig.CopyFlags(e2econfig.Flags, flag.CommandLine)
 	framework.RegisterCommonFlags(flag.CommandLine)

test/e2e_node/mounter/mounter.go

@@ -0,0 +1,97 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package mounter is a temporary copy of cluster/gce/gci/mounter for use in the e2e_node
+// binary itself. If that approach works out, cluster/gce/gci/mounter will be refactored
+// to remove the code duplication, otherwise this will get reverted.
+package mounter
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	// Location of the mount file to use
+	chrootCmd        = "chroot"
+	mountCmd         = "mount"
+	mountBin         = "/bin/mount"
+	rootfs           = "rootfs"
+	nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n"
+	rpcBindCmd       = "/sbin/rpcbind"
+	defaultRootfs    = "/home/kubernetes/containerized_mounter/rootfs"
+)
+
+func Main() {
+
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n")
+		return
+	}
+	path, _ := filepath.Split(os.Args[0])
+	rootfsPath := filepath.Join(path, rootfs)
+	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
+		rootfsPath = defaultRootfs
+	}
+	command := os.Args[1]
+	switch command {
+	case mountCmd:
+		mountErr := mountInChroot(rootfsPath, os.Args[2:])
+		if mountErr != nil {
+			fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr)
+			os.Exit(1)
+		}
+	default:
+		fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd)
+		os.Exit(1)
+
+	}
+}
+
+// mountInChroot runs mount within chroot with the passing root directory
+func mountInChroot(rootfsPath string, args []string) error {
+	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
+		return fmt.Errorf("path <%s> does not exist", rootfsPath)
+	}
+	args = append([]string{rootfsPath, mountBin}, args...)
+	output, err := exec.Command(chrootCmd, args...).CombinedOutput()
+	if err == nil {
+		return nil
+	}
+
+	if !strings.EqualFold(string(output), nfsRPCBindErrMsg) {
+		// Mount failed but not because of RPC bind error
+		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s", err, chrootCmd, args, string(output))
+	}
+
+	// Mount failed because it is NFS V3 and we need to run rpcBind
+	output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err)
+	}
+
+	// Rpcbind is running, try mounting again
+	output, err = exec.Command(chrootCmd, args...).CombinedOutput()
+
+	if err != nil {
+		return fmt.Errorf("mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err)
+	}
+
+	return nil
+}

test/e2e_node/plugins/gcp-credential-provider/pkg/main.go

@@ -0,0 +1,179 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcpcredentialsprovider
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"reflect"
+	"strings"
+	"time"
+
+	"gopkg.in/go-jose/go-jose.v2/jwt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
+)
+
+const (
+	metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token"
+
+	pluginModeEnvVar = "PLUGIN_MODE"
+)
+
+func Main() {
+	if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil {
+		klog.Fatalf("failed to get credentials: %v", err)
+	}
+}
+
+func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
+	provider := &provider{
+		client: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+		tokenEndpoint: tokenEndpoint,
+	}
+
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	var authRequest credentialproviderv1.CredentialProviderRequest
+	err = json.Unmarshal(data, &authRequest)
+	if err != nil {
+		return err
+	}
+
+	pluginUsingServiceAccount := os.Getenv(pluginModeEnvVar) == "serviceaccount"
+	if pluginUsingServiceAccount {
+		if len(authRequest.ServiceAccountToken) == 0 {
+			return errors.New("service account token is empty")
+		}
+		expectedAnnotations := map[string]string{
+			"domain.io/identity-id":   "123456",
+			"domain.io/identity-type": "serviceaccount",
+		}
+		if !reflect.DeepEqual(authRequest.ServiceAccountAnnotations, expectedAnnotations) {
+			return fmt.Errorf("unexpected service account annotations, want: %v, got: %v", expectedAnnotations, authRequest.ServiceAccountAnnotations)
+		}
+		// The service account token is not actually used for authentication by this test plugin.
+		// We extract the claims from the token to validate the audience.
+		// This is solely for testing assertions and is not an actual security layer.
+		// Post validation in this block, we proceed with the default flow for fetching credentials.
+		c, err := getClaims(authRequest.ServiceAccountToken)
+		if err != nil {
+			return err
+		}
+		// The audience in the token should match the audience configured in tokenAttributes.serviceAccountTokenAudience
+		// in CredentialProviderConfig.
+		if len(c.Audience) != 1 || c.Audience[0] != "test-audience" {
+			return fmt.Errorf("unexpected audience: %v", c.Audience)
+		}
+	} else {
+		if len(authRequest.ServiceAccountToken) > 0 {
+			return errors.New("service account token is not expected")
+		}
+		if len(authRequest.ServiceAccountAnnotations) > 0 {
+			return errors.New("service account annotations are not expected")
+		}
+	}
+
+	auth, err := provider.Provide(authRequest.Image)
+	if err != nil {
+		return err
+	}
+
+	response := &credentialproviderv1.CredentialProviderResponse{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "CredentialProviderResponse",
+			APIVersion: "credentialprovider.kubelet.k8s.io/v1",
+		},
+		CacheKeyType: credentialproviderv1.RegistryPluginCacheKeyType,
+		Auth:         auth,
+	}
+
+	if pluginUsingServiceAccount {
+		response.CacheKeyType = credentialproviderv1.GlobalPluginCacheKeyType
+	}
+
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		// The error from json.Marshal is intentionally not included so as to not leak credentials into the logs
+		return errors.New("error marshaling response")
+	}
+
+	return nil
+}
+
+// getClaims is used to extract claims from the service account token when the plugin is running in service account mode
+// This is solely for testing assertions and is not an actual security layer.
+// We get claims and validate the audience of the token (audience in the token matches the audience configured
+// in tokenAttributes.serviceAccountTokenAudience in CredentialProviderConfig).
+func getClaims(tokenData string) (claims, error) {
+	if strings.HasPrefix(strings.TrimSpace(tokenData), "{") {
+		return claims{}, errors.New("token is not a JWS")
+	}
+	parts := strings.Split(tokenData, ".")
+	if len(parts) != 3 {
+		return claims{}, errors.New("token is not a JWS")
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return claims{}, fmt.Errorf("error decoding token payload: %w", err)
+	}
+
+	var c claims
+	d := json.NewDecoder(strings.NewReader(string(payload)))
+	d.DisallowUnknownFields()
+	if err := d.Decode(&c); err != nil {
+		return claims{}, fmt.Errorf("error decoding token payload: %w", err)
+	}
+
+	return c, nil
+}
+
+type claims struct {
+	jwt.Claims
+	privateClaims
+}
+
+// copied from https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/serviceaccount/claims.go#L51-L67
+
+type privateClaims struct {
+	Kubernetes kubernetes `json:"kubernetes.io,omitempty"`
+}
+
+type kubernetes struct {
+	Namespace string           `json:"namespace,omitempty"`
+	Svcacct   ref              `json:"serviceaccount,omitempty"`
+	Pod       *ref             `json:"pod,omitempty"`
+	Secret    *ref             `json:"secret,omitempty"`
+	Node      *ref             `json:"node,omitempty"`
+	WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"`
+}
+
+type ref struct {
+	Name string `json:"name,omitempty"`
+	UID  string `json:"uid,omitempty"`
+}

test/e2e_node/plugins/gcp-credential-provider/pkg/main_test.go

@@ -0,0 +1,55 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcpcredentialsprovider
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeTokenServer struct {
+	token string
+}
+
+func (f *fakeTokenServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	fmt.Fprintf(w, `{"access_token": "%s"}`, f.token)
+}
+
+func Test_getCredentials(t *testing.T) {
+	server := httptest.NewServer(&fakeTokenServer{token: "abc123"})
+	defer server.Close()
+
+	in := bytes.NewBuffer([]byte(`{"kind":"CredentialProviderRequest","apiVersion":"credentialprovider.kubelet.k8s.io/v1","image":"gcr.io/foobar"}`))
+	out := bytes.NewBuffer(nil)
+
+	err := getCredentials(server.URL, in, out)
+	if err != nil {
+		t.Fatalf("unexpected error running getCredentials: %v", err)
+	}
+
+	expected := `{"kind":"CredentialProviderResponse","apiVersion":"credentialprovider.kubelet.k8s.io/v1","cacheKeyType":"Registry","auth":{"*.gcr.io":{"username":"_token","password":"abc123"},"*.pkg.dev":{"username":"_token","password":"abc123"},"container.cloud.google.com":{"username":"_token","password":"abc123"},"gcr.io":{"username":"_token","password":"abc123"}}}
+`
+
+	if out.String() != expected {
+		t.Logf("actual response: %v", out)
+		t.Logf("expected response: %v", expected)
+		t.Errorf("unexpected credential provider response")
+	}
+}

test/e2e_node/plugins/gcp-credential-provider/pkg/provider.go

@@ -0,0 +1,125 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package gcpcredentialsprovider is a temporary copy of the
+// gcp-credentials-provider command for use in the e2e_node binary itself. If
+// that approach works out, the command will be refactored to remove the code
+// duplication, otherwise this will get reverted.
+//
+// Originally copied from pkg/credentialproviders/gcp
+package gcpcredentialsprovider
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
+)
+
+const (
+	maxReadLength = 10 * 1 << 20 // 10MB
+)
+
+var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io", "*.pkg.dev"}
+
+// HTTPError wraps a non-StatusOK error code as an error.
+type HTTPError struct {
+	StatusCode int
+	URL        string
+}
+
+var _ error = &HTTPError{}
+
+// Error implements error
+func (h *HTTPError) Error() string {
+	return fmt.Sprintf("http status code: %d while fetching url %s",
+		h.StatusCode, h.URL)
+}
+
+// TokenBlob is used to decode the JSON blob containing an access token
+// that is returned by GCE metadata.
+type TokenBlob struct {
+	AccessToken string `json:"access_token"`
+}
+
+type provider struct {
+	client        *http.Client
+	tokenEndpoint string
+}
+
+func (p *provider) Provide(image string) (map[string]credentialproviderv1.AuthConfig, error) {
+	cfg := map[string]credentialproviderv1.AuthConfig{}
+
+	tokenJSONBlob, err := readURL(p.tokenEndpoint, p.client)
+	if err != nil {
+		return cfg, err
+	}
+
+	var parsedBlob TokenBlob
+	if err := json.Unmarshal(tokenJSONBlob, &parsedBlob); err != nil {
+		return cfg, err
+	}
+
+	authConfig := credentialproviderv1.AuthConfig{
+		Username: "_token",
+		Password: parsedBlob.AccessToken,
+	}
+
+	// Add our entry for each of the supported container registry URLs
+	for _, k := range containerRegistryUrls {
+		cfg[k] = authConfig
+	}
+	return cfg, nil
+}
+
+func readURL(url string, client *http.Client) (body []byte, err error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header = http.Header{
+		"Metadata-Flavor": []string{"Google"},
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, &HTTPError{
+			StatusCode: resp.StatusCode,
+			URL:        url,
+		}
+	}
+
+	limitedReader := &io.LimitedReader{R: resp.Body, N: maxReadLength}
+	contents, err := io.ReadAll(limitedReader)
+	if err != nil {
+		return nil, err
+	}
+
+	if limitedReader.N <= 0 {
+		return nil, errors.New("the read limit is reached")
+	}
+
+	return contents, nil
+}

test/e2e_node/remote/node_e2e.go

@@ -40,17 +40,45 @@ func init() {
 	RegisterTestSuite("default", &NodeE2ERemote{})
 }
 
+var ginkgoBin = CommandLine.String("ginkgo-binary", "", "Existing Ginkgo binary to be used on the target instead of building from source")
+var kubeletBin = CommandLine.String("kubelet-binary", "", "Existing kubelet binary to be used on the target instead of building from source")
+var e2eNodeBin = CommandLine.String("e2e-node-binary", "", "Existing e2e-node.test binary to be used on the target instead of building from source")
+
 // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests
 func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
-	// Build the executables
-	if err := builder.BuildGo(); err != nil {
-		return fmt.Errorf("failed to build the dependencies: %w", err)
+	requiredBins := map[string]struct {
+		file   string
+		target string
+	}{
+		"ginkgo":        {*ginkgoBin, "github.com/onsi/ginkgo/v2/ginkgo"},
+		"kubelet":       {*kubeletBin, "cmd/kubelet"},
+		"e2e_node.test": {*e2eNodeBin, "test/e2e_node/e2e_node.test"},
 	}
 
-	// Make sure we can find the newly built binaries
-	buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
-	if err != nil {
-		return fmt.Errorf("failed to locate kubernetes build output directory: %w", err)
+	// Build only targets for which we don't have a binary already.
+	var targets []string
+	for _, entry := range requiredBins {
+		if entry.file == "" {
+			targets = append(targets, entry.target)
+		}
+	}
+	if len(targets) > 0 {
+		// Build the missing executables required below.
+		if err := builder.BuildTargets(targets...); err != nil {
+			return fmt.Errorf("failed to build the dependencies: %w", err)
+		}
+
+		// Make sure we can find the newly built binaries
+		buildOutputDir, err := utils.GetK8sBuildOutputDir(builder.IsDockerizedBuild(), builder.GetTargetBuildArch())
+		if err != nil {
+			return fmt.Errorf("failed to locate kubernetes build output directory: %w", err)
+		}
+		for bin, entry := range requiredBins {
+			if entry.file == "" {
+				entry.file = filepath.Join(buildOutputDir, bin)
+			}
+			requiredBins[bin] = entry
+		}
 	}
 
 	rootDir, err := utils.GetK8sRootDir()
@@ -59,9 +87,8 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
 	}
 
 	// Copy binaries
-	requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo", "mounter", "gcp-credential-provider"}
-	for _, bin := range requiredBins {
-		source := filepath.Join(buildOutputDir, bin)
+	for bin, entry := range requiredBins {
+		source := entry.file
 		klog.V(2).Infof("Copying binaries from %s", source)
 		if _, err := os.Stat(source); err != nil {
 			return fmt.Errorf("failed to locate test binary %s: %w", bin, err)
@@ -72,6 +99,17 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
 		}
 	}
 
+	// When e2e_node.test is invoked through these symlinks, it behaves like these
+	// separate binaries.
+	e2eNodeBinary := "e2e_node.test"
+	for _, alias := range []string{"mounter", "gcp-credential-provider"} {
+		symlink := filepath.Join(tardir, alias)
+		klog.V(2).Infof("Creating symlink %s -> %s", symlink, e2eNodeBinary)
+		if err := os.Symlink(e2eNodeBinary, symlink); err != nil {
+			return fmt.Errorf("failed to create symlink %q: %w", alias, err)
+		}
+	}
+
 	// create a symlink of gcp-credential-provider binary to use for testing
 	// service account token for credential providers.
 	// feature-gate: KubeletServiceAccountTokenForCredentialProviders=true

test/e2e_node/remote/run_remote_suite.go

@@ -84,7 +84,7 @@ func RunRemoteTestSuite(testSuite TestSuite) {
 
 	// Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh
 	allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags)
-	fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags)
+	klog.Infof("Will use ginkgo flags as: %s", allGinkgoFlags)
 
 	var runner Runner
 	cfg := Config{

test/e2e_node/runner/node/node.go

@@ -0,0 +1,243 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package node implements `e2e_node.test remote <flags>`, i.e.
+// it gets built into e2e_node.test. This avoids shipping another binary
+// in release packages. Only remote execution of the E2E suite is
+// supported.
+//
+// It gets called by `kubetest2 noop -test=node -- <flags>`
+// when the flags include any of the "use or build binaries" variants.
+// This enables running E2E node tests without the Kubernetes source code.
+//
+// It provides the command line flags of the kubetest2 node tester (with some
+// changes, see below) and maps them to the way how `make test-e2e-node` would
+// have invoked test/e2e_node/runner/remote.
+//
+// It's derived from
+// https://github.com/kubernetes-sigs/kubetest2/blob/master/pkg/testers/node/node.go
+// (revision 558f16b589d15d031595af7d035330b8e87bcaaf).
+package node
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/kubernetes/test/e2e_node/remote"
+	_ "k8s.io/kubernetes/test/e2e_node/remote/gce" // Register remote execution via GCE.
+)
+
+// Tester is intentionally as similar as possible to
+// https://github.com/kubernetes-sigs/kubetest2/blob/master/pkg/testers/node/node.go
+// to make it more obvious that the command line is the same,
+// with a few flags added for binary paths.
+type Tester struct {
+	// Flags passed through by node tester.
+	GCPProject       string        `desc:"GCP Project to create VMs in."`
+	GCPZone          string        `desc:"GCP Zone to create VMs in."`
+	SkipRegex        string        `desc:"Regular expression of jobs to skip."`
+	FocusRegex       string        `desc:"Regular expression of jobs to focus on."`
+	TestArgs         string        `desc:"A space-separated list of arguments to pass to node e2e test."`
+	LabelFilter      string        `desc:"Label filter arguments to be passed to ginkgo."`
+	ImageConfigFile  string        `desc:"Path to a file containing image configuration."`
+	Images           string        `desc:"List of images to use when creating instances separated by commas"`
+	ImageProject     string        `desc:"A GCP Project containing an image to use when creating instances"`
+	InstanceType     string        `desc:"Machine/Instance type to use on AWS/GCP"`
+	InstanceMetadata string        `desc:"Instance Metadata to use for creating GCE instance"`
+	UserDataFile     string        `desc:"User Data to use for creating EC2 instance"`
+	Provider         string        `desc:"Cloud Provider to use for node tests. Valid options are ec2 and gce"`
+	ImageConfigDir   string        `desc:"Path to image config files."`
+	Parallelism      int           `desc:"The number of nodes to run in parallel."`
+	RuntimeConfig    string        `desc:"The runtime configuration for the API server. Format: a list of key=value pairs."`
+	Timeout          time.Duration `desc:"How long (in golang duration format) to wait for ginkgo tests to complete."`
+	DeleteInstances  bool          `desc:"Where to delete instances after running the test"`
+	NodeEnv          string        `desc:"Additional metadata keys to add to a gce instance"`
+
+	// Flags for pre-built binary support. Required, building from source is not supported.
+	GinkgoBinary  string `desc:"Existing Ginkgo binary to be used on the target instead of building from source"`
+	KubeletBinary string `desc:"Existing kubelet binary to be used on the target instead of building from source"`
+	E2ENodeBinary string `desc:"Existing e2e-node.test binary to be used on the target instead of building from source"`
+
+	// Flags for values determined by node tester.
+	SSHUser string `desc:"Used for ssh into the remote node."`
+	SSHKey  string `desc:"Used for ssh into the remote node."`
+}
+
+func NewDefaultTester() *Tester {
+	return &Tester{
+		Parallelism:     8,
+		DeleteInstances: true,
+	}
+}
+
+func (t *Tester) Execute(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet(`"e2e_node.test remote"`, flag.ExitOnError)
+
+	// Bind all exported fields to flags. This replaces github.com/octago/sflags/gen/gpflag.
+	value := reflect.ValueOf(t).Elem()
+	for field := range reflect.TypeOf(*t).Fields() {
+		if !field.IsExported() {
+			continue
+		}
+		name := fieldToFlagName(field.Name)
+		desc := field.Tag.Get("desc")
+		ptr := value.FieldByIndex(field.Index).Addr()
+		switch ptr := ptr.Interface().(type) {
+		case *string:
+			fs.StringVar(ptr, name, *ptr, desc)
+		case *int:
+			fs.IntVar(ptr, name, *ptr, desc)
+		case *bool:
+			fs.BoolVar(ptr, name, *ptr, desc)
+		case *time.Duration:
+			fs.DurationVar(ptr, name, *ptr, desc)
+		default:
+			return fmt.Errorf("unsupported config field type %T", ptr)
+		}
+	}
+
+	klog.InitFlags(fs)
+	if err := fs.Parse(args); err != nil {
+		return fmt.Errorf("parse flags: %v", err)
+	}
+	if err := t.validateFlags(); err != nil {
+		return fmt.Errorf("validate flags: %v", err)
+	}
+
+	return t.Test()
+}
+
+// fieldToFlagName converts to lower case and inserts hyphens between the boundary
+// between upper and lower characters. As a special case, e.g. "GCPProject" becomes
+// "gcp-project".
+func fieldToFlagName(in string) string {
+	parts := regexp.MustCompile(`[a-z0-9]+|[A-Z0-9]+`).FindAllString(in, -1)
+	for i, part := range parts {
+		if part[0] < 'A' || part[0] > 'Z' {
+			continue
+		}
+		if l := len(part); l > 1 {
+			// Split before the last upper character.
+			part = part[:l-1] + "-" + part[l-1:]
+		}
+		if i > 0 {
+			part = "-" + part
+		}
+		parts[i] = strings.ToLower(part)
+	}
+	return strings.Join(parts, "")
+}
+
+func (t *Tester) validateFlags() error {
+	if t.GinkgoBinary == "" {
+		return errors.New("required --ginkgo-binary path missing")
+	}
+	if t.KubeletBinary == "" {
+		return errors.New("required --kubelet-binary path missing")
+	}
+	if t.E2ENodeBinary == "" {
+		return errors.New("required --e2e-node-binary path missing")
+	}
+	if t.GCPZone == "" && t.Provider == "gce" {
+		return errors.New("required --gcp-zone")
+	}
+	return nil
+}
+
+// configureRemote configures the "remote" package through it's flags.
+// This corresponds to test/e2e_node/runner/remote/run_remote.go
+// as invoked by hack/make-rules/test-e2e-node.sh.
+func (t *Tester) configureRemote() (finalErr error) {
+	for name, value := range map[string]string{
+		"ssh-env": "gce", // Hard-coded as in https://github.com/kubernetes/kubernetes/blob/34341909b3e9a4854ab5d336b056b934bbbd9f16/hack/make-rules/test-e2e-node.sh#L218
+
+		// See test/e2e_node/remote/run_remote_suite.go and test/e2e_node/remote/gce/gce_runner.go
+		// for the definition of these flags or check `go run ./test/e2e_node/runner/remote/ -help`.
+		"project":           t.GCPProject,
+		"zone":              t.GCPZone,
+		"test_args":         t.TestArgs,
+		"node-env":          t.NodeEnv,
+		"delete-instances":  strconv.FormatBool(t.DeleteInstances),
+		"image-config-file": t.ImageConfigFile,
+		"image-config-dir":  t.ImageConfigDir,
+		"image-project":     t.ImageProject,
+		"images":            t.Images,
+		"instance-metadata": t.InstanceMetadata,
+		"instance-type":     t.InstanceType,
+		"test-timeout":      t.Timeout.String(),
+
+		"ginkgo-binary":   t.GinkgoBinary,
+		"kubelet-binary":  t.KubeletBinary,
+		"e2e-node-binary": t.E2ENodeBinary,
+
+		"ssh-user": t.SSHUser,
+		"ssh-key":  t.SSHKey,
+
+		// Not used by the remote runner and cannot be set because the flag is only defined in the command:
+		// https://github.com/kubernetes/kubernetes/blob/a5098cf9a1405b6b6ed6cf9e0e4e49270c5a0996/test/e2e_node/runner/remote/run_remote.go#L37-L41
+		//
+		// We accept the parameter for the sake of consistency and because it might make
+		// migrating jobs easier. Maybe it will also be supported in the future, if there
+		// turns out to be a need for it.
+		// "runtime-config":    t.RuntimeConfig,
+
+		// Combining multiple different parameters inside a single parameter is problematic
+		// because of quoting, but that is what the existing code uses. A better solution
+		// would be have a `ginkgo-flag` parameter that can be used once for each individual
+		// Ginkgo parameter.
+		"ginkgo-flags": fmt.Sprintf("--nodes=%d --label-filter=%q --skip=%q --focus=%q", t.Parallelism, t.LabelFilter, t.SkipRegex, t.FocusRegex),
+	} {
+		if err := remote.CommandLine.Set(name, value); err != nil {
+			return fmt.Errorf("set --%s to %q: %v", name, value, err)
+		}
+	}
+
+	return nil
+}
+
+func (t *Tester) Test() error {
+	if err := t.configureRemote(); err != nil {
+		return fmt.Errorf("configure remote E2E node execution: %v", err)
+	}
+
+	suite, err := remote.GetTestSuite("default")
+	if err != nil {
+		return fmt.Errorf("error looking up testsuite [%v] - registered test suites [%v]", err, remote.GetTestSuiteKeys())
+	}
+	remote.RunRemoteTestSuite(suite)
+	return nil
+}
+
+func Main(args []string) {
+	// No cancellation, there's nothing to clean up when killed.
+	ctx := context.Background()
+
+	t := NewDefaultTester()
+	if err := t.Execute(ctx, args); err != nil {
+		fmt.Fprintf(os.Stderr, "\"%s remote\"  failed: %v\n", os.Args[0], err)
+		os.Exit(1)
+	}
+}