diff --git a/pkg/credentialprovider/keyring.go b/pkg/credentialprovider/keyring.go index 0c5b3a0c934..63005f6322b 100644 --- a/pkg/credentialprovider/keyring.go +++ b/pkg/credentialprovider/keyring.go @@ -17,6 +17,9 @@ limitations under the License. package credentialprovider import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "net" "net/url" "path/filepath" @@ -24,7 +27,9 @@ import ( "strings" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" ) // DockerKeyring tracks a set of docker registry credentials, maintaining a @@ -35,13 +40,13 @@ import ( // most specific match for a given image // - iterating a map does not yield predictable results type DockerKeyring interface { - Lookup(image string) ([]AuthConfig, bool) + Lookup(image string) ([]TrackedAuthConfig, bool) } // BasicDockerKeyring is a trivial map-backed implementation of DockerKeyring type BasicDockerKeyring struct { index []string - creds map[string][]AuthConfig + creds map[string][]TrackedAuthConfig } // providersDockerKeyring is an implementation of DockerKeyring that @@ -50,6 +55,47 @@ type providersDockerKeyring struct { Providers []DockerConfigProvider } +// TrackedAuthConfig wraps the AuthConfig and adds information about the source +// of the credentials. +type TrackedAuthConfig struct { + AuthConfig + AuthConfigHash string + + Source *CredentialSource +} + +// NewTrackedAuthConfig initializes the TrackedAuthConfig structure by adding +// the source information to the supplied AuthConfig. It also counts a hash of the +// AuthConfig and keeps it in the returned structure. +// +// The supplied CredentialSource is only used when the "KubeletEnsureSecretPulledImages" +// is enabled, the same applies for counting the hash. +func NewTrackedAuthConfig(c *AuthConfig, src *CredentialSource) *TrackedAuthConfig { + if c == nil { + panic("cannot construct TrackedAuthConfig with a nil AuthConfig") + } + + authConfig := &TrackedAuthConfig{ + AuthConfig: *c, + } + + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + authConfig.Source = src + authConfig.AuthConfigHash = hashAuthConfig(c) + } + return authConfig +} + +type CredentialSource struct { + Secret SecretCoordinates +} + +type SecretCoordinates struct { + UID string + Namespace string + Name string +} + // AuthConfig contains authorization information for connecting to a Registry // This type mirrors "github.com/docker/docker/api/types.AuthConfig" type AuthConfig struct { @@ -72,11 +118,13 @@ type AuthConfig struct { RegistryToken string `json:"registrytoken,omitempty"` } -// Add add some docker config in basic docker keyring -func (dk *BasicDockerKeyring) Add(cfg DockerConfig) { +// Add inserts the docker config `cfg` into the basic docker keyring. It attaches +// the `src` information that describes where the docker config `cfg` comes from. +// `src` is nil if the docker config is globally available on the node. +func (dk *BasicDockerKeyring) Add(src *CredentialSource, cfg DockerConfig) { if dk.index == nil { dk.index = make([]string, 0) - dk.creds = make(map[string][]AuthConfig) + dk.creds = make(map[string][]TrackedAuthConfig) } for loc, ident := range cfg { creds := AuthConfig{ @@ -111,7 +159,9 @@ func (dk *BasicDockerKeyring) Add(cfg DockerConfig) { } else { key = parsed.Host } - dk.creds[key] = append(dk.creds[key], creds) + trackedCreds := NewTrackedAuthConfig(&creds, src) + + dk.creds[key] = append(dk.creds[key], *trackedCreds) dk.index = append(dk.index, key) } @@ -235,9 +285,9 @@ func URLsMatch(globURL *url.URL, targetURL *url.URL) (bool, error) { // Lookup implements the DockerKeyring method for fetching credentials based on image name. // Multiple credentials may be returned if there are multiple potentially valid credentials // available. This allows for rotation. -func (dk *BasicDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { +func (dk *BasicDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) { // range over the index as iterating over a map does not provide a predictable ordering - ret := []AuthConfig{} + ret := []TrackedAuthConfig{} for _, k := range dk.index { // both k and image are schemeless URLs because even though schemes are allowed // in the credential configurations, we remove them in Add. @@ -257,16 +307,18 @@ func (dk *BasicDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { } } - return []AuthConfig{}, false + return []TrackedAuthConfig{}, false } // Lookup implements the DockerKeyring method for fetching credentials // based on image name. -func (dk *providersDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { +func (dk *providersDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) { keyring := &BasicDockerKeyring{} for _, p := range dk.Providers { - keyring.Add(p.Provide(image)) + // TODO: the source should probably change once we depend on service accounts (KEP-4412). + // Perhaps `Provide()` should return the source modified to accommodate this? + keyring.Add(nil, p.Provide(image)) } return keyring.Lookup(image) @@ -274,13 +326,13 @@ func (dk *providersDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { // FakeKeyring a fake config credentials type FakeKeyring struct { - auth []AuthConfig + auth []TrackedAuthConfig ok bool } // Lookup implements the DockerKeyring method for fetching credentials based on image name // return fake auth and ok -func (f *FakeKeyring) Lookup(image string) ([]AuthConfig, bool) { +func (f *FakeKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) { return f.auth, f.ok } @@ -289,8 +341,8 @@ type UnionDockerKeyring []DockerKeyring // Lookup implements the DockerKeyring method for fetching credentials based on image name. // return each credentials -func (k UnionDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { - authConfigs := []AuthConfig{} +func (k UnionDockerKeyring) Lookup(image string) ([]TrackedAuthConfig, bool) { + authConfigs := []TrackedAuthConfig{} for _, subKeyring := range k { if subKeyring == nil { continue @@ -302,3 +354,14 @@ func (k UnionDockerKeyring) Lookup(image string) ([]AuthConfig, bool) { return authConfigs, (len(authConfigs) > 0) } + +func hashAuthConfig(creds *AuthConfig) string { + credBytes, err := json.Marshal(creds) + if err != nil { + return "" + } + + hash := sha256.New() + hash.Write([]byte(credBytes)) + return hex.EncodeToString(hash.Sum(nil)) +} diff --git a/pkg/credentialprovider/keyring_test.go b/pkg/credentialprovider/keyring_test.go index 8535a7b7271..ba37c48c323 100644 --- a/pkg/credentialprovider/keyring_test.go +++ b/pkg/credentialprovider/keyring_test.go @@ -21,6 +21,10 @@ import ( "fmt" "reflect" "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" ) func TestURLsMatch(t *testing.T) { @@ -222,7 +226,7 @@ func TestDockerKeyringForGlob(t *testing.T) { if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { - keyring.Add(cfg) + keyring.Add(nil, cfg) } creds, ok := keyring.Lookup(test.targetURL + "/foo/bar") @@ -290,7 +294,7 @@ func TestKeyringMiss(t *testing.T) { if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { - keyring.Add(cfg) + keyring.Add(nil, cfg) } _, ok := keyring.Lookup(test.lookupURL + "/foo/bar") @@ -318,7 +322,7 @@ func TestKeyringMissWithDockerHubCredentials(t *testing.T) { if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { - keyring.Add(cfg) + keyring.Add(nil, cfg) } val, ok := keyring.Lookup("world.mesos.org/foo/bar") @@ -344,7 +348,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) { if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { - keyring.Add(cfg) + keyring.Add(nil, cfg) } creds, ok := keyring.Lookup("google/docker-registry") @@ -353,7 +357,7 @@ func TestKeyringHitWithUnqualifiedDockerHub(t *testing.T) { return } if len(creds) > 1 { - t.Errorf("Got more hits than expected: %s", creds) + t.Errorf("Got more hits than expected: %v", creds) } val := creds[0] @@ -385,7 +389,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) { if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { - keyring.Add(cfg) + keyring.Add(nil, cfg) } creds, ok := keyring.Lookup("jenkins") @@ -394,7 +398,7 @@ func TestKeyringHitWithUnqualifiedLibraryDockerHub(t *testing.T) { return } if len(creds) > 1 { - t.Errorf("Got more hits than expected: %s", creds) + t.Errorf("Got more hits than expected: %v", creds) } val := creds[0] @@ -426,7 +430,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) { if cfg, err := ReadDockerConfigFileFromBytes([]byte(sampleDockerConfig)); err != nil { t.Errorf("Error processing json blob %q, %v", sampleDockerConfig, err) } else { - keyring.Add(cfg) + keyring.Add(nil, cfg) } creds, ok := keyring.Lookup(url + "/google/docker-registry") @@ -435,7 +439,7 @@ func TestKeyringHitWithQualifiedDockerHub(t *testing.T) { return } if len(creds) > 2 { - t.Errorf("Got more hits than expected: %s", creds) + t.Errorf("Got more hits than expected: %v", creds) } val := creds[0] @@ -498,20 +502,24 @@ func TestProvidersDockerKeyring(t *testing.T) { } func TestDockerKeyringLookup(t *testing.T) { + // turn on the ensure secret pulled images feature to get the hashes with the creds + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletEnsureSecretPulledImages, true) ada := AuthConfig{ Username: "ada", Password: "smash", // Fake value for testing. Email: "ada@example.com", } + adaHash := "353258b53f5e9a57b059eab3f05312fc35bbeb874f08ce101e7bf0bf46977423" grace := AuthConfig{ Username: "grace", Password: "squash", // Fake value for testing. Email: "grace@example.com", } + graceHash := "f949b3837a1eb733a951b6aeda0b3327c09ec50c917de9ca35818e8fbf567e29" dk := &BasicDockerKeyring{} - dk.Add(DockerConfig{ + dk.Add(nil, DockerConfig{ "bar.example.com/pong": DockerConfigEntry{ Username: grace.Username, Password: grace.Password, @@ -526,27 +534,27 @@ func TestDockerKeyringLookup(t *testing.T) { tests := []struct { image string - match []AuthConfig + match []TrackedAuthConfig ok bool }{ // direct match - {"bar.example.com", []AuthConfig{ada}, true}, + {"bar.example.com", []TrackedAuthConfig{{AuthConfig: ada, AuthConfigHash: adaHash}}, true}, // direct match deeper than other possible matches - {"bar.example.com/pong", []AuthConfig{grace, ada}, true}, + {"bar.example.com/pong", []TrackedAuthConfig{{AuthConfig: grace, AuthConfigHash: graceHash}, {AuthConfig: ada, AuthConfigHash: adaHash}}, true}, // no direct match, deeper path ignored - {"bar.example.com/ping", []AuthConfig{ada}, true}, + {"bar.example.com/ping", []TrackedAuthConfig{{AuthConfig: ada, AuthConfigHash: adaHash}}, true}, // match first part of path token - {"bar.example.com/pongz", []AuthConfig{grace, ada}, true}, + {"bar.example.com/pongz", []TrackedAuthConfig{{AuthConfig: grace, AuthConfigHash: graceHash}, {AuthConfig: ada, AuthConfigHash: adaHash}}, true}, // match regardless of sub-path - {"bar.example.com/pong/pang", []AuthConfig{grace, ada}, true}, + {"bar.example.com/pong/pang", []TrackedAuthConfig{{AuthConfig: grace, AuthConfigHash: graceHash}, {AuthConfig: ada, AuthConfigHash: adaHash}}, true}, // no host match - {"example.com", []AuthConfig{}, false}, - {"foo.example.com", []AuthConfig{}, false}, + {"example.com", []TrackedAuthConfig{}, false}, + {"foo.example.com", []TrackedAuthConfig{}, false}, } for i, tt := range tests { @@ -565,14 +573,17 @@ func TestDockerKeyringLookup(t *testing.T) { // by images that only match the hostname. // NOTE: the above covers the case of a more specific match trumping just hostname. func TestIssue3797(t *testing.T) { + // turn on the ensure secret pulled images feature to get the hashes with the creds + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletEnsureSecretPulledImages, true) rex := AuthConfig{ Username: "rex", Password: "tiny arms", // Fake value for testing. Email: "rex@example.com", } + rexHash := "899748fec74c8dd761845fca727f4249b05be275ff24026676fcd4351f656363" dk := &BasicDockerKeyring{} - dk.Add(DockerConfig{ + dk.Add(nil, DockerConfig{ "https://quay.io/v1/": DockerConfigEntry{ Username: rex.Username, Password: rex.Password, @@ -582,15 +593,15 @@ func TestIssue3797(t *testing.T) { tests := []struct { image string - match []AuthConfig + match []TrackedAuthConfig ok bool }{ // direct match - {"quay.io", []AuthConfig{rex}, true}, + {"quay.io", []TrackedAuthConfig{{AuthConfig: rex, AuthConfigHash: rexHash}}, true}, // partial matches - {"quay.io/foo", []AuthConfig{rex}, true}, - {"quay.io/foo/bar", []AuthConfig{rex}, true}, + {"quay.io/foo", []TrackedAuthConfig{{AuthConfig: rex, AuthConfigHash: rexHash}}, true}, + {"quay.io/foo/bar", []TrackedAuthConfig{{AuthConfig: rex, AuthConfigHash: rexHash}}, true}, } for i, tt := range tests { diff --git a/pkg/credentialprovider/plugin/plugins.go b/pkg/credentialprovider/plugin/plugins.go index 0260158ae47..fbadc80eddf 100644 --- a/pkg/credentialprovider/plugin/plugins.go +++ b/pkg/credentialprovider/plugin/plugins.go @@ -87,11 +87,12 @@ func NewExternalCredentialProviderDockerKeyring(podNamespace, podName, podUID, s return keyring } -func (k *externalCredentialProviderKeyring) Lookup(image string) ([]credentialprovider.AuthConfig, bool) { +func (k *externalCredentialProviderKeyring) Lookup(image string) ([]credentialprovider.TrackedAuthConfig, bool) { keyring := &credentialprovider.BasicDockerKeyring{} for _, p := range k.providers { - keyring.Add(p.Provide(image)) + // TODO: modify the credentialprovider.CredentialSource to contain the SA/pod information + keyring.Add(nil, p.Provide(image)) } return keyring.Lookup(image) diff --git a/pkg/credentialprovider/secrets/secrets.go b/pkg/credentialprovider/secrets/secrets.go index 423cd2bbf94..eab1e22ab40 100644 --- a/pkg/credentialprovider/secrets/secrets.go +++ b/pkg/credentialprovider/secrets/secrets.go @@ -27,32 +27,52 @@ import ( // then a DockerKeyring is built based on every hit and unioned with the defaultKeyring. // If they do not, then the default keyring is returned func MakeDockerKeyring(passedSecrets []v1.Secret, defaultKeyring credentialprovider.DockerKeyring) (credentialprovider.DockerKeyring, error) { - passedCredentials := []credentialprovider.DockerConfig{} - for _, passedSecret := range passedSecrets { + providerFromSecrets, err := secretsToTrackedDockerConfigs(passedSecrets) + if err != nil { + return nil, err + } + + if providerFromSecrets == nil { + return defaultKeyring, nil + } + + return credentialprovider.UnionDockerKeyring{providerFromSecrets, defaultKeyring}, nil +} + +func secretsToTrackedDockerConfigs(secrets []v1.Secret) (credentialprovider.DockerKeyring, error) { + provider := &credentialprovider.BasicDockerKeyring{} + validSecretsFound := 0 + for _, passedSecret := range secrets { if dockerConfigJSONBytes, dockerConfigJSONExists := passedSecret.Data[v1.DockerConfigJsonKey]; (passedSecret.Type == v1.SecretTypeDockerConfigJson) && dockerConfigJSONExists && (len(dockerConfigJSONBytes) > 0) { dockerConfigJSON := credentialprovider.DockerConfigJSON{} if err := json.Unmarshal(dockerConfigJSONBytes, &dockerConfigJSON); err != nil { return nil, err } - passedCredentials = append(passedCredentials, dockerConfigJSON.Auths) + coords := credentialprovider.SecretCoordinates{ + UID: string(passedSecret.UID), + Namespace: passedSecret.Namespace, + Name: passedSecret.Name} + + provider.Add(&credentialprovider.CredentialSource{Secret: coords}, dockerConfigJSON.Auths) + validSecretsFound++ } else if dockercfgBytes, dockercfgExists := passedSecret.Data[v1.DockerConfigKey]; (passedSecret.Type == v1.SecretTypeDockercfg) && dockercfgExists && (len(dockercfgBytes) > 0) { dockercfg := credentialprovider.DockerConfig{} if err := json.Unmarshal(dockercfgBytes, &dockercfg); err != nil { return nil, err } - passedCredentials = append(passedCredentials, dockercfg) + coords := credentialprovider.SecretCoordinates{ + UID: string(passedSecret.UID), + Namespace: passedSecret.Namespace, + Name: passedSecret.Name} + provider.Add(&credentialprovider.CredentialSource{Secret: coords}, dockercfg) + validSecretsFound++ } } - if len(passedCredentials) > 0 { - basicKeyring := &credentialprovider.BasicDockerKeyring{} - for _, currCredentials := range passedCredentials { - basicKeyring.Add(currCredentials) - } - return credentialprovider.UnionDockerKeyring{basicKeyring, defaultKeyring}, nil + if validSecretsFound == 0 { + return nil, nil } - - return defaultKeyring, nil + return provider, nil } diff --git a/pkg/credentialprovider/secrets/secrets_test.go b/pkg/credentialprovider/secrets/secrets_test.go index ad03cdd9dda..e96112fd3b3 100644 --- a/pkg/credentialprovider/secrets/secrets_test.go +++ b/pkg/credentialprovider/secrets/secrets_test.go @@ -20,107 +20,145 @@ import ( "reflect" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/kubernetes/pkg/features" ) -// fakeKeyring is a fake docker auth config keyring -type fakeKeyring struct { - auth []credentialprovider.AuthConfig +// FakeKeyring a fake config credentials +type FakeKeyring struct { + auth []credentialprovider.TrackedAuthConfig ok bool } -// Lookup implements the DockerKeyring method for fetching credentials based on image name. -// Returns fake results based on the auth and ok fields in fakeKeyring -func (f *fakeKeyring) Lookup(image string) ([]credentialprovider.AuthConfig, bool) { +// Lookup implements the DockerKeyring method for fetching credentials based on image name +// return fake auth and ok +func (f *FakeKeyring) Lookup(image string) ([]credentialprovider.TrackedAuthConfig, bool) { return f.auth, f.ok } func Test_MakeDockerKeyring(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletEnsureSecretPulledImages, true) + testcases := []struct { - name string - image string - defaultKeyring credentialprovider.DockerKeyring - pullSecrets []v1.Secret - authConfigs []credentialprovider.AuthConfig - found bool + name string + image string + defaultKeyring credentialprovider.DockerKeyring + pullSecrets []v1.Secret + expectedAuthConfigs []credentialprovider.TrackedAuthConfig + found bool }{ { - name: "with .dockerconfigjson and auth field", - image: "test.registry.io", - defaultKeyring: &fakeKeyring{}, + name: "with .dockerconfigjson and auth field", + image: "test.registry.io", pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{ v1.DockerConfigJsonKey: []byte(`{"auths": {"test.registry.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "user", - Password: "password", + Source: &credentialprovider.CredentialSource{ + Secret: credentialprovider.SecretCoordinates{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, + }, + AuthConfig: credentialprovider.AuthConfig{ + Username: "user", + Password: "password", + }, + AuthConfigHash: "a55436fc140d516560d072c5fe8700385ce9f41629abf65c1edcbcb39fac691d", }, }, found: true, }, { - name: "with .dockerconfig and auth field", - image: "test.registry.io", - defaultKeyring: &fakeKeyring{}, + name: "with .dockerconfig and auth field", + image: "test.registry.io", pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockercfg, Data: map[string][]byte{ v1.DockerConfigKey: []byte(`{"test.registry.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "user", - Password: "password", + Source: &credentialprovider.CredentialSource{ + Secret: credentialprovider.SecretCoordinates{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, + }, + AuthConfig: credentialprovider.AuthConfig{ + Username: "user", + Password: "password", + }, + AuthConfigHash: "a55436fc140d516560d072c5fe8700385ce9f41629abf65c1edcbcb39fac691d", }, }, found: true, }, { - name: "with .dockerconfigjson and username/password fields", - image: "test.registry.io", - defaultKeyring: &fakeKeyring{}, + name: "with .dockerconfigjson and username/password fields", + image: "test.registry.io", pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{ v1.DockerConfigJsonKey: []byte(`{"auths": {"test.registry.io": {"username": "user", "password": "password"}}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "user", - Password: "password", + Source: &credentialprovider.CredentialSource{ + Secret: credentialprovider.SecretCoordinates{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, + }, + AuthConfig: credentialprovider.AuthConfig{ + Username: "user", + Password: "password", + }, + AuthConfigHash: "a55436fc140d516560d072c5fe8700385ce9f41629abf65c1edcbcb39fac691d", }, }, found: true, }, { - name: "with .dockerconfig and username/password fields", - image: "test.registry.io", - defaultKeyring: &fakeKeyring{}, + name: "with .dockerconfig and username/password fields", + image: "test.registry.io", pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockercfg, Data: map[string][]byte{ v1.DockerConfigKey: []byte(`{"test.registry.io": {"username": "user", "password": "password"}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "user", - Password: "password", + Source: &credentialprovider.CredentialSource{ + Secret: credentialprovider.SecretCoordinates{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, + }, + AuthConfig: credentialprovider.AuthConfig{ + Username: "user", + Password: "password", + }, + AuthConfigHash: "a55436fc140d516560d072c5fe8700385ce9f41629abf65c1edcbcb39fac691d", }, }, found: true, @@ -128,56 +166,65 @@ func Test_MakeDockerKeyring(t *testing.T) { { name: "with .dockerconfigjson but with wrong Secret Type", image: "test.registry.io", - defaultKeyring: &fakeKeyring{}, + defaultKeyring: &FakeKeyring{}, pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockercfg, Data: map[string][]byte{ v1.DockerConfigJsonKey: []byte(`{"auths": {"test.registry.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}}`), }, }, }, - authConfigs: nil, - found: false, + found: false, }, { name: "with .dockerconfig but with wrong Secret Type", image: "test.registry.io", - defaultKeyring: &fakeKeyring{}, + defaultKeyring: &FakeKeyring{}, pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{ v1.DockerConfigKey: []byte(`{"test.registry.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}`), }, }, }, - authConfigs: nil, - found: false, + found: false, }, { name: "with not matcing .dockerconfigjson and default keyring", image: "test.registry.io", - defaultKeyring: &fakeKeyring{ - auth: []credentialprovider.AuthConfig{ + defaultKeyring: &FakeKeyring{ + auth: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, }, }, }, pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{ v1.DockerConfigJsonKey: []byte(`{"auths": {"foobar.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, + AuthConfigHash: "", }, }, found: true, @@ -185,26 +232,33 @@ func Test_MakeDockerKeyring(t *testing.T) { { name: "with not matching .dockerconfig and default keyring", image: "test.registry.io", - defaultKeyring: &fakeKeyring{ - auth: []credentialprovider.AuthConfig{ + defaultKeyring: &FakeKeyring{ + auth: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, }, }, }, pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockercfg, Data: map[string][]byte{ v1.DockerConfigKey: []byte(`{"foobar.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, + AuthConfigHash: "", }, }, found: true, @@ -212,19 +266,24 @@ func Test_MakeDockerKeyring(t *testing.T) { { name: "with no pull secrets but has default keyring", image: "test.registry.io", - defaultKeyring: &fakeKeyring{ - auth: []credentialprovider.AuthConfig{ + defaultKeyring: &FakeKeyring{ + auth: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, }, }, }, pullSecrets: []v1.Secret{}, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, + AuthConfigHash: "", }, }, found: false, @@ -232,30 +291,44 @@ func Test_MakeDockerKeyring(t *testing.T) { { name: "with pull secrets and has default keyring", image: "test.registry.io", - defaultKeyring: &fakeKeyring{ - auth: []credentialprovider.AuthConfig{ + defaultKeyring: &FakeKeyring{ + auth: []credentialprovider.TrackedAuthConfig{ { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, }, }, }, pullSecrets: []v1.Secret{ { + ObjectMeta: metav1.ObjectMeta{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{ v1.DockerConfigJsonKey: []byte(`{"auths": {"test.registry.io": {"auth": "dXNlcjpwYXNzd29yZA=="}}}`), }, }, }, - authConfigs: []credentialprovider.AuthConfig{ + expectedAuthConfigs: []credentialprovider.TrackedAuthConfig{ { - Username: "user", - Password: "password", + Source: &credentialprovider.CredentialSource{ + Secret: credentialprovider.SecretCoordinates{ + Name: "s1", Namespace: "ns1", UID: "uid1"}, + }, + AuthConfig: credentialprovider.AuthConfig{ + Username: "user", + Password: "password", + }, + AuthConfigHash: "a55436fc140d516560d072c5fe8700385ce9f41629abf65c1edcbcb39fac691d", }, { - Username: "default-user", - Password: "default-password", + AuthConfig: credentialprovider.AuthConfig{ + Username: "default-user", + Password: "default-password", + }, + AuthConfigHash: "", }, }, found: true, @@ -276,9 +349,9 @@ func Test_MakeDockerKeyring(t *testing.T) { t.Errorf("unexpected lookup for image: %s", testcase.image) } - if !reflect.DeepEqual(authConfigs, testcase.authConfigs) { + if !reflect.DeepEqual(authConfigs, testcase.expectedAuthConfigs) { // TODO: we may need better comparison as the result is unordered t.Logf("actual auth configs: %#v", authConfigs) - t.Logf("expected auth configs: %#v", testcase.authConfigs) + t.Logf("expected auth configs: %#v", testcase.expectedAuthConfigs) t.Error("auth configs did not match") } }) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 49e6a490e08..8c409dfc160 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -352,6 +352,13 @@ const ( // fallback to using it's cgroupDriver option. KubeletCgroupDriverFromCRI featuregate.Feature = "KubeletCgroupDriverFromCRI" + // owner: @stlaz + // kep: https://kep.k8s.io/2535 + // + // Enables tracking credentials for image pulls in order to authorize image + // access for different tenants. + KubeletEnsureSecretPulledImages featuregate.Feature = "KubeletEnsureSecretPulledImages" + // owner: @vinayakankugoyal // kep: http://kep.k8s.io/2862 // diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go index a92e791317c..a0b58205959 100644 --- a/pkg/features/versioned_kube_features.go +++ b/pkg/features/versioned_kube_features.go @@ -438,6 +438,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, }, + KubeletEnsureSecretPulledImages: { + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, + }, + KubeletFineGrainedAuthz: { {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 560ea0d573d..50b8ef06036 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1260,6 +1260,10 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/kubelet/config/v1alpha1.CredentialProvider": schema_k8sio_kubelet_config_v1alpha1_CredentialProvider(ref), "k8s.io/kubelet/config/v1alpha1.CredentialProviderConfig": schema_k8sio_kubelet_config_v1alpha1_CredentialProviderConfig(ref), "k8s.io/kubelet/config/v1alpha1.ExecEnvVar": schema_k8sio_kubelet_config_v1alpha1_ExecEnvVar(ref), + "k8s.io/kubelet/config/v1alpha1.ImagePullCredentials": schema_k8sio_kubelet_config_v1alpha1_ImagePullCredentials(ref), + "k8s.io/kubelet/config/v1alpha1.ImagePullIntent": schema_k8sio_kubelet_config_v1alpha1_ImagePullIntent(ref), + "k8s.io/kubelet/config/v1alpha1.ImagePullSecret": schema_k8sio_kubelet_config_v1alpha1_ImagePullSecret(ref), + "k8s.io/kubelet/config/v1alpha1.ImagePulledRecord": schema_k8sio_kubelet_config_v1alpha1_ImagePulledRecord(ref), "k8s.io/kubelet/config/v1beta1.CrashLoopBackOffConfig": schema_k8sio_kubelet_config_v1beta1_CrashLoopBackOffConfig(ref), "k8s.io/kubelet/config/v1beta1.CredentialProvider": schema_k8sio_kubelet_config_v1beta1_CredentialProvider(ref), "k8s.io/kubelet/config/v1beta1.CredentialProviderConfig": schema_k8sio_kubelet_config_v1beta1_CredentialProviderConfig(ref), @@ -64667,6 +64671,185 @@ func schema_k8sio_kubelet_config_v1alpha1_ExecEnvVar(ref common.ReferenceCallbac } } +func schema_k8sio_kubelet_config_v1alpha1_ImagePullCredentials(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ImagePullCredentials describe credentials that can be used to pull an image.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kubernetesSecrets": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "KuberneteSecretCoordinates is an index of coordinates of all the kubernetes secrets that were used to pull the image.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/kubelet/config/v1alpha1.ImagePullSecret"), + }, + }, + }, + }, + }, + "nodePodsAccessible": { + SchemaProps: spec.SchemaProps{ + Description: "NodePodsAccessible is a flag denoting the pull credentials are accessible by all the pods on the node, or that no credentials are needed for the pull.\n\nIf true, it is mutually exclusive with the `kubernetesSecrets` field.", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/kubelet/config/v1alpha1.ImagePullSecret"}, + } +} + +func schema_k8sio_kubelet_config_v1alpha1_ImagePullIntent(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ImagePullIntent is a record of the kubelet attempting to pull an image.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "image": { + SchemaProps: spec.SchemaProps{ + Description: "Image is the image spec from a Container's `image` field. The filename is a SHA-256 hash of this value. This is to avoid filename-unsafe characters like ':' and '/'.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"image"}, + }, + }, + } +} + +func schema_k8sio_kubelet_config_v1alpha1_ImagePullSecret(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ImagePullSecret is a representation of a Kubernetes secret object coordinates along with a credential hash of the pull secret credentials this object contains.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "uid": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "namespace": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "credentialHash": { + SchemaProps: spec.SchemaProps{ + Description: "CredentialHash is a SHA-256 retrieved by hashing the image pull credentials content of the secret specified by the UID/Namespace/Name coordinates.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"uid", "namespace", "name", "credentialHash"}, + }, + }, + } +} + +func schema_k8sio_kubelet_config_v1alpha1_ImagePulledRecord(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ImagePullRecord is a record of an image that was pulled by the kubelet.\n\nIf there are no records in the `kubernetesSecrets` field and both `nodeWideCredentials` and `anonymous` are `false`, credentials must be re-checked the next time an image represented by this record is being requested.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "lastUpdatedTime": { + SchemaProps: spec.SchemaProps{ + Description: "LastUpdatedTime is the time of the last update to this record", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + "imageRef": { + SchemaProps: spec.SchemaProps{ + Description: "ImageRef is a reference to the image represented by this file as received from the CRI. The filename is a SHA-256 hash of this value. This is to avoid filename-unsafe characters like ':' and '/'.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "credentialMapping": { + SchemaProps: spec.SchemaProps{ + Description: "CredentialMapping maps `image` to the set of credentials that it was previously pulled with. `image` in this case is the content of a pod's container `image` field that's got its tag/digest removed.\n\nExample:\n Container requests the `hello-world:latest@sha256:91fb4b041da273d5a3273b6d587d62d518300a6ad268b28628f74997b93171b2` image:\n \"credentialMapping\": {\n \"hello-world\": { \"nodePodsAccessible\": true }\n }", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/kubelet/config/v1alpha1.ImagePullCredentials"), + }, + }, + }, + }, + }, + }, + Required: []string{"lastUpdatedTime", "imageRef"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Time", "k8s.io/kubelet/config/v1alpha1.ImagePullCredentials"}, + } +} + func schema_k8sio_kubelet_config_v1beta1_CrashLoopBackOffConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -65112,6 +65295,33 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen Format: "int32", }, }, + "imagePullCredentialsVerificationPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "imagePullCredentialsVerificationPolicy determines how credentials should be verified when pod requests an image that is already present on the node:\n - NeverVerify\n - anyone on a node can use any image present on the node\n - NeverVerifyPreloadedImages\n - images that were pulled to the node by something else than the kubelet\n can be used without reverifying pull credentials\n - NeverVerifyAllowlistedImages\n - like \"NeverVerifyPreloadedImages\" but only node images from\n `preloadedImagesVerificationAllowlist` don't require reverification\n - AlwaysVerify\n - all images require credential reverification", + Type: []string{"string"}, + Format: "", + }, + }, + "preloadedImagesVerificationAllowlist": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "preloadedImagesVerificationAllowlist specifies a list of images that are exempted from credential reverification for the \"NeverVerifyAllowlistedImages\" `imagePullCredentialsVerificationPolicy`. The list accepts a full path segment wildcard suffix \"/*\". Only use image specs without an image tag or digest.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "eventRecordQPS": { SchemaProps: spec.SchemaProps{ Description: "eventRecordQPS is the maximum event creations per second. If 0, there is no limit enforced. The value cannot be a negative number. Default: 50", diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index 568b0be20df..b170def02dc 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -243,6 +243,7 @@ var ( "ImageGCLowThresholdPercent", "ImageMinimumGCAge.Duration", "ImageMaximumGCAge.Duration", + "ImagePullCredentialsVerificationPolicy", "KernelMemcgNotification", "KubeAPIBurst", "KubeAPIQPS", @@ -268,6 +269,7 @@ var ( "PodPidsLimit", "PodsPerCore", "Port", + "PreloadedImagesVerificationAllowlist[*]", "ProtectKernelDefaults", "ProviderID", "ReadOnlyPort", diff --git a/pkg/kubelet/apis/config/register.go b/pkg/kubelet/apis/config/register.go index d13cc6bdb63..307995d1bc3 100644 --- a/pkg/kubelet/apis/config/register.go +++ b/pkg/kubelet/apis/config/register.go @@ -40,6 +40,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &KubeletConfiguration{}, &SerializedNodeConfigSource{}, &CredentialProviderConfig{}, + &ImagePullIntent{}, + &ImagePulledRecord{}, ) return nil } diff --git a/pkg/kubelet/apis/config/register_test.go b/pkg/kubelet/apis/config/register_test.go index 339bafd8fad..ed7e89ad8de 100644 --- a/pkg/kubelet/apis/config/register_test.go +++ b/pkg/kubelet/apis/config/register_test.go @@ -37,6 +37,7 @@ func TestComponentConfigSetup(t *testing.T) { reflect.TypeOf(logsapi.LoggingConfiguration{}): true, reflect.TypeOf(tracingapi.TracingConfiguration{}): true, reflect.TypeOf(metav1.Duration{}): true, + reflect.TypeOf(metav1.Time{}): true, reflect.TypeOf(metav1.TypeMeta{}): true, reflect.TypeOf(v1.NodeConfigSource{}): true, reflect.TypeOf(v1.Taint{}): true, diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 0924dc63846..d295f16122c 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -155,6 +155,25 @@ type KubeletConfiguration struct { // pulls to burst to this number, while still not exceeding registryPullQPS. // Only used if registryPullQPS > 0. RegistryBurst int32 + // imagePullCredentialsVerificationPolicy determines how credentials should be + // verified when pod requests an image that is already present on the node: + // - NeverVerify + // - anyone on a node can use any image present on the node + // - NeverVerifyPreloadedImages + // - images that were pulled to the node by something else than the kubelet + // can be used without reverifying pull credentials + // - NeverVerifyAllowlistedImages + // - like "NeverVerifyPreloadedImages" but only node images from + // `preloadedImagesVerificationAllowlist` don't require reverification + // - AlwaysVerify + // - all images require credential reverification + ImagePullCredentialsVerificationPolicy string + // preloadedImagesVerificationAllowlist specifies a list of images that are + // exempted from credential reverification for the "NeverVerifyAllowlistedImages" + // `imagePullCredentialsVerificationPolicy`. + // The list accepts a full path segment wildcard suffix "/*". + // Only use image specs without an image tag or digest. + PreloadedImagesVerificationAllowlist []string // eventRecordQPS is the maximum event creations per second. If 0, there // is no limit enforced. EventRecordQPS int32 @@ -769,3 +788,93 @@ type CrashLoopBackOffConfig struct { // +optional MaxContainerRestartPeriod *metav1.Duration } + +// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced +// when pod is requesting an image that appears on the system +type ImagePullCredentialsVerificationPolicy string + +const ( + // NeverVerify will never require credential verification for images that + // already exist on the node + NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify" + // NeverVerifyPreloadedImages does not require credential verification for images + // pulled outside the kubelet process + NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages" + // NeverVerifyAllowlistedImages does not require credential verification for + // a list of images that were pulled outside the kubelet process + NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages" + // AlwaysVerify requires credential verification for accessing any image on the + // node irregardless how it was pulled + AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify" +) + +// ImagePullIntent is a record of the kubelet attempting to pull an image. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ImagePullIntent struct { + metav1.TypeMeta + + // Image is the image spec from a Container's `image` field. + // The filename is a SHA-256 hash of this value. This is to avoid filename-unsafe + // characters like ':' and '/'. + Image string +} + +// ImagePullRecord is a record of an image that was pulled by the kubelet. +// +// If there are no records in the `kubernetesSecrets` field and both `nodeWideCredentials` +// and `anonymous` are `false`, credentials must be re-checked the next time an +// image represented by this record is being requested. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ImagePulledRecord struct { + metav1.TypeMeta + + // LastUpdatedTime is the time of the last update to this record + LastUpdatedTime metav1.Time + + // ImageRef is a reference to the image represented by this file as received + // from the CRI. + // The filename is a SHA-256 hash of this value. This is to avoid filename-unsafe + // characters like ':' and '/'. + ImageRef string + + // CredentialMapping maps `image` to the set of credentials that it was + // previously pulled with. + // `image` in this case is the content of a pod's container `image` field that's + // got its tag/digest removed. + // + // Example: + // Container requests the `hello-world:latest@sha256:91fb4b041da273d5a3273b6d587d62d518300a6ad268b28628f74997b93171b2` image: + // "credentialMapping": { + // "hello-world": { "nodePodsAccessible": true } + // } + CredentialMapping map[string]ImagePullCredentials +} + +// ImagePullCredentials describe credentials that can be used to pull an image. +type ImagePullCredentials struct { + // KuberneteSecretCoordinates is an index of coordinates of all the kubernetes + // secrets that were used to pull the image. + // +optional + KubernetesSecrets []ImagePullSecret + + // NodePodsAccessible is a flag denoting the pull credentials are accessible + // by all the pods on the node, or that no credentials are needed for the pull. + // + // If true, it is mutually exclusive with the `kubernetesSecrets` field. + // +optional + NodePodsAccessible bool +} + +// ImagePullSecret is a representation of a Kubernetes secret object coordinates along +// with a credential hash of the pull secret credentials this object contains. +type ImagePullSecret struct { + UID string + Namespace string + Name string + + // CredentialHash is a SHA-256 retrieved by hashing the image pull credentials + // content of the secret specified by the UID/Namespace/Name coordinates. + CredentialHash string +} diff --git a/pkg/kubelet/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1alpha1/zz_generated.conversion.go index 3cf454ff17f..25cda8fd37f 100644 --- a/pkg/kubelet/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1alpha1/zz_generated.conversion.go @@ -63,6 +63,46 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.ImagePullCredentials)(nil), (*config.ImagePullCredentials)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ImagePullCredentials_To_config_ImagePullCredentials(a.(*configv1alpha1.ImagePullCredentials), b.(*config.ImagePullCredentials), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ImagePullCredentials)(nil), (*configv1alpha1.ImagePullCredentials)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ImagePullCredentials_To_v1alpha1_ImagePullCredentials(a.(*config.ImagePullCredentials), b.(*configv1alpha1.ImagePullCredentials), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.ImagePullIntent)(nil), (*config.ImagePullIntent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ImagePullIntent_To_config_ImagePullIntent(a.(*configv1alpha1.ImagePullIntent), b.(*config.ImagePullIntent), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ImagePullIntent)(nil), (*configv1alpha1.ImagePullIntent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ImagePullIntent_To_v1alpha1_ImagePullIntent(a.(*config.ImagePullIntent), b.(*configv1alpha1.ImagePullIntent), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.ImagePullSecret)(nil), (*config.ImagePullSecret)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ImagePullSecret_To_config_ImagePullSecret(a.(*configv1alpha1.ImagePullSecret), b.(*config.ImagePullSecret), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ImagePullSecret)(nil), (*configv1alpha1.ImagePullSecret)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ImagePullSecret_To_v1alpha1_ImagePullSecret(a.(*config.ImagePullSecret), b.(*configv1alpha1.ImagePullSecret), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.ImagePulledRecord)(nil), (*config.ImagePulledRecord)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ImagePulledRecord_To_config_ImagePulledRecord(a.(*configv1alpha1.ImagePulledRecord), b.(*config.ImagePulledRecord), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ImagePulledRecord)(nil), (*configv1alpha1.ImagePulledRecord)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ImagePulledRecord_To_v1alpha1_ImagePulledRecord(a.(*config.ImagePulledRecord), b.(*configv1alpha1.ImagePulledRecord), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*config.CredentialProvider)(nil), (*configv1alpha1.CredentialProvider)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_config_CredentialProvider_To_v1alpha1_CredentialProvider(a.(*config.CredentialProvider), b.(*configv1alpha1.CredentialProvider), scope) }); err != nil { @@ -158,3 +198,95 @@ func autoConvert_config_ExecEnvVar_To_v1alpha1_ExecEnvVar(in *config.ExecEnvVar, func Convert_config_ExecEnvVar_To_v1alpha1_ExecEnvVar(in *config.ExecEnvVar, out *configv1alpha1.ExecEnvVar, s conversion.Scope) error { return autoConvert_config_ExecEnvVar_To_v1alpha1_ExecEnvVar(in, out, s) } + +func autoConvert_v1alpha1_ImagePullCredentials_To_config_ImagePullCredentials(in *configv1alpha1.ImagePullCredentials, out *config.ImagePullCredentials, s conversion.Scope) error { + out.KubernetesSecrets = *(*[]config.ImagePullSecret)(unsafe.Pointer(&in.KubernetesSecrets)) + out.NodePodsAccessible = in.NodePodsAccessible + return nil +} + +// Convert_v1alpha1_ImagePullCredentials_To_config_ImagePullCredentials is an autogenerated conversion function. +func Convert_v1alpha1_ImagePullCredentials_To_config_ImagePullCredentials(in *configv1alpha1.ImagePullCredentials, out *config.ImagePullCredentials, s conversion.Scope) error { + return autoConvert_v1alpha1_ImagePullCredentials_To_config_ImagePullCredentials(in, out, s) +} + +func autoConvert_config_ImagePullCredentials_To_v1alpha1_ImagePullCredentials(in *config.ImagePullCredentials, out *configv1alpha1.ImagePullCredentials, s conversion.Scope) error { + out.KubernetesSecrets = *(*[]configv1alpha1.ImagePullSecret)(unsafe.Pointer(&in.KubernetesSecrets)) + out.NodePodsAccessible = in.NodePodsAccessible + return nil +} + +// Convert_config_ImagePullCredentials_To_v1alpha1_ImagePullCredentials is an autogenerated conversion function. +func Convert_config_ImagePullCredentials_To_v1alpha1_ImagePullCredentials(in *config.ImagePullCredentials, out *configv1alpha1.ImagePullCredentials, s conversion.Scope) error { + return autoConvert_config_ImagePullCredentials_To_v1alpha1_ImagePullCredentials(in, out, s) +} + +func autoConvert_v1alpha1_ImagePullIntent_To_config_ImagePullIntent(in *configv1alpha1.ImagePullIntent, out *config.ImagePullIntent, s conversion.Scope) error { + out.Image = in.Image + return nil +} + +// Convert_v1alpha1_ImagePullIntent_To_config_ImagePullIntent is an autogenerated conversion function. +func Convert_v1alpha1_ImagePullIntent_To_config_ImagePullIntent(in *configv1alpha1.ImagePullIntent, out *config.ImagePullIntent, s conversion.Scope) error { + return autoConvert_v1alpha1_ImagePullIntent_To_config_ImagePullIntent(in, out, s) +} + +func autoConvert_config_ImagePullIntent_To_v1alpha1_ImagePullIntent(in *config.ImagePullIntent, out *configv1alpha1.ImagePullIntent, s conversion.Scope) error { + out.Image = in.Image + return nil +} + +// Convert_config_ImagePullIntent_To_v1alpha1_ImagePullIntent is an autogenerated conversion function. +func Convert_config_ImagePullIntent_To_v1alpha1_ImagePullIntent(in *config.ImagePullIntent, out *configv1alpha1.ImagePullIntent, s conversion.Scope) error { + return autoConvert_config_ImagePullIntent_To_v1alpha1_ImagePullIntent(in, out, s) +} + +func autoConvert_v1alpha1_ImagePullSecret_To_config_ImagePullSecret(in *configv1alpha1.ImagePullSecret, out *config.ImagePullSecret, s conversion.Scope) error { + out.UID = in.UID + out.Namespace = in.Namespace + out.Name = in.Name + out.CredentialHash = in.CredentialHash + return nil +} + +// Convert_v1alpha1_ImagePullSecret_To_config_ImagePullSecret is an autogenerated conversion function. +func Convert_v1alpha1_ImagePullSecret_To_config_ImagePullSecret(in *configv1alpha1.ImagePullSecret, out *config.ImagePullSecret, s conversion.Scope) error { + return autoConvert_v1alpha1_ImagePullSecret_To_config_ImagePullSecret(in, out, s) +} + +func autoConvert_config_ImagePullSecret_To_v1alpha1_ImagePullSecret(in *config.ImagePullSecret, out *configv1alpha1.ImagePullSecret, s conversion.Scope) error { + out.UID = in.UID + out.Namespace = in.Namespace + out.Name = in.Name + out.CredentialHash = in.CredentialHash + return nil +} + +// Convert_config_ImagePullSecret_To_v1alpha1_ImagePullSecret is an autogenerated conversion function. +func Convert_config_ImagePullSecret_To_v1alpha1_ImagePullSecret(in *config.ImagePullSecret, out *configv1alpha1.ImagePullSecret, s conversion.Scope) error { + return autoConvert_config_ImagePullSecret_To_v1alpha1_ImagePullSecret(in, out, s) +} + +func autoConvert_v1alpha1_ImagePulledRecord_To_config_ImagePulledRecord(in *configv1alpha1.ImagePulledRecord, out *config.ImagePulledRecord, s conversion.Scope) error { + out.LastUpdatedTime = in.LastUpdatedTime + out.ImageRef = in.ImageRef + out.CredentialMapping = *(*map[string]config.ImagePullCredentials)(unsafe.Pointer(&in.CredentialMapping)) + return nil +} + +// Convert_v1alpha1_ImagePulledRecord_To_config_ImagePulledRecord is an autogenerated conversion function. +func Convert_v1alpha1_ImagePulledRecord_To_config_ImagePulledRecord(in *configv1alpha1.ImagePulledRecord, out *config.ImagePulledRecord, s conversion.Scope) error { + return autoConvert_v1alpha1_ImagePulledRecord_To_config_ImagePulledRecord(in, out, s) +} + +func autoConvert_config_ImagePulledRecord_To_v1alpha1_ImagePulledRecord(in *config.ImagePulledRecord, out *configv1alpha1.ImagePulledRecord, s conversion.Scope) error { + out.LastUpdatedTime = in.LastUpdatedTime + out.ImageRef = in.ImageRef + out.CredentialMapping = *(*map[string]configv1alpha1.ImagePullCredentials)(unsafe.Pointer(&in.CredentialMapping)) + return nil +} + +// Convert_config_ImagePulledRecord_To_v1alpha1_ImagePulledRecord is an autogenerated conversion function. +func Convert_config_ImagePulledRecord_To_v1alpha1_ImagePulledRecord(in *config.ImagePulledRecord, out *configv1alpha1.ImagePulledRecord, s conversion.Scope) error { + return autoConvert_config_ImagePulledRecord_To_v1alpha1_ImagePulledRecord(in, out, s) +} diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 6a7ea7fd468..04d71d00bd7 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -313,4 +313,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura obj.CrashLoopBackOff.MaxContainerRestartPeriod = &metav1.Duration{Duration: MaxContainerBackOff} } } + + if localFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + if obj.ImagePullCredentialsVerificationPolicy == "" { + obj.ImagePullCredentialsVerificationPolicy = kubeletconfigv1beta1.NeverVerifyPreloadedImages + } + } } diff --git a/pkg/kubelet/apis/config/v1beta1/defaults_test.go b/pkg/kubelet/apis/config/v1beta1/defaults_test.go index 391bdbe0b9d..9c79a088cc0 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults_test.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults_test.go @@ -77,6 +77,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeLeaseDurationSeconds: 40, ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageMaximumGCAge: metav1.Duration{}, + ImagePullCredentialsVerificationPolicy: "", ImageGCHighThresholdPercent: ptr.To[int32](85), ImageGCLowThresholdPercent: ptr.To[int32](80), ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", @@ -168,74 +169,75 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { CacheUnauthorizedTTL: zeroDuration, }, }, - RegistryPullQPS: ptr.To[int32](0), - RegistryBurst: 0, - EventRecordQPS: ptr.To[int32](0), - EventBurst: 0, - EnableDebuggingHandlers: ptr.To(false), - EnableContentionProfiling: false, - HealthzPort: ptr.To[int32](0), - HealthzBindAddress: "", - OOMScoreAdj: ptr.To[int32](0), - ClusterDomain: "", - ClusterDNS: []string{}, - StreamingConnectionIdleTimeout: zeroDuration, - NodeStatusUpdateFrequency: zeroDuration, - NodeStatusReportFrequency: zeroDuration, - NodeLeaseDurationSeconds: 0, - ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - ImageMinimumGCAge: zeroDuration, - ImageGCHighThresholdPercent: ptr.To[int32](0), - ImageGCLowThresholdPercent: ptr.To[int32](0), - VolumeStatsAggPeriod: zeroDuration, - KubeletCgroups: "", - SystemCgroups: "", - CgroupRoot: "", - CgroupsPerQOS: ptr.To(false), - CgroupDriver: "", - CPUManagerPolicy: "", - CPUManagerPolicyOptions: map[string]string{}, - CPUManagerReconcilePeriod: zeroDuration, - MemoryManagerPolicy: "", - TopologyManagerPolicy: "", - TopologyManagerScope: "", - QOSReserved: map[string]string{}, - RuntimeRequestTimeout: zeroDuration, - HairpinMode: "", - MaxPods: 0, - PodCIDR: "", - PodPidsLimit: ptr.To[int64](0), - ResolverConfig: ptr.To(""), - RunOnce: false, - CPUCFSQuota: ptr.To(false), - CPUCFSQuotaPeriod: &zeroDuration, - NodeStatusMaxImages: ptr.To[int32](0), - MaxOpenFiles: 0, - ContentType: "", - KubeAPIQPS: ptr.To[int32](0), - KubeAPIBurst: 0, - SerializeImagePulls: ptr.To(false), - MaxParallelImagePulls: nil, - EvictionHard: map[string]string{}, - EvictionSoft: map[string]string{}, - EvictionSoftGracePeriod: map[string]string{}, - EvictionPressureTransitionPeriod: zeroDuration, - EvictionMaxPodGracePeriod: 0, - EvictionMinimumReclaim: map[string]string{}, - MergeDefaultEvictionSettings: ptr.To(false), - PodsPerCore: 0, - EnableControllerAttachDetach: ptr.To(false), - ProtectKernelDefaults: false, - MakeIPTablesUtilChains: ptr.To(false), - IPTablesMasqueradeBit: ptr.To[int32](0), - IPTablesDropBit: ptr.To[int32](0), - FeatureGates: map[string]bool{}, - FailSwapOn: ptr.To(false), - MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""}, - ContainerLogMaxSize: "", - ContainerLogMaxFiles: ptr.To[int32](0), - ContainerLogMaxWorkers: ptr.To[int32](1), - ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second}, + RegistryPullQPS: ptr.To[int32](0), + RegistryBurst: 0, + EventRecordQPS: ptr.To[int32](0), + EventBurst: 0, + EnableDebuggingHandlers: ptr.To(false), + EnableContentionProfiling: false, + HealthzPort: ptr.To[int32](0), + HealthzBindAddress: "", + OOMScoreAdj: ptr.To[int32](0), + ClusterDomain: "", + ClusterDNS: []string{}, + StreamingConnectionIdleTimeout: zeroDuration, + NodeStatusUpdateFrequency: zeroDuration, + NodeStatusReportFrequency: zeroDuration, + NodeLeaseDurationSeconds: 0, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", + ImageMinimumGCAge: zeroDuration, + ImagePullCredentialsVerificationPolicy: "", + ImageGCHighThresholdPercent: ptr.To[int32](0), + ImageGCLowThresholdPercent: ptr.To[int32](0), + VolumeStatsAggPeriod: zeroDuration, + KubeletCgroups: "", + SystemCgroups: "", + CgroupRoot: "", + CgroupsPerQOS: ptr.To(false), + CgroupDriver: "", + CPUManagerPolicy: "", + CPUManagerPolicyOptions: map[string]string{}, + CPUManagerReconcilePeriod: zeroDuration, + MemoryManagerPolicy: "", + TopologyManagerPolicy: "", + TopologyManagerScope: "", + QOSReserved: map[string]string{}, + RuntimeRequestTimeout: zeroDuration, + HairpinMode: "", + MaxPods: 0, + PodCIDR: "", + PodPidsLimit: ptr.To[int64](0), + ResolverConfig: ptr.To(""), + RunOnce: false, + CPUCFSQuota: ptr.To(false), + CPUCFSQuotaPeriod: &zeroDuration, + NodeStatusMaxImages: ptr.To[int32](0), + MaxOpenFiles: 0, + ContentType: "", + KubeAPIQPS: ptr.To[int32](0), + KubeAPIBurst: 0, + SerializeImagePulls: ptr.To(false), + MaxParallelImagePulls: nil, + EvictionHard: map[string]string{}, + EvictionSoft: map[string]string{}, + EvictionSoftGracePeriod: map[string]string{}, + EvictionPressureTransitionPeriod: zeroDuration, + EvictionMaxPodGracePeriod: 0, + EvictionMinimumReclaim: map[string]string{}, + MergeDefaultEvictionSettings: ptr.To(false), + PodsPerCore: 0, + EnableControllerAttachDetach: ptr.To(false), + ProtectKernelDefaults: false, + MakeIPTablesUtilChains: ptr.To(false), + IPTablesMasqueradeBit: ptr.To[int32](0), + IPTablesDropBit: ptr.To[int32](0), + FeatureGates: map[string]bool{}, + FailSwapOn: ptr.To(false), + MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""}, + ContainerLogMaxSize: "", + ContainerLogMaxFiles: ptr.To[int32](0), + ContainerLogMaxWorkers: ptr.To[int32](1), + ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second}, ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy, SystemReserved: map[string]string{}, KubeReserved: map[string]string{}, @@ -307,6 +309,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageGCHighThresholdPercent: ptr.To[int32](0), ImageGCLowThresholdPercent: ptr.To[int32](0), + ImagePullCredentialsVerificationPolicy: "", VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute}, CgroupsPerQOS: ptr.To(false), CgroupDriver: "cgroupfs", @@ -407,54 +410,55 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { CacheUnauthorizedTTL: metav1.Duration{Duration: 60 * time.Second}, }, }, - RegistryPullQPS: ptr.To[int32](1), - RegistryBurst: 1, - EventRecordQPS: ptr.To[int32](1), - EventBurst: 1, - EnableDebuggingHandlers: ptr.To(true), - EnableContentionProfiling: true, - HealthzPort: ptr.To[int32](1), - HealthzBindAddress: "127.0.0.2", - OOMScoreAdj: ptr.To[int32](1), - ClusterDomain: "cluster-domain", - ClusterDNS: []string{"192.168.1.3"}, - StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeLeaseDurationSeconds: 1, - ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, - ImageGCHighThresholdPercent: ptr.To[int32](1), - ImageGCLowThresholdPercent: ptr.To[int32](1), - VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, - KubeletCgroups: "kubelet-cgroup", - SystemCgroups: "system-cgroup", - CgroupRoot: "root-cgroup", - CgroupsPerQOS: ptr.To(true), - CgroupDriver: "systemd", - CPUManagerPolicy: "cpu-manager-policy", - CPUManagerPolicyOptions: map[string]string{"key": "value"}, - CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, - MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, - TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, - TopologyManagerScope: v1beta1.PodTopologyManagerScope, - QOSReserved: map[string]string{"memory": "10%"}, - RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, - HairpinMode: v1beta1.HairpinVeth, - MaxPods: 1, - PodCIDR: "192.168.1.0/24", - PodPidsLimit: ptr.To[int64](1), - ResolverConfig: ptr.To("resolver-config"), - RunOnce: true, - CPUCFSQuota: ptr.To(true), - CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, - NodeStatusMaxImages: ptr.To[int32](1), - MaxOpenFiles: 1, - ContentType: "application/protobuf", - KubeAPIQPS: ptr.To[int32](1), - KubeAPIBurst: 1, - SerializeImagePulls: ptr.To(true), - MaxParallelImagePulls: ptr.To[int32](5), + RegistryPullQPS: ptr.To[int32](1), + RegistryBurst: 1, + EventRecordQPS: ptr.To[int32](1), + EventBurst: 1, + EnableDebuggingHandlers: ptr.To(true), + EnableContentionProfiling: true, + HealthzPort: ptr.To[int32](1), + HealthzBindAddress: "127.0.0.2", + OOMScoreAdj: ptr.To[int32](1), + ClusterDomain: "cluster-domain", + ClusterDNS: []string{"192.168.1.3"}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeLeaseDurationSeconds: 1, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", + ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, + ImageGCHighThresholdPercent: ptr.To[int32](1), + ImageGCLowThresholdPercent: ptr.To[int32](1), + PreloadedImagesVerificationAllowlist: []string{"test.test/repo/image"}, + VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, + KubeletCgroups: "kubelet-cgroup", + SystemCgroups: "system-cgroup", + CgroupRoot: "root-cgroup", + CgroupsPerQOS: ptr.To(true), + CgroupDriver: "systemd", + CPUManagerPolicy: "cpu-manager-policy", + CPUManagerPolicyOptions: map[string]string{"key": "value"}, + CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, + MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, + TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, + TopologyManagerScope: v1beta1.PodTopologyManagerScope, + QOSReserved: map[string]string{"memory": "10%"}, + RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, + HairpinMode: v1beta1.HairpinVeth, + MaxPods: 1, + PodCIDR: "192.168.1.0/24", + PodPidsLimit: ptr.To[int64](1), + ResolverConfig: ptr.To("resolver-config"), + RunOnce: true, + CPUCFSQuota: ptr.To(true), + CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, + NodeStatusMaxImages: ptr.To[int32](1), + MaxOpenFiles: 1, + ContentType: "application/protobuf", + KubeAPIQPS: ptr.To[int32](1), + KubeAPIBurst: 1, + SerializeImagePulls: ptr.To(true), + MaxParallelImagePulls: ptr.To[int32](5), EvictionHard: map[string]string{ "memory.available": "1Mi", "nodefs.available": "1%", @@ -563,54 +567,55 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { CacheUnauthorizedTTL: metav1.Duration{Duration: 60 * time.Second}, }, }, - RegistryPullQPS: ptr.To[int32](1), - RegistryBurst: 1, - EventRecordQPS: ptr.To[int32](1), - EventBurst: 1, - EnableDebuggingHandlers: ptr.To(true), - EnableContentionProfiling: true, - HealthzPort: ptr.To[int32](1), - HealthzBindAddress: "127.0.0.2", - OOMScoreAdj: ptr.To[int32](1), - ClusterDomain: "cluster-domain", - ClusterDNS: []string{"192.168.1.3"}, - StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeLeaseDurationSeconds: 1, - ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, - ImageGCHighThresholdPercent: ptr.To[int32](1), - ImageGCLowThresholdPercent: ptr.To[int32](1), - VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, - KubeletCgroups: "kubelet-cgroup", - SystemCgroups: "system-cgroup", - CgroupRoot: "root-cgroup", - CgroupsPerQOS: ptr.To(true), - CgroupDriver: "systemd", - CPUManagerPolicy: "cpu-manager-policy", - CPUManagerPolicyOptions: map[string]string{"key": "value"}, - CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, - MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, - TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, - TopologyManagerScope: v1beta1.PodTopologyManagerScope, - QOSReserved: map[string]string{"memory": "10%"}, - RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, - HairpinMode: v1beta1.HairpinVeth, - MaxPods: 1, - PodCIDR: "192.168.1.0/24", - PodPidsLimit: ptr.To[int64](1), - ResolverConfig: ptr.To("resolver-config"), - RunOnce: true, - CPUCFSQuota: ptr.To(true), - CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, - NodeStatusMaxImages: ptr.To[int32](1), - MaxOpenFiles: 1, - ContentType: "application/protobuf", - KubeAPIQPS: ptr.To[int32](1), - KubeAPIBurst: 1, - SerializeImagePulls: ptr.To(true), - MaxParallelImagePulls: ptr.To[int32](5), + RegistryPullQPS: ptr.To[int32](1), + RegistryBurst: 1, + EventRecordQPS: ptr.To[int32](1), + EventBurst: 1, + EnableDebuggingHandlers: ptr.To(true), + EnableContentionProfiling: true, + HealthzPort: ptr.To[int32](1), + HealthzBindAddress: "127.0.0.2", + OOMScoreAdj: ptr.To[int32](1), + ClusterDomain: "cluster-domain", + ClusterDNS: []string{"192.168.1.3"}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeLeaseDurationSeconds: 1, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", + ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, + ImageGCHighThresholdPercent: ptr.To[int32](1), + ImageGCLowThresholdPercent: ptr.To[int32](1), + PreloadedImagesVerificationAllowlist: []string{"test.test/repo/image"}, + VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, + KubeletCgroups: "kubelet-cgroup", + SystemCgroups: "system-cgroup", + CgroupRoot: "root-cgroup", + CgroupsPerQOS: ptr.To(true), + CgroupDriver: "systemd", + CPUManagerPolicy: "cpu-manager-policy", + CPUManagerPolicyOptions: map[string]string{"key": "value"}, + CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, + MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, + TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, + TopologyManagerScope: v1beta1.PodTopologyManagerScope, + QOSReserved: map[string]string{"memory": "10%"}, + RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, + HairpinMode: v1beta1.HairpinVeth, + MaxPods: 1, + PodCIDR: "192.168.1.0/24", + PodPidsLimit: ptr.To[int64](1), + ResolverConfig: ptr.To("resolver-config"), + RunOnce: true, + CPUCFSQuota: ptr.To(true), + CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, + NodeStatusMaxImages: ptr.To[int32](1), + MaxOpenFiles: 1, + ContentType: "application/protobuf", + KubeAPIQPS: ptr.To[int32](1), + KubeAPIBurst: 1, + SerializeImagePulls: ptr.To(true), + MaxParallelImagePulls: ptr.To[int32](5), EvictionHard: map[string]string{ "memory.available": "1Mi", "nodefs.available": "1%", diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index 73f3b14e0b9..377fc235395 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -417,6 +417,8 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in return err } out.RegistryBurst = in.RegistryBurst + out.ImagePullCredentialsVerificationPolicy = string(in.ImagePullCredentialsVerificationPolicy) + out.PreloadedImagesVerificationAllowlist = *(*[]string)(unsafe.Pointer(&in.PreloadedImagesVerificationAllowlist)) if err := v1.Convert_Pointer_int32_To_int32(&in.EventRecordQPS, &out.EventRecordQPS, s); err != nil { return err } @@ -622,6 +624,8 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in return err } out.RegistryBurst = in.RegistryBurst + out.ImagePullCredentialsVerificationPolicy = configv1beta1.ImagePullCredentialsVerificationPolicy(in.ImagePullCredentialsVerificationPolicy) + out.PreloadedImagesVerificationAllowlist = *(*[]string)(unsafe.Pointer(&in.PreloadedImagesVerificationAllowlist)) if err := v1.Convert_int32_To_Pointer_int32(&in.EventRecordQPS, &out.EventRecordQPS, s); err != nil { return err } diff --git a/pkg/kubelet/apis/config/validation/validation.go b/pkg/kubelet/apis/config/validation/validation.go index a4195a56a68..6f5197ca92c 100644 --- a/pkg/kubelet/apis/config/validation/validation.go +++ b/pkg/kubelet/apis/config/validation/validation.go @@ -32,6 +32,7 @@ import ( tracingapi "k8s.io/component-base/tracing/api/v1" "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + imagepullmanager "k8s.io/kubernetes/pkg/kubelet/images/pullmanager" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" utilfs "k8s.io/kubernetes/pkg/util/filesystem" utiltaints "k8s.io/kubernetes/pkg/util/taints" @@ -286,6 +287,32 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for hairpinMode (--hairpin-mode). Valid options are %q, %q or %q", kc.HairpinMode, kubeletconfig.HairpinNone, kubeletconfig.HairpinVeth, kubeletconfig.PromiscuousBridge)) } + + if localFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + switch kc.ImagePullCredentialsVerificationPolicy { + case string(kubeletconfig.NeverVerify), + string(kubeletconfig.NeverVerifyPreloadedImages), + string(kubeletconfig.NeverVerifyAllowlistedImages), + string(kubeletconfig.AlwaysVerify): + default: + allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for imagePullCredentialsVerificationPolicy. Valid options are %q, %q, %q or %q", + kc.ImagePullCredentialsVerificationPolicy, kubeletconfig.NeverVerify, kubeletconfig.NeverVerifyPreloadedImages, kubeletconfig.NeverVerifyAllowlistedImages, kubeletconfig.AlwaysVerify)) + } + + if len(kc.PreloadedImagesVerificationAllowlist) > 0 && kc.ImagePullCredentialsVerificationPolicy != string(kubeletconfig.NeverVerifyAllowlistedImages) { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: can't set `preloadedImagesVerificationAllowlist` if `imagePullCredentialsVertificationPolicy` is not \"NeverVerifyAllowlistedImages\"")) + } else if err := imagepullmanager.ValidateAllowlistImagesPatterns(kc.PreloadedImagesVerificationAllowlist); err != nil { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: invalid image pattern in `preloadedImagesVerificationAllowlist`: %w", err)) + } + } else { + if len(kc.ImagePullCredentialsVerificationPolicy) > 0 { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: `imagePullCredentialsVerificationPolicy` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled")) + } + if len(kc.PreloadedImagesVerificationAllowlist) > 0 { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: `preloadedImagesVerificationAllowlist` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled")) + } + } + if kc.ReservedSystemCPUs != "" { // --reserved-cpus does not support --system-reserved-cgroup or --kube-reserved-cgroup if kc.SystemReservedCgroup != "" || kc.KubeReservedCgroup != "" { diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index b09677228b5..08da05c2efc 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -728,6 +728,39 @@ func TestValidateKubeletConfiguration(t *testing.T) { return conf }, errMsg: "logging.format: Invalid value: \"invalid\": Unsupported log format", + }, { + name: "invalid imagePullCredentialsVerificationPolicy configuration", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": true} + conf.ImagePullCredentialsVerificationPolicy = "invalid" + return conf + }, + errMsg: `option "invalid" specified for imagePullCredentialsVerificationPolicy. Valid options are "NeverVerify", "NeverVerifyPreloadedImages", "NeverVerifyAllowlistedImages" or "AlwaysVerify"]`, + }, { + name: "invalid PreloadedImagesVerificationAllowlist configuration - featuregate enabled", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": true} + conf.ImagePullCredentialsVerificationPolicy = string(kubeletconfig.NeverVerify) + conf.PreloadedImagesVerificationAllowlist = []string{"test.test/repo"} + return conf + }, + errMsg: "can't set `preloadedImagesVerificationAllowlist` if `imagePullCredentialsVertificationPolicy` is not \"NeverVerifyAllowlistedImages\"]", + }, { + name: "invalid PreloadedImagesVerificationAllowlist configuration - featuregate disabled", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": false} + conf.ImagePullCredentialsVerificationPolicy = string(kubeletconfig.NeverVerify) + return conf + }, + errMsg: "invalid configuration: `imagePullCredentialsVerificationPolicy` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled", + }, { + name: "invalid PreloadedImagesVerificationAllowlist configuration", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": false} + conf.PreloadedImagesVerificationAllowlist = []string{"test.test/repo"} + return conf + }, + errMsg: "invalid configuration: `preloadedImagesVerificationAllowlist` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled", }, { name: "invalid FeatureGate", configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { diff --git a/pkg/kubelet/apis/config/zz_generated.deepcopy.go b/pkg/kubelet/apis/config/zz_generated.deepcopy.go index 5b4a1d1b8fa..fd7828cd14a 100644 --- a/pkg/kubelet/apis/config/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/config/zz_generated.deepcopy.go @@ -138,6 +138,101 @@ func (in *ExecEnvVar) DeepCopy() *ExecEnvVar { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePullCredentials) DeepCopyInto(out *ImagePullCredentials) { + *out = *in + if in.KubernetesSecrets != nil { + in, out := &in.KubernetesSecrets, &out.KubernetesSecrets + *out = make([]ImagePullSecret, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullCredentials. +func (in *ImagePullCredentials) DeepCopy() *ImagePullCredentials { + if in == nil { + return nil + } + out := new(ImagePullCredentials) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePullIntent) DeepCopyInto(out *ImagePullIntent) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullIntent. +func (in *ImagePullIntent) DeepCopy() *ImagePullIntent { + if in == nil { + return nil + } + out := new(ImagePullIntent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ImagePullIntent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePullSecret) DeepCopyInto(out *ImagePullSecret) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullSecret. +func (in *ImagePullSecret) DeepCopy() *ImagePullSecret { + if in == nil { + return nil + } + out := new(ImagePullSecret) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePulledRecord) DeepCopyInto(out *ImagePulledRecord) { + *out = *in + out.TypeMeta = in.TypeMeta + in.LastUpdatedTime.DeepCopyInto(&out.LastUpdatedTime) + if in.CredentialMapping != nil { + in, out := &in.CredentialMapping, &out.CredentialMapping + *out = make(map[string]ImagePullCredentials, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePulledRecord. +func (in *ImagePulledRecord) DeepCopy() *ImagePulledRecord { + if in == nil { + return nil + } + out := new(ImagePulledRecord) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ImagePulledRecord) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeletAnonymousAuthentication) DeepCopyInto(out *KubeletAnonymousAuthentication) { *out = *in @@ -219,6 +314,11 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { } out.Authentication = in.Authentication out.Authorization = in.Authorization + if in.PreloadedImagesVerificationAllowlist != nil { + in, out := &in.PreloadedImagesVerificationAllowlist, &out.PreloadedImagesVerificationAllowlist + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.ClusterDNS != nil { in, out := &in.ClusterDNS, &out.ClusterDNS *out = make([]string, len(*in)) diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index cb535a95932..87cad70bd8a 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/util/flowcontrol" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/volume" ) @@ -151,8 +152,11 @@ type StreamingRuntime interface { // ImageService interfaces allows to work with image service. type ImageService interface { // PullImage pulls an image from the network to local storage using the supplied - // secrets if necessary. It returns a reference (digest or ID) to the pulled image. - PullImage(ctx context.Context, image ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) + // secrets if necessary. + // It returns a reference (digest or ID) to the pulled image and the credentials + // that were used to pull the image. If the returned credentials are nil, the + // pull was anonymous. + PullImage(ctx context.Context, image ImageSpec, credentials []credentialprovider.TrackedAuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, *credentialprovider.TrackedAuthConfig, error) // GetImageRef gets the reference (digest or ID) of the image which has already been in // the local storage. It returns ("", nil) if the image isn't in the local storage. GetImageRef(ctx context.Context, image ImageSpec) (string, error) diff --git a/pkg/kubelet/container/testing/fake_runtime.go b/pkg/kubelet/container/testing/fake_runtime.go index f7889f06903..ca7002e4b91 100644 --- a/pkg/kubelet/container/testing/fake_runtime.go +++ b/pkg/kubelet/container/testing/fake_runtime.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/flowcontrol" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "k8s.io/kubernetes/pkg/credentialprovider" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/volume" ) @@ -308,7 +309,7 @@ func (f *FakeRuntime) GetContainerLogs(_ context.Context, pod *v1.Pod, container return f.Err } -func (f *FakeRuntime) PullImage(ctx context.Context, image kubecontainer.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) { +func (f *FakeRuntime) PullImage(ctx context.Context, image kubecontainer.ImageSpec, creds []credentialprovider.TrackedAuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, *credentialprovider.TrackedAuthConfig, error) { f.Lock() f.CalledFunctions = append(f.CalledFunctions, "PullImage") if f.Err == nil { @@ -319,9 +320,15 @@ func (f *FakeRuntime) PullImage(ctx context.Context, image kubecontainer.ImageSp f.ImageList = append(f.ImageList, i) } + // if credentials were supplied for the pull at least return the first in the list + var retCreds *credentialprovider.TrackedAuthConfig = nil + if len(creds) > 0 { + retCreds = &creds[0] + } + if !f.BlockImagePulls { f.Unlock() - return image.Image, f.Err + return image.Image, retCreds, f.Err } retErr := f.Err @@ -334,7 +341,8 @@ func (f *FakeRuntime) PullImage(ctx context.Context, image kubecontainer.ImageSp case <-ctx.Done(): case <-f.imagePullTokenBucket: } - return image.Image, retErr + + return image.Image, retCreds, retErr } // UnblockImagePulls unblocks a certain number of image pulls, if BlockImagePulls is true. diff --git a/pkg/kubelet/container/testing/runtime_mock.go b/pkg/kubelet/container/testing/runtime_mock.go index 8f91580eee5..1f021ea06d7 100644 --- a/pkg/kubelet/container/testing/runtime_mock.go +++ b/pkg/kubelet/container/testing/runtime_mock.go @@ -25,6 +25,8 @@ import ( corev1 "k8s.io/api/core/v1" + credentialprovider "k8s.io/kubernetes/pkg/credentialprovider" + flowcontrol "k8s.io/client-go/util/flowcontrol" io "io" @@ -990,32 +992,41 @@ func (_c *MockRuntime_ListPodSandboxMetrics_Call) RunAndReturn(run func(context. return _c } -// PullImage provides a mock function with given fields: ctx, image, pullSecrets, podSandboxConfig, serviceAccountName -func (_m *MockRuntime) PullImage(ctx context.Context, image container.ImageSpec, pullSecrets []corev1.Secret, podSandboxConfig *v1.PodSandboxConfig, serviceAccountName string) (string, error) { - ret := _m.Called(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) +// PullImage provides a mock function with given fields: ctx, image, credentials, podSandboxConfig +func (_m *MockRuntime) PullImage(ctx context.Context, image container.ImageSpec, credentials []credentialprovider.TrackedAuthConfig, podSandboxConfig *v1.PodSandboxConfig) (string, *credentialprovider.TrackedAuthConfig, error) { + ret := _m.Called(ctx, image, credentials, podSandboxConfig) if len(ret) == 0 { panic("no return value specified for PullImage") } var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) (string, error)); ok { - return rf(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) + var r1 *credentialprovider.TrackedAuthConfig + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []credentialprovider.TrackedAuthConfig, *v1.PodSandboxConfig) (string, *credentialprovider.TrackedAuthConfig, error)); ok { + return rf(ctx, image, credentials, podSandboxConfig) } - if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) string); ok { - r0 = rf(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) + if rf, ok := ret.Get(0).(func(context.Context, container.ImageSpec, []credentialprovider.TrackedAuthConfig, *v1.PodSandboxConfig) string); ok { + r0 = rf(ctx, image, credentials, podSandboxConfig) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) error); ok { - r1 = rf(ctx, image, pullSecrets, podSandboxConfig, serviceAccountName) + if rf, ok := ret.Get(1).(func(context.Context, container.ImageSpec, []credentialprovider.TrackedAuthConfig, *v1.PodSandboxConfig) *credentialprovider.TrackedAuthConfig); ok { + r1 = rf(ctx, image, credentials, podSandboxConfig) } else { - r1 = ret.Error(1) + if ret.Get(1) != nil { + r1 = ret.Get(1).(*credentialprovider.TrackedAuthConfig) + } } - return r0, r1 + if rf, ok := ret.Get(2).(func(context.Context, container.ImageSpec, []credentialprovider.TrackedAuthConfig, *v1.PodSandboxConfig) error); ok { + r2 = rf(ctx, image, credentials, podSandboxConfig) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } // MockRuntime_PullImage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PullImage' @@ -1026,26 +1037,25 @@ type MockRuntime_PullImage_Call struct { // PullImage is a helper method to define mock.On call // - ctx context.Context // - image container.ImageSpec -// - pullSecrets []corev1.Secret +// - credentials []credentialprovider.TrackedAuthConfig // - podSandboxConfig *v1.PodSandboxConfig -// - serviceAccountName string -func (_e *MockRuntime_Expecter) PullImage(ctx interface{}, image interface{}, pullSecrets interface{}, podSandboxConfig interface{}, serviceAccountName interface{}) *MockRuntime_PullImage_Call { - return &MockRuntime_PullImage_Call{Call: _e.mock.On("PullImage", ctx, image, pullSecrets, podSandboxConfig, serviceAccountName)} +func (_e *MockRuntime_Expecter) PullImage(ctx interface{}, image interface{}, credentials interface{}, podSandboxConfig interface{}) *MockRuntime_PullImage_Call { + return &MockRuntime_PullImage_Call{Call: _e.mock.On("PullImage", ctx, image, credentials, podSandboxConfig)} } -func (_c *MockRuntime_PullImage_Call) Run(run func(ctx context.Context, image container.ImageSpec, pullSecrets []corev1.Secret, podSandboxConfig *v1.PodSandboxConfig, serviceAccountName string)) *MockRuntime_PullImage_Call { +func (_c *MockRuntime_PullImage_Call) Run(run func(ctx context.Context, image container.ImageSpec, credentials []credentialprovider.TrackedAuthConfig, podSandboxConfig *v1.PodSandboxConfig)) *MockRuntime_PullImage_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(container.ImageSpec), args[2].([]corev1.Secret), args[3].(*v1.PodSandboxConfig), args[4].(string)) + run(args[0].(context.Context), args[1].(container.ImageSpec), args[2].([]credentialprovider.TrackedAuthConfig), args[3].(*v1.PodSandboxConfig)) }) return _c } -func (_c *MockRuntime_PullImage_Call) Return(_a0 string, _a1 error) *MockRuntime_PullImage_Call { - _c.Call.Return(_a0, _a1) +func (_c *MockRuntime_PullImage_Call) Return(_a0 string, _a1 *credentialprovider.TrackedAuthConfig, _a2 error) *MockRuntime_PullImage_Call { + _c.Call.Return(_a0, _a1, _a2) return _c } -func (_c *MockRuntime_PullImage_Call) RunAndReturn(run func(context.Context, container.ImageSpec, []corev1.Secret, *v1.PodSandboxConfig, string) (string, error)) *MockRuntime_PullImage_Call { +func (_c *MockRuntime_PullImage_Call) RunAndReturn(run func(context.Context, container.ImageSpec, []credentialprovider.TrackedAuthConfig, *v1.PodSandboxConfig) (string, *credentialprovider.TrackedAuthConfig, error)) *MockRuntime_PullImage_Call { _c.Call.Return(run) return _c } diff --git a/pkg/kubelet/images/helpers.go b/pkg/kubelet/images/helpers.go index f2524c67a73..a45509a9ec9 100644 --- a/pkg/kubelet/images/helpers.go +++ b/pkg/kubelet/images/helpers.go @@ -20,9 +20,9 @@ import ( "context" "fmt" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/util/flowcontrol" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "k8s.io/kubernetes/pkg/credentialprovider" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -44,9 +44,9 @@ type throttledImageService struct { limiter flowcontrol.RateLimiter } -func (ts throttledImageService) PullImage(ctx context.Context, image kubecontainer.ImageSpec, secrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) { +func (ts throttledImageService) PullImage(ctx context.Context, image kubecontainer.ImageSpec, credentials []credentialprovider.TrackedAuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, *credentialprovider.TrackedAuthConfig, error) { if ts.limiter.TryAccept() { - return ts.ImageService.PullImage(ctx, image, secrets, podSandboxConfig, serviceAccountName) + return ts.ImageService.PullImage(ctx, image, credentials, podSandboxConfig) } - return "", fmt.Errorf("pull QPS exceeded") + return "", nil, fmt.Errorf("pull QPS exceeded") } diff --git a/pkg/kubelet/images/image_gc_manager.go b/pkg/kubelet/images/image_gc_manager.go index b13ec8f470b..60650376222 100644 --- a/pkg/kubelet/images/image_gc_manager.go +++ b/pkg/kubelet/images/image_gc_manager.go @@ -57,6 +57,11 @@ const ( ImageGarbageCollectedTotalReasonSpace = "space" ) +// PostImageGCHook allows external sources to react to GC collect events. +// `remainingImages` is a list of images that were left on the system after garbage +// collection finished. +type PostImageGCHook func(remainingImages []string, gcStart time.Time) + // StatsProvider is an interface for fetching stats used during image garbage // collection. type StatsProvider interface { @@ -128,6 +133,8 @@ type realImageGCManager struct { // imageCache is the cache of latest image list. imageCache imageCache + postGCHooks []PostImageGCHook + // tracer for recording spans tracer trace.Tracer } @@ -181,7 +188,7 @@ type imageRecord struct { } // NewImageGCManager instantiates a new ImageGCManager object. -func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, recorder record.EventRecorder, nodeRef *v1.ObjectReference, policy ImageGCPolicy, tracerProvider trace.TracerProvider) (ImageGCManager, error) { +func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, postGCHooks []PostImageGCHook, recorder record.EventRecorder, nodeRef *v1.ObjectReference, policy ImageGCPolicy, tracerProvider trace.TracerProvider) (ImageGCManager, error) { // Validate policy. if policy.HighThresholdPercent < 0 || policy.HighThresholdPercent > 100 { return nil, fmt.Errorf("invalid HighThresholdPercent %d, must be in range [0-100]", policy.HighThresholdPercent) @@ -200,6 +207,7 @@ func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, r statsProvider: statsProvider, recorder: recorder, nodeRef: nodeRef, + postGCHooks: postGCHooks, tracer: tracer, } @@ -381,11 +389,13 @@ func (im *realImageGCManager) GarbageCollect(ctx context.Context, beganGC time.T if usagePercent >= im.policy.HighThresholdPercent { amountToFree := capacity*int64(100-im.policy.LowThresholdPercent)/100 - available klog.InfoS("Disk usage on image filesystem is over the high threshold, trying to free bytes down to the low threshold", "usage", usagePercent, "highThreshold", im.policy.HighThresholdPercent, "amountToFree", amountToFree, "lowThreshold", im.policy.LowThresholdPercent) - freed, err := im.freeSpace(ctx, amountToFree, freeTime, images) + remainingImages, freed, err := im.freeSpace(ctx, amountToFree, freeTime, images) if err != nil { return err } + im.runPostGCHooks(remainingImages, freeTime) + if freed < amountToFree { err := fmt.Errorf("Failed to garbage collect required amount of images. Attempted to free %d bytes, but only found %d bytes eligible to free.", amountToFree, freed) im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.FreeDiskSpaceFailed, err.Error()) @@ -396,6 +406,12 @@ func (im *realImageGCManager) GarbageCollect(ctx context.Context, beganGC time.T return nil } +func (im *realImageGCManager) runPostGCHooks(remainingImages []string, gcStartTime time.Time) { + for _, h := range im.postGCHooks { + h(remainingImages, gcStartTime) + } +} + func (im *realImageGCManager) freeOldImages(ctx context.Context, images []evictionInfo, freeTime, beganGC time.Time) ([]evictionInfo, error) { if im.policy.MaxAge == 0 { return images, nil @@ -430,29 +446,38 @@ func (im *realImageGCManager) freeOldImages(ctx context.Context, images []evicti func (im *realImageGCManager) DeleteUnusedImages(ctx context.Context) error { klog.InfoS("Attempting to delete unused images") freeTime := time.Now() + images, err := im.imagesInEvictionOrder(ctx, freeTime) if err != nil { return err } - _, err = im.freeSpace(ctx, math.MaxInt64, freeTime, images) - return err + + remainingImages, _, err := im.freeSpace(ctx, math.MaxInt64, freeTime, images) + if err != nil { + return err + } + + im.runPostGCHooks(remainingImages, freeTime) + return nil } // Tries to free bytesToFree worth of images on the disk. // -// Returns the number of bytes free and an error if any occurred. The number of -// bytes freed is always returned. +// Returns the images that are still available after the cleanup, the number of bytes freed +// and an error if any occurred. The number of bytes freed is always returned. // Note that error may be nil and the number of bytes free may be less // than bytesToFree. -func (im *realImageGCManager) freeSpace(ctx context.Context, bytesToFree int64, freeTime time.Time, images []evictionInfo) (int64, error) { +func (im *realImageGCManager) freeSpace(ctx context.Context, bytesToFree int64, freeTime time.Time, images []evictionInfo) ([]string, int64, error) { // Delete unused images until we've freed up enough space. var deletionErrors []error spaceFreed := int64(0) + var imagesLeft []string for _, image := range images { klog.V(5).InfoS("Evaluating image ID for possible garbage collection based on disk usage", "imageID", image.id, "runtimeHandler", image.imageRecord.runtimeHandlerUsedToPullImage) // Images that are currently in used were given a newer lastUsed. if image.lastUsed.Equal(freeTime) || image.lastUsed.After(freeTime) { klog.V(5).InfoS("Image ID was used too recently, not eligible for garbage collection", "imageID", image.id, "lastUsed", image.lastUsed, "freeTime", freeTime) + imagesLeft = append(imagesLeft, image.id) continue } @@ -460,11 +485,13 @@ func (im *realImageGCManager) freeSpace(ctx context.Context, bytesToFree int64, // In such a case, the image may have just been pulled down, and will be used by a container right away. if freeTime.Sub(image.firstDetected) < im.policy.MinAge { klog.V(5).InfoS("Image ID's age is less than the policy's minAge, not eligible for garbage collection", "imageID", image.id, "age", freeTime.Sub(image.firstDetected), "minAge", im.policy.MinAge) + imagesLeft = append(imagesLeft, image.id) continue } if err := im.freeImage(ctx, image, ImageGarbageCollectedTotalReasonSpace); err != nil { deletionErrors = append(deletionErrors, err) + imagesLeft = append(imagesLeft, image.id) continue } spaceFreed += image.size @@ -475,9 +502,9 @@ func (im *realImageGCManager) freeSpace(ctx context.Context, bytesToFree int64, } if len(deletionErrors) > 0 { - return spaceFreed, fmt.Errorf("wanted to free %d bytes, but freed %d bytes space with errors in image deletion: %v", bytesToFree, spaceFreed, errors.NewAggregate(deletionErrors)) + return nil, spaceFreed, fmt.Errorf("wanted to free %d bytes, but freed %d bytes space with errors in image deletion: %w", bytesToFree, spaceFreed, errors.NewAggregate(deletionErrors)) } - return spaceFreed, nil + return imagesLeft, spaceFreed, nil } func (im *realImageGCManager) freeImage(ctx context.Context, image evictionInfo, reason string) error { diff --git a/pkg/kubelet/images/image_gc_manager_test.go b/pkg/kubelet/images/image_gc_manager_test.go index ffe5297267f..332c12cb026 100644 --- a/pkg/kubelet/images/image_gc_manager_test.go +++ b/pkg/kubelet/images/image_gc_manager_test.go @@ -740,7 +740,7 @@ func TestGarbageCollectImageNotOldEnough(t *testing.T) { func getImagesAndFreeSpace(ctx context.Context, t *testing.T, assert *assert.Assertions, im *realImageGCManager, fakeRuntime *containertest.FakeRuntime, spaceToFree, expectedSpaceFreed int64, imagesLen int, freeTime time.Time) { images, err := im.imagesInEvictionOrder(ctx, freeTime) require.NoError(t, err) - spaceFreed, err := im.freeSpace(ctx, spaceToFree, freeTime, images) + _, spaceFreed, err := im.freeSpace(ctx, spaceToFree, freeTime, images) require.NoError(t, err) assert.EqualValues(expectedSpaceFreed, spaceFreed) assert.Len(fakeRuntime.ImageList, imagesLen) @@ -910,7 +910,7 @@ func TestValidateImageGCPolicy(t *testing.T) { } for _, tc := range testCases { - if _, err := NewImageGCManager(nil, nil, nil, nil, tc.imageGCPolicy, noopoteltrace.NewTracerProvider()); err != nil { + if _, err := NewImageGCManager(nil, nil, nil, nil, nil, tc.imageGCPolicy, noopoteltrace.NewTracerProvider()); err != nil { if err.Error() != tc.expectErr { t.Errorf("[%s:]Expected err:%v, but got:%v", tc.name, tc.expectErr, err.Error()) } diff --git a/pkg/kubelet/images/image_manager.go b/pkg/kubelet/images/image_manager.go index c1576f434c3..b76f87d19aa 100644 --- a/pkg/kubelet/images/image_manager.go +++ b/pkg/kubelet/images/image_manager.go @@ -25,14 +25,21 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/flowcontrol" "k8s.io/klog/v2" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" crierrors "k8s.io/cri-api/pkg/errors" + "k8s.io/kubernetes/pkg/credentialprovider" + credentialproviderplugin "k8s.io/kubernetes/pkg/credentialprovider/plugin" + credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" + "k8s.io/kubernetes/pkg/features" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/events" + "k8s.io/kubernetes/pkg/kubelet/images/pullmanager" "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/util/parsers" ) @@ -44,13 +51,15 @@ type ImagePodPullingTimeRecorder interface { // imageManager provides the functionalities for image pulling. type imageManager struct { - recorder record.EventRecorder - imageService kubecontainer.ImageService - backOff *flowcontrol.Backoff - prevPullErrMsg sync.Map + recorder record.EventRecorder + imageService kubecontainer.ImageService + imagePullManager pullmanager.ImagePullManager + backOff *flowcontrol.Backoff + prevPullErrMsg sync.Map // It will check the presence of the image, and report the 'image pulling', image pulled' events correspondingly. - puller imagePuller + puller imagePuller + nodeKeyring credentialprovider.DockerKeyring podPullingTimeRecorder ImagePodPullingTimeRecorder } @@ -58,7 +67,19 @@ type imageManager struct { var _ ImageManager = &imageManager{} // NewImageManager instantiates a new ImageManager object. -func NewImageManager(recorder record.EventRecorder, imageService kubecontainer.ImageService, imageBackOff *flowcontrol.Backoff, serialized bool, maxParallelImagePulls *int32, qps float32, burst int, podPullingTimeRecorder ImagePodPullingTimeRecorder) ImageManager { +func NewImageManager( + recorder record.EventRecorder, + nodeKeyring credentialprovider.DockerKeyring, + imageService kubecontainer.ImageService, + imagePullManager pullmanager.ImagePullManager, + imageBackOff *flowcontrol.Backoff, + serialized bool, + maxParallelImagePulls *int32, + qps float32, + burst int, + podPullingTimeRecorder ImagePodPullingTimeRecorder, +) ImageManager { + imageService = throttleImagePulling(imageService, qps, burst) var puller imagePuller @@ -70,6 +91,8 @@ func NewImageManager(recorder record.EventRecorder, imageService kubecontainer.I return &imageManager{ recorder: recorder, imageService: imageService, + imagePullManager: imagePullManager, + nodeKeyring: nodeKeyring, backOff: imageBackOff, puller: puller, podPullingTimeRecorder: podPullingTimeRecorder, @@ -78,33 +101,25 @@ func NewImageManager(recorder record.EventRecorder, imageService kubecontainer.I // imagePullPrecheck inspects the pull policy and checks for image presence accordingly, // returning (imageRef, error msg, err) and logging any errors. -func (m *imageManager) imagePullPrecheck(ctx context.Context, objRef *v1.ObjectReference, logPrefix string, pullPolicy v1.PullPolicy, spec *kubecontainer.ImageSpec, imgRef string) (imageRef string, msg string, err error) { +func (m *imageManager) imagePullPrecheck(ctx context.Context, objRef *v1.ObjectReference, logPrefix string, pullPolicy v1.PullPolicy, spec *kubecontainer.ImageSpec, requestedImage string) (imageRef string, msg string, err error) { switch pullPolicy { case v1.PullAlways: return "", msg, nil - case v1.PullIfNotPresent: + case v1.PullIfNotPresent, v1.PullNever: imageRef, err = m.imageService.GetImageRef(ctx, *spec) if err != nil { msg = fmt.Sprintf("Failed to inspect image %q: %v", imageRef, err) m.logIt(objRef, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, klog.Warning) return "", msg, ErrImageInspect } - return imageRef, msg, nil - case v1.PullNever: - imageRef, err = m.imageService.GetImageRef(ctx, *spec) - if err != nil { - msg = fmt.Sprintf("Failed to inspect image %q: %v", imageRef, err) - m.logIt(objRef, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, klog.Warning) - return "", msg, ErrImageInspect - } - if imageRef == "" { - msg = fmt.Sprintf("Container image %q is not present with pull policy of Never", imgRef) - m.logIt(objRef, v1.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, klog.Warning) - return "", msg, ErrImageNeverPull - } - return imageRef, msg, nil } - return + + if len(imageRef) == 0 && pullPolicy == v1.PullNever { + msg, err = m.imageNotPresentOnNeverPolicyError(logPrefix, objRef, requestedImage) + return "", msg, err + } + + return imageRef, msg, nil } // records an event using ref, event msg. log to glog using prefix, msg, logFn @@ -116,15 +131,30 @@ func (m *imageManager) logIt(objRef *v1.ObjectReference, eventtype, event, prefi } } -// EnsureImageExists pulls the image for the specified pod and imgRef, and returns +// imageNotPresentOnNeverPolicy error is a utility function that emits an event about +// an image not being present and returns the appropriate error to be passed on. +// +// Called in 2 scenarios: +// 1. image is not present with `imagePullPolicy: Never“ +// 2. image is present but cannot be accessed with the presented set of credentials +// +// We don't want to reveal the presence of an image if it cannot be accessed, hence we +// want the same behavior in both the above scenarios. +func (m *imageManager) imageNotPresentOnNeverPolicyError(logPrefix string, objRef *v1.ObjectReference, requestedImage string) (string, error) { + msg := fmt.Sprintf("Container image %q is not present with pull policy of Never", requestedImage) + m.logIt(objRef, v1.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, klog.Warning) + return msg, ErrImageNeverPull +} + +// EnsureImageExists pulls the image for the specified pod and requestedImage, and returns // (imageRef, error message, error). -func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectReference, pod *v1.Pod, imgRef string, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, podRuntimeHandler string, pullPolicy v1.PullPolicy) (imageRef, message string, err error) { - logPrefix := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, imgRef) +func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectReference, pod *v1.Pod, requestedImage string, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, podRuntimeHandler string, pullPolicy v1.PullPolicy) (imageRef, message string, err error) { + logPrefix := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, requestedImage) // If the image contains no tag or digest, a default tag should be applied. - image, err := applyDefaultImageTag(imgRef) + image, err := applyDefaultImageTag(requestedImage) if err != nil { - msg := fmt.Sprintf("Failed to apply default image tag %q: %v", imgRef, err) + msg := fmt.Sprintf("Failed to apply default image tag %q: %v", requestedImage, err) m.logIt(objRef, v1.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, klog.Warning) return "", msg, ErrInvalidImageName } @@ -143,19 +173,99 @@ func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectR RuntimeHandler: podRuntimeHandler, } - imageRef, message, err = m.imagePullPrecheck(ctx, objRef, logPrefix, pullPolicy, &spec, imgRef) + imageRef, message, err = m.imagePullPrecheck(ctx, objRef, logPrefix, pullPolicy, &spec, requestedImage) if err != nil { return "", message, err } - if imageRef != "" { - msg := fmt.Sprintf("Container image %q already present on machine", imgRef) - m.logIt(objRef, v1.EventTypeNormal, events.PulledImage, logPrefix, msg, klog.Info) - return imageRef, msg, nil + + repoToPull, _, _, err := parsers.ParseImageName(spec.Image) + if err != nil { + return "", err.Error(), err } - backOffKey := fmt.Sprintf("%s_%s", pod.UID, imgRef) + // construct the dynamic keyring using the providers we have in the kubelet + var podName, podNamespace, podUID string + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) { + sandboxMetadata := podSandboxConfig.GetMetadata() + + podName = sandboxMetadata.Name + podNamespace = sandboxMetadata.Namespace + podUID = sandboxMetadata.Uid + } + + externalCredentialProviderKeyring := credentialproviderplugin.NewExternalCredentialProviderDockerKeyring( + podNamespace, + podName, + podUID, + pod.Spec.ServiceAccountName) + + keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, credentialprovider.UnionDockerKeyring{m.nodeKeyring, externalCredentialProviderKeyring}) + if err != nil { + return "", err.Error(), err + } + + pullCredentials, _ := keyring.Lookup(repoToPull) + + if imageRef != "" { + if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + msg := fmt.Sprintf("Container image %q already present on machine", requestedImage) + m.logIt(objRef, v1.EventTypeNormal, events.PulledImage, logPrefix, msg, klog.Info) + return imageRef, msg, nil + } + + var imagePullSecrets []kubeletconfiginternal.ImagePullSecret + for _, s := range pullCredentials { + if s.Source == nil { + // we're only interested in creds that are not node accessible + continue + } + imagePullSecrets = append(imagePullSecrets, kubeletconfiginternal.ImagePullSecret{ + UID: string(s.Source.Secret.UID), + Name: s.Source.Secret.Name, + Namespace: s.Source.Secret.Namespace, + CredentialHash: s.AuthConfigHash, + }) + } + + pullRequired := m.imagePullManager.MustAttemptImagePull(requestedImage, imageRef, imagePullSecrets) + if !pullRequired { + msg := fmt.Sprintf("Container image %q already present on machine and can be accessed by the pod", requestedImage) + m.logIt(objRef, v1.EventTypeNormal, events.PulledImage, logPrefix, msg, klog.Info) + return imageRef, msg, nil + } + } + + if pullPolicy == v1.PullNever { + // The image is present as confirmed by imagePullPrecheck but it apparently + // wasn't accessible given the credentials check by the imagePullManager. + msg, err := m.imageNotPresentOnNeverPolicyError(logPrefix, objRef, requestedImage) + return "", msg, err + } + + return m.pullImage(ctx, logPrefix, objRef, pod.UID, requestedImage, spec, pullCredentials, podSandboxConfig) +} + +func (m *imageManager) pullImage(ctx context.Context, logPrefix string, objRef *v1.ObjectReference, podUID types.UID, image string, imgSpec kubecontainer.ImageSpec, pullCredentials []credentialprovider.TrackedAuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (imageRef, message string, err error) { + var pullSucceeded bool + var finalPullCredentials *credentialprovider.TrackedAuthConfig + + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + if err := m.imagePullManager.RecordPullIntent(image); err != nil { + return "", fmt.Sprintf("Failed to record image pull intent for container image %q: %v", image, err), err + } + + defer func() { + if pullSucceeded { + m.imagePullManager.RecordImagePulled(image, imageRef, trackedToImagePullCreds(finalPullCredentials)) + } else { + m.imagePullManager.RecordImagePullFailed(image) + } + }() + } + + backOffKey := fmt.Sprintf("%s_%s", podUID, image) if m.backOff.IsInBackOffSinceUpdate(backOffKey, m.backOff.Clock.Now()) { - msg := fmt.Sprintf("Back-off pulling image %q", imgRef) + msg := fmt.Sprintf("Back-off pulling image %q", image) m.logIt(objRef, v1.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, klog.Info) // Wrap the error from the actual pull if available. @@ -171,17 +281,17 @@ func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectR // Ensure that the map cannot grow indefinitely. m.prevPullErrMsg.Delete(backOffKey) - m.podPullingTimeRecorder.RecordImageStartedPulling(pod.UID) - m.logIt(objRef, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("Pulling image %q", imgRef), klog.Info) + m.podPullingTimeRecorder.RecordImageStartedPulling(podUID) + m.logIt(objRef, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("Pulling image %q", image), klog.Info) startTime := time.Now() + pullChan := make(chan pullResult) - m.puller.pullImage(ctx, spec, pullSecrets, pullChan, podSandboxConfig, pod.Spec.ServiceAccountName) + m.puller.pullImage(ctx, imgSpec, pullCredentials, pullChan, podSandboxConfig) imagePullResult := <-pullChan if imagePullResult.err != nil { - m.logIt(objRef, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", imgRef, imagePullResult.err), klog.Warning) + m.logIt(objRef, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", image, imagePullResult.err), klog.Warning) m.backOff.Next(backOffKey, m.backOff.Clock.Now()) - - msg, err := evalCRIPullErr(imgRef, imagePullResult.err) + msg, err := evalCRIPullErr(image, imagePullResult.err) // Store the actual pull error for providing that information during // the image pull back-off. @@ -189,12 +299,15 @@ func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectR return "", msg, err } - m.podPullingTimeRecorder.RecordImageFinishedPulling(pod.UID) + m.podPullingTimeRecorder.RecordImageFinishedPulling(podUID) imagePullDuration := time.Since(startTime).Truncate(time.Millisecond) m.logIt(objRef, v1.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q in %v (%v including waiting). Image size: %v bytes.", - imgRef, imagePullResult.pullDuration.Truncate(time.Millisecond), imagePullDuration, imagePullResult.imageSize), klog.Info) + image, imagePullResult.pullDuration.Truncate(time.Millisecond), imagePullDuration, imagePullResult.imageSize), klog.Info) metrics.ImagePullDuration.WithLabelValues(metrics.GetImageSizeBucket(imagePullResult.imageSize)).Observe(imagePullDuration.Seconds()) m.backOff.GC() + finalPullCredentials = imagePullResult.credentialsUsed + pullSucceeded = true + return imagePullResult.imageRef, "", nil } @@ -247,3 +360,23 @@ func applyDefaultImageTag(image string) (string, error) { } return image, nil } + +func trackedToImagePullCreds(trackedCreds *credentialprovider.TrackedAuthConfig) *kubeletconfiginternal.ImagePullCredentials { + ret := &kubeletconfiginternal.ImagePullCredentials{} + switch { + case trackedCreds == nil, trackedCreds.Source == nil: + ret.NodePodsAccessible = true + default: + sourceSecret := trackedCreds.Source.Secret + ret.KubernetesSecrets = []kubeletconfiginternal.ImagePullSecret{ + { + UID: sourceSecret.UID, + Name: sourceSecret.Name, + Namespace: sourceSecret.Namespace, + CredentialHash: trackedCreds.AuthConfigHash, + }, + } + } + + return ret +} diff --git a/pkg/kubelet/images/image_manager_test.go b/pkg/kubelet/images/image_manager_test.go index 41a1f49f237..2f945348300 100644 --- a/pkg/kubelet/images/image_manager_test.go +++ b/pkg/kubelet/images/image_manager_test.go @@ -30,14 +30,19 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/util/flowcontrol" + "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" crierrors "k8s.io/cri-api/pkg/errors" "k8s.io/kubernetes/pkg/controller/testutil" + "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" . "k8s.io/kubernetes/pkg/kubelet/container" ctest "k8s.io/kubernetes/pkg/kubelet/container/testing" + "k8s.io/kubernetes/pkg/kubelet/images/pullmanager" "k8s.io/kubernetes/test/utils/ktesting" testingclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" @@ -53,17 +58,29 @@ type pullerExpects struct { } type pullerTestCase struct { - testName string - containerImage string - policy v1.PullPolicy - inspectErr error - pullerErr error - qps float32 - burst int - expected []pullerExpects + testName string + containerImage string + policy v1.PullPolicy + pullSecrets []v1.Secret + allowedCredentials map[string][]kubeletconfiginternal.ImagePullSecret // image -> allowedCredentials; nil means allow all + inspectErr error + pullerErr error + qps float32 + burst int + expected []pullerExpects + enableFeatures []featuregate.Feature } func pullerTestCases() []pullerTestCase { + return append( + noFGPullerTestCases(), + ensureSecretImagesTestCases()..., + ) +} + +// noFGPullerTestCases returns all test cases that test the default behavior without any +// feature gate required +func noFGPullerTestCases() []pullerTestCase { return []pullerTestCase{ { // pull missing image testName: "image missing, pull", @@ -82,7 +99,7 @@ func pullerTestCases() []pullerTestCase { }}, { // image present, don't pull - testName: "image present, don't pull ", + testName: "image present, allow all, don't pull ", containerImage: "present_image", policy: v1.PullIfNotPresent, inspectErr: nil, @@ -335,6 +352,142 @@ func pullerTestCases() []pullerTestCase { } } +// ensureSecretImages returns test cases specific for the KubeletEnsureSecretPulledImages +// featuregate plus a copy of all non-featuregated tests, but it requests the featuregate +// to be enabled there, too +func ensureSecretImagesTestCases() []pullerTestCase { + testCases := []pullerTestCase{ + { + testName: "[KubeletEnsureSecretPulledImages] image present, unknown to image pull manager, pull", + containerImage: "present_image", + policy: v1.PullIfNotPresent, + allowedCredentials: map[string][]kubeletconfiginternal.ImagePullSecret{ + "another_image": {{Namespace: "testns", Name: "testname", UID: "testuid"}}, + }, + pullSecrets: []v1.Secret{makeDockercfgSecretForRepo(metav1.ObjectMeta{Namespace: "testns", Name: "testname", UID: "testuid"}, "docker.io/library/present_image")}, + inspectErr: nil, + pullerErr: nil, + qps: 0.0, + burst: 0, + enableFeatures: []featuregate.Feature{features.KubeletEnsureSecretPulledImages}, + expected: []pullerExpects{ + {[]string{"GetImageRef", "PullImage", "GetImageSize"}, nil, true, true, + []v1.Event{ + {Reason: "Pulling"}, + {Reason: "Pulled"}, + }, ""}, + {[]string{"GetImageRef", "PullImage", "GetImageSize"}, nil, true, true, + []v1.Event{ + {Reason: "Pulling"}, + {Reason: "Pulled"}, + }, ""}, + {[]string{"GetImageRef", "PullImage", "GetImageSize"}, nil, true, true, + []v1.Event{ + {Reason: "Pulling"}, + {Reason: "Pulled"}, + }, ""}, + }}, + { + testName: "[KubeletEnsureSecretPulledImages] image present, unknown secret to image pull manager, pull", + containerImage: "present_image", + policy: v1.PullIfNotPresent, + allowedCredentials: map[string][]kubeletconfiginternal.ImagePullSecret{ + "present_image": {{Namespace: "testns", Name: "testname", UID: "testuid"}}, + }, + pullSecrets: []v1.Secret{makeDockercfgSecretForRepo(metav1.ObjectMeta{Namespace: "testns", Name: "testname", UID: "someothertestuid"}, "docker.io/library/present_image")}, + inspectErr: nil, + pullerErr: nil, + qps: 0.0, + burst: 0, + enableFeatures: []featuregate.Feature{features.KubeletEnsureSecretPulledImages}, + expected: []pullerExpects{ + {[]string{"GetImageRef", "PullImage", "GetImageSize"}, nil, true, true, + []v1.Event{ + {Reason: "Pulling"}, + {Reason: "Pulled"}, + }, ""}, + {[]string{"GetImageRef", "PullImage", "GetImageSize"}, nil, true, true, + []v1.Event{ + {Reason: "Pulling"}, + {Reason: "Pulled"}, + }, ""}, + {[]string{"GetImageRef", "PullImage", "GetImageSize"}, nil, true, true, + []v1.Event{ + {Reason: "Pulling"}, + {Reason: "Pulled"}, + }, ""}, + }, + }, + { + testName: "[KubeletEnsureSecretPulledImages] image present, unknown secret to image pull manager, never pull policy -> fail", + containerImage: "present_image", + policy: v1.PullNever, + allowedCredentials: map[string][]kubeletconfiginternal.ImagePullSecret{ + "present_image": {{Namespace: "testns", Name: "testname", UID: "testuid"}}, + }, + pullSecrets: []v1.Secret{makeDockercfgSecretForRepo(metav1.ObjectMeta{Namespace: "testns", Name: "testname", UID: "someothertestuid"}, "docker.io/library/present_image")}, + inspectErr: nil, + pullerErr: nil, + qps: 0.0, + burst: 0, + enableFeatures: []featuregate.Feature{features.KubeletEnsureSecretPulledImages}, + expected: []pullerExpects{ + {[]string{"GetImageRef"}, ErrImageNeverPull, false, false, + []v1.Event{ + {Reason: "ErrImageNeverPull"}, + }, ""}, + {[]string{"GetImageRef"}, ErrImageNeverPull, false, false, + []v1.Event{ + {Reason: "ErrImageNeverPull"}, + }, ""}, + {[]string{"GetImageRef"}, ErrImageNeverPull, false, false, + []v1.Event{ + {Reason: "ErrImageNeverPull"}, + }, ""}, + }, + }, + { + testName: "[KubeletEnsureSecretPulledImages] image present, a secret matches one of known to the image pull manager, don't pull", + containerImage: "present_image", + policy: v1.PullIfNotPresent, + allowedCredentials: map[string][]kubeletconfiginternal.ImagePullSecret{ + "present_image": {{Namespace: "testns", Name: "testname", UID: "testuid"}}, + }, + pullSecrets: []v1.Secret{ + makeDockercfgSecretForRepo(metav1.ObjectMeta{Namespace: "testns", Name: "testname", UID: "someothertestuid"}, "docker.io/library/present_image"), + makeDockercfgSecretForRepo(metav1.ObjectMeta{Namespace: "testns", Name: "testname", UID: "testuid"}, "docker.io/library/present_image"), + }, + inspectErr: nil, + pullerErr: nil, + qps: 0.0, + burst: 0, + enableFeatures: []featuregate.Feature{features.KubeletEnsureSecretPulledImages}, + expected: []pullerExpects{ + {[]string{"GetImageRef"}, nil, false, false, + []v1.Event{ + {Reason: "Pulled"}, + }, ""}, + {[]string{"GetImageRef"}, nil, false, false, + []v1.Event{ + {Reason: "Pulled"}, + }, ""}, + {[]string{"GetImageRef"}, nil, false, false, + []v1.Event{ + {Reason: "Pulled"}, + }, ""}, + }, + }, + } + + for _, tc := range noFGPullerTestCases() { + tc.testName = "[KubeletEnsureSecretPulledImages] " + tc.testName + tc.enableFeatures = append(tc.enableFeatures, features.KubeletEnsureSecretPulledImages) + testCases = append(testCases, tc) + } + + return testCases +} + type mockPodPullingTimeRecorder struct { sync.Mutex startedPullingRecorded bool @@ -360,7 +513,46 @@ func (m *mockPodPullingTimeRecorder) reset() { m.finishedPullingRecorded = false } -func pullerTestEnv(t *testing.T, c pullerTestCase, serialized bool, maxParallelImagePulls *int32) (puller ImageManager, fakeClock *testingclock.FakeClock, fakeRuntime *ctest.FakeRuntime, container *v1.Container, fakePodPullingTimeRecorder *mockPodPullingTimeRecorder, fakeRecorder *testutil.FakeRecorder) { +type mockImagePullManager struct { + pullmanager.NoopImagePullManager + + imageAllowlist map[string]sets.Set[kubeletconfiginternal.ImagePullSecret] + allowAll bool +} + +func (m *mockImagePullManager) MustAttemptImagePull(image, _ string, podSecrets []kubeletconfiginternal.ImagePullSecret) bool { + if m.allowAll == true { + return false + } + + cachedSecrets, ok := m.imageAllowlist[image] + if !ok { + return true + } + + // cut off all the hashes and only determine the match based on the secret coords to simplify testing + for _, s := range podSecrets { + if cachedSecrets.Has(kubeletconfiginternal.ImagePullSecret{Namespace: s.Namespace, Name: s.Name, UID: s.UID}) { + return false + } + } + + return true +} + +func pullerTestEnv( + t *testing.T, + c pullerTestCase, + serialized bool, + maxParallelImagePulls *int32, +) ( + puller ImageManager, + fakeClock *testingclock.FakeClock, + fakeRuntime *ctest.FakeRuntime, + container *v1.Container, + fakePodPullingTimeRecorder *mockPodPullingTimeRecorder, + fakeRecorder *testutil.FakeRecorder, +) { container = &v1.Container{ Name: "container_name", Image: c.containerImage, @@ -380,7 +572,19 @@ func pullerTestEnv(t *testing.T, c pullerTestCase, serialized bool, maxParallelI fakePodPullingTimeRecorder = &mockPodPullingTimeRecorder{} - puller = NewImageManager(fakeRecorder, fakeRuntime, backOff, serialized, maxParallelImagePulls, c.qps, c.burst, fakePodPullingTimeRecorder) + pullManager := &mockImagePullManager{allowAll: true} + if c.allowedCredentials != nil { + pullManager.allowAll = false + pullManager.imageAllowlist = make(map[string]sets.Set[kubeletconfiginternal.ImagePullSecret]) + for image, secrets := range c.allowedCredentials { + pullManager.imageAllowlist[image] = sets.New(secrets...) + } + } + + for _, fg := range c.enableFeatures { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, fg, true) + } + puller = NewImageManager(fakeRecorder, &credentialprovider.BasicDockerKeyring{}, fakeRuntime, pullManager, backOff, serialized, maxParallelImagePulls, c.qps, c.burst, fakePodPullingTimeRecorder) return } @@ -405,7 +609,7 @@ func TestParallelPuller(t *testing.T) { fakeRuntime.CalledFunctions = nil fakeClock.Step(time.Second) - _, msg, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, nil, nil, "", container.ImagePullPolicy) + _, msg, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, c.pullSecrets, nil, "", container.ImagePullPolicy) fakeRuntime.AssertCalls(expected.calls) assert.Equal(t, expected.err, err) assert.Equal(t, expected.shouldRecordStartedPullingTime, fakePodPullingTimeRecorder.startedPullingRecorded) @@ -438,7 +642,7 @@ func TestSerializedPuller(t *testing.T) { fakeRuntime.CalledFunctions = nil fakeClock.Step(time.Second) - _, msg, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, nil, nil, "", container.ImagePullPolicy) + _, msg, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, c.pullSecrets, nil, "", container.ImagePullPolicy) fakeRuntime.AssertCalls(expected.calls) assert.Equal(t, expected.err, err) assert.Equal(t, expected.shouldRecordStartedPullingTime, fakePodPullingTimeRecorder.startedPullingRecorded) @@ -502,7 +706,7 @@ func TestPullAndListImageWithPodAnnotations(t *testing.T) { fakeRuntime.ImageList = []Image{} fakeClock.Step(time.Second) - _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, nil, nil, "", container.ImagePullPolicy) + _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, c.pullSecrets, nil, "", container.ImagePullPolicy) fakeRuntime.AssertCalls(c.expected[0].calls) assert.Equal(t, c.expected[0].err, err, "tick=%d", 0) assert.Equal(t, c.expected[0].shouldRecordStartedPullingTime, fakePodPullingTimeRecorder.startedPullingRecorded) @@ -559,7 +763,7 @@ func TestPullAndListImageWithRuntimeHandlerInImageCriAPIFeatureGate(t *testing.T fakeRuntime.ImageList = []Image{} fakeClock.Step(time.Second) - _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, nil, nil, runtimeHandler, container.ImagePullPolicy) + _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, c.pullSecrets, nil, runtimeHandler, container.ImagePullPolicy) fakeRuntime.AssertCalls(c.expected[0].calls) assert.Equal(t, c.expected[0].err, err, "tick=%d", 0) assert.Equal(t, c.expected[0].shouldRecordStartedPullingTime, fakePodPullingTimeRecorder.startedPullingRecorded) @@ -618,7 +822,7 @@ func TestMaxParallelImagePullsLimit(t *testing.T) { for i := 0; i < maxParallelImagePulls; i++ { wg.Add(1) go func() { - _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, nil, nil, "", container.ImagePullPolicy) + _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, testCase.pullSecrets, nil, "", container.ImagePullPolicy) assert.NoError(t, err) wg.Done() }() @@ -630,7 +834,7 @@ func TestMaxParallelImagePullsLimit(t *testing.T) { for i := 0; i < 2; i++ { wg.Add(1) go func() { - _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, nil, nil, "", container.ImagePullPolicy) + _, _, err := puller.EnsureImageExists(ctx, nil, pod, container.Image, testCase.pullSecrets, nil, "", container.ImagePullPolicy) assert.NoError(t, err) wg.Done() }() @@ -731,7 +935,7 @@ func TestImagePullPrecheck(t *testing.T) { fakeRecorder.Events = []*v1.Event{} fakeClock.Step(time.Second) - _, _, err := puller.EnsureImageExists(ctx, &v1.ObjectReference{}, pod, container.Image, nil, nil, "", container.ImagePullPolicy) + _, _, err := puller.EnsureImageExists(ctx, &v1.ObjectReference{}, pod, container.Image, c.pullSecrets, nil, "", container.ImagePullPolicy) fakeRuntime.AssertCalls(expected.calls) var recorderEvents []v1.Event for _, event := range fakeRecorder.Events { @@ -747,3 +951,13 @@ func TestImagePullPrecheck(t *testing.T) { }) } } + +func makeDockercfgSecretForRepo(sMeta metav1.ObjectMeta, repo string) v1.Secret { + return v1.Secret{ + ObjectMeta: sMeta, + Type: v1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + v1.DockerConfigJsonKey: []byte(`{"auths": {"` + repo + `": {"auth": "dXNlcjpwYXNzd29yZA=="}}}`), + }, + } +} diff --git a/pkg/kubelet/images/puller.go b/pkg/kubelet/images/puller.go index 26432669ced..3d5df1dd124 100644 --- a/pkg/kubelet/images/puller.go +++ b/pkg/kubelet/images/puller.go @@ -20,21 +20,22 @@ import ( "context" "time" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "k8s.io/kubernetes/pkg/credentialprovider" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) type pullResult struct { - imageRef string - imageSize uint64 - err error - pullDuration time.Duration + imageRef string + imageSize uint64 + err error + pullDuration time.Duration + credentialsUsed *credentialprovider.TrackedAuthConfig } type imagePuller interface { - pullImage(context.Context, kubecontainer.ImageSpec, []v1.Secret, chan<- pullResult, *runtimeapi.PodSandboxConfig, string) + pullImage(context.Context, kubecontainer.ImageSpec, []credentialprovider.TrackedAuthConfig, chan<- pullResult, *runtimeapi.PodSandboxConfig) } var _, _ imagePuller = ¶llelImagePuller{}, &serialImagePuller{} @@ -51,24 +52,25 @@ func newParallelImagePuller(imageService kubecontainer.ImageService, maxParallel return ¶llelImagePuller{imageService, make(chan struct{}, *maxParallelImagePulls)} } -func (pip *parallelImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) { +func (pip *parallelImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, credentials []credentialprovider.TrackedAuthConfig, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) { go func() { if pip.tokens != nil { pip.tokens <- struct{}{} defer func() { <-pip.tokens }() } startTime := time.Now() - imageRef, err := pip.imageService.PullImage(ctx, spec, pullSecrets, podSandboxConfig, serviceAccountName) + imageRef, creds, err := pip.imageService.PullImage(ctx, spec, credentials, podSandboxConfig) var size uint64 if err == nil && imageRef != "" { // Getting the image size with best effort, ignoring the error. size, _ = pip.imageService.GetImageSize(ctx, spec) } pullChan <- pullResult{ - imageRef: imageRef, - imageSize: size, - err: err, - pullDuration: time.Since(startTime), + imageRef: imageRef, + imageSize: size, + err: err, + pullDuration: time.Since(startTime), + credentialsUsed: creds, } }() } @@ -88,29 +90,27 @@ func newSerialImagePuller(imageService kubecontainer.ImageService) imagePuller { } type imagePullRequest struct { - ctx context.Context - spec kubecontainer.ImageSpec - pullSecrets []v1.Secret - pullChan chan<- pullResult - podSandboxConfig *runtimeapi.PodSandboxConfig - serviceAccountName string + ctx context.Context + spec kubecontainer.ImageSpec + credentials []credentialprovider.TrackedAuthConfig + pullChan chan<- pullResult + podSandboxConfig *runtimeapi.PodSandboxConfig } -func (sip *serialImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) { +func (sip *serialImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, credentials []credentialprovider.TrackedAuthConfig, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) { sip.pullRequests <- &imagePullRequest{ - ctx: ctx, - spec: spec, - pullSecrets: pullSecrets, - pullChan: pullChan, - podSandboxConfig: podSandboxConfig, - serviceAccountName: serviceAccountName, + ctx: ctx, + spec: spec, + credentials: credentials, + pullChan: pullChan, + podSandboxConfig: podSandboxConfig, } } func (sip *serialImagePuller) processImagePullRequests() { for pullRequest := range sip.pullRequests { startTime := time.Now() - imageRef, err := sip.imageService.PullImage(pullRequest.ctx, pullRequest.spec, pullRequest.pullSecrets, pullRequest.podSandboxConfig, pullRequest.serviceAccountName) + imageRef, creds, err := sip.imageService.PullImage(pullRequest.ctx, pullRequest.spec, pullRequest.credentials, pullRequest.podSandboxConfig) var size uint64 if err == nil && imageRef != "" { // Getting the image size with best effort, ignoring the error. @@ -120,8 +120,9 @@ func (sip *serialImagePuller) processImagePullRequests() { imageRef: imageRef, imageSize: size, err: err, - // Note: pullDuration includes credential resolution and getting the image size. - pullDuration: time.Since(startTime), + // Note: pullDuration includes getting the image size. + pullDuration: time.Since(startTime), + credentialsUsed: creds, } } } diff --git a/pkg/kubelet/images/pullmanager/doc.go b/pkg/kubelet/images/pullmanager/doc.go new file mode 100644 index 00000000000..dbce837b6df --- /dev/null +++ b/pkg/kubelet/images/pullmanager/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// pullmanager package keeps the implementation of the image pull manager and +// image credential verification policies +package pullmanager diff --git a/pkg/kubelet/images/pullmanager/fs_pullrecords.go b/pkg/kubelet/images/pullmanager/fs_pullrecords.go new file mode 100644 index 00000000000..6dcee5f13c4 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/fs_pullrecords.go @@ -0,0 +1,302 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/errors" + kubeletconfigv1alpha1 "k8s.io/kubelet/config/v1alpha1" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" + kubeletconfigvint1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1alpha1" +) + +const ( + cacheFilesSHA256Prefix = "sha256-" + tmpFilesSuffix = ".tmp" +) + +var _ PullRecordsAccessor = &fsPullRecordsAccessor{} + +// fsPullRecordsAccessor uses the filesystem to read/write ImagePullIntent/ImagePulledRecord +// records. +type fsPullRecordsAccessor struct { + pullingDir string + pulledDir string + + encoder runtime.Encoder + decoder runtime.Decoder +} + +// NewFSPullRecordsAccessor returns an accessor for the ImagePullIntent/ImagePulledRecord +// records with a filesystem as the backing database. +func NewFSPullRecordsAccessor(kubeletDir string) (*fsPullRecordsAccessor, error) { + kubeletConfigEncoder, kubeletConfigDecoder, err := createKubeletConfigSchemeEncoderDecoder() + if err != nil { + return nil, err + } + + accessor := &fsPullRecordsAccessor{ + pullingDir: filepath.Join(kubeletDir, "image_manager", "pulling"), + pulledDir: filepath.Join(kubeletDir, "image_manager", "pulled"), + + encoder: kubeletConfigEncoder, + decoder: kubeletConfigDecoder, + } + + if err := os.MkdirAll(accessor.pullingDir, 0700); err != nil { + return nil, err + } + + if err := os.MkdirAll(accessor.pulledDir, 0700); err != nil { + return nil, err + } + + return accessor, nil +} + +func (f *fsPullRecordsAccessor) WriteImagePullIntent(image string) error { + intent := kubeletconfiginternal.ImagePullIntent{ + Image: image, + } + + intentBytes := bytes.NewBuffer([]byte{}) + if err := f.encoder.Encode(&intent, intentBytes); err != nil { + return err + } + + return writeFile(f.pullingDir, cacheFilename(image), intentBytes.Bytes()) +} + +func (f *fsPullRecordsAccessor) ListImagePullIntents() ([]*kubeletconfiginternal.ImagePullIntent, error) { + var intents []*kubeletconfiginternal.ImagePullIntent + // walk the pulling directory for any pull intent records + err := processDirFiles(f.pullingDir, + func(filePath string, fileContent []byte) error { + intent, err := decodeIntent(f.decoder, fileContent) + if err != nil { + return fmt.Errorf("failed to deserialize content of file %q into ImagePullIntent: %w", filePath, err) + } + intents = append(intents, intent) + + return nil + }) + return intents, err +} + +func (f *fsPullRecordsAccessor) ImagePullIntentExists(image string) (bool, error) { + intentRecordPath := filepath.Join(f.pullingDir, cacheFilename(image)) + intentBytes, err := os.ReadFile(intentRecordPath) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + + intent, err := decodeIntent(f.decoder, intentBytes) + if err != nil { + return false, err + } + + return intent.Image == image, nil +} + +func (f *fsPullRecordsAccessor) DeleteImagePullIntent(image string) error { + err := os.Remove(filepath.Join(f.pullingDir, cacheFilename(image))) + if os.IsNotExist(err) { + return nil + } + return err +} + +func (f *fsPullRecordsAccessor) GetImagePulledRecord(imageRef string) (*kubeletconfiginternal.ImagePulledRecord, bool, error) { + recordBytes, err := os.ReadFile(filepath.Join(f.pulledDir, cacheFilename(imageRef))) + if os.IsNotExist(err) { + return nil, false, nil + } else if err != nil { + return nil, false, err + } + + pulledRecord, err := decodePulledRecord(f.decoder, recordBytes) + if err != nil { + return nil, true, err + } + if pulledRecord.ImageRef != imageRef { + return nil, false, nil + } + return pulledRecord, true, err +} + +func (f *fsPullRecordsAccessor) ListImagePulledRecords() ([]*kubeletconfiginternal.ImagePulledRecord, error) { + var pullRecords []*kubeletconfiginternal.ImagePulledRecord + err := processDirFiles(f.pulledDir, + func(filePath string, fileContent []byte) error { + pullRecord, err := decodePulledRecord(f.decoder, fileContent) + if err != nil { + return fmt.Errorf("failed to deserialize content of file %q into ImagePulledRecord: %w", filePath, err) + } + pullRecords = append(pullRecords, pullRecord) + return nil + }) + + return pullRecords, err +} + +func (f *fsPullRecordsAccessor) WriteImagePulledRecord(pulledRecord *kubeletconfiginternal.ImagePulledRecord) error { + recordBytes := bytes.NewBuffer([]byte{}) + if err := f.encoder.Encode(pulledRecord, recordBytes); err != nil { + return fmt.Errorf("failed to serialize ImagePulledRecord: %w", err) + } + + return writeFile(f.pulledDir, cacheFilename(pulledRecord.ImageRef), recordBytes.Bytes()) +} + +func (f *fsPullRecordsAccessor) DeleteImagePulledRecord(imageRef string) error { + err := os.Remove(filepath.Join(f.pulledDir, cacheFilename(imageRef))) + if os.IsNotExist(err) { + return nil + } + return err +} + +func cacheFilename(image string) string { + return fmt.Sprintf("%s%x", cacheFilesSHA256Prefix, sha256.Sum256([]byte(image))) +} + +// writeFile writes `content` to the file with name `filename` in directory `dir`. +// It assures write atomicity by creating a temporary file first and only after +// a successful write, it move the temp file in place of the target. +func writeFile(dir, filename string, content []byte) error { + // create target folder if it does not exists yet + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("failed to create directory %q: %w", dir, err) + } + + targetPath := filepath.Join(dir, filename) + tmpPath := targetPath + tmpFilesSuffix + if err := os.WriteFile(tmpPath, content, 0600); err != nil { + _ = os.Remove(tmpPath) // attempt a delete in case the file was at least partially written + return fmt.Errorf("failed to create temporary file %q: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, targetPath); err != nil { + _ = os.Remove(tmpPath) // attempt a cleanup + return err + } + return nil +} + +// processDirFiles reads files in a given directory and peforms `fileAction` action on those. +func processDirFiles(dirName string, fileAction func(filePath string, fileContent []byte) error) error { + var walkErrors []error + err := filepath.WalkDir(dirName, func(path string, d fs.DirEntry, err error) error { + if err != nil { + walkErrors = append(walkErrors, err) + return nil + } + + if path == dirName { + return nil + } + + if d.IsDir() { + return filepath.SkipDir + } + + // skip files we didn't write or .tmp files + if filename := d.Name(); !strings.HasPrefix(filename, cacheFilesSHA256Prefix) || strings.HasSuffix(filename, tmpFilesSuffix) { + return nil + } + + fileContent, err := os.ReadFile(path) + if err != nil { + walkErrors = append(walkErrors, fmt.Errorf("failed to read %q: %w", path, err)) + return nil + } + + if err := fileAction(path, fileContent); err != nil { + walkErrors = append(walkErrors, err) + return nil + } + + return nil + }) + if err != nil { + walkErrors = append(walkErrors, err) + } + + return errors.NewAggregate(walkErrors) +} + +// createKubeletCOnfigSchemeEncoderDecoder creates strict-encoding encoder and +// decoder for the internal and alpha kubelet config APIs. +func createKubeletConfigSchemeEncoderDecoder() (runtime.Encoder, runtime.Decoder, error) { + const mediaType = runtime.ContentTypeJSON + + scheme := runtime.NewScheme() + if err := kubeletconfigvint1alpha1.AddToScheme(scheme); err != nil { + return nil, nil, err + } + if err := kubeletconfiginternal.AddToScheme(scheme); err != nil { + return nil, nil, err + } + + // use the strict scheme to fail on unknown fields + codecs := serializer.NewCodecFactory(scheme, serializer.EnableStrict) + + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) + if !ok { + return nil, nil, fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) + } + return codecs.EncoderForVersion(info.Serializer, kubeletconfigv1alpha1.SchemeGroupVersion), codecs.UniversalDecoder(), nil +} + +func decodeIntent(d runtime.Decoder, objBytes []byte) (*kubeletconfiginternal.ImagePullIntent, error) { + obj, _, err := d.Decode(objBytes, nil, nil) + if err != nil { + return nil, err + } + + intentObj, ok := obj.(*kubeletconfiginternal.ImagePullIntent) + if !ok { + return nil, fmt.Errorf("failed to convert object to *ImagePullIntent: %T", obj) + } + + return intentObj, nil +} + +func decodePulledRecord(d runtime.Decoder, objBytes []byte) (*kubeletconfiginternal.ImagePulledRecord, error) { + obj, _, err := d.Decode(objBytes, nil, nil) + if err != nil { + return nil, err + } + + pulledRecord, ok := obj.(*kubeletconfiginternal.ImagePulledRecord) + if !ok { + return nil, fmt.Errorf("failed to convert object to *ImagePulledRecord: %T", obj) + } + + return pulledRecord, nil +} diff --git a/pkg/kubelet/images/pullmanager/image_pull_manager.go b/pkg/kubelet/images/pullmanager/image_pull_manager.go new file mode 100644 index 00000000000..a6b54d52e9f --- /dev/null +++ b/pkg/kubelet/images/pullmanager/image_pull_manager.go @@ -0,0 +1,538 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/util/parsers" +) + +var _ ImagePullManager = &PullManager{} + +// writeRecordWhileMatchingLimit is a limit at which we stop writing yet-uncached +// records that we found when we were checking if an image pull must be attempted. +// This is to prevent unbounded writes in cases of high namespace turnover. +const writeRecordWhileMatchingLimit = 100 + +// PullManager is an implementation of the ImagePullManager. It +// tracks images pulled by the kubelet by creating records about ongoing and +// successful pulls. +// It tracks the credentials used with each successful pull in order to be able +// to distinguish tenants requesting access to an image that exists on the kubelet's +// node. +type PullManager struct { + recordsAccessor PullRecordsAccessor + + imagePolicyEnforcer ImagePullPolicyEnforcer + + imageService kubecontainer.ImageService + + intentAccessors *StripedLockSet // image -> sync.Mutex + intentCounters *sync.Map // image -> number of current in-flight pulls + + pulledAccessors *StripedLockSet // imageRef -> sync.Mutex +} + +func NewImagePullManager(ctx context.Context, recordsAccessor PullRecordsAccessor, imagePullPolicy ImagePullPolicyEnforcer, imageService kubecontainer.ImageService, lockStripesNum int32) (*PullManager, error) { + m := &PullManager{ + recordsAccessor: recordsAccessor, + + imagePolicyEnforcer: imagePullPolicy, + + imageService: imageService, + + intentAccessors: NewStripedLockSet(lockStripesNum), + intentCounters: &sync.Map{}, + + pulledAccessors: NewStripedLockSet(lockStripesNum), + } + + m.initialize(ctx) + + return m, nil +} + +func (f *PullManager) RecordPullIntent(image string) error { + f.intentAccessors.Lock(image) + defer f.intentAccessors.Unlock(image) + + if err := f.recordsAccessor.WriteImagePullIntent(image); err != nil { + return fmt.Errorf("failed to record image pull intent: %w", err) + } + + f.incrementIntentCounterForImage(image) + return nil +} + +func (f *PullManager) RecordImagePulled(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) { + if err := f.writePulledRecordIfChanged(image, imageRef, credentials); err != nil { + klog.ErrorS(err, "failed to write image pulled record", "imageRef", imageRef) + return + } + + // Notice we don't decrement in case of record write error, which leaves dangling + // imagePullIntents and refCount in the intentCounters map. + // This is done so that the successfully pulled image is still considered as pulled by the kubelet. + // The kubelet will attempt to turn the imagePullIntent into a pulled record again when + // it's restarted. + f.decrementImagePullIntent(image) +} + +// writePulledRecordIfChanged writes an ImagePulledRecord into the f.pulledDir directory. +// `image` is an image from a container of a Pod object. +// `imageRef` is a reference to the `image“ as used by the CRI. +// `credentials` is a set of credentials that should be written to a new/merged into +// an existing record. +// +// If `credentials` is nil, it marks a situation where an image was pulled under +// unknown circumstances. We should record the image as tracked but no credentials +// should be written in order to force credential verification when the image is +// accessed the next time. +func (f *PullManager) writePulledRecordIfChanged(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) error { + f.pulledAccessors.Lock(imageRef) + defer f.pulledAccessors.Unlock(imageRef) + + sanitizedImage, err := trimImageTagDigest(image) + if err != nil { + return fmt.Errorf("invalid image name %q: %w", image, err) + } + + pulledRecord, _, err := f.recordsAccessor.GetImagePulledRecord(imageRef) + if err != nil { + klog.InfoS("failed to retrieve an ImagePulledRecord", "image", image, "err", err) + pulledRecord = nil + } + + var pulledRecordChanged bool + if pulledRecord == nil { + pulledRecordChanged = true + pulledRecord = &kubeletconfiginternal.ImagePulledRecord{ + LastUpdatedTime: metav1.Time{Time: time.Now()}, + ImageRef: imageRef, + CredentialMapping: make(map[string]kubeletconfiginternal.ImagePullCredentials), + } + // just the existence of the pulled record for a given imageRef is enough + // for us to consider it kubelet-pulled. The kubelet should fail safe + // if it does not find a credential record for the specific image, and it + // must require credential validation + if credentials != nil { + pulledRecord.CredentialMapping[sanitizedImage] = *credentials + } + } else { + pulledRecord, pulledRecordChanged = pulledRecordMergeNewCreds(pulledRecord, sanitizedImage, credentials) + } + + if !pulledRecordChanged { + return nil + } + + return f.recordsAccessor.WriteImagePulledRecord(pulledRecord) +} + +func (f *PullManager) RecordImagePullFailed(image string) { + f.decrementImagePullIntent(image) +} + +// decrementImagePullIntent decreses the number of how many times image pull +// intent for a given `image` was requested, and removes the ImagePullIntent file +// if the reference counter for the image reaches zero. +func (f *PullManager) decrementImagePullIntent(image string) { + f.intentAccessors.Lock(image) + defer f.intentAccessors.Unlock(image) + + if f.getIntentCounterForImage(image) <= 1 { + if err := f.recordsAccessor.DeleteImagePullIntent(image); err != nil { + klog.ErrorS(err, "failed to remove image pull intent", "image", image) + return + } + // only delete the intent counter once the file was deleted to be consistent + // with the records + f.intentCounters.Delete(image) + return + } + + f.decrementIntentCounterForImage(image) +} + +func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []kubeletconfiginternal.ImagePullSecret) bool { + if len(imageRef) == 0 { + return true + } + + var imagePulledByKubelet bool + var pulledRecord *kubeletconfiginternal.ImagePulledRecord + + err := func() error { + // don't allow changes to the files we're using for our decision + f.pulledAccessors.Lock(imageRef) + defer f.pulledAccessors.Unlock(imageRef) + f.intentAccessors.Lock(image) + defer f.intentAccessors.Unlock(image) + + var err error + var exists bool + pulledRecord, exists, err = f.recordsAccessor.GetImagePulledRecord(imageRef) + switch { + case err != nil: + return err + case exists: + imagePulledByKubelet = true + case pulledRecord != nil: + imagePulledByKubelet = true + default: + // optimized check - we can check the intent number, however, if it's zero + // it may only mean kubelet restarted since writing the intent record and + // we must fall back to the actual cache + imagePulledByKubelet = f.getIntentCounterForImage(image) > 0 + if imagePulledByKubelet { + break + } + + if exists, err := f.recordsAccessor.ImagePullIntentExists(image); err != nil { + return fmt.Errorf("failed to check existence of an image pull intent: %w", err) + } else if exists { + imagePulledByKubelet = true + } + } + + return nil + }() + + if err != nil { + klog.ErrorS(err, "Unable to access cache records about image pulls") + return true + } + + if !f.imagePolicyEnforcer.RequireCredentialVerificationForImage(image, imagePulledByKubelet) { + return false + } + + if pulledRecord == nil { + // we have no proper records of the image being pulled in the past, we can short-circuit here + return true + } + + sanitizedImage, err := trimImageTagDigest(image) + if err != nil { + klog.ErrorS(err, "failed to parse image name, forcing image credentials reverification", "image", sanitizedImage) + return true + } + + cachedCreds, ok := pulledRecord.CredentialMapping[sanitizedImage] + if !ok { + return true + } + + if cachedCreds.NodePodsAccessible { + // anyone on this node can access the image + return false + } + + if len(cachedCreds.KubernetesSecrets) == 0 { + return true + } + + for _, podSecret := range podSecrets { + for _, cachedSecret := range cachedCreds.KubernetesSecrets { + + // we need to check hash len in case hashing failed while storing the record in the keyring + hashesMatch := len(cachedSecret.CredentialHash) > 0 && podSecret.CredentialHash == cachedSecret.CredentialHash + secretCoordinatesMatch := podSecret.UID == cachedSecret.UID && + podSecret.Namespace == cachedSecret.Namespace && + podSecret.Name == cachedSecret.Name + + if hashesMatch { + if !secretCoordinatesMatch && len(cachedCreds.KubernetesSecrets) < writeRecordWhileMatchingLimit { + // While we're only matching at this point, we want to ensure this secret is considered valid in the future + // and so we make an additional write to the cache. + // writePulledRecord() is a noop in case the secret with the updated hash already appears in the cache. + if err := f.writePulledRecordIfChanged(image, imageRef, &kubeletconfiginternal.ImagePullCredentials{KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{podSecret}}); err != nil { + klog.ErrorS(err, "failed to write an image pulled record", "image", image, "imageRef", imageRef) + } + } + return false + } + + if secretCoordinatesMatch { + if !hashesMatch && len(cachedCreds.KubernetesSecrets) < writeRecordWhileMatchingLimit { + // While we're only matching at this point, we want to ensure the updated credentials are considered valid in the future + // and so we make an additional write to the cache. + // writePulledRecord() is a noop in case the hash got updated in the meantime. + if err := f.writePulledRecordIfChanged(image, imageRef, &kubeletconfiginternal.ImagePullCredentials{KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{podSecret}}); err != nil { + klog.ErrorS(err, "failed to write an image pulled record", "image", image, "imageRef", imageRef) + } + return false + } + } + } + } + + return true +} + +func (f *PullManager) PruneUnknownRecords(imageList []string, until time.Time) { + f.pulledAccessors.GlobalLock() + defer f.pulledAccessors.GlobalUnlock() + + pulledRecords, err := f.recordsAccessor.ListImagePulledRecords() + if err != nil { + klog.ErrorS(err, "there were errors listing ImagePulledRecords, garbage collection will proceed with incomplete records list") + } + + imagesInUse := sets.New(imageList...) + for _, imageRecord := range pulledRecords { + if !imageRecord.LastUpdatedTime.Time.Before(until) { + // the image record was only updated after the GC started + continue + } + + if imagesInUse.Has(imageRecord.ImageRef) { + continue + } + + if err := f.recordsAccessor.DeleteImagePulledRecord(imageRecord.ImageRef); err != nil { + klog.ErrorS(err, "failed to remove an ImagePulledRecord", "imageRef", imageRecord.ImageRef) + } + } + +} + +// initialize gathers all the images from pull intent records that exist +// from the previous kubelet runs. +// If the CRI reports any of the above images as already pulled, we turn the +// pull intent into a pulled record and the original pull intent is deleted. +// +// This method is not thread-safe and it should only be called upon the creation +// of the PullManager. +func (f *PullManager) initialize(ctx context.Context) { + pullIntents, err := f.recordsAccessor.ListImagePullIntents() + if err != nil { + klog.ErrorS(err, "there were errors listing ImagePullIntents, continuing with an incomplete records list") + } + + if len(pullIntents) == 0 { + return + } + + imageObjs, err := f.imageService.ListImages(ctx) + if err != nil { + klog.ErrorS(err, "failed to list images") + } + + inFlightPulls := sets.New[string]() + for _, intent := range pullIntents { + inFlightPulls.Insert(intent.Image) + } + + // Each of the images known to the CRI might consist of multiple tags and digests, + // which is what we track in the ImagePullIntent - we need to go through all of these + // for each image. + for _, imageObj := range imageObjs { + existingRecordedImages := searchForExistingTagDigest(inFlightPulls, imageObj) + + for _, image := range existingRecordedImages.UnsortedList() { + + if err := f.writePulledRecordIfChanged(image, imageObj.ID, nil); err != nil { + klog.ErrorS(err, "failed to write an image pull record", "imageRef", imageObj.ID) + continue + } + + if err := f.recordsAccessor.DeleteImagePullIntent(image); err != nil { + klog.V(2).InfoS("failed to remove image pull intent file", "imageName", image, "error", err) + } + } + } + +} + +func (f *PullManager) incrementIntentCounterForImage(image string) { + f.intentCounters.Store(image, f.getIntentCounterForImage(image)+1) +} +func (f *PullManager) decrementIntentCounterForImage(image string) { + f.intentCounters.Store(image, f.getIntentCounterForImage(image)-1) +} + +func (f *PullManager) getIntentCounterForImage(image string) int32 { + intentNumAny, ok := f.intentCounters.Load(image) + if !ok { + return 0 + } + intentNum, ok := intentNumAny.(int32) + if !ok { + panic(fmt.Sprintf("expected the intentCounters sync map to only contain int32 values, got %T", intentNumAny)) + } + return intentNum +} + +// searchForExistingTagDigest loops through the `image` RepoDigests and RepoTags +// and tries to find all image digests/tags in `inFlightPulls`, which is a map of +// containerImage -> pulling intent path. +func searchForExistingTagDigest(inFlightPulls sets.Set[string], image kubecontainer.Image) sets.Set[string] { + existingRecordedImages := sets.New[string]() + for _, digest := range image.RepoDigests { + if ok := inFlightPulls.Has(digest); ok { + existingRecordedImages.Insert(digest) + } + } + + for _, tag := range image.RepoTags { + if ok := inFlightPulls.Has(tag); ok { + existingRecordedImages.Insert(tag) + } + } + + return existingRecordedImages +} + +type kubeSecretCoordinates struct { + UID string + Namespace string + Name string +} + +// pulledRecordMergeNewCreds merges the credentials from `newCreds` into the `orig` +// record for the `imageNoTagDigest` image. +// `imageNoTagDigest` is the content of the `image` field from a pod's container +// after any tag or digest were removed from it. +// +// NOTE: pulledRecordMergeNewCreds() may be often called in the read path of +// PullManager.MustAttemptImagePul() and so it's desirable to limit allocations +// (e.g. DeepCopy()) until it is necessary. +func pulledRecordMergeNewCreds(orig *kubeletconfiginternal.ImagePulledRecord, imageNoTagDigest string, newCreds *kubeletconfiginternal.ImagePullCredentials) (*kubeletconfiginternal.ImagePulledRecord, bool) { + if newCreds == nil { + // no new credential information to record + return orig, false + } + + if !newCreds.NodePodsAccessible && len(newCreds.KubernetesSecrets) == 0 { + // we don't have any secret credentials or node-wide access to record + // TODO(stlaz,aramase): add in a serviceaccount dimension check + return orig, false + } + selectedCreds, found := orig.CredentialMapping[imageNoTagDigest] + if !found { + ret := orig.DeepCopy() + if ret.CredentialMapping == nil { + ret.CredentialMapping = make(map[string]kubeletconfiginternal.ImagePullCredentials) + } + ret.CredentialMapping[imageNoTagDigest] = *newCreds + ret.LastUpdatedTime = metav1.Time{Time: time.Now()} + return ret, true + } + + if selectedCreds.NodePodsAccessible { + return orig, false + } + + if newCreds.NodePodsAccessible { + selectedCreds.NodePodsAccessible = true + selectedCreds.KubernetesSecrets = nil + + ret := orig.DeepCopy() + ret.CredentialMapping[imageNoTagDigest] = selectedCreds + ret.LastUpdatedTime = metav1.Time{Time: time.Now()} + return ret, true + } + + var secretsChanged bool + selectedCreds.KubernetesSecrets, secretsChanged = mergePullSecrets(selectedCreds.KubernetesSecrets, newCreds.KubernetesSecrets) + if !secretsChanged { + return orig, false + } + + ret := orig.DeepCopy() + ret.CredentialMapping[imageNoTagDigest] = selectedCreds + ret.LastUpdatedTime = metav1.Time{Time: time.Now()} + return ret, true +} + +// mergePullSecrets merges two slices of ImagePullSecret object into one while +// keeping the objects unique per `Namespace, Name, UID` key. +// +// In case an object from the `new` slice has the same `Namespace, Name, UID` combination +// as an object from `orig`, the result will use the CredentialHash value of the +// object from `new`. +// +// The returned slice is sorted by Namespace, Name and UID (in this order). Also +// returns an indicator whether the set of input secrets chaged. +func mergePullSecrets(orig, new []kubeletconfiginternal.ImagePullSecret) ([]kubeletconfiginternal.ImagePullSecret, bool) { + credSet := make(map[kubeSecretCoordinates]string) + for _, secret := range orig { + credSet[kubeSecretCoordinates{ + UID: secret.UID, + Namespace: secret.Namespace, + Name: secret.Name, + }] = secret.CredentialHash + } + + changed := false + for _, s := range new { + key := kubeSecretCoordinates{UID: s.UID, Namespace: s.Namespace, Name: s.Name} + if existingHash, ok := credSet[key]; !ok || existingHash != s.CredentialHash { + changed = true + credSet[key] = s.CredentialHash + } + } + if !changed { + return orig, false + } + + ret := make([]kubeletconfiginternal.ImagePullSecret, 0, len(credSet)) + for coords, hash := range credSet { + ret = append(ret, kubeletconfiginternal.ImagePullSecret{ + UID: coords.UID, + Namespace: coords.Namespace, + Name: coords.Name, + CredentialHash: hash, + }) + } + // we don't need to use the stable version because secret coordinates used for ordering are unique in the set + slices.SortFunc(ret, imagePullSecretLess) + + return ret, true +} + +// imagePullSecretLess is a helper function to define ordering in a slice of +// ImagePullSecret objects. +func imagePullSecretLess(a, b kubeletconfiginternal.ImagePullSecret) int { + if cmp := strings.Compare(a.Namespace, b.Namespace); cmp != 0 { + return cmp + } + + if cmp := strings.Compare(a.Name, b.Name); cmp != 0 { + return cmp + } + + return strings.Compare(a.UID, b.UID) +} + +// trimImageTagDigest removes the tag and digest from an image name +func trimImageTagDigest(containerImage string) (string, error) { + imageName, _, _, err := parsers.ParseImageName(containerImage) + return imageName, err +} diff --git a/pkg/kubelet/images/pullmanager/image_pull_manager_test.go b/pkg/kubelet/images/pullmanager/image_pull_manager_test.go new file mode 100644 index 00000000000..47e4411448b --- /dev/null +++ b/pkg/kubelet/images/pullmanager/image_pull_manager_test.go @@ -0,0 +1,1053 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "io/fs" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/container" + ctesting "k8s.io/kubernetes/pkg/kubelet/container/testing" + "k8s.io/kubernetes/test/utils/ktesting" +) + +func Test_pulledRecordMergeNewCreds(t *testing.T) { + testTime := metav1.Time{Time: time.Date(2022, 2, 22, 12, 00, 00, 00, time.Local)} + testRecord := &kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testImageRef", + LastUpdatedTime: testTime, + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "test-image1": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid1", Namespace: "namespace1", Name: "name1", CredentialHash: "hash1"}, + }, + }, + "test-image2": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid1", Namespace: "namespace1", Name: "name1", CredentialHash: "hash1"}, + {UID: "uid2", Namespace: "namespace2", Name: "name2", CredentialHash: "hash2"}, + }, + }, + "test-nodewide": { + NodePodsAccessible: true, + }, + }, + } + + tests := []struct { + name string + current *kubeletconfiginternal.ImagePulledRecord + image string + credsForMerging *kubeletconfiginternal.ImagePullCredentials + expectedRecord *kubeletconfiginternal.ImagePulledRecord + wantUpdate bool + }{ + { + name: "create a new image record", + image: "new-image", + current: testRecord.DeepCopy(), + credsForMerging: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "newuid", Namespace: "newnamespace", Name: "newname", CredentialHash: "newhash"}, + }, + }, + expectedRecord: withImageRecord(testRecord.DeepCopy(), "new-image", + kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "newuid", Namespace: "newnamespace", Name: "newname", CredentialHash: "newhash"}, + }, + }, + ), + wantUpdate: true, + }, + { + name: "merge with an existing image secret", + image: "test-image1", + current: testRecord.DeepCopy(), + credsForMerging: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "newuid", Namespace: "newnamespace", Name: "newname", CredentialHash: "newhash"}, + }, + }, + expectedRecord: withImageRecord(testRecord.DeepCopy(), "test-image1", + kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid1", Namespace: "namespace1", Name: "name1", CredentialHash: "hash1"}, + {UID: "newuid", Namespace: "newnamespace", Name: "newname", CredentialHash: "newhash"}, + }, + }, + ), + wantUpdate: true, + }, + + { + name: "merge with existing image record secrets", + image: "test-image2", + current: testRecord.DeepCopy(), + credsForMerging: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "newuid", Namespace: "namespace1", Name: "newname", CredentialHash: "newhash"}, + }, + }, + expectedRecord: withImageRecord(testRecord.DeepCopy(), "test-image2", + kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid1", Namespace: "namespace1", Name: "name1", CredentialHash: "hash1"}, + {UID: "newuid", Namespace: "namespace1", Name: "newname", CredentialHash: "newhash"}, + {UID: "uid2", Namespace: "namespace2", Name: "name2", CredentialHash: "hash2"}, + }, + }, + ), + wantUpdate: true, + }, + { + name: "node-accessible overrides all secrets", + image: "test-image2", + current: testRecord.DeepCopy(), + credsForMerging: &kubeletconfiginternal.ImagePullCredentials{ + NodePodsAccessible: true, + }, + expectedRecord: withImageRecord(testRecord.DeepCopy(), "test-image2", + kubeletconfiginternal.ImagePullCredentials{ + NodePodsAccessible: true, + }, + ), + wantUpdate: true, + }, + { + name: "new creds have the same secret coordinates but a different hash", + image: "test-image2", + current: testRecord.DeepCopy(), + credsForMerging: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid2", Namespace: "namespace2", Name: "name2", CredentialHash: "newhash"}, + }, + }, + expectedRecord: withImageRecord(testRecord.DeepCopy(), "test-image2", + kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid1", Namespace: "namespace1", Name: "name1", CredentialHash: "hash1"}, + {UID: "uid2", Namespace: "namespace2", Name: "name2", CredentialHash: "newhash"}, + }, + }, + ), + wantUpdate: true, + }, + { + name: "new creds have the same hash but a different coordinates", + image: "test-image2", + current: testRecord.DeepCopy(), + credsForMerging: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid3", Namespace: "namespace2", Name: "name3", CredentialHash: "hash2"}, + }, + }, + expectedRecord: withImageRecord(testRecord.DeepCopy(), "test-image2", + kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "uid1", Namespace: "namespace1", Name: "name1", CredentialHash: "hash1"}, + {UID: "uid2", Namespace: "namespace2", Name: "name2", CredentialHash: "hash2"}, + {UID: "uid3", Namespace: "namespace2", Name: "name3", CredentialHash: "hash2"}, + }, + }, + ), + wantUpdate: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRecord, gotUpdate := pulledRecordMergeNewCreds(tt.current, tt.image, tt.credsForMerging) + if gotUpdate != tt.wantUpdate { + t.Errorf("pulledRecordMergeNewCreds() gotUpdate = %v, wantUpdate %v", gotUpdate, tt.wantUpdate) + } + if origTime, newTime := tt.expectedRecord.LastUpdatedTime, gotRecord.LastUpdatedTime; tt.wantUpdate && !origTime.Before(&newTime) { + t.Errorf("expected the new update time to be after the original update time: %v > %v", origTime, newTime) + } + // make the new update time equal to the expected time for the below comparison now + gotRecord.LastUpdatedTime = tt.expectedRecord.LastUpdatedTime + + if !reflect.DeepEqual(gotRecord, tt.expectedRecord) { + t.Errorf("pulledRecordMergeNewCreds() difference between got/expected: %v", cmp.Diff(tt.expectedRecord, gotRecord)) + } + }) + } +} + +func TestFileBasedImagePullManager_MustAttemptImagePull(t *testing.T) { + tests := []struct { + name string + imagePullPolicy ImagePullPolicyEnforcer + podSecrets []kubeletconfiginternal.ImagePullSecret + image string + imageRef string + pulledFiles []string + pullingFiles []string + expectedPullRecord *kubeletconfiginternal.ImagePulledRecord + want bool + expectedCacheWrite bool + }{ + { + name: "image exists and is recorded with pod's exact secret", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + want: false, + }, + { + name: "image exists and is recorded, no pod secrets", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + image: "docker.io/testing/test:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + want: true, + }, + { + name: "image exists and is recorded with the same secret but different credential hash", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "differenthash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + expectedPullRecord: &kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "docker.io/testing/test": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "differenthash"}, + }, + }, + }, + }, + want: false, + expectedCacheWrite: true, + }, + { + name: "image exists and is recorded with a different secret with a different UID", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "different uid", Namespace: "default", Name: "pull-secret", CredentialHash: "differenthash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + want: true, + }, + { + name: "image exists and is recorded with a different secret", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "differentns", Name: "pull-secret", CredentialHash: "differenthash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + want: true, + }, + { + name: "image exists and is recorded with a different secret with the same credential hash", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "differentns", Name: "pull-secret", CredentialHash: "testsecrethash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + expectedPullRecord: &kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "docker.io/testing/test": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash"}, + {UID: "testsecretuid", Namespace: "differentns", Name: "pull-secret", CredentialHash: "testsecrethash"}, + }, + }, + }, + }, + want: false, + expectedCacheWrite: true, + }, + { + name: "image exists but the pull is recorded with a different image name but with the exact same secret", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash", + }, + }, + image: "docker.io/testing/different:latest", + imageRef: "testimageref", + pulledFiles: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + want: true, + }, + { + name: "image exists and is recorded with empty credential mapping", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + image: "docker.io/testing/test:latest", + imageRef: "testemptycredmapping", + pulledFiles: []string{"sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991"}, + want: true, + }, + { + name: "image does not exist and there are no records of it", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + image: "docker.io/testing/test:latest", + imageRef: "", + want: true, + }, + { + name: "image exists and there are no records of it with NeverVerifyPreloadedImages pull policy", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + image: "docker.io/testing/test:latest", + imageRef: "testexistingref", + want: false, + }, + { + name: "image exists and there are no records of it with AlwaysVerify pull policy", + imagePullPolicy: AlwaysVerifyImagePullPolicy(), + image: "docker.io/testing/test:latest", + imageRef: "testexistingref", + want: true, + }, + { + name: "image exists but is only recorded via pulling intent", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testexistingref", + pullingFiles: []string{"sha256-aef2af226629a35d5f3ef0fdbb29fdbebf038d0acd8850590e8c48e1e283aa56"}, + want: true, + }, + { + name: "image exists but is only recorded via pulling intent - NeverVerify policy", + imagePullPolicy: NeverVerifyImagePullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testexistingref", + pullingFiles: []string{"sha256-aef2af226629a35d5f3ef0fdbb29fdbebf038d0acd8850590e8c48e1e283aa56"}, + want: false, + }, + { + name: "image exists and is recorded as node-accessible, no pod secrets", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + image: "docker.io/testing/test:latest", + imageRef: "testimage-anonpull", + pulledFiles: []string{"sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a"}, + want: false, + }, + { + name: "image exists and is recorded as node-accessible, request with pod secrets", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "testimage-anonpull", + pulledFiles: []string{"sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a"}, + want: false, + }, + { + name: "image exists and is recorded with empty hash as its hashing originally failed, the same fail for a different pod secret", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "differentns", Name: "pull-secret", CredentialHash: "", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "test-brokenhash", + pulledFiles: []string{"sha256-38a8906435c4dd5f4258899d46621bfd8eea3ad6ff494ee3c2f17ef0321625bd"}, + want: true, + }, + { + name: "image exists and is recorded with empty hash as its hashing originally failed, the same fail for the same pod secret", + imagePullPolicy: NeverVerifyPreloadedPullPolicy(), + podSecrets: []kubeletconfiginternal.ImagePullSecret{ + { + UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "", + }, + }, + image: "docker.io/testing/test:latest", + imageRef: "test-brokenhash", + pulledFiles: []string{"sha256-38a8906435c4dd5f4258899d46621bfd8eea3ad6ff494ee3c2f17ef0321625bd"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder() + require.NoError(t, err) + + testDir := t.TempDir() + pullingDir := filepath.Join(testDir, "pulling") + pulledDir := filepath.Join(testDir, "pulled") + + copyTestData(t, pullingDir, "pulling", tt.pullingFiles) + copyTestData(t, pulledDir, "pulled", tt.pulledFiles) + + fsRecordAccessor := &testWriteCountingFSPullRecordsAccessor{ + fsPullRecordsAccessor: fsPullRecordsAccessor{ + pullingDir: pullingDir, + pulledDir: pulledDir, + encoder: encoder, + decoder: decoder, + }, + } + + f := &PullManager{ + recordsAccessor: fsRecordAccessor, + imagePolicyEnforcer: tt.imagePullPolicy, + intentAccessors: NewStripedLockSet(10), + intentCounters: &sync.Map{}, + pulledAccessors: NewStripedLockSet(10), + } + if got := f.MustAttemptImagePull(tt.image, tt.imageRef, tt.podSecrets); got != tt.want { + t.Errorf("FileBasedImagePullManager.MustAttemptImagePull() = %v, want %v", got, tt.want) + } + + if tt.expectedCacheWrite != (fsRecordAccessor.imagePulledRecordsWrites != 0) { + t.Errorf("expected zero cache writes, got: %v", fsRecordAccessor.imagePulledRecordsWrites) + } + + if tt.expectedPullRecord != nil { + got, found, err := fsRecordAccessor.GetImagePulledRecord(tt.imageRef) + if err != nil && !found { + t.Fatalf("failed to get an expected ImagePulledRecord") + } + got.LastUpdatedTime = tt.expectedPullRecord.LastUpdatedTime + + if !reflect.DeepEqual(got, tt.expectedPullRecord) { + t.Errorf("expected ImagePulledRecord != got; diff: %s", cmp.Diff(tt.expectedPullRecord, got)) + } + } + }) + } +} + +type testWriteCountingFSPullRecordsAccessor struct { + imagePulledRecordsWrites int + + fsPullRecordsAccessor +} + +func (a *testWriteCountingFSPullRecordsAccessor) WriteImagePulledRecord(pulledRecord *kubeletconfiginternal.ImagePulledRecord) error { + a.imagePulledRecordsWrites += 1 + return a.fsPullRecordsAccessor.WriteImagePulledRecord(pulledRecord) +} + +func TestFileBasedImagePullManager_RecordPullIntent(t *testing.T) { + tests := []struct { + name string + inputImage string + wantFile string + startCounter int32 + wantCounter int32 + }{ + { + name: "first pull", + inputImage: "repo.repo/test/test:latest", + wantFile: "sha256-7d8c031e2f1aeaa71649ca3e0b64c9902370ed460ef57fb07582a87a5a1e1c02", + wantCounter: 1, + }, + { + name: "first pull", + inputImage: "repo.repo/test/test:latest", + wantFile: "sha256-7d8c031e2f1aeaa71649ca3e0b64c9902370ed460ef57fb07582a87a5a1e1c02", + startCounter: 1, + wantCounter: 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder() + require.NoError(t, err) + + testDir := t.TempDir() + pullingDir := filepath.Join(testDir, "pulling") + + fsRecordAccessor := &fsPullRecordsAccessor{ + pullingDir: pullingDir, + encoder: encoder, + decoder: decoder, + } + + f := &PullManager{ + recordsAccessor: fsRecordAccessor, + intentAccessors: NewStripedLockSet(10), + intentCounters: &sync.Map{}, + } + + if tt.startCounter > 0 { + f.intentCounters.Store(tt.inputImage, tt.startCounter) + } + + _ = f.RecordPullIntent(tt.inputImage) + + expectFilename := filepath.Join(pullingDir, tt.wantFile) + require.FileExists(t, expectFilename) + require.Equal(t, tt.wantCounter, f.getIntentCounterForImage(tt.inputImage), "pull intent counter does not match") + + expected := kubeletconfiginternal.ImagePullIntent{ + Image: tt.inputImage, + } + + gotBytes, err := os.ReadFile(expectFilename) + if err != nil { + t.Fatalf("failed to read the expected file: %v", err) + } + + var got kubeletconfiginternal.ImagePullIntent + if _, _, err := decoder.Decode(gotBytes, nil, &got); err != nil { + t.Fatalf("failed to unmarshal the created file data: %v", err) + } + + if !reflect.DeepEqual(expected, got) { + t.Errorf("expected ImagePullIntent != got; diff: %s", cmp.Diff(expected, got)) + } + }) + } +} + +func TestFileBasedImagePullManager_RecordImagePulled(t *testing.T) { + tests := []struct { + name string + image string + imageRef string + creds *kubeletconfiginternal.ImagePullCredentials + pullsInFlight int32 + existingPulling []string + existingPulled []string + expectPullingRemoved string + expectPulled []string + checkedPullFile string + expectedPullRecord kubeletconfiginternal.ImagePulledRecord + expectUpdated bool + }{ + { + name: "new pull record", + image: "repo.repo/test/test:v1", + imageRef: "testimageref", + creds: &kubeletconfiginternal.ImagePullCredentials{NodePodsAccessible: true}, + expectPulled: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + existingPulling: []string{"sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11"}, + pullsInFlight: 1, + expectPullingRemoved: "sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11", + checkedPullFile: "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + expectUpdated: true, + expectedPullRecord: kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "repo.repo/test/test": { + NodePodsAccessible: true, + }, + }, + }, + }, + { + name: "new pull record, more puls in-flight", + image: "repo.repo/test/test:v1", + imageRef: "testimageref", + creds: &kubeletconfiginternal.ImagePullCredentials{NodePodsAccessible: true}, + expectPulled: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + existingPulling: []string{"sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11"}, + pullsInFlight: 2, + checkedPullFile: "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + expectUpdated: true, + expectedPullRecord: kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "repo.repo/test/test": { + NodePodsAccessible: true, + }, + }, + }, + }, + { + name: "merge into existing record", + image: "repo.repo/test/test:v1", + imageRef: "testimageref", + creds: &kubeletconfiginternal.ImagePullCredentials{NodePodsAccessible: true}, + existingPulled: []string{ + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + expectPulled: []string{ + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + existingPulling: []string{"sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11"}, + pullsInFlight: 1, + expectPullingRemoved: "sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11", + checkedPullFile: "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + expectUpdated: true, + expectedPullRecord: kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "repo.repo/test/test": { + NodePodsAccessible: true, + }, + "docker.io/testing/test": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash"}, + }, + }, + }, + }, + }, + { + name: "merge into existing record - existing key in creds mapping", + image: "docker.io/testing/test:something", + imageRef: "testimageref", + creds: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{{UID: "newuid", Namespace: "newns", Name: "newname", CredentialHash: "somehash"}}, + }, + existingPulled: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + pullsInFlight: 1, + expectPulled: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + existingPulling: []string{"sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af"}, + expectPullingRemoved: "sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af", + checkedPullFile: "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + expectUpdated: true, + expectedPullRecord: kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "docker.io/testing/test": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash"}, + {UID: "newuid", Namespace: "newns", Name: "newname", CredentialHash: "somehash"}, + }, + }, + }, + }, + }, + { + name: "existing record stays unchanged", + image: "docker.io/testing/test:something", + imageRef: "testimageref", + creds: &kubeletconfiginternal.ImagePullCredentials{ + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{{UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash"}}, + }, + existingPulled: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + expectPulled: []string{"sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064"}, + existingPulling: []string{"sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af"}, + pullsInFlight: 1, + expectPullingRemoved: "sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af", + checkedPullFile: "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + expectUpdated: false, + expectedPullRecord: kubeletconfiginternal.ImagePulledRecord{ + ImageRef: "testimageref", + CredentialMapping: map[string]kubeletconfiginternal.ImagePullCredentials{ + "docker.io/testing/test": { + KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{ + {UID: "testsecretuid", Namespace: "default", Name: "pull-secret", CredentialHash: "testsecrethash"}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder() + require.NoError(t, err) + + testDir := t.TempDir() + pullingDir := filepath.Join(testDir, "pulling") + pulledDir := filepath.Join(testDir, "pulled") + + copyTestData(t, pullingDir, "pulling", tt.existingPulling) + copyTestData(t, pulledDir, "pulled", tt.existingPulled) + + fsRecordAccessor := &fsPullRecordsAccessor{ + pullingDir: pullingDir, + pulledDir: pulledDir, + encoder: encoder, + decoder: decoder, + } + + f := &PullManager{ + recordsAccessor: fsRecordAccessor, + intentAccessors: NewStripedLockSet(10), + intentCounters: &sync.Map{}, + pulledAccessors: NewStripedLockSet(10), + } + f.intentCounters.Store(tt.image, tt.pullsInFlight) + origIntentCounter := f.getIntentCounterForImage(tt.image) + f.RecordImagePulled(tt.image, tt.imageRef, tt.creds) + require.Equal(t, f.getIntentCounterForImage(tt.image), origIntentCounter-1, "intent counter for %s was not decremented", tt.image) + + for _, fname := range tt.expectPulled { + expectFilename := filepath.Join(pulledDir, fname) + require.FileExists(t, expectFilename) + } + + if len(tt.expectPullingRemoved) > 0 { + dontExpectFilename := filepath.Join(pullingDir, tt.expectPullingRemoved) + require.NoFileExists(t, dontExpectFilename) + } + + pulledBytes, err := os.ReadFile(filepath.Join(pulledDir, tt.checkedPullFile)) + if err != nil { + t.Fatalf("failed to read the expected image pulled record: %v", err) + } + + got, err := decodePulledRecord(decoder, pulledBytes) + if err != nil { + t.Fatalf("failed to deserialize the image pulled record: %v", err) + } + + if tt.expectUpdated { + require.True(t, got.LastUpdatedTime.After(time.Now().Add(-1*time.Minute)), "expected the record to be updated but it didn't - last update time %s", got.LastUpdatedTime.String()) + } else { + require.True(t, got.LastUpdatedTime.Before(&metav1.Time{Time: time.Now().Add(-240 * time.Minute)}), "expected the record to NOT be updated but it was - last update time %s", got.LastUpdatedTime.String()) + } + got.LastUpdatedTime = tt.expectedPullRecord.LastUpdatedTime + + if !reflect.DeepEqual(got, &tt.expectedPullRecord) { + t.Errorf("expected ImagePulledRecord != got; diff: %s", cmp.Diff(tt.expectedPullRecord, got)) + } + }) + } +} + +func TestFileBasedImagePullManager_initialize(t *testing.T) { + imageService := &ctesting.FakeRuntime{ + ImageList: []container.Image{ + { + ID: "testimageref1", + RepoTags: []string{"repo.repo/test/test:docker", "docker.io/testing/test:something"}, + }, + { + ID: "testimageref2", + RepoTags: []string{"repo.repo/test/test:v2", "repo.repo/test/test:test2"}, + RepoDigests: []string{"repo.repo/test/test@dgst2"}, + }, + { + ID: "testimageref", + RepoTags: []string{"repo.repo/test/test:v1", "repo.repo/test/test:test1"}, + RepoDigests: []string{"repo.repo/test/test@dgst1"}, + }, + { + ID: "testimageref3", + RepoTags: []string{"repo.repo/test/test:v3", "repo.repo/test/test:test3"}, + }, + { + ID: "testimageref4", + RepoDigests: []string{"repo.repo/test/test@dgst4", "repo.repo/test/notatest@dgst44"}, + }, + }, + } + + tests := []struct { + name string + existingIntents []string + existingPulledRecords []string + expectedIntents sets.Set[string] + expectedPulled sets.Set[string] + }{ + { + name: "no pulling/pulled records", + }, + { + name: "only pulled records", + existingPulledRecords: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + expectedPulled: sets.New( + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991"), + }, + { + name: "pulling intent that matches an existing image - no matching pulled record", + existingPulledRecords: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + existingIntents: []string{ + "sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af", + }, + expectedPulled: sets.New( + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + "sha256-d77ed7480bc819274ea7a4dba5b2699b2d3f73c6e578762df42e5a8224771096", + ), + }, + { + name: "pulling intent that matches an existing image - a pull record matches", + existingPulledRecords: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + existingIntents: []string{ + "sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11", + }, + expectedPulled: sets.New( + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + ), + }, + { + name: "multiple pulling intents that match existing images", + existingPulledRecords: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + existingIntents: []string{ + "sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11", + "sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af", + }, + expectedPulled: sets.New( + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + "sha256-d77ed7480bc819274ea7a4dba5b2699b2d3f73c6e578762df42e5a8224771096", + ), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testCtx := ktesting.Init(t) + + encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder() + require.NoError(t, err) + + testDir := t.TempDir() + pullingDir := filepath.Join(testDir, "pulling") + pulledDir := filepath.Join(testDir, "pulled") + + if err := os.MkdirAll(pullingDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(pulledDir, 0700); err != nil { + t.Fatal(err) + } + + copyTestData(t, pullingDir, "pulling", tt.existingIntents) + copyTestData(t, pulledDir, "pulled", tt.existingPulledRecords) + + fsRecordAccessor := &fsPullRecordsAccessor{ + pullingDir: pullingDir, + pulledDir: pulledDir, + encoder: encoder, + decoder: decoder, + } + + f := &PullManager{ + recordsAccessor: fsRecordAccessor, + imageService: imageService, + intentAccessors: NewStripedLockSet(10), + intentCounters: &sync.Map{}, + pulledAccessors: NewStripedLockSet(10), + } + f.initialize(testCtx) + + gotIntents := sets.New[string]() + + if err := processDirFiles(pullingDir, func(filePath string, fileContent []byte) error { + gotIntents.Insert(filepath.Base(filePath)) + return nil + }); err != nil { + t.Fatalf("there was an error processing file in the test output pulling dir: %v", err) + } + + gotPulled := sets.New[string]() + if err := processDirFiles(pulledDir, func(filePath string, fileContent []byte) error { + gotPulled.Insert(filepath.Base(filePath)) + return nil + }); err != nil { + t.Fatalf("there was an error processing file in the test output pulled dir: %v", err) + } + + if !gotIntents.Equal(tt.expectedIntents) { + t.Errorf("difference between expected and received pull intent files: %v", cmp.Diff(tt.expectedIntents, gotIntents)) + } + + if !gotPulled.Equal(tt.expectedPulled) { + t.Errorf("difference between expected and received pull record files: %v", cmp.Diff(tt.expectedPulled, gotPulled)) + } + }) + } +} + +func TestFileBasedImagePullManager_PruneUnknownRecords(t *testing.T) { + tests := []struct { + name string + imageList []string + gcStartTime time.Time + pulledFiles []string + wantFiles sets.Set[string] + }{ + { + name: "all images present", + imageList: []string{"testimage-anonpull", "testimageref", "testemptycredmapping"}, + gcStartTime: time.Date(2024, 12, 25, 00, 01, 00, 00, time.UTC), + pulledFiles: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + wantFiles: sets.New( + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + ), + }, + { + name: "remove all records on empty list from the GC", + imageList: []string{}, + gcStartTime: time.Date(2024, 12, 25, 00, 01, 00, 00, time.UTC), + pulledFiles: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + }, + { + name: "remove all records on list of untracked images from the GC", + imageList: []string{"untracked1", "different-untracked"}, + gcStartTime: time.Date(2024, 12, 25, 00, 01, 00, 00, time.UTC), + pulledFiles: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + }, + { + name: "remove records without a match in the image list from the GC", + imageList: []string{"testimage-anonpull", "untracked1", "testimageref", "different-untracked"}, + gcStartTime: time.Date(2024, 12, 25, 00, 01, 00, 00, time.UTC), + pulledFiles: []string{ + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + "sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991", + }, + wantFiles: sets.New( + "sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a", + "sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064", + ), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder() + require.NoError(t, err) + + testDir := t.TempDir() + pulledDir := filepath.Join(testDir, "pulled") + if err := os.MkdirAll(pulledDir, 0700); err != nil { + t.Fatalf("failed to create testing dir %q: %v", pulledDir, err) + } + + copyTestData(t, pulledDir, "pulled", tt.pulledFiles) + + fsRecordAccessor := &fsPullRecordsAccessor{ + pulledDir: pulledDir, + encoder: encoder, + decoder: decoder, + } + + f := &PullManager{ + recordsAccessor: fsRecordAccessor, + pulledAccessors: NewStripedLockSet(10), + } + f.PruneUnknownRecords(tt.imageList, tt.gcStartTime) + + filesLeft := sets.New[string]() + err = filepath.Walk(pulledDir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + if path == pulledDir { + return nil + } + + filesLeft.Insert(info.Name()) + return nil + }) + if err != nil { + t.Fatalf("failed to walk the pull dir after prune: %v", err) + } + + if !tt.wantFiles.Equal(filesLeft) { + t.Errorf("expected equal sets, diff: %s", cmp.Diff(tt.wantFiles, filesLeft)) + } + }) + } +} + +func copyTestData(t *testing.T, dstDir string, testdataDir string, src []string) { + for _, f := range src { + testBytes, err := os.ReadFile(filepath.Join("testdata", testdataDir, f)) + if err != nil { + t.Fatalf("failed to read test data: %v", err) + } + if err := writeFile(dstDir, f, testBytes); err != nil { + t.Fatalf("failed to write test data: %v", err) + } + } +} + +func withImageRecord(r *kubeletconfiginternal.ImagePulledRecord, image string, record kubeletconfiginternal.ImagePullCredentials) *kubeletconfiginternal.ImagePulledRecord { + r.CredentialMapping[image] = record + return r +} diff --git a/pkg/kubelet/images/pullmanager/image_pull_policies.go b/pkg/kubelet/images/pullmanager/image_pull_policies.go new file mode 100644 index 00000000000..2642275a3d5 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/image_pull_policies.go @@ -0,0 +1,171 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "fmt" + "strings" + + dockerref "github.com/distribution/reference" + + "k8s.io/apimachinery/pkg/util/sets" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" +) + +// ImagePullPolicyEnforcer defines a class of functions implementing a credential +// verification policies for image pulls. These function determines whether the +// implemented policy requires credential verification based on image name, local +// image presence and existence of records about previous image pulls. +// +// `image` is an image name from a Pod's container "image" field. +// `imagePresent` informs whether the `image` is present on the node. +// `imagePulledByKubelet` marks that ImagePulledRecord or ImagePullingIntent records +// for the `image` exist on the node, meaning it was pulled by the kubelet somewhere +// in the past. +type ImagePullPolicyEnforcer interface { + RequireCredentialVerificationForImage(image string, imagePulledByKubelet bool) bool +} + +// ImagePullPolicyEnforcerFunc is a function type that implements the ImagePullPolicyEnforcer interface +type ImagePullPolicyEnforcerFunc func(image string, imagePulledByKubelet bool) bool + +func (e ImagePullPolicyEnforcerFunc) RequireCredentialVerificationForImage(image string, imagePulledByKubelet bool) bool { + return e(image, imagePulledByKubelet) +} + +func NewImagePullCredentialVerificationPolicy(policy kubeletconfiginternal.ImagePullCredentialsVerificationPolicy, imageAllowList []string) (ImagePullPolicyEnforcer, error) { + switch policy { + case kubeletconfiginternal.NeverVerify: + return NeverVerifyImagePullPolicy(), nil + case kubeletconfiginternal.NeverVerifyPreloadedImages: + return NeverVerifyPreloadedPullPolicy(), nil + case kubeletconfiginternal.NeverVerifyAllowlistedImages: + return NewNeverVerifyAllowListedPullPolicy(imageAllowList) + case kubeletconfiginternal.AlwaysVerify: + return AlwaysVerifyImagePullPolicy(), nil + default: + return nil, fmt.Errorf("unknown image pull credential verification policy: %v", policy) + } +} + +func NeverVerifyImagePullPolicy() ImagePullPolicyEnforcerFunc { + return func(image string, imagePulledByKubelet bool) bool { + return false + } +} + +func NeverVerifyPreloadedPullPolicy() ImagePullPolicyEnforcerFunc { + return func(image string, imagePulledByKubelet bool) bool { + return imagePulledByKubelet + } +} + +func AlwaysVerifyImagePullPolicy() ImagePullPolicyEnforcerFunc { + return func(image string, imagePulledByKubelet bool) bool { + return true + } +} + +type NeverVerifyAllowlistedImages struct { + absoluteURLs sets.Set[string] + prefixes []string +} + +func NewNeverVerifyAllowListedPullPolicy(allowList []string) (*NeverVerifyAllowlistedImages, error) { + policy := &NeverVerifyAllowlistedImages{ + absoluteURLs: sets.New[string](), + } + for _, pattern := range allowList { + normalizedPattern, isWildcard, err := getAllowlistImagePattern(pattern) + if err != nil { + return nil, err + } + + if isWildcard { + policy.prefixes = append(policy.prefixes, normalizedPattern) + } else { + policy.absoluteURLs.Insert(normalizedPattern) + } + } + + return policy, nil +} + +func (p *NeverVerifyAllowlistedImages) RequireCredentialVerificationForImage(image string, imagePulledByKubelet bool) bool { + return !p.imageMatches(image) +} + +func (p *NeverVerifyAllowlistedImages) imageMatches(image string) bool { + if p.absoluteURLs.Has(image) { + return true + } + for _, prefix := range p.prefixes { + if strings.HasPrefix(image, prefix) { + return true + } + } + return false +} + +func ValidateAllowlistImagesPatterns(patterns []string) error { + for _, p := range patterns { + if _, _, err := getAllowlistImagePattern(p); err != nil { + return err + } + } + return nil +} + +func getAllowlistImagePattern(pattern string) (string, bool, error) { + if pattern != strings.TrimSpace(pattern) { + return "", false, fmt.Errorf("leading/trailing spaces are not allowed: %s", pattern) + } + + trimmedPattern := pattern + isWildcard := false + if strings.HasSuffix(pattern, "/*") { + isWildcard = true + trimmedPattern = strings.TrimSuffix(trimmedPattern, "*") + } + + if len(trimmedPattern) == 0 { + return "", false, fmt.Errorf("the supplied pattern is too short: %s", pattern) + } + + if strings.ContainsRune(trimmedPattern, '*') { + return "", false, fmt.Errorf("not a valid wildcard pattern, only patterns ending with '/*' are allowed: %s", pattern) + } + + if isWildcard { + if len(trimmedPattern) == 1 { + return "", false, fmt.Errorf("at least registry hostname is required") + } + } else { // not a wildcard + image, err := dockerref.ParseNormalizedNamed(trimmedPattern) + if err != nil { + return "", false, fmt.Errorf("failed to parse as an image name: %w", err) + } + + if trimmedPattern != image.Name() { // image.Name() returns the image name without tag/digest + return "", false, fmt.Errorf("neither tag nor digest is accepted in an image reference: %s", pattern) + } + + return trimmedPattern, false, nil + } + + return trimmedPattern, true, nil +} diff --git a/pkg/kubelet/images/pullmanager/image_pull_policies_test.go b/pkg/kubelet/images/pullmanager/image_pull_policies_test.go new file mode 100644 index 00000000000..eb8af5ee0f6 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/image_pull_policies_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "reflect" + "testing" +) + +func TestNeverVerifyPreloadedPullPolicy(t *testing.T) { + tests := []struct { + name string + imageRecordsExist bool + want bool + }{ + { + name: "there are no records about the image being pulled", + imageRecordsExist: false, + want: false, + }, + { + name: "there are records about the image being pulled", + imageRecordsExist: true, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NeverVerifyPreloadedPullPolicy()("test-image", tt.imageRecordsExist); got != tt.want { + t.Errorf("NeverVerifyPreloadedPullPolicy() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewNeverVerifyAllowListedPullPolicy(t *testing.T) { + tests := []struct { + name string + imageRecordsExist bool + allowlist []string + expectedAbsolutes int + expectedWildcards int + want bool + wantErr bool + }{ + { + name: "there are no records about the image being pulled, not in allowlist", + imageRecordsExist: false, + want: true, + allowlist: []string{"test.io/test/image1", "test.io/test/image2", "test.io/test/image3"}, + expectedAbsolutes: 3, + }, + { + name: "there are records about the image being pulled, not in allowlist", + imageRecordsExist: true, + want: true, + allowlist: []string{"test.io/test/image1", "test.io/test/image3", "test.io/test/image2", "test.io/test/image3"}, + expectedAbsolutes: 3, + }, + { + name: "there are no records about the image being pulled, appears in allowlist", + imageRecordsExist: false, + want: false, + allowlist: []string{"test.io/test/image1", "test.io/test/image2", "test.io/test/test-image", "test.io/test/image3"}, + expectedAbsolutes: 4, + }, + { + name: "there are records about the image being pulled, appears in allowlist", + imageRecordsExist: true, + want: false, + allowlist: []string{"test.io/test/image1", "test.io/test/image2", "test.io/test/test-image", "test.io/test/image3"}, + expectedAbsolutes: 4, + }, + { + name: "invalid allowlist pattern - wildcard in the middle", + wantErr: true, + allowlist: []string{"image.repo/pokus*/imagename"}, + }, + { + name: "invalid allowlist pattern - trailing non-segment wildcard middle", + wantErr: true, + allowlist: []string{"image.repo/pokus*"}, + }, + { + name: "invalid allowlist pattern - wildcard path segment in the middle", + wantErr: true, + allowlist: []string{"image.repo/*/imagename"}, + }, + { + name: "invalid allowlist pattern - only wildcard segment", + wantErr: true, + allowlist: []string{"/*"}, + }, + { + name: "invalid allowlist pattern - ends with a '/'", + wantErr: true, + allowlist: []string{"image.repo/"}, + }, + { + name: "invalid allowlist pattern - empty", + wantErr: true, + allowlist: []string{""}, + }, + { + name: "invalid allowlist pattern - asterisk", + wantErr: true, + allowlist: []string{"*"}, + }, + { + name: "invalid allowlist pattern - image with a tag", + wantErr: true, + allowlist: []string{"test.io/test/image1:tagged"}, + }, + { + name: "invalid allowlist pattern - image with a digest", + wantErr: true, + allowlist: []string{"test.io/test/image1@sha256:38a8906435c4dd5f4258899d46621bfd8eea3ad6ff494ee3c2f17ef0321625bd"}, + }, + { + name: "invalid allowlist pattern - trailing whitespace", + wantErr: true, + allowlist: []string{"test.io/test/image1 "}, + }, + { + name: "there are no records about the image being pulled, not in allowlist - different repo wildcard", + imageRecordsExist: false, + want: true, + allowlist: []string{"test.io/test/image1", "test.io/test/image2", "different.repo/test/*"}, + expectedAbsolutes: 2, + expectedWildcards: 1, + }, + { + name: "there are no records about the image being pulled, not in allowlist - matches org wildcard", + imageRecordsExist: false, + want: false, + allowlist: []string{"test.io/test/image1", "test.io/test/image2", "test.io/test/*"}, + expectedAbsolutes: 2, + expectedWildcards: 1, + }, + { + name: "there are no records about the image being pulled, not in allowlist - matches repo wildcard", + imageRecordsExist: false, + want: false, + allowlist: []string{"test.io/test/image1", "test.io/test/image2", "test.io/*"}, + expectedAbsolutes: 2, + expectedWildcards: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + policyEnforcer, err := NewNeverVerifyAllowListedPullPolicy(tt.allowlist) + if tt.wantErr != (err != nil) { + t.Fatalf("wanted error: %t, got: %v", tt.wantErr, err) + } + + if err != nil { + return + } + + if len(policyEnforcer.absoluteURLs) != tt.expectedAbsolutes { + t.Errorf("expected %d of absolute image URLs in the allowlist policy, got %d: %v", tt.expectedAbsolutes, len(policyEnforcer.absoluteURLs), policyEnforcer.absoluteURLs) + } + + if len(policyEnforcer.prefixes) != tt.expectedWildcards { + t.Errorf("expected %d of wildcard image URLs in the allowlist policy, got %d: %v", tt.expectedWildcards, len(policyEnforcer.prefixes), policyEnforcer.prefixes) + } + + got := policyEnforcer.RequireCredentialVerificationForImage("test.io/test/test-image", tt.imageRecordsExist) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewNeverVerifyAllowListedPullPolicy() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/kubelet/images/pullmanager/interfaces.go b/pkg/kubelet/images/pullmanager/interfaces.go new file mode 100644 index 00000000000..3fc05c3a975 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/interfaces.go @@ -0,0 +1,110 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "time" + + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" +) + +// ImagePullManager keeps the state of images that were pulled and which are +// currently still being pulled. +// It should keep an internal state of images currently being pulled by the kubelet +// in order to determine whether to destroy a "pulling" record should an image +// pull fail. +type ImagePullManager interface { + // RecordPullIntent records an intent to pull an image and should be called + // before a pull of the image occurs. + // + // RecordPullIntent() should be called before every image pull. Each call of + // RecordPullIntent() must match exactly one call of RecordImagePulled()/RecordImagePullFailed(). + // + // `image` is the content of the pod's container `image` field. + RecordPullIntent(image string) error + // RecordImagePulled writes a record of an image being successfully pulled + // with ImagePullCredentials. + // + // `credentials` must not be nil and must contain either exactly one Kubernetes + // Secret coordinates in the `.KubernetesSecrets` slice or set `.NodePodsAccessible` + // to `true`. + // + // `image` is the content of the pod's container `image` field. + RecordImagePulled(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) + // RecordImagePullFailed should be called if an image failed to pull. + // + // Internally, it lowers its reference counter for the given image. If the + // counter reaches zero, the pull intent record for the image is removed. + // + // `image` is the content of the pod's container `image` field. + RecordImagePullFailed(image string) + // MustAttemptImagePull evaluates the policy for the image specified in + // `image` and if the policy demands verification, it checks the internal + // cache to see if there's a record of pulling the image with the presented + // set of credentials or if the image can be accessed by any of the node's pods. + // + // Returns true if the policy demands verification and no record of the pull + // was found in the cache. + // + // `image` is the content of the pod's container `image` field. + MustAttemptImagePull(image, imageRef string, credentials []kubeletconfiginternal.ImagePullSecret) bool + // PruneUnknownRecords deletes all of the cache ImagePulledRecords for each of the images + // whose imageRef does not appear in the `imageList` iff such an record was last updated + // _before_ the `until` timestamp. + // + // This method is only expected to be called by the kubelet's image garbage collector. + // `until` is a timestamp created _before_ the `imageList` was requested from the CRI. + PruneUnknownRecords(imageList []string, until time.Time) +} + +// PullRecordsAccessor allows unified access to ImagePullIntents/ImagePulledRecords +// irregardless of the backing database implementation +type PullRecordsAccessor interface { + // ListImagePullIntents lists all the ImagePullIntents in the database. + // ImagePullIntents that cannot be decoded will not appear in the list. + // Returns nil and an error if there was a problem reading from the database. + // + // This method may return partial success in case there were errors listing + // the results. A list of records that were successfully read and an aggregated + // error is returned in that case. + ListImagePullIntents() ([]*kubeletconfiginternal.ImagePullIntent, error) + // ImagePullIntentExists returns whether a valid ImagePullIntent is present + // for the given image. + ImagePullIntentExists(image string) (bool, error) + // WriteImagePullIntent writes a an intent record for the image into the database + WriteImagePullIntent(image string) error + // DeleteImagePullIntent removes an `image` intent record from the database + DeleteImagePullIntent(image string) error + + // ListImagePulledRecords lists the database ImagePulledRecords. + // Records that cannot be decoded will be ignored. + // Returns an error if there was a problem reading from the database. + // + // This method may return partial success in case there were errors listing + // the results. A list of records that were successfully read and an aggregated + // error is returned in that case. + ListImagePulledRecords() ([]*kubeletconfiginternal.ImagePulledRecord, error) + // GetImagePulledRecord fetches an ImagePulledRecord for the given `imageRef`. + // If a file for the `imageRef` is present but the contents cannot be decoded, + // it returns a exists=true with err equal to the decoding error. + GetImagePulledRecord(imageRef string) (record *kubeletconfiginternal.ImagePulledRecord, exists bool, err error) + // WriteImagePulledRecord writes an ImagePulledRecord into the database. + WriteImagePulledRecord(record *kubeletconfiginternal.ImagePulledRecord) error + // DeleteImagePulledRecord removes an ImagePulledRecord for `imageRef` from the + // database. + DeleteImagePulledRecord(imageRef string) error +} diff --git a/pkg/kubelet/images/pullmanager/locks.go b/pkg/kubelet/images/pullmanager/locks.go new file mode 100644 index 00000000000..a1cae9c64b3 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/locks.go @@ -0,0 +1,67 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "hash/fnv" + "sync" +) + +// StripedLockSet allows context locking based on string keys, where each key +// is mapped to a an index in a size-limited slice of locks. +type StripedLockSet struct { + locks []sync.Mutex + size int32 +} + +// NewStripedLockSet creates a StripedLockSet with `size` number of locks to be +// used for locking context based on string keys. +// The size will be normalized to stay in the <1, 31> interval. +func NewStripedLockSet(size int32) *StripedLockSet { + size = max(size, 1) // make sure we're at least at size 1 + + return &StripedLockSet{ + locks: make([]sync.Mutex, min(31, size)), + size: size, + } +} + +func (s *StripedLockSet) Lock(key string) { + s.locks[keyToID(key, s.size)].Lock() +} + +func (s *StripedLockSet) Unlock(key string) { + s.locks[keyToID(key, s.size)].Unlock() +} + +func (s *StripedLockSet) GlobalLock() { + for i := range s.locks { + s.locks[i].Lock() + } +} + +func (s *StripedLockSet) GlobalUnlock() { + for i := range s.locks { + s.locks[i].Unlock() + } +} + +func keyToID(key string, sliceSize int32) uint32 { + h := fnv.New32() + h.Write([]byte(key)) + return h.Sum32() % uint32(sliceSize) +} diff --git a/pkg/kubelet/images/pullmanager/noop_pull_manager.go b/pkg/kubelet/images/pullmanager/noop_pull_manager.go new file mode 100644 index 00000000000..51838d79163 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/noop_pull_manager.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pullmanager + +import ( + "time" + + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" +) + +var _ ImagePullManager = &NoopImagePullManager{} + +type NoopImagePullManager struct{} + +func (m *NoopImagePullManager) RecordPullIntent(_ string) error { return nil } +func (m *NoopImagePullManager) RecordImagePulled(_, _ string, _ *kubeletconfiginternal.ImagePullCredentials) { +} +func (m *NoopImagePullManager) RecordImagePullFailed(image string) {} +func (m *NoopImagePullManager) MustAttemptImagePull(_, _ string, _ []kubeletconfiginternal.ImagePullSecret) bool { + return false +} +func (m *NoopImagePullManager) PruneUnknownRecords(_ []string, _ time.Time) {} diff --git a/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-38a8906435c4dd5f4258899d46621bfd8eea3ad6ff494ee3c2f17ef0321625bd b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-38a8906435c4dd5f4258899d46621bfd8eea3ad6ff494ee3c2f17ef0321625bd new file mode 100644 index 00000000000..e4c408f6aea --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-38a8906435c4dd5f4258899d46621bfd8eea3ad6ff494ee3c2f17ef0321625bd @@ -0,0 +1 @@ +{"kind":"ImagePulledRecord","apiVersion":"kubelet.config.k8s.io/v1alpha1","lastUpdatedTime":"2024-10-21T12:26:40Z","imageRef":"test-brokenhash","credentialMapping":{"docker.io/testing/test":{"kubernetesSecrets":[{"uid":"testsecretuid","namespace":"default","name":"pull-secret","credentialHash":""}]}}} \ No newline at end of file diff --git a/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a new file mode 100644 index 00000000000..658ad7f6d18 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-a2eace2182b24cdbbb730798e47b10709b9ef5e0f0c1624a3bc06c8ca987727a @@ -0,0 +1 @@ +{"kind":"ImagePulledRecord","apiVersion":"kubelet.config.k8s.io/v1alpha1","lastUpdatedTime":"2024-10-21T12:26:40Z","imageRef":"testimage-anonpull","credentialMapping":{"docker.io/testing/test":{"nodePodsAccessible":true}}} \ No newline at end of file diff --git a/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064 b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064 new file mode 100644 index 00000000000..c80e23d50c5 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-b3c0cc4278800b03a308ceb2611161430df571ca733122f0a40ac8b9792a9064 @@ -0,0 +1 @@ +{"kind":"ImagePulledRecord","apiVersion":"kubelet.config.k8s.io/v1alpha1","lastUpdatedTime":"2024-10-21T12:26:40Z","imageRef":"testimageref","credentialMapping":{"docker.io/testing/test":{"kubernetesSecrets":[{"uid":"testsecretuid","namespace":"default","name":"pull-secret","credentialHash":"testsecrethash"}]}}} \ No newline at end of file diff --git a/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991 b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991 new file mode 100644 index 00000000000..b79d714f851 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulled/sha256-f8778b6393eaf39315e767a58cbeacf2c4b270d94b4d6926ee993d9e49444991 @@ -0,0 +1 @@ +{"kind":"ImagePulledRecord","apiVersion":"kubelet.config.k8s.io/v1alpha1","lastUpdatedTime":"2024-10-21T12:26:40Z","imageRef":"testemptycredmapping"} \ No newline at end of file diff --git a/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-aef2af226629a35d5f3ef0fdbb29fdbebf038d0acd8850590e8c48e1e283aa56 b/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-aef2af226629a35d5f3ef0fdbb29fdbebf038d0acd8850590e8c48e1e283aa56 new file mode 100644 index 00000000000..8096c438df0 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-aef2af226629a35d5f3ef0fdbb29fdbebf038d0acd8850590e8c48e1e283aa56 @@ -0,0 +1 @@ +{"kind":"ImagePullIntent","apiVersion":"kubelet.config.k8s.io/v1alpha1","image":"docker.io/testing/test:latest"} \ No newline at end of file diff --git a/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11 b/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11 new file mode 100644 index 00000000000..5f5989df822 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-ee81caca15454863449fb55a1d942904d56d5ed9f9b20a7cb3453944ea2c7e11 @@ -0,0 +1 @@ +{"kind":"ImagePullIntent","apiVersion":"kubelet.config.k8s.io/v1alpha1","image":"repo.repo/test/test:v1"} \ No newline at end of file diff --git a/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af b/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af new file mode 100644 index 00000000000..46a6ccde690 --- /dev/null +++ b/pkg/kubelet/images/pullmanager/testdata/pulling/sha256-f24acc752be18b93b0504c86312bbaf482c9efb0c45e925bbccb0a591cebd7af @@ -0,0 +1 @@ +{"kind":"ImagePullIntent","apiVersion":"kubelet.config.k8s.io/v1alpha1","image":"docker.io/testing/test:something"} \ No newline at end of file diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6eed8bc42f2..b03edd716db 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -759,7 +759,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } } - runtime, err := kuberuntime.NewKubeGenericRuntimeManager( + runtime, postImageGCHooks, err := kuberuntime.NewKubeGenericRuntimeManager( kubecontainer.FilterEventRecorder(kubeDeps.Recorder), klet.livenessManager, klet.readinessManager, @@ -776,6 +776,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeCfg.MaxParallelImagePulls, float32(kubeCfg.RegistryPullQPS), int(kubeCfg.RegistryBurst), + kubeCfg.ImagePullCredentialsVerificationPolicy, + kubeCfg.PreloadedImagesVerificationAllowlist, imageCredentialProviderConfigFile, imageCredentialProviderBinDir, singleProcessOOMKill, @@ -881,7 +883,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.containerDeletor = newPodContainerDeletor(klet.containerRuntime, max(containerGCPolicy.MaxPerPodContainer, minDeadContainerInPod)) // setup imageManager - imageManager, err := images.NewImageGCManager(klet.containerRuntime, klet.StatsProvider, kubeDeps.Recorder, nodeRef, imageGCPolicy, kubeDeps.TracerProvider) + imageManager, err := images.NewImageGCManager(klet.containerRuntime, klet.StatsProvider, postImageGCHooks, kubeDeps.Recorder, nodeRef, imageGCPolicy, kubeDeps.TracerProvider) if err != nil { return nil, fmt.Errorf("failed to initialize image manager: %v", err) } diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 2a5652e7471..b431135d2f8 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -322,7 +322,7 @@ func newTestKubeletWithImageList( HighThresholdPercent: 90, LowThresholdPercent: 80, } - imageGCManager, err := images.NewImageGCManager(fakeRuntime, kubelet.StatsProvider, fakeRecorder, fakeNodeRef, fakeImageGCPolicy, noopoteltrace.NewTracerProvider()) + imageGCManager, err := images.NewImageGCManager(fakeRuntime, kubelet.StatsProvider, nil, fakeRecorder, fakeNodeRef, fakeImageGCPolicy, noopoteltrace.NewTracerProvider()) assert.NoError(t, err) kubelet.imageManager = &fakeImageGCManager{ fakeImageService: fakeRuntime, @@ -3394,7 +3394,7 @@ func TestSyncPodSpans(t *testing.T) { imageSvc, err := remote.NewRemoteImageService(endpoint, 15*time.Second, tp, &logger) assert.NoError(t, err) - kubelet.containerRuntime, err = kuberuntime.NewKubeGenericRuntimeManager( + kubelet.containerRuntime, _, err = kuberuntime.NewKubeGenericRuntimeManager( kubelet.recorder, kubelet.livenessManager, kubelet.readinessManager, @@ -3411,6 +3411,8 @@ func TestSyncPodSpans(t *testing.T) { kubeCfg.MaxParallelImagePulls, float32(kubeCfg.RegistryPullQPS), int(kubeCfg.RegistryBurst), + string(kubeletconfiginternal.NeverVerify), + nil, "", "", nil, diff --git a/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go b/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go index ade8c114251..6d41de84a90 100644 --- a/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/images" + imagepullmanager "k8s.io/kubernetes/pkg/kubelet/images/pullmanager" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/logs" proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" @@ -94,7 +95,7 @@ func (f *fakePodPullingTimeRecorder) RecordImageStartedPulling(podUID types.UID) func (f *fakePodPullingTimeRecorder) RecordImageFinishedPulling(podUID types.UID) {} -func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, machineInfo *cadvisorapi.MachineInfo, osInterface kubecontainer.OSInterface, runtimeHelper kubecontainer.RuntimeHelper, keyring credentialprovider.DockerKeyring, tracer trace.Tracer) (*kubeGenericRuntimeManager, error) { +func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, machineInfo *cadvisorapi.MachineInfo, osInterface kubecontainer.OSInterface, runtimeHelper kubecontainer.RuntimeHelper, tracer trace.Tracer) (*kubeGenericRuntimeManager, error) { ctx := context.Background() recorder := &record.FakeRecorder{} logManager, err := logs.NewContainerLogManager(runtimeService, osInterface, "1", 2, 10, metav1.Duration{Duration: 10 * time.Second}) @@ -113,7 +114,6 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS runtimeHelper: runtimeHelper, runtimeService: runtimeService, imageService: imageService, - keyring: keyring, seccompProfileRoot: fakeSeccompProfileRoot, internalLifecycle: cm.NewFakeInternalContainerLifecycle(), logReduction: logreduction.NewLogReduction(identicalErrorDelay), @@ -134,7 +134,9 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS kubeRuntimeManager.runtimeName = typedVersion.RuntimeName kubeRuntimeManager.imagePuller = images.NewImageManager( kubecontainer.FilterEventRecorder(recorder), + &credentialprovider.BasicDockerKeyring{}, kubeRuntimeManager, + &imagepullmanager.NoopImagePullManager{}, flowcontrol.NewBackOff(time.Second, 300*time.Second), false, ptr.To[int32](0), // No limit on max parallel image pulls, diff --git a/pkg/kubelet/kuberuntime/kuberuntime_image.go b/pkg/kubelet/kuberuntime/kuberuntime_image.go index 22c49d12c11..2e374261387 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_image.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_image.go @@ -19,62 +19,35 @@ package kuberuntime import ( "context" - v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/credentialprovider" - credentialproviderplugin "k8s.io/kubernetes/pkg/credentialprovider/plugin" - credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" + crededentialprovider "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/util/parsers" ) // PullImage pulls an image from the network to local storage using the supplied // secrets if necessary. -func (m *kubeGenericRuntimeManager) PullImage(ctx context.Context, image kubecontainer.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig, serviceAccountName string) (string, error) { +func (m *kubeGenericRuntimeManager) PullImage(ctx context.Context, image kubecontainer.ImageSpec, credentials []crededentialprovider.TrackedAuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, *crededentialprovider.TrackedAuthConfig, error) { img := image.Image - repoToPull, _, _, err := parsers.ParseImageName(img) - if err != nil { - return "", err - } - - // construct the dynamic keyring using the providers we have in the kubelet - var podName, podNamespace, podUID string - if utilfeature.DefaultFeatureGate.Enabled(features.KubeletServiceAccountTokenForCredentialProviders) { - sandboxMetadata := podSandboxConfig.GetMetadata() - - podName = sandboxMetadata.Name - podNamespace = sandboxMetadata.Namespace - podUID = sandboxMetadata.Uid - } - - externalCredentialProviderKeyring := credentialproviderplugin.NewExternalCredentialProviderDockerKeyring(podNamespace, podName, podUID, serviceAccountName) - - keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, credentialprovider.UnionDockerKeyring{m.keyring, externalCredentialProviderKeyring}) - if err != nil { - return "", err - } - imgSpec := toRuntimeAPIImageSpec(image) - creds, withCredentials := keyring.Lookup(repoToPull) - if !withCredentials { + if len(credentials) == 0 { klog.V(3).InfoS("Pulling image without credentials", "image", img) imageRef, err := m.imageService.PullImage(ctx, imgSpec, nil, podSandboxConfig) if err != nil { klog.ErrorS(err, "Failed to pull image", "image", img) - return "", err + return "", nil, err } - return imageRef, nil + return imageRef, nil, nil } var pullErrs []error - for _, currentCreds := range creds { + for _, currentCreds := range credentials { auth := &runtimeapi.AuthConfig{ Username: currentCreds.Username, Password: currentCreds.Password, @@ -87,13 +60,13 @@ func (m *kubeGenericRuntimeManager) PullImage(ctx context.Context, image kubecon imageRef, err := m.imageService.PullImage(ctx, imgSpec, auth, podSandboxConfig) // If there was no error, return success if err == nil { - return imageRef, nil + return imageRef, ¤tCreds, nil } pullErrs = append(pullErrs, err) } - return "", utilerrors.NewAggregate(pullErrs) + return "", nil, utilerrors.NewAggregate(pullErrs) } // GetImageRef gets the ID of the image which has already been in diff --git a/pkg/kubelet/kuberuntime/kuberuntime_image_test.go b/pkg/kubelet/kuberuntime/kuberuntime_image_test.go index 3dd38b6b7d4..6368cf113d2 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_image_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_image_test.go @@ -21,15 +21,20 @@ import ( "encoding/json" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/flowcontrol" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/kubernetes/pkg/credentialprovider" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/images" + imagepullmanager "k8s.io/kubernetes/pkg/kubelet/images/pullmanager" + "k8s.io/utils/ptr" ) func TestPullImage(t *testing.T) { @@ -37,9 +42,10 @@ func TestPullImage(t *testing.T) { _, _, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") + imageRef, creds, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) assert.NoError(t, err) assert.Equal(t, "busybox", imageRef) + assert.Nil(t, creds) // as this was an anonymous pull images, err := fakeManager.ListImages(ctx) assert.NoError(t, err) @@ -52,36 +58,17 @@ func TestPullImageWithError(t *testing.T) { _, fakeImageService, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - // trying to pull an image with an invalid name should return an error - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: ":invalid"}, nil, nil, "") - assert.Error(t, err) - assert.Equal(t, "", imageRef) - fakeImageService.InjectError("PullImage", fmt.Errorf("test-error")) - imageRef, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") + imageRef, creds, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) assert.Error(t, err) assert.Equal(t, "", imageRef) + assert.Nil(t, creds) images, err := fakeManager.ListImages(ctx) assert.NoError(t, err) assert.Empty(t, images) } -func TestPullImageWithInvalidImageName(t *testing.T) { - _, fakeImageService, fakeManager, err := createTestRuntimeManager() - assert.NoError(t, err) - - imageList := []string{"FAIL", "http://fail", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} - fakeImageService.SetFakeImages(imageList) - for _, val := range imageList { - ctx := context.Background() - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: val}, nil, nil, "") - assert.Error(t, err) - assert.Equal(t, "", imageRef) - - } -} - func TestListImages(t *testing.T) { ctx := context.Background() _, fakeImageService, fakeManager, err := createTestRuntimeManager() @@ -196,7 +183,7 @@ func TestRemoveImage(t *testing.T) { _, fakeImageService, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") + _, _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) assert.NoError(t, err) assert.Len(t, fakeImageService.Images, 1) @@ -219,7 +206,7 @@ func TestRemoveImageWithError(t *testing.T) { _, fakeImageService, fakeManager, err := createTestRuntimeManager() assert.NoError(t, err) - _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil, "") + _, _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: "busybox"}, nil, nil) assert.NoError(t, err) assert.Len(t, fakeImageService.Images, 1) @@ -280,13 +267,13 @@ func TestPullWithSecrets(t *testing.T) { expectedAuth *runtimeapi.AuthConfig }{ "no matching secrets": { - "ubuntu", + "ubuntu:latest", []v1.Secret{}, credentialprovider.DockerConfig(map[string]credentialprovider.DockerConfigEntry{}), nil, }, "default keyring secrets": { - "ubuntu", + "ubuntu:latest", []v1.Secret{}, credentialprovider.DockerConfig(map[string]credentialprovider.DockerConfigEntry{ "index.docker.io/v1/": {Username: "built-in", Password: "password", Provider: nil}, @@ -294,7 +281,7 @@ func TestPullWithSecrets(t *testing.T) { &runtimeapi.AuthConfig{Username: "built-in", Password: "password"}, }, "default keyring secrets unused": { - "ubuntu", + "ubuntu:latest", []v1.Secret{}, credentialprovider.DockerConfig(map[string]credentialprovider.DockerConfigEntry{ "extraneous": {Username: "built-in", Password: "password", Provider: nil}, @@ -302,7 +289,7 @@ func TestPullWithSecrets(t *testing.T) { nil, }, "builtin keyring secrets, but use passed": { - "ubuntu", + "ubuntu:latest", []v1.Secret{{Type: v1.SecretTypeDockercfg, Data: map[string][]byte{v1.DockerConfigKey: dockercfgContent}}}, credentialprovider.DockerConfig(map[string]credentialprovider.DockerConfigEntry{ "index.docker.io/v1/": {Username: "built-in", Password: "password", Provider: nil}, @@ -310,7 +297,7 @@ func TestPullWithSecrets(t *testing.T) { &runtimeapi.AuthConfig{Username: "passed-user", Password: "passed-password"}, }, "builtin keyring secrets, but use passed with new docker config": { - "ubuntu", + "ubuntu:latest", []v1.Secret{{Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{v1.DockerConfigJsonKey: dockerConfigJSONContent}}}, credentialprovider.DockerConfig(map[string]credentialprovider.DockerConfigEntry{ "index.docker.io/v1/": {Username: "built-in", Password: "password", Provider: nil}, @@ -320,11 +307,35 @@ func TestPullWithSecrets(t *testing.T) { } for description, test := range tests { builtInKeyRing := &credentialprovider.BasicDockerKeyring{} - builtInKeyRing.Add(test.builtInDockerConfig) - _, fakeImageService, fakeManager, err := customTestRuntimeManager(builtInKeyRing) + builtInKeyRing.Add(nil, test.builtInDockerConfig) + + _, fakeImageService, fakeManager, err := createTestRuntimeManager() require.NoError(t, err) - _, err = fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: test.imageName}, test.passedSecrets, nil, "") + fsRecordAccessor, err := imagepullmanager.NewFSPullRecordsAccessor(t.TempDir()) + if err != nil { + t.Fatal("failed to setup an file pull records accessor") + } + + imagePullManager, err := imagepullmanager.NewImagePullManager(context.Background(), fsRecordAccessor, imagepullmanager.AlwaysVerifyImagePullPolicy(), fakeManager, 10) + if err != nil { + t.Fatal("failed to setup an image pull manager") + } + + fakeManager.imagePuller = images.NewImageManager( + fakeManager.recorder, + builtInKeyRing, + fakeManager, + imagePullManager, + flowcontrol.NewBackOff(time.Second, 300*time.Second), + false, + ptr.To[int32](0), // No limit on max parallel image pulls, + 0, // Disable image pull throttling by setting QPS to 0, + 0, + &fakePodPullingTimeRecorder{}, + ) + + _, _, err = fakeManager.imagePuller.EnsureImageExists(ctx, nil, makeTestPod("testpod", "testpod-ns", "testpod-uid", []v1.Container{}), test.imageName, test.passedSecrets, nil, "", v1.PullAlways) require.NoError(t, err) fakeImageService.AssertImagePulledWithAuth(t, &runtimeapi.ImageSpec{Image: test.imageName, Annotations: make(map[string]string)}, test.expectedAuth, description) } @@ -355,12 +366,12 @@ func TestPullWithSecretsWithError(t *testing.T) { }{ { name: "invalid docker secret", - imageName: "ubuntu", + imageName: "ubuntu:latest", passedSecrets: []v1.Secret{{Type: v1.SecretTypeDockercfg, Data: map[string][]byte{v1.DockerConfigKey: []byte("invalid")}}}, }, { name: "secret provided, pull failed", - imageName: "ubuntu", + imageName: "ubuntu:latest", passedSecrets: []v1.Secret{ {Type: v1.SecretTypeDockerConfigJson, Data: map[string][]byte{v1.DockerConfigKey: dockerConfigJSON}}, }, @@ -375,7 +386,30 @@ func TestPullWithSecretsWithError(t *testing.T) { fakeImageService.InjectError("PullImage", fmt.Errorf("test-error")) } - imageRef, err := fakeManager.PullImage(ctx, kubecontainer.ImageSpec{Image: test.imageName}, test.passedSecrets, nil, "") + fsRecordAccessor, err := imagepullmanager.NewFSPullRecordsAccessor(t.TempDir()) + if err != nil { + t.Fatal("failed to setup an file pull records accessor") + } + + imagePullManager, err := imagepullmanager.NewImagePullManager(context.Background(), fsRecordAccessor, imagepullmanager.AlwaysVerifyImagePullPolicy(), fakeManager, 10) + if err != nil { + t.Fatal("failed to setup an image pull manager") + } + + fakeManager.imagePuller = images.NewImageManager( + fakeManager.recorder, + &credentialprovider.BasicDockerKeyring{}, + fakeManager, + imagePullManager, + flowcontrol.NewBackOff(time.Second, 300*time.Second), + false, + ptr.To[int32](0), // No limit on max parallel image pulls, + 0, // Disable image pull throttling by setting QPS to 0, + 0, + &fakePodPullingTimeRecorder{}, + ) + + imageRef, _, err := fakeManager.imagePuller.EnsureImageExists(ctx, nil, makeTestPod("testpod", "testpod-ns", "testpod-uid", []v1.Container{}), test.imageName, test.passedSecrets, nil, "", v1.PullAlways) assert.Error(t, err) assert.Equal(t, "", imageRef) @@ -398,7 +432,7 @@ func TestPullThenListWithAnnotations(t *testing.T) { }, } - _, err = fakeManager.PullImage(ctx, imageSpec, nil, nil, "") + _, _, err = fakeManager.PullImage(ctx, imageSpec, nil, nil) assert.NoError(t, err) images, err := fakeManager.ListImages(ctx) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 1b42c5acd1c..988e754b84e 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -51,10 +51,12 @@ import ( "k8s.io/kubernetes/pkg/credentialprovider/plugin" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/allocation" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/images" + imagepullmanager "k8s.io/kubernetes/pkg/kubelet/images/pullmanager" runtimeutil "k8s.io/kubernetes/pkg/kubelet/kuberuntime/util" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/logs" @@ -67,6 +69,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/cache" "k8s.io/kubernetes/pkg/kubelet/util/format" sc "k8s.io/kubernetes/pkg/securitycontext" + "k8s.io/utils/ptr" ) const ( @@ -107,9 +110,6 @@ type kubeGenericRuntimeManager struct { // Container GC manager containerGC *containerGC - // Keyring for pulling images - keyring credentialprovider.DockerKeyring - // Runner of lifecycle events. runner kubecontainer.HandlerRunner @@ -207,6 +207,8 @@ func NewKubeGenericRuntimeManager( maxParallelImagePulls *int32, imagePullQPS float32, imagePullBurst int, + imagePullsCredentialVerificationPolicy string, + preloadedImagesCredentialVerificationWhitelist []string, imageCredentialProviderConfigFile string, imageCredentialProviderBinDir string, singleProcessOOMKill *bool, @@ -226,7 +228,7 @@ func NewKubeGenericRuntimeManager( tracerProvider trace.TracerProvider, tokenManager *token.Manager, getServiceAccount plugin.GetServiceAccountFunc, -) (KubeGenericRuntime, error) { +) (KubeGenericRuntime, []images.PostImageGCHook, error) { ctx := context.Background() runtimeService = newInstrumentedRuntimeService(runtimeService) imageService = newInstrumentedImageManagerService(imageService) @@ -261,7 +263,7 @@ func NewKubeGenericRuntimeManager( typedVersion, err := kubeRuntimeManager.getTypedVersion(ctx) if err != nil { klog.ErrorS(err, "Get runtime version failed") - return nil, err + return nil, nil, err } // Only matching kubeRuntimeAPIVersion is supported now @@ -270,7 +272,7 @@ func NewKubeGenericRuntimeManager( klog.ErrorS(err, "This runtime api version is not supported", "apiVersion", typedVersion.Version, "supportedAPIVersion", kubeRuntimeAPIVersion) - return nil, ErrVersionNotSupported + return nil, nil, ErrVersionNotSupported } kubeRuntimeManager.runtimeName = typedVersion.RuntimeName @@ -285,11 +287,37 @@ func NewKubeGenericRuntimeManager( os.Exit(1) } } - kubeRuntimeManager.keyring = credentialprovider.NewDefaultDockerKeyring() + var imageGCHooks []images.PostImageGCHook + var imagePullManager imagepullmanager.ImagePullManager = &imagepullmanager.NoopImagePullManager{} + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + imagePullCredentialsVerificationPolicy, err := imagepullmanager.NewImagePullCredentialVerificationPolicy( + kubeletconfiginternal.ImagePullCredentialsVerificationPolicy(imagePullsCredentialVerificationPolicy), + preloadedImagesCredentialVerificationWhitelist) + + if err != nil { + return nil, nil, err + } + + fsRecordAccessor, err := imagepullmanager.NewFSPullRecordsAccessor(rootDirectory) + if err != nil { + return nil, nil, fmt.Errorf("failed to setup the FSPullRecordsAccessor: %w", err) + } + + imagePullManager, err = imagepullmanager.NewImagePullManager(ctx, fsRecordAccessor, imagePullCredentialsVerificationPolicy, kubeRuntimeManager, ptr.Deref(maxParallelImagePulls, 0)) + if err != nil { + return nil, nil, fmt.Errorf("failed to create image pull manager: %w", err) + } + + imageGCHooks = append(imageGCHooks, imagePullManager.PruneUnknownRecords) + } + + nodeKeyring := credentialprovider.NewDefaultDockerKeyring() kubeRuntimeManager.imagePuller = images.NewImageManager( kubecontainer.FilterEventRecorder(recorder), + nodeKeyring, kubeRuntimeManager, + imagePullManager, imageBackOff, serializeImagePulls, maxParallelImagePulls, @@ -307,7 +335,7 @@ func NewKubeGenericRuntimeManager( versionCacheTTL, ) - return kubeRuntimeManager, nil + return kubeRuntimeManager, imageGCHooks, nil } // Type returns the type of the container runtime. diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index e02b955f0e3..cd309597a17 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -49,7 +49,6 @@ import ( apitest "k8s.io/cri-api/pkg/apis/testing" crierror "k8s.io/cri-api/pkg/errors" podutil "k8s.io/kubernetes/pkg/api/v1/pod" - "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" cmtesting "k8s.io/kubernetes/pkg/kubelet/cm/testing" @@ -68,10 +67,6 @@ var ( ) func createTestRuntimeManager() (*apitest.FakeRuntimeService, *apitest.FakeImageService, *kubeGenericRuntimeManager, error) { - return customTestRuntimeManager(&credentialprovider.BasicDockerKeyring{}) -} - -func customTestRuntimeManager(keyring *credentialprovider.BasicDockerKeyring) (*apitest.FakeRuntimeService, *apitest.FakeImageService, *kubeGenericRuntimeManager, error) { fakeRuntimeService := apitest.NewFakeRuntimeService() fakeImageService := apitest.NewFakeImageService() // Only an empty machineInfo is needed here, because in unit test all containers are besteffort, @@ -82,7 +77,7 @@ func customTestRuntimeManager(keyring *credentialprovider.BasicDockerKeyring) (* MemoryCapacity: uint64(memoryCapacityQuantity.Value()), } osInterface := &containertest.FakeOS{} - manager, err := newFakeKubeRuntimeManager(fakeRuntimeService, fakeImageService, machineInfo, osInterface, &containertest.FakeRuntimeHelper{}, keyring, noopoteltrace.NewTracerProvider().Tracer("")) + manager, err := newFakeKubeRuntimeManager(fakeRuntimeService, fakeImageService, machineInfo, osInterface, &containertest.FakeRuntimeHelper{}, noopoteltrace.NewTracerProvider().Tracer("")) return fakeRuntimeService, fakeImageService, manager, err } diff --git a/staging/src/k8s.io/kubelet/config/v1alpha1/register.go b/staging/src/k8s.io/kubelet/config/v1alpha1/register.go index b12ce03ec03..00a6541cb32 100644 --- a/staging/src/k8s.io/kubelet/config/v1alpha1/register.go +++ b/staging/src/k8s.io/kubelet/config/v1alpha1/register.go @@ -38,6 +38,8 @@ var ( func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &CredentialProviderConfig{}, + &ImagePullIntent{}, + &ImagePulledRecord{}, ) return nil } diff --git a/staging/src/k8s.io/kubelet/config/v1alpha1/types.go b/staging/src/k8s.io/kubelet/config/v1alpha1/types.go index 6de4250c6cb..188d5e6ec75 100644 --- a/staging/src/k8s.io/kubelet/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1alpha1/types.go @@ -96,3 +96,75 @@ type ExecEnvVar struct { Name string `json:"name"` Value string `json:"value"` } + +// ImagePullIntent is a record of the kubelet attempting to pull an image. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ImagePullIntent struct { + metav1.TypeMeta `json:",inline"` + + // Image is the image spec from a Container's `image` field. + // The filename is a SHA-256 hash of this value. This is to avoid filename-unsafe + // characters like ':' and '/'. + Image string `json:"image"` +} + +// ImagePullRecord is a record of an image that was pulled by the kubelet. +// +// If there are no records in the `kubernetesSecrets` field and both `nodeWideCredentials` +// and `anonymous` are `false`, credentials must be re-checked the next time an +// image represented by this record is being requested. +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ImagePulledRecord struct { + metav1.TypeMeta `json:",inline"` + + // LastUpdatedTime is the time of the last update to this record + LastUpdatedTime metav1.Time `json:"lastUpdatedTime"` + + // ImageRef is a reference to the image represented by this file as received + // from the CRI. + // The filename is a SHA-256 hash of this value. This is to avoid filename-unsafe + // characters like ':' and '/'. + ImageRef string `json:"imageRef"` + + // CredentialMapping maps `image` to the set of credentials that it was + // previously pulled with. + // `image` in this case is the content of a pod's container `image` field that's + // got its tag/digest removed. + // + // Example: + // Container requests the `hello-world:latest@sha256:91fb4b041da273d5a3273b6d587d62d518300a6ad268b28628f74997b93171b2` image: + // "credentialMapping": { + // "hello-world": { "nodePodsAccessible": true } + // } + CredentialMapping map[string]ImagePullCredentials `json:"credentialMapping,omitempty"` +} + +// ImagePullCredentials describe credentials that can be used to pull an image. +type ImagePullCredentials struct { + // KuberneteSecretCoordinates is an index of coordinates of all the kubernetes + // secrets that were used to pull the image. + // +optional + // +listType=set + KubernetesSecrets []ImagePullSecret `json:"kubernetesSecrets"` + + // NodePodsAccessible is a flag denoting the pull credentials are accessible + // by all the pods on the node, or that no credentials are needed for the pull. + // + // If true, it is mutually exclusive with the `kubernetesSecrets` field. + // +optional + NodePodsAccessible bool `json:"nodePodsAccessible,omitempty"` +} + +// ImagePullSecret is a representation of a Kubernetes secret object coordinates along +// with a credential hash of the pull secret credentials this object contains. +type ImagePullSecret struct { + UID string `json:"uid"` + Namespace string `json:"namespace"` + Name string `json:"name"` + + // CredentialHash is a SHA-256 retrieved by hashing the image pull credentials + // content of the secret specified by the UID/Namespace/Name coordinates. + CredentialHash string `json:"credentialHash"` +} diff --git a/staging/src/k8s.io/kubelet/config/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1alpha1/zz_generated.deepcopy.go index 6203d545493..f3faff02acf 100644 --- a/staging/src/k8s.io/kubelet/config/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1alpha1/zz_generated.deepcopy.go @@ -109,3 +109,98 @@ func (in *ExecEnvVar) DeepCopy() *ExecEnvVar { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePullCredentials) DeepCopyInto(out *ImagePullCredentials) { + *out = *in + if in.KubernetesSecrets != nil { + in, out := &in.KubernetesSecrets, &out.KubernetesSecrets + *out = make([]ImagePullSecret, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullCredentials. +func (in *ImagePullCredentials) DeepCopy() *ImagePullCredentials { + if in == nil { + return nil + } + out := new(ImagePullCredentials) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePullIntent) DeepCopyInto(out *ImagePullIntent) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullIntent. +func (in *ImagePullIntent) DeepCopy() *ImagePullIntent { + if in == nil { + return nil + } + out := new(ImagePullIntent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ImagePullIntent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePullSecret) DeepCopyInto(out *ImagePullSecret) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullSecret. +func (in *ImagePullSecret) DeepCopy() *ImagePullSecret { + if in == nil { + return nil + } + out := new(ImagePullSecret) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImagePulledRecord) DeepCopyInto(out *ImagePulledRecord) { + *out = *in + out.TypeMeta = in.TypeMeta + in.LastUpdatedTime.DeepCopyInto(&out.LastUpdatedTime) + if in.CredentialMapping != nil { + in, out := &in.CredentialMapping, &out.CredentialMapping + *out = make(map[string]ImagePullCredentials, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePulledRecord. +func (in *ImagePulledRecord) DeepCopy() *ImagePulledRecord { + if in == nil { + return nil + } + out := new(ImagePulledRecord) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ImagePulledRecord) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index edbee00b45f..b08cd91ad51 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -83,6 +83,25 @@ const ( StaticMemoryManagerPolicy = "Static" ) +// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced +// when pod is requesting an image that appears on the system +type ImagePullCredentialsVerificationPolicy string + +const ( + // NeverVerify will never require credential verification for images that + // already exist on the node + NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify" + // NeverVerifyPreloadedImages does not require credential verification for images + // pulled outside the kubelet process + NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages" + // NeverVerifyAllowlistedImages does not require credential verification for + // a list of images that were pulled outside the kubelet process + NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages" + // AlwaysVerify requires credential verification for accessing any image on the + // node irregardless how it was pulled + AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // KubeletConfiguration contains the configuration for the Kubelet @@ -210,6 +229,28 @@ type KubeletConfiguration struct { // Default: 10 // +optional RegistryBurst int32 `json:"registryBurst,omitempty"` + // imagePullCredentialsVerificationPolicy determines how credentials should be + // verified when pod requests an image that is already present on the node: + // - NeverVerify + // - anyone on a node can use any image present on the node + // - NeverVerifyPreloadedImages + // - images that were pulled to the node by something else than the kubelet + // can be used without reverifying pull credentials + // - NeverVerifyAllowlistedImages + // - like "NeverVerifyPreloadedImages" but only node images from + // `preloadedImagesVerificationAllowlist` don't require reverification + // - AlwaysVerify + // - all images require credential reverification + // +optional + ImagePullCredentialsVerificationPolicy ImagePullCredentialsVerificationPolicy `json:"imagePullCredentialsVerificationPolicy,omitempty"` + // preloadedImagesVerificationAllowlist specifies a list of images that are + // exempted from credential reverification for the "NeverVerifyAllowlistedImages" + // `imagePullCredentialsVerificationPolicy`. + // The list accepts a full path segment wildcard suffix "/*". + // Only use image specs without an image tag or digest. + // +optional + // +listType=set + PreloadedImagesVerificationAllowlist []string `json:"preloadedImagesVerificationAllowlist,omitempty"` // eventRecordQPS is the maximum event creations per second. If 0, there // is no limit enforced. The value cannot be a negative number. // Default: 50 diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go index 87d56899c81..1c3f792da05 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -229,6 +229,11 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = new(int32) **out = **in } + if in.PreloadedImagesVerificationAllowlist != nil { + in, out := &in.PreloadedImagesVerificationAllowlist, &out.PreloadedImagesVerificationAllowlist + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.EventRecordQPS != nil { in, out := &in.EventRecordQPS, &out.EventRecordQPS *out = new(int32) diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 81de567551a..dd9a6c3f389 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -651,6 +651,12 @@ lockToDefault: false preRelease: Alpha version: "1.32" +- name: KubeletEnsureSecretPulledImages + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.33" - name: KubeletFineGrainedAuthz versionedSpecs: - default: false