From 3c1ad42773d325295d82fcf545ea1cb707c7fc2b Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 11 Dec 2025 15:51:20 -0500 Subject: [PATCH 1/5] Add a helper for the operation-counting unit tests in nftables (This will make it easier to keep the counts in sync when we change things.) --- pkg/proxy/nftables/proxier_test.go | 118 +++++++++++++++-------------- 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index db2b4a1bce3..4b64d22ffda 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -3808,6 +3808,20 @@ func TestInternalExternalMasquerade(t *testing.T) { } } +// assertNumOperations asserts that the last transaction on nft had a number of operations +// equal to the sum of the values in ops[]. (ops is multiple values just for ease of +// explaining the math; see the callers below.) +func assertNumOperations(t *testing.T, nft *knftables.Fake, ops ...int) { + t.Helper() + expectedOps := 0 + for _, n := range ops { + expectedOps += n + } + if nft.LastTransaction.NumOperations() != expectedOps { + t.Errorf("Expected transaction with %d operations, got %d", expectedOps, nft.LastTransaction.NumOperations()) + } +} + // Test calling syncProxyRules() multiple times with various changes func TestSyncProxyRulesRepeated(t *testing.T) { nft, fp := NewFakeProxier(v1.IPv4Protocol) @@ -3953,12 +3967,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add 1 element to cluster-ips and service-ips = 2 operations - // add+flush 2 chains for service and endpoint, add 2 rules in each = 8 operations - // 10 operations total. - if nft.LastTransaction.NumOperations() != 10 { - t.Errorf("Expected 10 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // add 1 element each to cluster-ips and service-ips + 4, // add+flush service chain, add 2 rules + 4, // add+flush endpoint chain, add 2 rules + ) // Delete a service; its chains will be flushed, but not immediately deleted. fp.OnServiceDelete(svc2) @@ -3987,12 +4000,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // delete 1 element from cluster-ips and service-ips = 2 operations - // flush 2 chains for service and endpoint = 2 operations - // 4 operations total. - if nft.LastTransaction.NumOperations() != 4 { - t.Errorf("Expected 4 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // delete 1 element each from cluster-ips and service-ips + 2, // flush 2 chains for service and endpoint + ) // Fake the passage of time and confirm that the stale chains get deleted. ageStaleChains() @@ -4018,10 +4029,8 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // delete stale chains happens in a separate transaction, nothing else changed => last transaction will have 0 operations. - if nft.LastTransaction.NumOperations() != 0 { - t.Errorf("Expected 0 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + // delete stale chains happens in a separate transaction, nothing else changed + assertNumOperations(t, nft, 0) // Add a service, sync, then add its endpoints. makeServiceMap(fp, @@ -4060,10 +4069,9 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add 1 element to cluster-ips and no-endpoint-services = 2 operations - if nft.LastTransaction.NumOperations() != 2 { - t.Errorf("Expected 2 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // add 1 element each to cluster-ips and no-endpoint-services + ) populateEndpointSlices(fp, makeTestEndpointSlice("ns4", "svc4", 1, func(eps *discovery.EndpointSlice) { @@ -4109,11 +4117,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add 1 element to service-ips, remove 1 element from no-endpoint-services = 2 operations - // add+flush 2 chains for service and endpoint, add 2 rules in each = 8 operations - if nft.LastTransaction.NumOperations() != 10 { - t.Errorf("Expected 10 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services + 4, // add+flush service chain, add 2 rules + 4, // add+flush endpoint chain, add 2 rules + ) // Change an endpoint of an existing service. eps3update := eps3.DeepCopy() @@ -4153,11 +4161,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add+flush 2 chains for service and endpoint, add 2 rules in each = 8 operations - // flush old endpoint chain = 1 operation - if nft.LastTransaction.NumOperations() != 9 { - t.Errorf("Expected 9 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 4, // add+flush service chain, add 2 rules + 4, // add+flush endpoint chain, add 2 rules + 1, // flush old endpoint chain + ) // (Ensure the old svc3 chain gets deleted in the next sync.) ageStaleChains() @@ -4201,10 +4209,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add+flush 3 chains for 1 service and 2 endpoints, add 2 rules in each = 12 operations - if nft.LastTransaction.NumOperations() != 12 { - t.Errorf("Expected 12 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + // (The first endpoint chain is unchanged, but the code recreates it anyway.) + assertNumOperations(t, nft, + 4, // add+flush service chain, add 2 rules + 4, // add+flush original endpoint chain, add 2 rules + 4, // add+flush new endpoint chain, add 2 rules + ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. eps3update3 := eps3update2.DeepCopy() @@ -4238,11 +4248,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // remove 1 element from service-ips, add 1 element to no-endpoint-services = 2 operations - // flush 3 chains = 3 operations - if nft.LastTransaction.NumOperations() != 5 { - t.Errorf("Expected 5 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services + 3, // flush 3 chains + ) expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80", "endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80", "endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80") gotStaleChains := sets.StringKeySet(fp.staleChains) @@ -4285,11 +4294,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // remove 1 element from no-endpoint-services, add 1 element to service-ips = 2 operations - // add+flush 3 chains for 1 service and 2 endpoints, add 2 rules in each = 12 operations - if nft.LastTransaction.NumOperations() != 14 { - t.Errorf("Expected 14 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips + 4, // add+flush service chain, add 2 rules + 8, // add+flush 2 endpoint chains, add 2 rules to each + ) if len(fp.staleChains) != 0 { t.Errorf("unexpected stale chains: %v", fp.staleChains) @@ -4309,9 +4318,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { // Sync with no new changes, so same expected rules as last time fp.syncProxyRules() assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - if nft.LastTransaction.NumOperations() != 0 { - t.Errorf("Expected 0 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, 0) } func TestSyncProxyRulesStartup(t *testing.T) { @@ -4437,17 +4444,12 @@ func TestSyncProxyRulesStartup(t *testing.T) { add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // initial transaction consists of: - // 1. nft setup, total ops = setupOps - // 2. services setup (should skip adding existing set/map elements and endpoint chains+rules) - // - add svc3 IP to the cluster-ips, and to the no-endpoint-services set = 2 ops - // - add+flush 2 service chains + 2 rules each = 8 ops - // - add+flush svc1 endpoint chain + 2 rules = 4 ops - // total: 14 ops - if nft.LastTransaction.NumOperations() != setupOps+14 { - fmt.Println(nft.LastTransaction) - t.Errorf("Expected %v trasaction operations, got %d", setupOps+14, nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + setupOps, // nft setup + 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set + 8, // add+flush 2 service chains + 2 rules each + 4, // add+flush svc1 endpoint chain + 2 rules + ) } func TestNoEndpointsMetric(t *testing.T) { From 75aab220b46cc7a79e34727047b77859faa3af7b Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Mon, 2 Mar 2026 10:57:21 -0500 Subject: [PATCH 2/5] Add NodeName to all EndpointSlices in nftables proxier unit tests Previously it was leaving NodeName unset in many cases. Give all of the endpoints an explicit NodeName, making them explicitly local in all the test cases that don't care either way, and explicitly non-local in those test cases that did care but were previously just relying on the fact that a nil NodeName would be treated as remote. --- pkg/proxy/nftables/proxier_test.go | 51 ++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index 4b64d22ffda..f66c3f8005f 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -296,6 +296,7 @@ func TestOverallNFTablesRules(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -303,12 +304,14 @@ func TestOverallNFTablesRules(t *testing.T) { Protocol: ptr.To(v1.ProtocolTCP), }} }), - // create Local LoadBalancer endpoints. Note that since we aren't setting - // its NodeName, this endpoint will be considered non-local and ignored. + // create Local LoadBalancer endpoints. Note that we set NodeName to the + // "wrong" value, so this endpoint will be considered non-local and + // ignored. makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.2"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -321,6 +324,7 @@ func TestOverallNFTablesRules(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.3"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -333,6 +337,7 @@ func TestOverallNFTablesRules(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.4"}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{"10.180.0.5"}, NodeName: ptr.To(testNodeName), @@ -348,6 +353,7 @@ func TestOverallNFTablesRules(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.3"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -770,6 +776,7 @@ func TestLoadBalancer(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{epIP}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To(svcPortName.Port), @@ -980,7 +987,7 @@ func TestNodePorts(t *testing.T) { } eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{epIP1}, - NodeName: nil, + NodeName: ptr.To("other-node"), }, { Addresses: []string{epIP2}, NodeName: ptr.To(testNodeName), @@ -1106,6 +1113,7 @@ func TestExternalTrafficPolicyLocal(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{epIP1}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{epIP2}, NodeName: ptr.To(testNodeName), @@ -1221,7 +1229,7 @@ func TestExternalTrafficPolicyCluster(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{epIP1}, - NodeName: nil, + NodeName: ptr.To("other-node"), }, { Addresses: []string{epIP2}, NodeName: ptr.To(testNodeName), @@ -1603,6 +1611,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.1"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p11"), @@ -1614,6 +1623,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.2"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p12"), @@ -1645,6 +1655,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.1"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p11-2"), @@ -1659,6 +1670,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.1"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p11"), @@ -1673,6 +1685,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.1"}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{"10.1.1.2"}, NodeName: ptr.To(testNodeName), @@ -1728,6 +1741,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.3"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p13"), @@ -1743,6 +1757,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.1"}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{"10.1.1.2"}, NodeName: ptr.To(testNodeName), @@ -1761,6 +1776,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.3"}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{"10.1.1.4"}, NodeName: ptr.To(testNodeName), @@ -1779,6 +1795,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.2.2.1"}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{"10.2.2.2"}, NodeName: ptr.To(testNodeName), @@ -1856,8 +1873,10 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.1"}, + NodeName: ptr.To("other-node"), }, { Addresses: []string{"10.1.1.11"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p11"), @@ -1869,6 +1888,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.1.1.2"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p12"), @@ -1884,6 +1904,7 @@ func TestUpdateEndpointsMap(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.3.3.3"}, + NodeName: ptr.To("other-node"), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p33"), @@ -3864,6 +3885,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.0.1.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -3875,6 +3897,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.0.2.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p8080"), @@ -3927,6 +3950,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.0.3.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -4078,6 +4102,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.0.4.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -4172,7 +4197,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { // Add an endpoint to a service. eps3update2 := eps3update.DeepCopy() - eps3update2.Endpoints = append(eps3update2.Endpoints, discovery.Endpoint{Addresses: []string{"10.0.3.3"}}) + eps3update2.Endpoints = append(eps3update2.Endpoints, discovery.Endpoint{ + Addresses: []string{"10.0.3.3"}, + NodeName: ptr.To(testNodeName), + }) fp.OnEndpointSliceUpdate(eps3update, eps3update2) fp.syncProxyRules() @@ -4394,8 +4422,14 @@ func TestSyncProxyRulesStartup(t *testing.T) { makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{ - {Addresses: []string{"10.0.1.1"}}, - {Addresses: []string{"10.0.1.2"}}, + { + Addresses: []string{"10.0.1.1"}, + NodeName: ptr.To(testNodeName), + }, + { + Addresses: []string{"10.0.1.2"}, + NodeName: ptr.To(testNodeName), + }, } eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -4407,6 +4441,7 @@ func TestSyncProxyRulesStartup(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.0.2.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p8080"), @@ -4672,6 +4707,7 @@ func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.1"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), @@ -4891,6 +4927,7 @@ func TestBadIPs(t *testing.T) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.00.001"}, + NodeName: ptr.To(testNodeName), }} eps.Ports = []discovery.EndpointPort{{ Name: ptr.To("p80"), From aa3a30d134eaca561869dfb0607afc86ca4116e6 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 14:02:25 -0500 Subject: [PATCH 3/5] Do clusterIP masquerading centrally rather than per-service --- pkg/proxy/nftables/proxier.go | 56 ++++++++++++++---------------- pkg/proxy/nftables/proxier_test.go | 48 ++++--------------------- 2 files changed, 34 insertions(+), 70 deletions(-) diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index e975c791ec8..967c2849e41 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -614,6 +614,33 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { Type: "inet_proto . inet_service : verdict", Comment: ptr.To("NodePort traffic"), }) + + if proxier.masqueradeAll { + tx.Add(&knftables.Rule{ + Chain: servicesChain, + Rule: knftables.Concat( + ipX, "daddr", "@", clusterIPsSet, + proxier.masqueradeRule, + ), + Comment: ptr.To("masquerade all clusterIP traffic"), + }) + } else if proxier.localDetector.IsImplemented() { + // This masquerades off-cluster traffic to a service VIP. The + // idea is that you can establish a static route for your + // Service range, routing to any node, and that node will + // bridge into the Service for you. Since that might bounce + // off-node, we masquerade here. + tx.Add(&knftables.Rule{ + Chain: servicesChain, + Rule: knftables.Concat( + ipX, "daddr", "@", clusterIPsSet, + proxier.localDetector.IfNotLocalNFT(), + proxier.masqueradeRule, + ), + Comment: ptr.To("masquerade clusterIP traffic from outside cluster"), + }) + } + tx.Add(&knftables.Rule{ Chain: servicesChain, Rule: knftables.Concat( @@ -1430,35 +1457,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { continue } - // Set up internal traffic handling. - if hasInternalEndpoints { - if proxier.masqueradeAll { - tx.Add(&knftables.Rule{ - Chain: internalTrafficChain, - Rule: knftables.Concat( - ipX, "daddr", svcInfo.ClusterIP(), - proxier.masqueradeRule, - ), - Comment: ptr.To("masquerade all service traffic"), - }) - } else if proxier.localDetector.IsImplemented() { - // This masquerades off-cluster traffic to a service VIP. The - // idea is that you can establish a static route for your - // Service range, routing to any node, and that node will - // bridge into the Service for you. Since that might bounce - // off-node, we masquerade here. - tx.Add(&knftables.Rule{ - Chain: internalTrafficChain, - Rule: knftables.Concat( - ipX, "daddr", svcInfo.ClusterIP(), - proxier.localDetector.IfNotLocalNFT(), - proxier.masqueradeRule, - ), - Comment: ptr.To("masquerade traffic from outside cluster"), - }) - } - } - // Set up external traffic handling (if any "external" destinations are // enabled). All captured traffic for all external destinations should // jump to externalTrafficChain, which will handle some special cases and diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index f66c3f8005f..d35766c9cf5 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -188,6 +188,7 @@ var baseRules = dedent.Dedent(` add rule ip kube-proxy nat-prerouting jump services add rule ip kube-proxy nodeport-endpoints-check ip daddr @nodeport-ips meta l4proto . th dport vmap @no-endpoint-nodeports add rule ip kube-proxy reject-chain reject + add rule ip kube-proxy services ip daddr @cluster-ips ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade clusterIP traffic from outside cluster" add rule ip kube-proxy services ip daddr . meta l4proto . th dport vmap @service-ips add rule ip kube-proxy services ip daddr @nodeport-ips meta l4proto . th dport vmap @service-nodeports @@ -368,7 +369,6 @@ func TestOverallNFTablesRules(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 @@ -380,7 +380,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 - add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 } add chain ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 ip saddr 10.0.0.0/8 goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit pod traffic" @@ -402,7 +401,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc3 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 } add chain ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" @@ -417,7 +415,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 , 1 : goto endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 } add chain ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" @@ -436,7 +433,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc5 add set ip kube-proxy affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 { type ipv4_addr ; flags dynamic,timeout ; timeout 10800s ; } add chain ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 - add rule ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 ip daddr 172.30.0.45 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 ip saddr @affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 goto endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 add rule ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 } add chain ip kube-proxy external-HVFWP5L3-ns5/svc5/tcp/p80 @@ -3916,14 +3912,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -3970,21 +3964,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -3993,7 +3984,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element each to cluster-ips and service-ips - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush endpoint chain, add 2 rules ) @@ -4007,7 +3998,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4017,7 +4007,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4039,14 +4028,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4077,14 +4064,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4121,21 +4106,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4144,7 +4126,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush endpoint chain, add 2 rules ) @@ -4164,14 +4146,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 @@ -4179,7 +4159,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4187,7 +4166,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush endpoint chain, add 2 rules 1, // flush old endpoint chain ) @@ -4213,14 +4192,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4230,7 +4207,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4239,7 +4215,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush original endpoint chain, add 2 rules 4, // add+flush new endpoint chain, add 2 rules ) @@ -4258,7 +4234,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4269,7 +4244,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4298,14 +4272,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4315,7 +4287,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4324,7 +4295,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 8, // add+flush 2 endpoint chains, add 2 rules to each ) @@ -4365,12 +4336,10 @@ func TestSyncProxyRulesStartup(t *testing.T) { add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 @@ -4462,7 +4431,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy no-endpoint-services { 172.30.0.43 . tcp . 80 comment "ns3/svc3:p80" : goto reject-chain } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4472,7 +4440,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4482,7 +4449,7 @@ func TestSyncProxyRulesStartup(t *testing.T) { assertNumOperations(t, nft, setupOps, // nft setup 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set - 8, // add+flush 2 service chains + 2 rules each + 6, // add+flush 2 service chains + 1 rule each 4, // add+flush svc1 endpoint chain + 2 rules ) } @@ -4942,7 +4909,6 @@ func TestBadIPs(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } add chain ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 From e17963cb99dfdd9fe22befe18232931fbda64466 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 10:38:51 -0500 Subject: [PATCH 4/5] Do nftables hairpin handling centrally rather than per-endpoint --- pkg/proxy/nftables/helpers_test.go | 26 ++++++++ pkg/proxy/nftables/proxier.go | 45 ++++++++++--- pkg/proxy/nftables/proxier_test.go | 103 +++++++++++++++-------------- 3 files changed, 115 insertions(+), 59 deletions(-) diff --git a/pkg/proxy/nftables/helpers_test.go b/pkg/proxy/nftables/helpers_test.go index 274af26a569..d2a9c4859b0 100644 --- a/pkg/proxy/nftables/helpers_test.go +++ b/pkg/proxy/nftables/helpers_test.go @@ -162,6 +162,7 @@ type nftablesTracer struct { // additional info about rules we've hit markMasq bool masquerade bool + dnat bool } // newNFTablesTracer creates an nftablesTracer. nodeIPs are the IP to treat as local node @@ -275,6 +276,8 @@ var endpointVMapEntryRegexp = regexp.MustCompile(`\d+ : goto (\S+)`) var masqMarkRegexp = regexp.MustCompile(`^mark set mark or 0x[[:xdigit:]]+$`) var masqCheckRegexp = regexp.MustCompile(`^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+`) var masqueradeRegexp = regexp.MustCompile(`^masquerade fully-random$`) +var dnatCheckRegexp = regexp.MustCompile(`^ct status dnat`) +var hairpinCheckRegexp = regexp.MustCompile(`^ip6* saddr \. ip6* daddr \. meta l4proto \. th dport @hairpin-connections`) var jumpRegexp = regexp.MustCompile(`^(jump|goto) (\S+)$`) var returnRegexp = regexp.MustCompile(`^return$`) var verdictRegexp = regexp.MustCompile(`^(drop|reject)$`) @@ -310,6 +313,28 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP // some of the regexes might match parts of others. switch { + case dnatCheckRegexp.MatchString(rule): + // `^ct status dnat` + // Part of the hairpin check + match := dnatCheckRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if !tracer.dnat { + rule = "" + break + } + + case hairpinCheckRegexp.MatchString(rule): + // `^ip6* saddr \. ip6* daddr \. meta l4proto \. th dport @hairpin-connections` + // Tests whether the packet is hairpinning back to its client. + match := hairpinCheckRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if sourceIP != destIP { + rule = "" + break + } + // HACK: we don't actually bother doing the full lookup; we + // assume that if src and dst matched, it's hairpin. + case destIPOnlyLookupRegexp.MatchString(rule): // `^ip6* daddr @(\S+)` // Tests whether destIP is a member of the indicated set. @@ -552,6 +577,7 @@ func tracePacket(t *testing.T, nft *knftables.Fake, sourceIP, protocol, destIP, if err != nil { t.Errorf("failed to parse host port '%s': %s", tracer.outputs[0], err.Error()) } + tracer.dnat = true } // Run filter-forward, return if packet is terminated. diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index 967c2849e41..0b0ab84196a 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -93,7 +93,8 @@ const ( firewallCheckChain = "firewall-check" // masquerading - masqueradingChain = "masquerading" + masqueradingChain = "masquerading" + hairpinConnectionsSet = "hairpin-connections" ) // NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies. @@ -197,6 +198,7 @@ type Proxier struct { noEndpointServices *nftElementStorage noEndpointNodePorts *nftElementStorage serviceNodePorts *nftElementStorage + hairpinConnections *nftElementStorage } // Proxier implements proxy.Provider @@ -267,6 +269,7 @@ func NewProxier(ctx context.Context, noEndpointServices: newNFTElementStorage("map", noEndpointServicesMap), noEndpointNodePorts: newNFTElementStorage("map", noEndpointNodePortsMap), serviceNodePorts: newNFTElementStorage("map", serviceNodePortsMap), + hairpinConnections: newNFTElementStorage("set", hairpinConnectionsSet), } logger.V(2).Info("NFTables sync params", "minSyncPeriod", minSyncPeriod, "syncPeriod", syncPeriod, "maxSyncPeriod", proxyutil.FullSyncPeriod) @@ -463,6 +466,13 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { ensureChain(chain, tx, createdChains, false) } + // add hairpin-connections set + tx.Add(&knftables.Set{ + Name: hairpinConnectionsSet, + Type: ipvX_addr + " . " + ipvX_addr + " . inet_proto . inet_service", + Comment: ptr.To("service hairpin connections"), + }) + // Add the rules in the masquerading chain tx.Add(&knftables.Rule{ Chain: masqueradingChain, @@ -472,6 +482,14 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { "masquerade fully-random", ), }) + tx.Add(&knftables.Rule{ + Chain: masqueradingChain, + Rule: knftables.Concat( + "ct status dnat", + ipX, "saddr", ".", ipX, "daddr", ".", "meta l4proto", ".", "th dport", "@", hairpinConnectionsSet, + "masquerade fully-random", + ), + }) // add cluster-ips set. tx.Add(&knftables.Set{ @@ -676,6 +694,7 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { proxier.noEndpointServices.readOrReset(tx, proxier.nftables, proxier.logger) proxier.noEndpointNodePorts.readOrReset(tx, proxier.nftables, proxier.logger) proxier.serviceNodePorts.readOrReset(tx, proxier.nftables, proxier.logger) + proxier.hairpinConnections.readOrReset(tx, proxier.nftables, proxier.logger) } // Sync is called to synchronize the proxier state to nftables as soon as possible. @@ -1205,6 +1224,19 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { activeAffinitySets.Insert(epInfo.affinitySetName) } + + // Add local endpoints to the hairpin set. + if epInfo.IsLocal() { + proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ + Set: hairpinConnectionsSet, + Key: []string{ + epInfo.IP(), + epInfo.IP(), + protocol, + strconv.Itoa(epInfo.Port()), + }, + }) + } } } @@ -1620,16 +1652,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { } endpointChain := epInfo.chainName - // Handle traffic that loops back to the originator with SNAT. - tx.Add(&knftables.Rule{ - Chain: endpointChain, - Rule: knftables.Concat( - ipX, "saddr", epInfo.IP(), - proxier.masqueradeRule, - ), - Comment: ptr.To("masquerade hairpin traffic"), - }) - // Handle session affinity if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { tx.Add(&knftables.Rule{ @@ -1686,6 +1708,7 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { proxier.noEndpointServices.cleanupLeftoverKeys(tx) proxier.noEndpointNodePorts.cleanupLeftoverKeys(tx) proxier.serviceNodePorts.cleanupLeftoverKeys(tx) + proxier.hairpinConnections.cleanupLeftoverKeys(tx) // Sync rules. proxier.logger.V(2).Info("Reloading service nftables data", diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index d35766c9cf5..9de165f27bc 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -136,6 +136,7 @@ func NewFakeProxier(ipFamily v1.IPFamily) (*knftables.Fake, *Proxier) { noEndpointServices: newNFTElementStorage("map", noEndpointServicesMap), noEndpointNodePorts: newNFTElementStorage("map", noEndpointNodePortsMap), serviceNodePorts: newNFTElementStorage("map", serviceNodePortsMap), + hairpinConnections: newNFTElementStorage("set", hairpinConnectionsSet), } p.setInitialized(true) p.syncRunner = runner.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, 30*time.Second, time.Minute) @@ -148,6 +149,7 @@ var baseRules = dedent.Dedent(` add set ip kube-proxy cluster-ips { type ipv4_addr ; comment "Active ClusterIPs" ; } add set ip kube-proxy nodeport-ips { type ipv4_addr ; comment "IPs that accept NodePort traffic" ; } + add set ip kube-proxy hairpin-connections { type ipv4_addr . ipv4_addr . inet_proto . inet_service ; comment "service hairpin connections" ; } add map ip kube-proxy firewall-ips { type ipv4_addr . inet_proto . inet_service : verdict ; comment "destinations that are subject to LoadBalancerSourceRanges" ; } add map ip kube-proxy no-endpoint-nodeports { type inet_proto . inet_service : verdict ; comment "vmap to drop or reject packets to service nodeports with no endpoints" ; } @@ -183,6 +185,7 @@ var baseRules = dedent.Dedent(` add rule ip kube-proxy filter-output ct state new jump cluster-ips-check add rule ip kube-proxy firewall-check ip daddr . meta l4proto . th dport vmap @firewall-ips add rule ip kube-proxy masquerading mark and 0x4000 != 0 mark set mark xor 0x4000 masquerade fully-random + add rule ip kube-proxy masquerading ct status dnat ip saddr . ip daddr . meta l4proto . th dport @hairpin-connections masquerade fully-random add rule ip kube-proxy nat-output jump services add rule ip kube-proxy nat-postrouting jump masquerading add rule ip kube-proxy nat-prerouting jump services @@ -372,11 +375,11 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 ip saddr 10.180.0.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.1 . 10.180.0.1 . tcp . 80 } # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 @@ -386,7 +389,6 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local mark set mark or 0x4000 comment "masquerade local traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit local traffic" add chain ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 - add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 ip saddr 10.180.0.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 meta l4proto tcp dnat to 10.180.0.2:80 add element ip kube-proxy cluster-ips { 172.30.0.42 } @@ -406,12 +408,12 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 goto service-4AT6LBPK-ns3/svc3/tcp/p80 add chain ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 ip saddr 10.180.0.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3003 : goto external-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.3 . 10.180.0.3 . tcp . 80 } # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 @@ -420,15 +422,14 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 goto service-LAUZTJTB-ns4/svc4/tcp/p80 add chain ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 - add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 ip saddr 10.180.0.5 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 meta l4proto tcp dnat to 10.180.0.5:80 add chain ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 - add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 ip saddr 10.180.0.4 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 meta l4proto tcp dnat to 10.180.0.4:80 add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add element ip kube-proxy service-ips { 192.168.99.33 . tcp . 80 : goto external-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.5 . 10.180.0.5 . tcp . 80 } # svc5 add set ip kube-proxy affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 { type ipv4_addr ; flags dynamic,timeout ; timeout 10800s ; } @@ -440,7 +441,6 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-HVFWP5L3-ns5/svc5/tcp/p80 goto service-HVFWP5L3-ns5/svc5/tcp/p80 add chain ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 ip saddr 10.180.0.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 update @affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 { ip saddr } add rule ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 @@ -452,6 +452,7 @@ func TestOverallNFTablesRules(t *testing.T) { add element ip kube-proxy service-ips { 5.6.7.8 . tcp . 80 : goto external-HVFWP5L3-ns5/svc5/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3002 : goto external-HVFWP5L3-ns5/svc5/tcp/p80 } add element ip kube-proxy firewall-ips { 5.6.7.8 . tcp . 80 : goto firewall-HVFWP5L3-ns5/svc5/tcp/p80 } + # 10.180.0.3:80 is already in hairpin-connections, so we don't want another "add element" line... # svc6 add element ip kube-proxy cluster-ips { 172.30.0.46 } @@ -3910,17 +3911,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -3962,30 +3963,30 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, - 2, // add 1 element each to cluster-ips and service-ips + 3, // add 1 element each to cluster-ips, service-ips, and hairpin-connections 3, // add+flush service chain, add 1 rule - 4, // add+flush endpoint chain, add 2 rules + 3, // add+flush endpoint chain, add 1 rule ) // Delete a service; its chains will be flushed, but not immediately deleted. @@ -3996,11 +3997,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 @@ -4009,12 +4011,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, - 2, // delete 1 element each from cluster-ips and service-ips + 3, // delete 1 element each from cluster-ips, service-ips, hairpin-connections 2, // flush 2 chains for service and endpoint ) @@ -4026,17 +4027,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -4062,17 +4063,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } @@ -4104,30 +4105,31 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services + 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 4, // add+flush endpoint chain, add 2 rules + 3, // add+flush endpoint chain, add 1 rule ) // Change an endpoint of an existing service. @@ -4144,30 +4146,31 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.2 . 10.0.3.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, + 2, // remove 1 old element from hairpin-connections, add 1 new one 3, // add+flush service chain, add 1 rule - 4, // add+flush endpoint chain, add 2 rules + 3, // add+flush endpoint chain, add 1 rule 1, // flush old endpoint chain ) @@ -4190,34 +4193,35 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.2 . 10.0.3.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.3 . 10.0.3.3 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 ip saddr 10.0.3.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, + 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 4, // add+flush original endpoint chain, add 2 rules - 4, // add+flush new endpoint chain, add 2 rules + 3, // add+flush original endpoint chain, add 1 rule + 3, // add+flush new endpoint chain, add 1 rule ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. @@ -4232,11 +4236,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy no-endpoint-services { 172.30.0.43 . tcp . 80 comment "ns3/svc3:p80" : goto reject-chain } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 @@ -4246,12 +4251,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services + 2, // remove 2 elements from hairpin-connections 3, // flush 3 chains ) @@ -4270,33 +4275,34 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.2 . 10.0.3.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.3 . 10.0.3.3 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 ip saddr 10.0.3.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips + 2, // add 2 elements to hairpin-connections 3, // add+flush service chain, add 1 rule - 8, // add+flush 2 endpoint chains, add 2 rules to each + 6, // add+flush 2 endpoint chains, add 1 rule to each ) if len(fp.staleChains) != 0 { @@ -4337,17 +4343,17 @@ func TestSyncProxyRulesStartup(t *testing.T) { add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } `)) if err != nil { @@ -4429,28 +4435,29 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } add element ip kube-proxy no-endpoint-services { 172.30.0.43 . tcp . 80 comment "ns3/svc3:p80" : goto reject-chain } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.1.2 . 10.0.1.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 - add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 ip saddr 10.0.1.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, setupOps, // nft setup + 1, // add new svc1 endpoint to hairpin-connections 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set 6, // add+flush 2 service chains + 1 rule each - 4, // add+flush svc1 endpoint chain + 2 rules + 3, // add+flush svc1 endpoint chain + 1 rule ) } @@ -4916,7 +4923,6 @@ func TestBadIPs(t *testing.T) { add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 goto service-ULMVA6XW-ns1/svc1/tcp/p80 add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 ip saddr 10.180.0.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add chain ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 @@ -4928,6 +4934,7 @@ func TestBadIPs(t *testing.T) { add element ip kube-proxy service-ips { 1.2.3.4 . tcp . 80 : goto external-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3001 : goto external-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy firewall-ips { 1.2.3.4 . tcp . 80 : goto firewall-ULMVA6XW-ns1/svc1/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.1 . 10.180.0.1 . tcp . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) From 475f9622c8eab73915a4df9fc87b34a61cf12fdb Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 11:14:17 -0500 Subject: [PATCH 5/5] Squash nftables endpoint chains into service vmap We only need a separate chain for the endpoints if the service uses affinity. --- pkg/proxy/nftables/helpers_test.go | 40 +++++- pkg/proxy/nftables/proxier.go | 210 ++++++++++++++--------------- pkg/proxy/nftables/proxier_test.go | 163 +++++----------------- 3 files changed, 176 insertions(+), 237 deletions(-) diff --git a/pkg/proxy/nftables/helpers_test.go b/pkg/proxy/nftables/helpers_test.go index d2a9c4859b0..daf49b88fd5 100644 --- a/pkg/proxy/nftables/helpers_test.go +++ b/pkg/proxy/nftables/helpers_test.go @@ -270,8 +270,10 @@ var sourceAddrRegexp = regexp.MustCompile(`^ip6* saddr (!= )?(\S+)`) var sourceAddrLookupRegexp = regexp.MustCompile(`^ip6* saddr (!= )?\{([^}]*)\}`) var sourceAddrLocalRegexp = regexp.MustCompile(`^fib saddr type local`) -var endpointVMAPRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) +var endpointVMapRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) var endpointVMapEntryRegexp = regexp.MustCompile(`\d+ : goto (\S+)`) +var endpointDNATRegexp = regexp.MustCompile(`^dnat ip6* addr \. port to numgen random mod \d+ map \{(.*)\}$`) +var endpointDNATEntryRegexp = regexp.MustCompile(`\d+ : (\S+) \. (\d+)`) var masqMarkRegexp = regexp.MustCompile(`^mark set mark or 0x[[:xdigit:]]+$`) var masqCheckRegexp = regexp.MustCompile(`^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+`) @@ -281,7 +283,8 @@ var hairpinCheckRegexp = regexp.MustCompile(`^ip6* saddr \. ip6* daddr \. meta l var jumpRegexp = regexp.MustCompile(`^(jump|goto) (\S+)$`) var returnRegexp = regexp.MustCompile(`^return$`) var verdictRegexp = regexp.MustCompile(`^(drop|reject)$`) -var dnatRegexp = regexp.MustCompile(`^meta l4proto (tcp|udp|sctp) dnat to (\S+)$`) +var l4protoRegexp = regexp.MustCompile(`^meta l4proto (tcp|udp|sctp)`) +var dnatRegexp = regexp.MustCompile(`^dnat to (\S+)$`) var ignoredRegexp = regexp.MustCompile(strings.Join( []string{ @@ -449,6 +452,15 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP break } + case l4protoRegexp.MatchString(rule): + // `meta l4proto (tcp|udp|sctp)` + match := l4protoRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if match[1] != protocol { + rule = "" + break + } + case masqMarkRegexp.MatchString(rule): // `^mark set mark or 0x[[:xdigit:]]+$` // Mark for masquerade. @@ -515,20 +527,20 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP return false case dnatRegexp.MatchString(rule): - // `meta l4proto (tcp|udp|sctp) dnat to (\S+)` + // `^dnat to (\S+)$` // DNAT to an endpoint IP and terminate processing. match := dnatRegexp.FindStringSubmatch(rule) - destEndpoint := match[2] + destEndpoint := match[1] tracer.matches = append(tracer.matches, ruleObj.Rule) tracer.outputs = append(tracer.outputs, destEndpoint) return true - case endpointVMAPRegexp.MatchString(rule): + case endpointVMapRegexp.MatchString(rule): // `^numgen random mod \d+ vmap \{(.*)\}$` // Selects a random endpoint and jumps to it. For tracePacket's // purposes, we jump to *all* of the endpoints. - match := endpointVMAPRegexp.FindStringSubmatch(rule) + match := endpointVMapRegexp.FindStringSubmatch(rule) elements := match[1] for _, match = range endpointVMapEntryRegexp.FindAllStringSubmatch(elements, -1) { @@ -543,6 +555,22 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP } return true + case endpointDNATRegexp.MatchString(rule): + // `^dnat ip6* addr \. port to numgen random mod \d+ map \{(.*)\}$` + // Selects a random endpoint and DNATs to it. For tracePacket's + // purposes, we DNAT to *all* of the endpoints. + match := endpointDNATRegexp.FindStringSubmatch(rule) + elements := match[1] + + for _, match = range endpointDNATEntryRegexp.FindAllStringSubmatch(elements, -1) { + // `\d+ : (\S+) \. (\d+)` + endpointIP, endpointPort := match[1], match[2] + + tracer.matches = append(tracer.matches, ruleObj.Rule) + tracer.outputs = append(tracer.outputs, net.JoinHostPort(endpointIP, endpointPort)) + } + return true + default: tracer.t.Errorf("unmatched rule: %s", ruleObj.Rule) rule = "" diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index 0b0ab84196a..7e1e7c3299f 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -155,7 +155,6 @@ type Proxier struct { // updating nftables with some partial data after kube-proxy restart. endpointSlicesSynced bool servicesSynced bool - syncedOnce bool lastFullSync time.Time needFullSync bool initialized int32 @@ -1080,7 +1079,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { doFullSync := proxier.needFullSync || (time.Since(proxier.lastFullSync) > proxyutil.FullSyncPeriod) defer func() { - proxier.syncedOnce = true metrics.SyncProxyRulesLatency.WithLabelValues(string(proxier.ipFamily)).Observe(metrics.SinceInSeconds(start)) if !doFullSync { metrics.SyncPartialProxyRulesLatency.WithLabelValues(string(proxier.ipFamily)).Observe(metrics.SinceInSeconds(start)) @@ -1215,28 +1213,34 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { !serviceUpdateResult.UpdatedServices.Has(svcName.NamespacedName) && !endpointUpdateResult.UpdatedServices.Has(svcName.NamespacedName) - // Note the endpoint chains that will be used - for _, ep := range allLocallyReachableEndpoints { - if epInfo, ok := ep.(*endpointInfo); ok { - ensureChain(epInfo.chainName, tx, activeChains, skipServiceUpdate || - proxier.epChainSkipUpdate(existingChains, existingAffinitySets, svcInfo, epInfo)) - // Note the affinity sets that will be used - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - activeAffinitySets.Insert(epInfo.affinitySetName) - } + // We only use separate endpoint chains when adding affinity rules + serviceUsesAffinity := svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP - // Add local endpoints to the hairpin set. - if epInfo.IsLocal() { - proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ - Set: hairpinConnectionsSet, - Key: []string{ - epInfo.IP(), - epInfo.IP(), - protocol, - strconv.Itoa(epInfo.Port()), - }, - }) - } + // Note the endpoints that will be used + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } + + // If using affinity, note the endpoint chain and affinity set + // names, and ensure that the chain exists. + if serviceUsesAffinity { + ensureChain(epInfo.chainName, tx, activeChains, skipServiceUpdate) + activeAffinitySets.Insert(epInfo.affinitySetName) + } + + // Add local endpoints to the hairpin set. + if epInfo.IsLocal() { + proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ + Set: hairpinConnectionsSet, + Key: []string{ + epInfo.IP(), + epInfo.IP(), + protocol, + strconv.Itoa(epInfo.Port()), + }, + }) } } @@ -1626,34 +1630,32 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { } } - // If Cluster policy is in use, create the chain and create rules jumping - // from clusterPolicyChain to the clusterEndpoints - if usesClusterPolicyChain { - proxier.writeServiceToEndpointRules(tx, svcInfo, clusterPolicyChain, clusterEndpoints) - } - - // If Local policy is in use, create rules jumping from localPolicyChain - // to the localEndpoints - if usesLocalPolicyChain { - proxier.writeServiceToEndpointRules(tx, svcInfo, localPolicyChain, localEndpoints) - } - - // Generate the per-endpoint chains - for _, ep := range allLocallyReachableEndpoints { - epInfo, ok := ep.(*endpointInfo) - if !ok { - proxier.logger.Error(nil, "Failed to cast endpointInfo", "endpointInfo", ep) - continue + // Write the endpoint rules and/or chains + if !serviceUsesAffinity { + if usesClusterPolicyChain { + proxier.writeServiceToEndpointDNATs(tx, svcInfo, clusterPolicyChain, clusterEndpoints) + } + if usesLocalPolicyChain { + proxier.writeServiceToEndpointDNATs(tx, svcInfo, localPolicyChain, localEndpoints) + } + } else { + if usesClusterPolicyChain { + proxier.writeServiceToEndpointJumps(tx, svcInfo, clusterPolicyChain, clusterEndpoints) + } + if usesLocalPolicyChain { + proxier.writeServiceToEndpointJumps(tx, svcInfo, localPolicyChain, localEndpoints) } - if proxier.epChainSkipUpdate(existingChains, existingAffinitySets, svcInfo, epInfo) { - // If the EP chain is already updated, we can skip it. - continue - } - endpointChain := epInfo.chainName + // And generate the per-endpoint chains and affinity sets + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } - // Handle session affinity - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { + endpointChain := epInfo.chainName + + // Handle session affinity tx.Add(&knftables.Rule{ Chain: endpointChain, Rule: knftables.Concat( @@ -1661,16 +1663,16 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { "{", ipX, "saddr", "}", ), }) - } - // DNAT to final destination. - tx.Add(&knftables.Rule{ - Chain: endpointChain, - Rule: knftables.Concat( - "meta l4proto", protocol, - "dnat to", epInfo.String(), - ), - }) + // DNAT to final destination. + tx.Add(&knftables.Rule{ + Chain: endpointChain, + Rule: knftables.Concat( + "meta l4proto", protocol, + "dnat to", epInfo.String(), + ), + }) + } } } @@ -1767,55 +1769,12 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { return } -// epChainSkipUpdate returns true if the EP chain doesn't need to be updated. -func (proxier *Proxier) epChainSkipUpdate(existingChains, existingAffinitySets sets.Set[string], svcInfo *servicePortInfo, epInfo *endpointInfo) bool { - if proxier.syncedOnce { - // We only skip updating EP chains during the first sync to speed up kube-proxy restart, otherwise return false. - return false - } - if existingChains == nil || existingAffinitySets == nil { - // listing existing objects failed, can't skip updating - return false - } - // EP chain can have up to 3 rules: - // - loopback masquerade rule - // - includes the endpoint IP - // - affinity rule when session affinity is set to ClusterIP - // - includes the affinity set name - // - DNAT rule - // - includes the endpoint IP + port - // EP chain name includes the endpoint IP + port => loopback and DNAT rules are pre-defined by the chain name. - // When session affinity is set to ClusterIP, the affinity set is created for local endpoints. - // Therefore, we can check that sessions affinity hasn't changed by checking if the affinity set exists. - wantAffinitySet := svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP - return existingChains.Has(epInfo.chainName) && wantAffinitySet == existingAffinitySets.Has(epInfo.affinitySetName) -} - -func (proxier *Proxier) writeServiceToEndpointRules(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { - // First write session affinity rules, if applicable. - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - ipX := "ip" - if proxier.ipFamily == v1.IPv6Protocol { - ipX = "ip6" - } - - for _, ep := range endpoints { - epInfo, ok := ep.(*endpointInfo) - if !ok { - continue - } - - tx.Add(&knftables.Rule{ - Chain: svcChain, - Rule: knftables.Concat( - ipX, "saddr", "@", epInfo.affinitySetName, - "goto", epInfo.chainName, - ), - }) - } +func (proxier *Proxier) writeServiceToEndpointDNATs(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { + ipX := "ip" + if proxier.ipFamily == v1.IPv6Protocol { + ipX = "ip6" } - // Now write loadbalancing rule var elements []string for i, ep := range endpoints { epInfo, ok := ep.(*endpointInfo) @@ -1824,12 +1783,53 @@ func (proxier *Proxier) writeServiceToEndpointRules(tx *knftables.Transaction, s } elements = append(elements, - strconv.Itoa(i), ":", "goto", epInfo.chainName, + strconv.Itoa(i), ":", epInfo.IP(), ".", strconv.Itoa(epInfo.Port()), ) if i != len(endpoints)-1 { elements = append(elements, ",") } } + tx.Add(&knftables.Rule{ + Chain: svcChain, + Rule: knftables.Concat( + "meta l4proto", strings.ToLower(string(svcInfo.Protocol())), + "dnat", ipX, "addr . port to", + "numgen random mod", len(endpoints), "map", + "{", elements, "}", + ), + }) +} + +func (proxier *Proxier) writeServiceToEndpointJumps(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { + ipX := "ip" + if proxier.ipFamily == v1.IPv6Protocol { + ipX = "ip6" + } + + var elements []string + // Write the affinity rules, construct the vmap elements + for i, ep := range endpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } + + tx.Add(&knftables.Rule{ + Chain: svcChain, + Rule: knftables.Concat( + ipX, "saddr", "@", epInfo.affinitySetName, + "goto", epInfo.chainName, + ), + }) + + elements = append(elements, + strconv.Itoa(i), ":", "goto", epInfo.chainName, + ) + if i != len(endpoints)-1 { + elements = append(elements, ",") + } + } + // Now write the vmap, for the case where no affinity rule matched tx.Add(&knftables.Rule{ Chain: svcChain, Rule: knftables.Concat( diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index 9de165f27bc..1a591114026 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -372,10 +372,7 @@ func TestOverallNFTablesRules(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } - - add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.1 . 80 } add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } @@ -383,13 +380,11 @@ func TestOverallNFTablesRules(t *testing.T) { # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 - add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 } + add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.2 . 80 } add chain ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 ip saddr 10.0.0.0/8 goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit pod traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local mark set mark or 0x4000 comment "masquerade local traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit local traffic" - add chain ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 - add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 meta l4proto tcp dnat to 10.180.0.2:80 add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 80 : goto service-42NFTM6N-ns2/svc2/tcp/p80 } @@ -403,12 +398,10 @@ func TestOverallNFTablesRules(t *testing.T) { # svc3 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 } + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.3 . 80 } add chain ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 goto service-4AT6LBPK-ns3/svc3/tcp/p80 - add chain ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } @@ -417,14 +410,10 @@ func TestOverallNFTablesRules(t *testing.T) { # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 , 1 : goto endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 } + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.180.0.4 . 80 , 1 : 10.180.0.5 . 80 } add chain ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 goto service-LAUZTJTB-ns4/svc4/tcp/p80 - add chain ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 - add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 meta l4proto tcp dnat to 10.180.0.5:80 - add chain ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 - add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 meta l4proto tcp dnat to 10.180.0.4:80 add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } @@ -3915,14 +3904,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -3968,25 +3953,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 3, // add 1 element each to cluster-ips, service-ips, and hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule ) // Delete a service; its chains will be flushed, but not immediately deleted. @@ -4001,22 +3979,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 3, // delete 1 element each from cluster-ips, service-ips, hairpin-connections - 2, // flush 2 chains for service and endpoint + 1, // flush service chain ) // Fake the passage of time and confirm that the stale chains get deleted. @@ -4031,14 +4004,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // delete stale chains happens in a separate transaction, nothing else changed @@ -4067,14 +4036,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } `) @@ -4110,26 +4075,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule ) // Change an endpoint of an existing service. @@ -4151,27 +4109,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.2 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 old element from hairpin-connections, add 1 new one 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule - 1, // flush old endpoint chain ) // (Ensure the old svc3 chain gets deleted in the next sync.) @@ -4199,29 +4148,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.3.2 . 80 , 1 : 10.0.3.3 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush original endpoint chain, add 1 rule - 3, // add+flush new endpoint chain, add 1 rule ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. @@ -4240,27 +4179,21 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services 2, // remove 2 elements from hairpin-connections - 3, // flush 3 chains + 1, // flush service chain ) - expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80", "endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80", "endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80") + expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80") gotStaleChains := sets.StringKeySet(fp.staleChains) if !expectedStaleChains.Equal(gotStaleChains) { t.Errorf("expected stale chains %v, got %v", expectedStaleChains, gotStaleChains) @@ -4281,28 +4214,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.3.2 . 80 , 1 : 10.0.3.3 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips 2, // add 2 elements to hairpin-connections 3, // add+flush service chain, add 1 rule - 6, // add+flush 2 endpoint chains, add 1 rule to each ) if len(fp.staleChains) != 0 { @@ -4338,15 +4262,11 @@ func TestSyncProxyRulesStartup(t *testing.T) { // put a part of desired state to nftables err := nft.ParseDump(baseRules + dedent.Dedent(` add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy cluster-ips { 172.30.0.42 } @@ -4440,16 +4360,10 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 - add chain ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 - add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.1.1 . 80 , 1 : 10.0.1.2 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, @@ -4457,7 +4371,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { 1, // add new svc1 endpoint to hairpin-connections 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set 6, // add+flush 2 service chains + 1 rule each - 3, // add+flush svc1 endpoint chain + 1 rule ) } @@ -4916,14 +4829,12 @@ func TestBadIPs(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.1 . 80 } add chain ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 goto service-ULMVA6XW-ns1/svc1/tcp/p80 - add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add chain ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 ip saddr != { 203.0.113.0/25 } drop