From 70561f02538ed843b7dc6538ee9ee4829b0912e8 Mon Sep 17 00:00:00 2001 From: docktofuture Date: Wed, 20 Nov 2024 14:30:32 +0100 Subject: [PATCH 1/2] add dual-stack support for node-cache --- cmd/node-cache/app/cache_app.go | 191 +++++++++++++++++++++----------- cmd/node-cache/main.go | 7 -- 2 files changed, 124 insertions(+), 74 deletions(-) diff --git a/cmd/node-cache/app/cache_app.go b/cmd/node-cache/app/cache_app.go index d8c1dab41..3bd674c46 100644 --- a/cmd/node-cache/app/cache_app.go +++ b/cmd/node-cache/app/cache_app.go @@ -22,10 +22,10 @@ import ( "github.com/coredns/coredns/coremain" clog "github.com/coredns/coredns/plugin/pkg/log" - "k8s.io/dns/cmd/kube-dns/app/options" "k8s.io/dns/pkg/dns/config" "k8s.io/dns/pkg/netif" + utiliptables "k8s.io/kubernetes/pkg/util/iptables" utilnet "k8s.io/utils/net" ) @@ -63,14 +63,16 @@ type iptablesRule struct { // CacheApp contains all the config required to run node-cache. type CacheApp struct { - iptables utiliptables.Interface - iptablesRules []iptablesRule - params *ConfigParams - netifHandle *netif.NetifManager - kubednsConfig *options.KubeDNSConfig - exitChan chan struct{} // Channel to terminate background goroutines - clusterDNSIP net.IP - selfProcess *os.Process + iptablesV4 utiliptables.Interface + iptablesV6 utiliptables.Interface + iptablesRulesV4 []iptablesRule + iptablesRulesV6 []iptablesRule + params *ConfigParams + netifHandle *netif.NetifManager + kubednsConfig *options.KubeDNSConfig + exitChan chan struct{} // Channel to terminate background goroutines + clusterDNSIP net.IP + selfProcess *os.Process } func isLockedErr(err error) bool { @@ -100,61 +102,82 @@ func (c *CacheApp) Init() { c.params.SetupIptables = setupIptables } -// isIPv6 return if the node-cache is working in IPv6 mode -// LocalIPs are guaranteed to have the same family -func (c *CacheApp) isIPv6() bool { - if len(c.params.LocalIPs) > 0 { - return utilnet.IsIPv6(c.params.LocalIPs[0]) - } - return false -} - func (c *CacheApp) initIptables() { // using the localIPStr param since we need ip strings here for _, localIP := range strings.Split(c.params.LocalIPStr, ",") { - c.iptablesRules = append(c.iptablesRules, []iptablesRule{ - // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, - // need these additional filter table rules. - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // Additional filter table rules for traffic frpm localIp:localPort - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // skip connection tracking for healthcheck requests generated by liveness probe to health plugin - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - }...) - } - c.iptables = newIPTables(c.isIPv6()) -} + if utilnet.IsIPv6(net.ParseIP(localIP)) { + c.iptablesRulesV6 = append(c.iptablesRulesV6, []iptablesRule{ + // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, + // need these additional filter table rules. + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // Additional filter table rules for traffic frpm localIp:localPort + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // skip connection tracking for healthcheck requests generated by liveness probe to health plugin + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + }...) + } else { + c.iptablesRulesV4 = append(c.iptablesRulesV4, []iptablesRule{ + // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, + // need these additional filter table rules. + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // Additional filter table rules for traffic frpm localIp:localPort + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // skip connection tracking for healthcheck requests generated by liveness probe to health plugin + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + }...) + } -func newIPTables(isIPv6 bool) utiliptables.Interface { - protocol := utiliptables.ProtocolIPv4 - if isIPv6 { - protocol = utiliptables.ProtocolIPv6 } - return utiliptables.New(protocol) + c.iptablesV4 = utiliptables.New(utiliptables.ProtocolIPv4) + c.iptablesV6 = utiliptables.New(utiliptables.ProtocolIPv6) } func handleIPTablesError(err error) { @@ -181,23 +204,41 @@ func (c *CacheApp) TeardownNetworking() error { err = c.netifHandle.RemoveDummyDevice(c.params.InterfaceName) } if c.params.SetupIptables { - for _, rule := range c.iptablesRules { + for _, rule := range c.iptablesRulesV4 { + exists := true + for exists == true { + // check in a loop in case the same rule got added multiple times. + err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...) + if err != nil { + clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err) + handleIPTablesError(err) + } + exists, err = c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + if err != nil { + clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err) + handleIPTablesError(err) + } + } + // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist + err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...) + } + for _, rule := range c.iptablesRulesV6 { exists := true for exists == true { // check in a loop in case the same rule got added multiple times. - err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...) + err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...) if err != nil { - clog.Errorf("Failed deleting iptables rule %v, error - %v", rule, err) + clog.Errorf("Failed deleting iptablesV6 rule %v, error - %v", rule, err) handleIPTablesError(err) } - exists, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + exists, err = c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) if err != nil { - clog.Errorf("Failed checking iptables rule after deletion, rule - %v, error - %v", rule, err) + clog.Errorf("Failed checking iptablesV6 rule after deletion, rule - %v, error - %v", rule, err) handleIPTablesError(err) } } // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist - err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...) + err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...) } } return err @@ -205,8 +246,24 @@ func (c *CacheApp) TeardownNetworking() error { func (c *CacheApp) setupNetworking() { if c.params.SetupIptables { - for _, rule := range c.iptablesRules { - exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + for _, rule := range c.iptablesRulesV4 { + exists, err := c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + switch { + case exists: + // debug messages can be printed by including "debug" plugin in coreFile. + clog.Debugf("iptables rule %v for nodelocaldns already exists", rule) + continue + case err == nil: + clog.Infof("Added back nodelocaldns rule - %v", rule) + continue + default: + // iptables check/rule add failed with error since control reached here. + clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err) + handleIPTablesError(err) + } + } + for _, rule := range c.iptablesRulesV6 { + exists, err := c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) switch { case exists: // debug messages can be printed by including "debug" plugin in coreFile. diff --git a/cmd/node-cache/main.go b/cmd/node-cache/main.go index 154b7eb30..ebbb7be7d 100644 --- a/cmd/node-cache/main.go +++ b/cmd/node-cache/main.go @@ -25,7 +25,6 @@ import ( corednsmain "github.com/coredns/coredns/coremain" clog "github.com/coredns/coredns/plugin/pkg/log" - utilnet "k8s.io/utils/net" "github.com/coredns/caddy" // blank imports to make sure the plugin code is pulled in from vendor when building node-cache image @@ -101,12 +100,6 @@ func parseAndValidateFlags() (*app.ConfigParams, error) { params.LocalIPs = append(params.LocalIPs, newIP) } - // validate all the IPs have the same IP family - for _, ip := range params.LocalIPs { - if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) { - return params, fmt.Errorf("unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0])) - } - } // lookup specified dns port f := flag.Lookup("dns.port") if f == nil { From 12f43ea9a4f86d4e222b52ee3858b106a9478bdc Mon Sep 17 00:00:00 2001 From: docktofuture Date: Mon, 20 Oct 2025 14:41:10 +0200 Subject: [PATCH 2/2] address review feedback --- cmd/node-cache/app/cache_app.go | 233 ++++++++++++-------------------- 1 file changed, 89 insertions(+), 144 deletions(-) diff --git a/cmd/node-cache/app/cache_app.go b/cmd/node-cache/app/cache_app.go index 3bd674c46..b351ea8dd 100644 --- a/cmd/node-cache/app/cache_app.go +++ b/cmd/node-cache/app/cache_app.go @@ -63,18 +63,21 @@ type iptablesRule struct { // CacheApp contains all the config required to run node-cache. type CacheApp struct { - iptablesV4 utiliptables.Interface - iptablesV6 utiliptables.Interface - iptablesRulesV4 []iptablesRule - iptablesRulesV6 []iptablesRule - params *ConfigParams - netifHandle *netif.NetifManager - kubednsConfig *options.KubeDNSConfig - exitChan chan struct{} // Channel to terminate background goroutines - clusterDNSIP net.IP - selfProcess *os.Process + iptables [2]utiliptables.Interface // 0 - ipv4, 1 - ipv6 + iptablesRules [2][]iptablesRule // 0 - ipv4, 1 - ipv6 + params *ConfigParams + netifHandle *netif.NetifManager + kubednsConfig *options.KubeDNSConfig + exitChan chan struct{} // Channel to terminate background goroutines + clusterDNSIP net.IP + selfProcess *os.Process } +const ( + ipv4 int = iota + ipv6 +) + func isLockedErr(err error) bool { return strings.Contains(err.Error(), "holding the xtables lock") } @@ -102,82 +105,54 @@ func (c *CacheApp) Init() { c.params.SetupIptables = setupIptables } +func isIPv6ToIndex(isIPv6 bool) int { + if isIPv6 { + return ipv6 + } + return ipv4 +} + func (c *CacheApp) initIptables() { // using the localIPStr param since we need ip strings here for _, localIP := range strings.Split(c.params.LocalIPStr, ",") { - if utilnet.IsIPv6(net.ParseIP(localIP)) { - c.iptablesRulesV6 = append(c.iptablesRulesV6, []iptablesRule{ - // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, - // need these additional filter table rules. - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // Additional filter table rules for traffic frpm localIp:localPort - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // skip connection tracking for healthcheck requests generated by liveness probe to health plugin - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - }...) - } else { - c.iptablesRulesV4 = append(c.iptablesRulesV4, []iptablesRule{ - // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, - // need these additional filter table rules. - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // Additional filter table rules for traffic frpm localIp:localPort - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // skip connection tracking for healthcheck requests generated by liveness probe to health plugin - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - }...) + i := isIPv6ToIndex(utilnet.IsIPv6(net.ParseIP(localIP))) + rules := []iptablesRule{ + // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, + // need these additional filter table rules. + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // Additional filter table rules for traffic frpm localIp:localPort + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // skip connection tracking for healthcheck requests generated by liveness probe to health plugin + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, } - + c.iptablesRules[i] = append(c.iptablesRules[i], rules...) } - c.iptablesV4 = utiliptables.New(utiliptables.ProtocolIPv4) - c.iptablesV6 = utiliptables.New(utiliptables.ProtocolIPv6) + c.iptables[ipv4] = utiliptables.New(utiliptables.ProtocolIPv4) + c.iptables[ipv6] = utiliptables.New(utiliptables.ProtocolIPv6) } func handleIPTablesError(err error) { @@ -204,41 +179,25 @@ func (c *CacheApp) TeardownNetworking() error { err = c.netifHandle.RemoveDummyDevice(c.params.InterfaceName) } if c.params.SetupIptables { - for _, rule := range c.iptablesRulesV4 { - exists := true - for exists == true { - // check in a loop in case the same rule got added multiple times. - err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...) - if err != nil { - clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err) - handleIPTablesError(err) - } - exists, err = c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) - if err != nil { - clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err) - handleIPTablesError(err) + for _, ipVersion := range []int{ipv4, ipv6} { + for _, rule := range c.iptablesRules[ipVersion] { + exists := true + for exists == true { + // check in a loop in case the same rule got added multiple times. + err = c.iptables[ipVersion].DeleteRule(rule.table, rule.chain, rule.args...) + if err != nil { + clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err) + handleIPTablesError(err) + } + exists, err = c.iptables[ipVersion].EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + if err != nil { + clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err) + handleIPTablesError(err) + } } + // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist + err = c.iptables[ipVersion].DeleteRule(rule.table, rule.chain, rule.args...) } - // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist - err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...) - } - for _, rule := range c.iptablesRulesV6 { - exists := true - for exists == true { - // check in a loop in case the same rule got added multiple times. - err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...) - if err != nil { - clog.Errorf("Failed deleting iptablesV6 rule %v, error - %v", rule, err) - handleIPTablesError(err) - } - exists, err = c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) - if err != nil { - clog.Errorf("Failed checking iptablesV6 rule after deletion, rule - %v, error - %v", rule, err) - handleIPTablesError(err) - } - } - // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist - err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...) } } return err @@ -246,36 +205,22 @@ func (c *CacheApp) TeardownNetworking() error { func (c *CacheApp) setupNetworking() { if c.params.SetupIptables { - for _, rule := range c.iptablesRulesV4 { - exists, err := c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) - switch { - case exists: - // debug messages can be printed by including "debug" plugin in coreFile. - clog.Debugf("iptables rule %v for nodelocaldns already exists", rule) - continue - case err == nil: - clog.Infof("Added back nodelocaldns rule - %v", rule) - continue - default: - // iptables check/rule add failed with error since control reached here. - clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err) - handleIPTablesError(err) - } - } - for _, rule := range c.iptablesRulesV6 { - exists, err := c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) - switch { - case exists: - // debug messages can be printed by including "debug" plugin in coreFile. - clog.Debugf("iptables rule %v for nodelocaldns already exists", rule) - continue - case err == nil: - clog.Infof("Added back nodelocaldns rule - %v", rule) - continue - default: - // iptables check/rule add failed with error since control reached here. - clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err) - handleIPTablesError(err) + for _, ipVersion := range []int{ipv4, ipv6} { + for _, rule := range c.iptablesRules[ipVersion] { + exists, err := c.iptables[ipVersion].EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + switch { + case exists: + // debug messages can be printed by including "debug" plugin in coreFile. + clog.Debugf("iptables rule %v for nodelocaldns already exists", rule) + continue + case err == nil: + clog.Infof("Added back nodelocaldns rule - %v", rule) + continue + default: + // iptables check/rule add failed with error since control reached here. + clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err) + handleIPTablesError(err) + } } } }