Skip to content

Add dual-stack support for node-cache #657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+51 −56
Open

Add dual-stack support for node-cache #657

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
70561f0
add dual-stack support for node-cache
DockToFuture Nov 20, 2024
12f43ea
address review feedback
DockToFuture Oct 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 51 additions & 49 deletions cmd/node-cache/app/cache_app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ import (

"github.com/coredns/coredns/coremain"
clog "github.com/coredns/coredns/plugin/pkg/log"

"k8s.io/dns/cmd/kube-dns/app/options"
"k8s.io/dns/pkg/dns/config"
"k8s.io/dns/pkg/netif"

utiliptables "k8s.io/kubernetes/pkg/util/iptables"
utilnet "k8s.io/utils/net"
)
Expand Down Expand Up @@ -63,8 +63,8 @@ type iptablesRule struct {

// CacheApp contains all the config required to run node-cache.
type CacheApp struct {
iptables utiliptables.Interface
iptablesRules []iptablesRule
iptables [2]utiliptables.Interface // 0 - ipv4, 1 - ipv6
iptablesRules [2][]iptablesRule // 0 - ipv4, 1 - ipv6
params *ConfigParams
netifHandle *netif.NetifManager
kubednsConfig *options.KubeDNSConfig
Expand All @@ -73,6 +73,11 @@ type CacheApp struct {
selfProcess *os.Process
}

const (
ipv4 int = iota
ipv6
)

func isLockedErr(err error) bool {
return strings.Contains(err.Error(), "holding the xtables lock")
}
Expand Down Expand Up @@ -100,19 +105,18 @@ func (c *CacheApp) Init() {
c.params.SetupIptables = setupIptables
}

// isIPv6 return if the node-cache is working in IPv6 mode
// LocalIPs are guaranteed to have the same family
func (c *CacheApp) isIPv6() bool {
if len(c.params.LocalIPs) > 0 {
return utilnet.IsIPv6(c.params.LocalIPs[0])
func isIPv6ToIndex(isIPv6 bool) int {
if isIPv6 {
return ipv6
}
return false
return ipv4
}

func (c *CacheApp) initIptables() {
// using the localIPStr param since we need ip strings here
for _, localIP := range strings.Split(c.params.LocalIPStr, ",") {
c.iptablesRules = append(c.iptablesRules, []iptablesRule{
i := isIPv6ToIndex(utilnet.IsIPv6(net.ParseIP(localIP)))
rules := []iptablesRule{
// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP,
"--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
Expand Down Expand Up @@ -144,17 +148,11 @@ func (c *CacheApp) initIptables() {
"--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP,
"--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}},
}...)
}
c.iptables = newIPTables(c.isIPv6())
}

func newIPTables(isIPv6 bool) utiliptables.Interface {
protocol := utiliptables.ProtocolIPv4
if isIPv6 {
protocol = utiliptables.ProtocolIPv6
}
c.iptablesRules[i] = append(c.iptablesRules[i], rules...)
}
return utiliptables.New(protocol)
c.iptables[ipv4] = utiliptables.New(utiliptables.ProtocolIPv4)
c.iptables[ipv6] = utiliptables.New(utiliptables.ProtocolIPv6)
}

func handleIPTablesError(err error) {
Expand All @@ -181,44 +179,48 @@ func (c *CacheApp) TeardownNetworking() error {
err = c.netifHandle.RemoveDummyDevice(c.params.InterfaceName)
}
if c.params.SetupIptables {
for _, rule := range c.iptablesRules {
exists := true
for exists == true {
// check in a loop in case the same rule got added multiple times.
err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
if err != nil {
clog.Errorf("Failed deleting iptables rule %v, error - %v", rule, err)
handleIPTablesError(err)
}
exists, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
if err != nil {
clog.Errorf("Failed checking iptables rule after deletion, rule - %v, error - %v", rule, err)
handleIPTablesError(err)
for _, ipVersion := range []int{ipv4, ipv6} {
for _, rule := range c.iptablesRules[ipVersion] {
exists := true
for exists == true {
// check in a loop in case the same rule got added multiple times.
err = c.iptables[ipVersion].DeleteRule(rule.table, rule.chain, rule.args...)
if err != nil {
clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err)
handleIPTablesError(err)
}
exists, err = c.iptables[ipVersion].EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
if err != nil {
clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err)
handleIPTablesError(err)
}
}
// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
err = c.iptables[ipVersion].DeleteRule(rule.table, rule.chain, rule.args...)
}
// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
}
}
return err
}

func (c *CacheApp) setupNetworking() {
if c.params.SetupIptables {
for _, rule := range c.iptablesRules {
exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
switch {
case exists:
// debug messages can be printed by including "debug" plugin in coreFile.
clog.Debugf("iptables rule %v for nodelocaldns already exists", rule)
continue
case err == nil:
clog.Infof("Added back nodelocaldns rule - %v", rule)
continue
default:
// iptables check/rule add failed with error since control reached here.
clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err)
handleIPTablesError(err)
for _, ipVersion := range []int{ipv4, ipv6} {
for _, rule := range c.iptablesRules[ipVersion] {
exists, err := c.iptables[ipVersion].EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
switch {
case exists:
// debug messages can be printed by including "debug" plugin in coreFile.
clog.Debugf("iptables rule %v for nodelocaldns already exists", rule)
continue
case err == nil:
clog.Infof("Added back nodelocaldns rule - %v", rule)
continue
default:
// iptables check/rule add failed with error since control reached here.
clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err)
handleIPTablesError(err)
}
}
}
}
Expand Down
7 changes: 0 additions & 7 deletions cmd/node-cache/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (

corednsmain "github.com/coredns/coredns/coremain"
clog "github.com/coredns/coredns/plugin/pkg/log"
utilnet "k8s.io/utils/net"

"github.com/coredns/caddy"
// blank imports to make sure the plugin code is pulled in from vendor when building node-cache image
Expand Down Expand Up @@ -101,12 +100,6 @@ func parseAndValidateFlags() (*app.ConfigParams, error) {
params.LocalIPs = append(params.LocalIPs, newIP)
}

// validate all the IPs have the same IP family
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we sure the condition validated here is not assumed anywhere else? This seems to require testing. The deleted comment above func (c *CacheApp) isIPv6() bool says

// LocalIPs are guaranteed to have the same family

for _, ip := range params.LocalIPs {
if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) {
return params, fmt.Errorf("unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0]))
}
}
// lookup specified dns port
f := flag.Lookup("dns.port")
if f == nil {
Expand Down