diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index 7991c6f67c..f38d834c8b 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -1047,8 +1047,12 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { } }() + n.Lock() + config := n.config + n.Unlock() + // Remove port mappings. Do not stop endpoint delete on unmap failure - n.releasePorts(ep) + n.releasePorts(ep, config.DefaultBindingIP) // Try removal of link. Discard error: it is a best effort. // Also make sure defer does not see this error either. diff --git a/drivers/bridge/port_mapping.go b/drivers/bridge/port_mapping.go index 4dab8a0c89..b5b6d7c831 100644 --- a/drivers/bridge/port_mapping.go +++ b/drivers/bridge/port_mapping.go @@ -33,13 +33,16 @@ func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, cont b := c.GetCopy() if err := n.allocatePort(&b, containerIP, defHostIP, ulPxyEnabled); err != nil { // On allocation failure, release previously allocated ports. On cleanup error, just log a warning message - if cuErr := n.releasePortsInternal(bs); cuErr != nil { + if cuErr := n.releasePortsInternal(bs, containerIP, defHostIP); cuErr != nil { logrus.Warnf("Upon allocation failure for %v, failed to clear previously allocated port bindings: %v", b, cuErr) } return nil, err } bs = append(bs, b) } + if err := n.portMapper.SetupOutgoingRoute(defHostIP, containerIP); err != nil { + return nil, err + } return bs, nil } @@ -84,6 +87,11 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos return err } + // setup outgoing route + if err := n.portMapper.SetupOutgoingRoute(bnd.HostIP, containerIP); err != nil { + return err + } + // Save the host port (regardless it was or not specified in the binding) switch netAddr := host.(type) { case *net.TCPAddr: @@ -98,31 +106,42 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos } } -func (n *bridgeNetwork) releasePorts(ep *bridgeEndpoint) error { - return n.releasePortsInternal(ep.portMapping) +func (n *bridgeNetwork) releasePorts(ep *bridgeEndpoint, reqDefBindIP net.IP) error { + defHostIP := defaultBindingIP + + if reqDefBindIP != nil { + defHostIP = reqDefBindIP + } + + return n.releasePortsInternal(ep.portMapping, ep.addr.IP, defHostIP) } -func (n *bridgeNetwork) releasePortsInternal(bindings []types.PortBinding) error { +func (n *bridgeNetwork) releasePortsInternal(bindings []types.PortBinding, containerIP, defaultHostIP net.IP) error { var errorBuf bytes.Buffer // Attempt to release all port bindings, do not stop on failure for _, m := range bindings { - if err := n.releasePort(m); err != nil { + if err := n.releasePort(m, containerIP); err != nil { errorBuf.WriteString(fmt.Sprintf("\ncould not release %v because of %v", m, err)) } } + n.portMapper.DestroyOutgoingRoute(defaultHostIP, containerIP) + if errorBuf.Len() != 0 { return errors.New(errorBuf.String()) } return nil } -func (n *bridgeNetwork) releasePort(bnd types.PortBinding) error { +func (n *bridgeNetwork) releasePort(bnd types.PortBinding, containerIP net.IP) error { // Construct the host side transport address host, err := bnd.HostAddr() if err != nil { return err } + + n.portMapper.DestroyOutgoingRoute(bnd.HostIP, containerIP) + return n.portMapper.Unmap(host) } diff --git a/drivers/bridge/setup_ip_tables.go b/drivers/bridge/setup_ip_tables.go index 6b1c5dcb11..d49aaaf0e2 100644 --- a/drivers/bridge/setup_ip_tables.go +++ b/drivers/bridge/setup_ip_tables.go @@ -105,20 +105,11 @@ type iptRule struct { func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairpin, enable bool) error { var ( - address = addr.String() - natRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}} hpNatRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}} outRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}} inRule = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}} ) - // Set NAT. - if ipmasq { - if err := programChainRule(natRule, "NAT", enable); err != nil { - return err - } - } - // In hairpin mode, masquerade traffic from localhost if hairpin { if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable); err != nil { diff --git a/iptables/iptables.go b/iptables/iptables.go index be7725f85d..eef2246482 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -221,6 +221,32 @@ func (c *ChainInfo) Forward(action Action, ip net.IP, port int, proto, destAddr return nil } +func (c *ChainInfo) ForwardOutgoing(action Action, source net.IP, destination net.IP) error { + // SNAT or MASQUERADE + if source.IsGlobalUnicast() { + if output, err := Raw("-t", string(Nat), string(action), "POSTROUTING", + "-p", "all", + "-s", destination.String(), + "-j", "SNAT", + "--to-source", source.String()); err != nil { + return err + } else if len(output) != 0 { + return ChainError{Chain: "FORWARD", Output: output} + } + } else { + if output, err := Raw("-t", string(Nat), string(action), "POSTROUTING", + "-p", "all", + "-s", destination.String(), + "-j", "MASQUERADE"); err != nil { + return err + } else if len(output) != 0 { + return ChainError{Chain: "FORWARD", Output: output} + } + } + + return nil +} + // Link adds reciprocal ACCEPT rule for two supplied IP addresses. // Traffic is allowed from ip1 to ip2 and vice-versa func (c *ChainInfo) Link(action Action, ip1, ip2 net.IP, port int, proto string, bridgeName string) error { diff --git a/portmapper/mapper.go b/portmapper/mapper.go index d125fa8d4b..39b31cc18b 100644 --- a/portmapper/mapper.go +++ b/portmapper/mapper.go @@ -60,6 +60,16 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string) pm.bridgeName = bridgeName } +// sets correct outgoing route +func (pm *PortMapper) SetupOutgoingRoute(hostIP net.IP, containerIP net.IP) error { + return pm.forwardOutgoing(iptables.Append, hostIP, containerIP) +} + +// releases used outgoing route +func (pm *PortMapper) DestroyOutgoingRoute(hostIP net.IP, containerIP net.IP) error { + return pm.forwardOutgoing(iptables.Delete, hostIP, containerIP) +} + // Map maps the specified container transport address to the host's network address and transport port func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) { return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy) @@ -183,6 +193,7 @@ func (pm *PortMapper) Unmap(host net.Addr) error { case *net.UDPAddr: return pm.Allocator.ReleasePort(a.IP, "udp", a.Port) } + return nil } @@ -226,3 +237,11 @@ func (pm *PortMapper) forward(action iptables.Action, proto string, sourceIP net } return pm.chain.Forward(action, sourceIP, sourcePort, proto, containerIP, containerPort, pm.bridgeName) } + +func (pm *PortMapper) forwardOutgoing(action iptables.Action, sourceIP net.IP, destIP net.IP) error { + if pm.chain == nil { + return nil + } + + return pm.chain.ForwardOutgoing(action, sourceIP, destIP) +}