Skip to content

Commit

Permalink
CNI to support transparent mode (#279)
Browse files Browse the repository at this point in the history
* added changes in azure cni to support transparent mode

* cni for calico policy controller

* removed unused parameter

* minor fix

* addressed review comments

* addressed review comments

* modified vethname generation and the hostbveth prefix

* removed setting arp for default gw

* minor fix
  • Loading branch information
tamilmani1989 authored and sharmasushant committed Jan 5, 2019
1 parent f816f8e commit b7f6742
Show file tree
Hide file tree
Showing 7 changed files with 201 additions and 14 deletions.
6 changes: 3 additions & 3 deletions cni/network/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
result *cniTypesCurr.Result
azIpamResult *cniTypesCurr.Result
err error
vethName string
nwCfg *cni.NetworkConfig
epInfo *network.EndpointInfo
iface *cniTypesCurr.Interface
Expand Down Expand Up @@ -388,17 +389,16 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
// Network already exists.
subnetPrefix := nwInfo.Subnets[0].Prefix.String()
log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix)
nwCfg.Ipam.Subnet = subnetPrefix

// Call into IPAM plugin to allocate an address for the endpoint.
nwCfg.Ipam.Subnet = subnetPrefix
result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg)
if err != nil {
err = plugin.Errorf("Failed to allocate address: %v", err)
return err
}

ipconfig := result.IPs[0]

iface := &cniTypesCurr.Interface{Name: args.IfName}
result.Interfaces = append(result.Interfaces, iface)

Expand Down Expand Up @@ -456,7 +456,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {

// A runtime must not call ADD twice (without a corresponding DEL) for the same
// (network name, container id, name of the interface inside the container)
vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName)
vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName)
setEndpointOptions(cnsNetworkConfig, epInfo, vethName)

// Create the endpoint.
Expand Down
11 changes: 11 additions & 0 deletions netlink/ip.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,17 @@ import (
"golang.org/x/sys/unix"
)

const (
RT_SCOPE_UNIVERSE = 0
RT_SCOPE_SITE = 200
RT_SCOPE_LINK = 253
RT_SCOPE_HOST = 254
RT_SCOPE_NOWHERE = 255
)
const (
RTPROT_KERNEL = 2
)

// GetIpAddressFamily returns the address family of an IP address.
func GetIpAddressFamily(ip net.IP) int {
if len(ip) <= net.IPv4len {
Expand Down
9 changes: 6 additions & 3 deletions network/endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,12 @@ type EndpointInfo struct {

// RouteInfo contains information about an IP route.
type RouteInfo struct {
Dst net.IPNet
Gw net.IP
DevName string
Dst net.IPNet
Src net.IP
Gw net.IP
Protocol int
DevName string
Scope int
}

// NewEndpoint creates a new endpoint in the network.
Expand Down
26 changes: 22 additions & 4 deletions network/endpoint_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,19 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
}

if vlanid != 0 {
log.Printf("OVS client")
epClient = NewOVSEndpointClient(
nw.extIf,
epInfo,
hostIfName,
contIfName,
vlanid)
} else {
} else if nw.Mode != opModeTransparent {
log.Printf("Bridge client")
epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode)
} else {
log.Printf("Transparent client")
epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode)
}

// Cleanup on failure.
Expand Down Expand Up @@ -207,8 +212,10 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error {
if ep.VlanID != 0 {
epInfo := ep.getInfo()
epClient = NewOVSEndpointClient(nw.extIf, epInfo, ep.HostIfName, "", ep.VlanID)
} else {
} else if nw.Mode != opModeTransparent {
epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode)
} else {
epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode)
}

epClient.DeleteEndpointRules(ep)
Expand Down Expand Up @@ -246,7 +253,7 @@ func addRoutes(interfaceName string, routes []RouteInfo) error {
if !strings.Contains(strings.ToLower(err.Error()), "file exists") {
return err
} else {
log.Printf("route already exists")
log.Printf("[net] route already exists")
}
}
}
Expand All @@ -259,7 +266,7 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error {
interfaceIf, _ := net.InterfaceByName(interfaceName)

for _, route := range routes {
log.Printf("[ovs] Deleting IP route %+v from link %v.", route, interfaceName)
log.Printf("[net] Deleting IP route %+v from link %v.", route, interfaceName)

if route.DevName != "" {
devIf, _ := net.InterfaceByName(route.DevName)
Expand Down Expand Up @@ -416,3 +423,14 @@ func updateRoutes(existingEp *EndpointInfo, targetEp *EndpointInfo) error {

return nil
}

func getDefaultGateway(routes []RouteInfo) net.IP {
_, defDstIP, _ := net.ParseCIDR("0.0.0.0/0")
for _, route := range routes {
if route.Dst.String() == defDstIP.String() {
return route.Gw
}
}

return nil
}
7 changes: 4 additions & 3 deletions network/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@ import (

const (
// Operational modes.
opModeBridge = "bridge"
opModeTunnel = "tunnel"
opModeDefault = opModeTunnel
opModeBridge = "bridge"
opModeTunnel = "tunnel"
opModeTransparent = "transparent"
opModeDefault = opModeTunnel
)

// ExternalInterface is a host network interface that bridges containers to external networks.
Expand Down
3 changes: 2 additions & 1 deletion network/network_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt
if opt != nil && opt[VlanIDKey] != nil {
vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string))
}

case opModeTransparent:
break
default:
return nil, errNetworkModeInvalid
}
Expand Down
153 changes: 153 additions & 0 deletions network/transparent_endpointclient_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package network

import (
"fmt"
"net"

"github.com/Azure/azure-container-networking/log"
"github.com/Azure/azure-container-networking/netlink"
"github.com/Azure/azure-container-networking/network/epcommon"
"github.com/Azure/azure-container-networking/platform"
)

const (
FAKE_GW_IP = "169.254.1.1/32"
DEFAULT_GW = "0.0.0.0/0"
)

type TransparentEndpointClient struct {
bridgeName string
hostPrimaryIfName string
hostVethName string
containerVethName string
hostPrimaryMac net.HardwareAddr
containerMac net.HardwareAddr
hostVethMac net.HardwareAddr
mode string
}

func NewTransparentEndpointClient(
extIf *externalInterface,
hostVethName string,
containerVethName string,
mode string,
) *TransparentEndpointClient {

client := &TransparentEndpointClient{
bridgeName: extIf.BridgeName,
hostPrimaryIfName: extIf.Name,
hostVethName: hostVethName,
containerVethName: containerVethName,
hostPrimaryMac: extIf.MacAddress,
mode: mode,
}

return client
}

func setArpProxy(ifName string) error {
cmd := fmt.Sprintf("echo 1 > /proc/sys/net/ipv4/conf/%v/proxy_arp", ifName)
_, err := platform.ExecuteCommand(cmd)
return err
}

func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error {
if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil {
return err
}

containerIf, err := net.InterfaceByName(client.containerVethName)
if err != nil {
return err
}

client.containerMac = containerIf.HardwareAddr

hostVethIf, err := net.InterfaceByName(client.hostVethName)
if err != nil {
return err
}

client.hostVethMac = hostVethIf.HardwareAddr

return nil
}

func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error {
var routeInfoList []RouteInfo

// ip route add <podip> dev <hostveth>
// This route is needed for incoming packets to pod to route via hostveth
for _, ipAddr := range epInfo.IPAddresses {
var routeInfo RouteInfo
ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)}
log.Printf("[net] Adding route for the ip %v", ipNet.String())
routeInfo.Dst = ipNet
routeInfoList = append(routeInfoList, routeInfo)
if err := addRoutes(client.hostVethName, routeInfoList); err != nil {
return err
}
}

log.Printf("calling setArpProxy for %v", client.hostVethName)
if err := setArpProxy(client.hostVethName); err != nil {
log.Printf("setArpProxy failed with: %v", err)
return err
}

return nil
}

func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) {
var routeInfoList []RouteInfo

// ip route del <podip> dev <hostveth>
// Deleting the route set up for routing the incoming packets to pod
for _, ipAddr := range ep.IPAddresses {
var routeInfo RouteInfo
ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)}
log.Printf("[net] Deleting route for the ip %v", ipNet.String())
routeInfo.Dst = ipNet
routeInfoList = append(routeInfoList, routeInfo)
deleteRoutes(client.hostVethName, routeInfoList)
}
}

func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error {
// Move the container interface to container's network namespace.
log.Printf("[net] Setting link %v netns %v.", client.containerVethName, epInfo.NetNsPath)
if err := netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil {
return err
}

return nil
}

func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error {
if err := epcommon.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil {
return err
}

client.containerVethName = epInfo.IfName

return nil
}

func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error {
if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil {
return err
}

return addRoutes(client.containerVethName, epInfo.Routes)
}

func (client *TransparentEndpointClient) DeleteEndpoints(ep *endpoint) error {
log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName)
err := netlink.DeleteLink(ep.HostIfName)
if err != nil {
log.Printf("[net] Failed to delete veth pair %v: %v.", ep.HostIfName, err)
return err
}

return nil
}

0 comments on commit b7f6742

Please sign in to comment.