From 49f007faec079546de2c37fca40e153123baddb4 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 16 Jan 2020 12:34:09 +0100 Subject: [PATCH 1/2] add support for `ips` in ipam config --- plugin/ipam/cni.go | 81 ++++++++++++++--- ...0_cni_plugin_ip_address_assignment_test.sh | 91 +++++++++++++++++++ 2 files changed, 157 insertions(+), 15 deletions(-) create mode 100755 test/880_cni_plugin_ip_address_assignment_test.sh diff --git a/plugin/ipam/cni.go b/plugin/ipam/cni.go index c0f97b5b0f..472213f60e 100644 --- a/plugin/ipam/cni.go +++ b/plugin/ipam/cni.go @@ -2,6 +2,7 @@ package ipamplugin import ( "encoding/json" + "errors" "fmt" "net" @@ -35,31 +36,80 @@ func (i *Ipam) Allocate(args *skel.CmdArgs) (types.Result, error) { if containerID == "" { return nil, fmt.Errorf("Weave CNI Allocate: blank container name") } - var ipnet *net.IPNet - if conf.Subnet == "" { - ipnet, err = i.weave.AllocateIP(containerID, false) + var ipconfigs []*current.IPConfig + + if len(conf.IPs) > 0 { + // configuration includes desired IPs + + var ips []net.IP + for _, ip := range conf.IPs { + ip4 := net.ParseIP(ip).To4() + ip16 := net.ParseIP(ip).To16() + + if ip4 == nil && ip16 == nil { + return nil, errors.New("provided value is not an IP") + } + + if ip4 == nil && ip16 != nil { + return nil, errors.New("allocation of ipv6 addresses is not implemented") + } + + ips = append(ips, ip4) + } + + for j := range ips { + ipnet := &net.IPNet{ + IP: ips[j], + Mask: ips[j].DefaultMask(), + } + + err := i.weave.ClaimIP(containerID, ipnet, false) + if err != nil { + return nil, err + } + + ipconfigs = append(ipconfigs, ¤t.IPConfig{ + Version: "4", + Address: *ipnet, + Gateway: conf.Gateway, + }) + } + } else if conf.Subnet == "" { + // configuration doesn't include Subnet or IPs, so ask the allocator for an IP + ipnet, err := i.weave.AllocateIP(containerID, false) + if err != nil { + return nil, err + } + + ipconfigs = append(ipconfigs, ¤t.IPConfig{ + Version: "4", + Address: *ipnet, + Gateway: conf.Gateway, + }) } else { - var subnet *net.IPNet - subnet, err = types.ParseCIDR(conf.Subnet) + // configuration includes desired Subnet + + subnet, err := types.ParseCIDR(conf.Subnet) if err != nil { return nil, fmt.Errorf("subnet given in config, but not parseable: %s", err) } - ipnet, err = i.weave.AllocateIPInSubnet(containerID, subnet, false) - } + ipnet, err := i.weave.AllocateIPInSubnet(containerID, subnet, false) + if err != nil { + return nil, err + } - if err != nil { - return nil, err - } - result := ¤t.Result{ - IPs: []*current.IPConfig{{ + ipconfigs = append(ipconfigs, ¤t.IPConfig{ Version: "4", Address: *ipnet, Gateway: conf.Gateway, - }}, - Routes: conf.Routes, + }) } - return result, nil + + return ¤t.Result{ + IPs: ipconfigs, + Routes: conf.Routes, + }, nil } func (i *Ipam) CmdDel(args *skel.CmdArgs) error { @@ -74,6 +124,7 @@ type ipamConf struct { Subnet string `json:"subnet,omitempty"` Gateway net.IP `json:"gateway,omitempty"` Routes []*types.Route `json:"routes"` + IPs []string `json:"ips,omitempty"` } type netConf struct { diff --git a/test/880_cni_plugin_ip_address_assignment_test.sh b/test/880_cni_plugin_ip_address_assignment_test.sh new file mode 100755 index 0000000000..1aaa3bf6ca --- /dev/null +++ b/test/880_cni_plugin_ip_address_assignment_test.sh @@ -0,0 +1,91 @@ +#! /bin/bash + +. "$(dirname "$0")/config.sh" + +start_suite "Test CNI plugin" + +cni_connect() { + pid=$(container_pid $1 $2) + id=$(docker_on $1 inspect -f '{{.Id}}' $2) + run_on $1 sudo CNI_COMMAND=ADD CNI_CONTAINERID=$id CNI_IFNAME=eth0 \ + CNI_NETNS=/proc/$pid/ns/net CNI_PATH=/opt/cni/bin /opt/cni/bin/weave-net +} + +run_on $HOST1 sudo mkdir -p /opt/cni/bin +# setup-cni is a subset of 'weave setup', without doing any 'docker pull's +weave_on $HOST1 setup-cni +weave_on $HOST1 launch + +C0=$(docker_on $HOST1 run --net=none --name=c0 --privileged -dt $SMALL_IMAGE /bin/sh) +C1=$(docker_on $HOST1 run --net=none --name=c1 --privileged -dt $SMALL_IMAGE /bin/sh) +C2=$(docker_on $HOST1 run --net=none --name=c2 --privileged -dt $SMALL_IMAGE /bin/sh) + +# Enable unsolicited ARPs so that ping after the address reuse does not time out +exec_on $HOST1 c1 sysctl -w net.ipv4.conf.all.arp_accept=1 + +cni_connect $HOST1 c0 <&1) +EXPECTED_OUTPUT=$(docker inspect --format="{{.Id}}" weave) + +assert_raises "[ $EXPECTED_OUTPUT == $ACTUAL_OUTPUT ]" + +assert "$SSH $HOST1 \"curl -s -X GET 127.0.0.1:6784/ip/$C1 | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'\"" "$C1IP" +assert "$SSH $HOST1 \"curl -s -X GET 127.0.0.1:6784/ip/$C2 | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'\"" "$C2IP" +assert "$SSH $HOST1 \"curl -s -X GET 127.0.0.1:6784/ip/$C3 | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'\"" "$C3IP" + + +end_suite From 32445eb4070f59f05b3e54523ccfdcb0fa54b6d5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 25 Jan 2021 18:50:43 +0100 Subject: [PATCH 2/2] basic test to confirm static ip allocation works --- ipam/claim.go | 3 ++ ...0_cni_plugin_ip_address_assignment_test.sh | 32 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/ipam/claim.go b/ipam/claim.go index 153eb7d1f0..6bcab4fe90 100644 --- a/ipam/claim.go +++ b/ipam/claim.go @@ -128,6 +128,9 @@ func (c *claim) Try(alloc *Allocator) bool { // but we also cannot prove otherwise, so we let it reclaim the address: alloc.debugln("Re-Claimed", c.cidr, "for ID", c.ident, "having existing ID as", existingIdent) c.sendResult(nil) + case c.ident == "weave:expose": + alloc.debugln("Ignoring weave:expose") + c.sendResult(nil) default: // Addr already owned by container on this machine c.sendResult(fmt.Errorf("address %s is already owned by %s", c.cidr.String(), existingIdent)) diff --git a/test/880_cni_plugin_ip_address_assignment_test.sh b/test/880_cni_plugin_ip_address_assignment_test.sh index 1aaa3bf6ca..86e79f461b 100755 --- a/test/880_cni_plugin_ip_address_assignment_test.sh +++ b/test/880_cni_plugin_ip_address_assignment_test.sh @@ -2,7 +2,7 @@ . "$(dirname "$0")/config.sh" -start_suite "Test CNI plugin" +start_suite "Test CNI plugin with static IP allocation" cni_connect() { pid=$(container_pid $1 $2) @@ -13,8 +13,8 @@ cni_connect() { run_on $HOST1 sudo mkdir -p /opt/cni/bin # setup-cni is a subset of 'weave setup', without doing any 'docker pull's -weave_on $HOST1 setup-cni -weave_on $HOST1 launch +weave_on $HOST1 setup-cni --log-level=debug +weave_on $HOST1 launch --log-level=debug C0=$(docker_on $HOST1 run --net=none --name=c0 --privileged -dt $SMALL_IMAGE /bin/sh) C1=$(docker_on $HOST1 run --net=none --name=c1 --privileged -dt $SMALL_IMAGE /bin/sh) @@ -70,16 +70,32 @@ C0IP=$(container_ip $HOST1 c0) C1IP=$(container_ip $HOST1 c1) C2IP=$(container_ip $HOST1 c2) -echo $C0IP -echo $C1IP -echo $C2IP +assert_raises "[ "10.32.1.30" == $C0IP ]" +assert_raises "[ "10.32.1.40" == $C1IP ]" +assert_raises "[ "10.32.1.42" == $C2IP ]" + +BRIP=$(container_ip $HOST1 weave:expose) +# Check the bridge IP is different from the container IPs +assert_raises "[ $BRIP != $C0IP ]" +assert_raises "[ $BRIP != $C1IP ]" +assert_raises "[ $BRIP != $C2IP ]" + +# Containers should be able to reach one another +assert_raises "exec_on $HOST1 c0 $PING $C1IP" +assert_raises "exec_on $HOST1 c1 $PING $C2IP" +assert_raises "exec_on $HOST1 c2 $PING $C1IP" + +# Containers should not have a default route to the world +assert_raises "exec_on $HOST1 c0 sh -c '! $PING 8.8.8.8'" +assert_raises "exec_on $HOST1 c1 sh -c '! $PING 8.8.8.8'" +assert_raises "exec_on $HOST1 c2 sh -c '! $PING 8.8.8.8'" # Ensure existing containers can reclaim their IP addresses after CNI has been used -- see #2548 stop_weave_on $HOST1 # Ensure no warning is printed to the standard error: -ACTUAL_OUTPUT=$(CHECKPOINT_DISABLE="$CHECKPOINT_DISABLE" $WEAVE launch 2>&1) -EXPECTED_OUTPUT=$(docker inspect --format="{{.Id}}" weave) +ACTUAL_OUTPUT=$(CHECKPOINT_DISABLE="$CHECKPOINT_DISABLE" DOCKER_HOST=tcp://$HOST1:$DOCKER_PORT $WEAVE launch 2>&1) +EXPECTED_OUTPUT=$($SSH $HOST1 docker inspect --format="{{.Id}}" weave) assert_raises "[ $EXPECTED_OUTPUT == $ACTUAL_OUTPUT ]"