diff --git a/clr-k8s-examples/9-multi-network/Dockerfile b/clr-k8s-examples/9-multi-network/Dockerfile index 3247aca..cb9f2f4 100644 --- a/clr-k8s-examples/9-multi-network/Dockerfile +++ b/clr-k8s-examples/9-multi-network/Dockerfile @@ -16,10 +16,18 @@ RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/sr WORKDIR /go/src/github.com/intel/sriov-network-device-plugin RUN make +# Build vfioveth plugin +FROM busybox as vfioveth +RUN wget -O /bin/vfioveth https://raw.githubusercontent.com/clearlinux/cloud-native-setup/master/clr-k8s-examples/9-multi-network/cni/vfioveth && \ + wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \ + chmod +x /bin/vfioveth /bin/jq + # Final image FROM centos/systemd WORKDIR /tmp/cni/bin COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus . COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov . +COPY --from=vfioveth /bin/vfioveth . +COPY --from=vfioveth /bin/jq . WORKDIR /usr/bin COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp . diff --git a/clr-k8s-examples/9-multi-network/cni/vfioveth b/clr-k8s-examples/9-multi-network/cni/vfioveth new file mode 100755 index 0000000..d9b6876 --- /dev/null +++ b/clr-k8s-examples/9-multi-network/cni/vfioveth @@ -0,0 +1,67 @@ +#!/bin/bash -x + +set -o errexit +set -o pipefail +set -o nounset + +exec 3>&1 +exec &>>/var/log/$(basename $0).log + +PATH="$CNI_PATH:$(dirname "${BASH_SOURCE[0]}"):$PATH" +CNI_CONF=$(cat /dev/stdin) + +get_peer_name() { + echo "$1-vdev" +} + +get_mac_with_vfpci() { + local pf=$(readlink /sys/devices/pci*/*/$1/physfn | awk '{print substr($1,4)}') + local pfName=$(ls /sys/devices/pci*/*/$pf/net/ | head -1) + local idx=$(ls -l /sys/devices/pci*/*/$pf | awk -v vf=$1 'substr($11,4)==vf {print substr($9,7)}') + local mac=$(ip link show dev $pfName | awk -v idx="$idx" '$1=="vf" && $2==idx {print substr($4,1,17)}') + echo $mac +} + +ipam() { + local plugin=$(echo $CNI_CONF | jq -r '.ipam.type') + local res=$(echo $"$CNI_CONF" | "$plugin" | jq -c '.') + echo $res +} + +add_pair_ns() { + vfpci=$(echo $CNI_CONF | jq -r '.deviceID') + mac=$(get_mac_with_vfpci $vfpci) + peer=$(get_peer_name $CNI_IFNAME) + ip=$1 + + mkdir -p /var/run/netns/ + ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID + + ip netns exec $CNI_CONTAINERID ip link add $CNI_IFNAME type veth peer name $peer + ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up + ip netns exec $CNI_CONTAINERID ip link set $peer up + ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME +} + +delete_pair_ns() { + ip netns exec $CNI_CONTAINERID ip link del $CNI_IFNAME +} + +case $CNI_COMMAND in +ADD) + res=$(ipam) + ip=$(echo $res | jq -r '.ip4.ip') + add_pair_ns $ip + echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3 + ;; +DEL) + set +o errexit + ipam + delete_pair_ns + set -o errexit + ;; +*) + echo "CNI_COMMAND=[ADD|DEL] only supported" + exit 1 + ;; +esac diff --git a/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml b/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml index b4fdd93..b6c1a77 100644 --- a/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml +++ b/clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml @@ -96,6 +96,8 @@ data: { "name": "multus-cni-network", "type": "multus", + "logFile": "/var/log/multus.log", + "logLevel": "debug", "kubeconfig": "/etc/cni/net.d/multus-kubeconfig", "delegates": [ $MASTER_PLUGIN_JSON @@ -132,10 +134,10 @@ spec: spec: initContainers: - name: multus - image: ngick8stesting/aio-cni:k8s-1.13 + image: krsna1729/multus-sriov:k8s-1.13 command: [ "bash", "-c" ] args: - - cp /tmp/cni/bin/{multus,sriov} /host/opt/cni/bin/; + - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/; /tmp/multus/install-multus-conf.sh; /tmp/multus/install-certs.sh; systemctl stop kubelet; @@ -160,7 +162,7 @@ spec: mountPath: /run/systemd containers: - name: sriovdp - image: ngick8stesting/aio-cni:k8s-1.13 + image: krsna1729/multus-sriov:k8s-1.13 command: [ "sh", "-c" ] args: - /usr/bin/sriovdp --logtostderr -v 10; diff --git a/clr-k8s-examples/9-multi-network/sriov-conf.yaml b/clr-k8s-examples/9-multi-network/sriov-conf.yaml index 68b853d..9f201c5 100644 --- a/clr-k8s-examples/9-multi-network/sriov-conf.yaml +++ b/clr-k8s-examples/9-multi-network/sriov-conf.yaml @@ -1,3 +1,6 @@ +# If you set "deviceType": "vfio", +# - uncomment line `bind_vfs_vfio $pf` in systemd/sriov.sh +# - set "type": "vfioveth" in test/sriov/0-sriov-net.yaml --- kind: ConfigMap apiVersion: v1 diff --git a/clr-k8s-examples/9-multi-network/systemd/sriov.sh b/clr-k8s-examples/9-multi-network/systemd/sriov.sh index 708747d..e5cb755 100755 --- a/clr-k8s-examples/9-multi-network/systemd/sriov.sh +++ b/clr-k8s-examples/9-multi-network/systemd/sriov.sh @@ -1,14 +1,50 @@ #!/bin/bash # Usage: sriov.sh ens785f0 ens785f1 ... -for pf in "$@"; do +set -o errexit +set -o pipefail +set -o nounset + +reset_pf() { + local pf=$1 echo "Resetting $pf" echo 0 | tee /sys/class/net/$pf/device/sriov_numvfs +} - NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs) +set_pf() { + local pf=$1 + local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs) echo "Enabling $NUM_VFS for $pf" echo $NUM_VFS | tee /sys/class/net/$pf/device/sriov_numvfs ip link set $pf up - #for ((i = 0 ; i < ${NUM_VFS} ; i++ )); do ip link set $pf vf $i spoofchk off; done - for ((i = 0; i < ${NUM_VFS}; i++)); do ip link set dev $pf vf $i state enable; done +} + +bind_vfs_vfio() { + local pf=$1 + local pci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}') + echo "Binding VFs of PF $pf ($pci) to vfio-pci" + for i in $(ls -l /sys/devices/pci*/*/$pci | awk '"virtfn"==substr($9,1,6) {print substr($11,4)}'); do + echo $i | tee /sys/bus/pci*/*/$i/driver/unbind + echo vfio-pci | tee /sys/devices/pci*/*/$i/driver_override + echo $i | tee /sys/bus/pci/drivers/vfio-pci/bind + done +} + +setup_vfs() { + local pf=$1 + local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs) + echo "Setting up VFs of PF $pf" + for ((i = 0; i < ${NUM_VFS}; i++)); do + ip link set dev $pf vf $i state enable + ip link set dev $pf vf $i mac \ + $(printf '00:80:86:%02X:%02X:%02X\n' $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256))) + # ip link set $pf vf $i spoofchk off + done +} + +for pf in "$@"; do + reset_pf $pf + set_pf $pf + # bind_vfs_vfio $pf + setup_vfs $pf done