diff --git a/clr-k8s-examples/9-multi-network/Dockerfile b/clr-k8s-examples/9-multi-network/Dockerfile index cb9f2f4..6cc7a3c 100644 --- a/clr-k8s-examples/9-multi-network/Dockerfile +++ b/clr-k8s-examples/9-multi-network/Dockerfile @@ -18,9 +18,9 @@ RUN make # Build vfioveth plugin FROM busybox as vfioveth -RUN wget -O /bin/vfioveth https://raw.githubusercontent.com/clearlinux/cloud-native-setup/master/clr-k8s-examples/9-multi-network/cni/vfioveth && \ - wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \ - chmod +x /bin/vfioveth /bin/jq +RUN wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 +COPY cni/vfioveth /bin/vfioveth +RUN chmod +x /bin/vfioveth /bin/jq # Final image FROM centos/systemd diff --git a/clr-k8s-examples/9-multi-network/README.md b/clr-k8s-examples/9-multi-network/README.md index 909e836..4169841 100644 --- a/clr-k8s-examples/9-multi-network/README.md +++ b/clr-k8s-examples/9-multi-network/README.md @@ -23,7 +23,7 @@ To install and configure `multus-cni` on all nodes, along with `sriov-cni` and ```bash kubectl apply -f . -kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov" +kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov_*" ``` ## Tests @@ -34,7 +34,7 @@ To test if default connectivity is working ```bash kubectl apply -f test/pod.yaml -kubectl exec test -- ip a # should see one interface only +kubectl exec test -- ip a # should see one interface only ``` ### Bridge @@ -43,8 +43,8 @@ To test multus with second interface created by `bridge` plugin ```bash kubectl apply -f test/bridge -kubectl exec test-bridge -- ip a # should see two interfaces -ip a show mynet # bridge created if it doesnt exist already +kubectl exec test-bridge -- ip a # should see two interfaces +ip a show mynet # bridge created on host if it doesnt exist already ``` ### SR-IOV @@ -53,5 +53,9 @@ To test multus with second interface created by `sriov` plugin ```bash kubectl apply -f test/sriov -kubectl exec test-sriov -- ip a # second interface is a VF + +kubectl exec test-sriov -- ip a # second interface is a VF + +kubectl exec test-sriov-dpdk -- ip a # veth pair with details of VF +kubectl exec test-sriov-dpdk -- ls -l /dev/vfio ``` diff --git a/clr-k8s-examples/9-multi-network/sriov-conf.yaml b/clr-k8s-examples/9-multi-network/sriov-conf.yaml index 9f201c5..01b12aa 100644 --- a/clr-k8s-examples/9-multi-network/sriov-conf.yaml +++ b/clr-k8s-examples/9-multi-network/sriov-conf.yaml @@ -1,6 +1,3 @@ -# If you set "deviceType": "vfio", -# - uncomment line `bind_vfs_vfio $pf` in systemd/sriov.sh -# - set "type": "vfioveth" in test/sriov/0-sriov-net.yaml --- kind: ConfigMap apiVersion: v1 @@ -13,10 +10,16 @@ data: "resourceList": [ { - "resourceName": "sriov", - "rootDevices": ["07:00.0", "07:00.1"], + "resourceName": "sriov_netdevice", + "rootDevices": ["07:00.0"], "sriovMode": true, "deviceType": "netdevice" + }, + { + "resourceName": "sriov_vfio", + "rootDevices": ["07:00.1"], + "sriovMode": true, + "deviceType": "vfio" } ] } diff --git a/clr-k8s-examples/9-multi-network/systemd/sriov.service b/clr-k8s-examples/9-multi-network/systemd/sriov.service index 97ad684..b775d1e 100644 --- a/clr-k8s-examples/9-multi-network/systemd/sriov.service +++ b/clr-k8s-examples/9-multi-network/systemd/sriov.service @@ -1,9 +1,10 @@ [Unit] -Description=Create VFs on ens785f0 ens785f1 interfaces +Description=Create VFs on ens785f0 (netdev) ens785f1 (vfio) interfaces [Service] Type=oneshot -ExecStart=/usr/bin/sriov.sh ens785f0 ens785f1 +ExecStart=/usr/bin/sriov.sh ens785f0 +ExecStart=/usr/bin/sriov.sh -b ens785f1 [Install] WantedBy=default.target diff --git a/clr-k8s-examples/9-multi-network/systemd/sriov.sh b/clr-k8s-examples/9-multi-network/systemd/sriov.sh index e5cb755..3a06060 100755 --- a/clr-k8s-examples/9-multi-network/systemd/sriov.sh +++ b/clr-k8s-examples/9-multi-network/systemd/sriov.sh @@ -1,10 +1,26 @@ #!/bin/bash -# Usage: sriov.sh ens785f0 ens785f1 ... set -o errexit set -o pipefail set -o nounset +OPTIND=1 +bind="false" + +while getopts ":b" opt; do + case ${opt} in + b) + bind="true" + ;; + \?) + echo "Usage: sriov.sh [-b] ens785f0 ens785f1 ..." + echo "-b Bind to vfio-pci" + exit + ;; + esac +done +shift $((OPTIND - 1)) + reset_pf() { local pf=$1 echo "Resetting $pf" @@ -20,6 +36,7 @@ set_pf() { } bind_vfs_vfio() { + if [ $bind != "true" ]; then return; fi local pf=$1 local pci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}') echo "Binding VFs of PF $pf ($pci) to vfio-pci" @@ -45,6 +62,6 @@ setup_vfs() { for pf in "$@"; do reset_pf $pf set_pf $pf - # bind_vfs_vfio $pf + bind_vfs_vfio $pf setup_vfs $pf done diff --git a/clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml b/clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml index 0d4e8d2..244d7ad 100644 --- a/clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml +++ b/clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml @@ -4,7 +4,7 @@ kind: NetworkAttachmentDefinition metadata: name: sriov-net annotations: - k8s.v1.cni.cncf.io/resourceName: intel.com/sriov + k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_netdevice spec: config: '{ "type": "sriov", @@ -17,4 +17,23 @@ spec: "gateway": "198.19.0.1" } }' +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net-dpdk + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_vfio +spec: + config: '{ + "type": "vfioveth", + "name": "sriov-net", + "ipam": { + "type": "host-local", + "subnet": "198.19.0.0/24", + "rangeStart": "198.19.0.100", + "rangeEnd": "198.19.0.200", + "gateway": "198.19.0.1" + } +}' diff --git a/clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml b/clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml index cd8cd4b..a4404b1 100644 --- a/clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml +++ b/clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Pod metadata: @@ -11,4 +12,20 @@ spec: command: [ "top" ] resources: limits: - intel.com/sriov: '1' + intel.com/sriov_netdevice: '1' +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-sriov-dpdk + annotations: + k8s.v1.cni.cncf.io/networks: sriov-net-dpdk +spec: + containers: + - name: busy + image: busybox + command: [ "top" ] + resources: + limits: + intel.com/sriov_vfio: '1' +