Show both netdevice and vfio examples

Addressing comments -
1a. added flag to `sriov.sh` to indicate bind to vfio-pci
1b. systemd unit sets up two pools 1 per mode
2a. updated `sriov-conf.yaml` to use the two pools
3a. included network examples for both resource pools
3b. included pod examples for both resource pools

Signed-off-by: Saikrishna Edupuganti <saikrishna.edupuganti@intel.com>

clr-k8s-examples/9-multi-network/Dockerfile

@@ -18,9 +18,9 @@ RUN make
 
 # Build vfioveth plugin
 FROM busybox as vfioveth
-RUN wget -O /bin/vfioveth https://raw.githubusercontent.com/clearlinux/cloud-native-setup/master/clr-k8s-examples/9-multi-network/cni/vfioveth && \
-    wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
-    chmod +x /bin/vfioveth /bin/jq
+RUN wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+COPY cni/vfioveth /bin/vfioveth
+RUN chmod +x /bin/vfioveth /bin/jq
 
 # Final image
 FROM centos/systemd

clr-k8s-examples/9-multi-network/README.md

@@ -23,7 +23,7 @@ To install and configure `multus-cni` on all nodes, along with `sriov-cni` and
 
 ```bash
 kubectl apply -f .
-kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov"
+kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov_*"
 ```
 
 ## Tests
@@ -34,7 +34,7 @@ To test if default connectivity is working
 
 ```bash
 kubectl apply -f test/pod.yaml
-kubectl exec test -- ip a        # should see one interface only
+kubectl exec test -- ip a                        # should see one interface only
 ```
 
 ### Bridge
@@ -43,8 +43,8 @@ To test multus with second interface created by `bridge` plugin
 
 ```bash
 kubectl apply -f test/bridge
-kubectl exec test-bridge -- ip a # should see two interfaces
-ip a show mynet                  # bridge created if it doesnt exist already
+kubectl exec test-bridge -- ip a                 # should see two interfaces
+ip a show mynet                                  # bridge created on host if it doesnt exist already
 ```
 
 ### SR-IOV
@@ -53,5 +53,9 @@ To test multus with second interface created by `sriov` plugin
 
 ```bash
 kubectl apply -f test/sriov
-kubectl exec test-sriov -- ip a  # second interface is a VF
+
+kubectl exec test-sriov -- ip a                  # second interface is a VF
+
+kubectl exec test-sriov-dpdk -- ip a             # veth pair with details of VF
+kubectl exec test-sriov-dpdk -- ls -l /dev/vfio
 ```

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -1,6 +1,3 @@
-# If you set "deviceType": "vfio",
-# - uncomment line `bind_vfs_vfio $pf` in systemd/sriov.sh
-# - set "type": "vfioveth" in test/sriov/0-sriov-net.yaml
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -13,10 +10,16 @@ data:
       "resourceList":
       [
         {
-          "resourceName": "sriov",
-          "rootDevices": ["07:00.0", "07:00.1"],
+          "resourceName": "sriov_netdevice",
+          "rootDevices": ["07:00.0"],
           "sriovMode": true,
           "deviceType": "netdevice"
+        },
+        {
+          "resourceName": "sriov_vfio",
+          "rootDevices": ["07:00.1"],
+          "sriovMode": true,
+          "deviceType": "vfio"
         }
       ]
     }

clr-k8s-examples/9-multi-network/systemd/sriov.service

@@ -1,9 +1,10 @@
 [Unit]
-Description=Create VFs on ens785f0 ens785f1 interfaces
+Description=Create VFs on ens785f0 (netdev) ens785f1 (vfio) interfaces
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/sriov.sh ens785f0 ens785f1
+ExecStart=/usr/bin/sriov.sh ens785f0
+ExecStart=/usr/bin/sriov.sh -b ens785f1
 
 [Install]
 WantedBy=default.target

clr-k8s-examples/9-multi-network/systemd/sriov.sh

@@ -1,10 +1,26 @@
 #!/bin/bash
-# Usage: sriov.sh ens785f0 ens785f1 ...
 
 set -o errexit
 set -o pipefail
 set -o nounset
 
+OPTIND=1
+bind="false"
+
+while getopts ":b" opt; do
+	case ${opt} in
+	b)
+		bind="true"
+		;;
+	\?)
+		echo "Usage: sriov.sh [-b] ens785f0 ens785f1 ..."
+		echo "-b	Bind to vfio-pci"
+		exit
+		;;
+	esac
+done
+shift $((OPTIND - 1))
+
 reset_pf() {
 	local pf=$1
 	echo "Resetting $pf"
@@ -20,6 +36,7 @@ set_pf() {
 }
 
 bind_vfs_vfio() {
+	if [ $bind != "true" ]; then return; fi
 	local pf=$1
 	local pci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}')
 	echo "Binding VFs of PF $pf ($pci) to vfio-pci"
@@ -45,6 +62,6 @@ setup_vfs() {
 for pf in "$@"; do
 	reset_pf $pf
 	set_pf $pf
-	# bind_vfs_vfio $pf
+	bind_vfs_vfio $pf
 	setup_vfs $pf
 done

clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml

@@ -4,7 +4,7 @@ kind: NetworkAttachmentDefinition
 metadata:
   name: sriov-net
   annotations:
-    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_netdevice
 spec:
   config: '{
     "type": "sriov",
@@ -17,4 +17,23 @@ spec:
         "gateway": "198.19.0.1"
     }
 }'
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-dpdk
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_vfio
+spec:
+  config: '{
+    "type": "vfioveth",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
 

clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Pod
 metadata:
@@ -11,4 +12,20 @@ spec:
     command: [ "top" ]
     resources:
       limits:
-        intel.com/sriov: '1'
+        intel.com/sriov_netdevice: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov-dpdk
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+