mirror of
https://github.com/clearlinux/cloud-native-setup.git
synced 2026-09-01 19:31:46 +00:00
Enable sriov-network-device-plugin vfio mode
Adding a cni to mirror MAC address of VF on a veth pair with results from ipam applied to enable DPDK apps to configure themselves when operating sriov-network-device-plugin in vfio mode Updated helper sriov.sh to allow for binding VFs to vfio-pci on boot Signed-off-by: Saikrishna Edupuganti <saikrishna.edupuganti@intel.com>
This commit is contained in:
Saikrishna
committed by
Manohar Castelino
Manohar Castelino
parent
72659b3ba3
commit
fc99db500b
@@ -16,10 +16,18 @@ RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/sr
|
||||
WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
|
||||
RUN make
|
||||
|
||||
# Build vfioveth plugin
|
||||
FROM busybox as vfioveth
|
||||
RUN wget -O /bin/vfioveth https://raw.githubusercontent.com/clearlinux/cloud-native-setup/master/clr-k8s-examples/9-multi-network/cni/vfioveth && \
|
||||
wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
|
||||
chmod +x /bin/vfioveth /bin/jq
|
||||
|
||||
# Final image
|
||||
FROM centos/systemd
|
||||
WORKDIR /tmp/cni/bin
|
||||
COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
|
||||
COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
|
||||
COPY --from=vfioveth /bin/vfioveth .
|
||||
COPY --from=vfioveth /bin/jq .
|
||||
WORKDIR /usr/bin
|
||||
COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
#!/bin/bash -x
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
|
||||
exec 3>&1
|
||||
exec &>>/var/log/$(basename $0).log
|
||||
|
||||
PATH="$CNI_PATH:$(dirname "${BASH_SOURCE[0]}"):$PATH"
|
||||
CNI_CONF=$(cat /dev/stdin)
|
||||
|
||||
get_peer_name() {
|
||||
echo "$1-vdev"
|
||||
}
|
||||
|
||||
get_mac_with_vfpci() {
|
||||
local pf=$(readlink /sys/devices/pci*/*/$1/physfn | awk '{print substr($1,4)}')
|
||||
local pfName=$(ls /sys/devices/pci*/*/$pf/net/ | head -1)
|
||||
local idx=$(ls -l /sys/devices/pci*/*/$pf | awk -v vf=$1 'substr($11,4)==vf {print substr($9,7)}')
|
||||
local mac=$(ip link show dev $pfName | awk -v idx="$idx" '$1=="vf" && $2==idx {print substr($4,1,17)}')
|
||||
echo $mac
|
||||
}
|
||||
|
||||
ipam() {
|
||||
local plugin=$(echo $CNI_CONF | jq -r '.ipam.type')
|
||||
local res=$(echo $"$CNI_CONF" | "$plugin" | jq -c '.')
|
||||
echo $res
|
||||
}
|
||||
|
||||
add_pair_ns() {
|
||||
vfpci=$(echo $CNI_CONF | jq -r '.deviceID')
|
||||
mac=$(get_mac_with_vfpci $vfpci)
|
||||
peer=$(get_peer_name $CNI_IFNAME)
|
||||
ip=$1
|
||||
|
||||
mkdir -p /var/run/netns/
|
||||
ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID
|
||||
|
||||
ip netns exec $CNI_CONTAINERID ip link add $CNI_IFNAME type veth peer name $peer
|
||||
ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up
|
||||
ip netns exec $CNI_CONTAINERID ip link set $peer up
|
||||
ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME
|
||||
}
|
||||
|
||||
delete_pair_ns() {
|
||||
ip netns exec $CNI_CONTAINERID ip link del $CNI_IFNAME
|
||||
}
|
||||
|
||||
case $CNI_COMMAND in
|
||||
ADD)
|
||||
res=$(ipam)
|
||||
ip=$(echo $res | jq -r '.ip4.ip')
|
||||
add_pair_ns $ip
|
||||
echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
|
||||
;;
|
||||
DEL)
|
||||
set +o errexit
|
||||
ipam
|
||||
delete_pair_ns
|
||||
set -o errexit
|
||||
;;
|
||||
*)
|
||||
echo "CNI_COMMAND=[ADD|DEL] only supported"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -96,6 +96,8 @@ data:
|
||||
{
|
||||
"name": "multus-cni-network",
|
||||
"type": "multus",
|
||||
"logFile": "/var/log/multus.log",
|
||||
"logLevel": "debug",
|
||||
"kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
|
||||
"delegates": [
|
||||
$MASTER_PLUGIN_JSON
|
||||
@@ -132,10 +134,10 @@ spec:
|
||||
spec:
|
||||
initContainers:
|
||||
- name: multus
|
||||
image: ngick8stesting/aio-cni:k8s-1.13
|
||||
image: krsna1729/multus-sriov:k8s-1.13
|
||||
command: [ "bash", "-c" ]
|
||||
args:
|
||||
- cp /tmp/cni/bin/{multus,sriov} /host/opt/cni/bin/;
|
||||
- cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
|
||||
/tmp/multus/install-multus-conf.sh;
|
||||
/tmp/multus/install-certs.sh;
|
||||
systemctl stop kubelet;
|
||||
@@ -160,7 +162,7 @@ spec:
|
||||
mountPath: /run/systemd
|
||||
containers:
|
||||
- name: sriovdp
|
||||
image: ngick8stesting/aio-cni:k8s-1.13
|
||||
image: krsna1729/multus-sriov:k8s-1.13
|
||||
command: [ "sh", "-c" ]
|
||||
args:
|
||||
- /usr/bin/sriovdp --logtostderr -v 10;
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# If you set "deviceType": "vfio",
|
||||
# - uncomment line `bind_vfs_vfio $pf` in systemd/sriov.sh
|
||||
# - set "type": "vfioveth" in test/sriov/0-sriov-net.yaml
|
||||
---
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
|
||||
@@ -1,14 +1,50 @@
|
||||
#!/bin/bash
|
||||
# Usage: sriov.sh ens785f0 ens785f1 ...
|
||||
|
||||
for pf in "$@"; do
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
|
||||
reset_pf() {
|
||||
local pf=$1
|
||||
echo "Resetting $pf"
|
||||
echo 0 | tee /sys/class/net/$pf/device/sriov_numvfs
|
||||
}
|
||||
|
||||
NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
|
||||
set_pf() {
|
||||
local pf=$1
|
||||
local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
|
||||
echo "Enabling $NUM_VFS for $pf"
|
||||
echo $NUM_VFS | tee /sys/class/net/$pf/device/sriov_numvfs
|
||||
ip link set $pf up
|
||||
#for ((i = 0 ; i < ${NUM_VFS} ; i++ )); do ip link set $pf vf $i spoofchk off; done
|
||||
for ((i = 0; i < ${NUM_VFS}; i++)); do ip link set dev $pf vf $i state enable; done
|
||||
}
|
||||
|
||||
bind_vfs_vfio() {
|
||||
local pf=$1
|
||||
local pci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}')
|
||||
echo "Binding VFs of PF $pf ($pci) to vfio-pci"
|
||||
for i in $(ls -l /sys/devices/pci*/*/$pci | awk '"virtfn"==substr($9,1,6) {print substr($11,4)}'); do
|
||||
echo $i | tee /sys/bus/pci*/*/$i/driver/unbind
|
||||
echo vfio-pci | tee /sys/devices/pci*/*/$i/driver_override
|
||||
echo $i | tee /sys/bus/pci/drivers/vfio-pci/bind
|
||||
done
|
||||
}
|
||||
|
||||
setup_vfs() {
|
||||
local pf=$1
|
||||
local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
|
||||
echo "Setting up VFs of PF $pf"
|
||||
for ((i = 0; i < ${NUM_VFS}; i++)); do
|
||||
ip link set dev $pf vf $i state enable
|
||||
ip link set dev $pf vf $i mac \
|
||||
$(printf '00:80:86:%02X:%02X:%02X\n' $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)))
|
||||
# ip link set $pf vf $i spoofchk off
|
||||
done
|
||||
}
|
||||
|
||||
for pf in "$@"; do
|
||||
reset_pf $pf
|
||||
set_pf $pf
|
||||
# bind_vfs_vfio $pf
|
||||
setup_vfs $pf
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user