The main content is from here:
https://github.com/intel/sriov-network-device-plugin
Enable IOMMU
#Enable IOMMU
sudo su
vi /etc/default/grub
GRUB_DEFAULT=0
GRUB_TIMEOUT_STYLE=hidden
GRUB_TIMEOUT=0
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="maybe-ubiquity intel_iommu=on iommu=pt"
GRUB_CMDLINE_LINUX=""
Update grub and verify it
sudo update-grub
cd /boot/grub
vi grub.cfg
Enable SR-IOV PF card interfaces with DHCP
sudo vi /etc/netplan/00-installer-config.yaml
network:
ethernets:
enp4s0:
addresses:
- 140.96.27.152/24
gateway4: 140.96.27.254
nameservers:
addresses:
- 140.96.254.99
- 140.96.254.100
search:
- itri.ds
enp5s0f0:
dhcp4: yes
enp5s0f1:
dhcp4: yes
Install Golang for compiling SR-IOV's driver
In current development, Golang version in the environment is v1.14
cd /tmp
wget https://dl.google.com/go/go1.14.1.linux-amd64.tar.gz
tar xvf go1.14.1.linux-amd64.tar.gz
sudo mv go /usr/local
We can add the following content in ~/.profile or ~/.bashrc
#Golang path
export PATH=$PATH:/usr/local/go/bin
export GOPATH=$HOME/go
export PATH=$PATH:$GOPATH/bin
Then, renewing the shell sessions
source ~/.profile
Creating SR-IOV Virtual Functions
enable_vf.sh
#!/bin/bash
# -> enable VF
#
#
echo ""
SRIOV_CARD_PF1=enp5s0f0
#SRIOV_CARD_PF2=enp5s0f1
echo "PF status:"
echo "---"
sudo ethtool ${SRIOV_CARD_PF1}
echo ""
#sudo ethtool ${SRIOV_CARD_PF2}
echo "---"
echo ""
echo "VF status:"
echo "---"
if [ "$1" = "0" ]; then
# disable VF
sudo sh -c "echo 0 > /sys/class/net/${SRIOV_CARD_PF1}/device/sriov_numvfs"
#sudo sh -c "echo 0 > /sys/class/net/${SRIOV_CARD_PF2}/device/sriov_numvfs"
else
# enable VF
sudo sh -c "echo 6 > /sys/class/net/${SRIOV_CARD_PF1}/device/sriov_numvfs"
#sudo sh -c "echo 4 > /sys/class/net/${SRIOV_CARD_PF2}/device/sriov_numvfs"
fi
sudo lshw -class network -businfo
echo ""
Build SR-IOV CNI
Compile SR-IOV-CNI (supported from release 2.0+):
$ git clone https://github.com/intel/sriov-cni.git
$ cd sriov-cni
$ make -j8
$ sudo cp build/sriov /opt/cni/bin
Build and run SR-IOV network device plugin
If you want to build the docker image locally then follow the following steps:
Clone the sriov-network-device-plugin
$ git clone https://github.com/intel/sriov-network-device-plugin.git
$ cd sriov-network-device-plugin
Build docker image binary using make
$ make image
Install one compatible CNI meta plugin
Install Multus
$ git clone https://github.com/intel/multus-cni.git && cd multus-cni
$ cat ./images/multus-daemonset.yml | kubectl apply -f -
$ kubectl get pods --all-namespaces | grep -i multus
Network Object CRDs
Multus uses Custom Resource Definitions(CRDs) for defining additional network attachments. These network attachment CRDs follow the standards defined by K8s Network Plumbing Working Group(NPWG). Please refer to Multus documentation for more information.
Deploy the Device Plugin
The images directory contains example Dockerfile, sample specs along build scripts to deploy the SR-IOV device plugin as daemonset. Please see README.md for more information about the Docker images.
# Create ConfigMap
$ kubectl create -f deployments/configMap.yaml
# Create sriov-device-plugin-daemonset ( When rebooted, we need to delete and apply this yaml again )
$ kubectl create -f deployments/k8s-v1.16/sriovdp-daemonset.yaml
Create the SR-IOV Network CRD
$ cd sriov-network-device-plugin
$ kubectl create -f deployments/sriov-crd.yaml
On successful run, the allocatable resource list for the node should be updated with resources discovered by the plugin as shown below. Note that the resource name is appended with the -resource-prefix i.e. "intel.com/intel_sriov_netdevice".
$ kubectl get node node1 -o json | jq '.status.allocatable'
{
"cpu": "8",
"ephemeral-storage": "885084245370",
"hugepages-1Gi": "0",
"hugepages-2Mi": "0",
"intel.com/intel_sriov_dpdk": "0",
"intel.com/intel_sriov_netdevice": "6",
"intel.com/mlnx_sriov_rdma": "0",
"memory": "16203124Ki",
"pods": "110"
}
Check SRIOV Network in details
$ kubectl get network-attachment-definitions.k8s.cni.cncf.io --all-namespaces
NAMESPACE NAME AGE
default sriov-net1 14h
$ kubectl describe network-attachment-definitions.k8s.cni.cncf.io sriov-net1
Name: sriov-net1
Namespace: default
Labels: <none>
Annotations: k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_netdevice
API Version: k8s.cni.cncf.io/v1
Kind: NetworkAttachmentDefinition
Metadata:
Creation Timestamp: 2020-09-09T09:58:36Z
Generation: 1
Managed Fields:
API Version: k8s.cni.cncf.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.:
f:k8s.v1.cni.cncf.io/resourceName:
f:spec:
.:
f:config:
Manager: kubectl-create
Operation: Update
Time: 2020-09-09T09:58:36Z
Resource Version: 29832
Self Link: /apis/k8s.cni.cncf.io/v1/namespaces/default/network-attachment-definitions/sriov-net1
UID: b1dd915b-9d70-4a33-9aee-66afdf7551c2
Spec:
Config: { "type": "sriov", "cniVersion": "0.3.1", "name": "sriov-network", "ipam": { "type": "host-local", "subnet": "10.56.217.0/24", "routes": [{ "dst": "0.0.0.0/0" }], "gateway": "10.56.217.1" } }
Events: <none>
# Trouble shooting
$ kubectl describe pods <pod name>
$ kubectl get <pod name> -o yaml
Python Script to get SR-IOV interface's IP address
#!/usr/bin/python
from kubernetes import client, config
import json
def main():
try:
config.load_kube_config()
v1 = client.CoreV1Api()
print("Listing nodes with their IPs:")
ret = v1.list_namespaced_pod('default')
for i in ret.items:
#print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
annot = i.metadata.annotations
sriov_network_status = annot['k8s.v1.cni.cncf.io/networks-status']
d = json.loads(sriov_network_status)
for item in d:
if ('sriov' in item['name']) or ('macvlan' in item['name']):
for ip in item['ips']:
print(i.metadata.name, item['ips'])
except Exception as e:
pass
return isExt
if __name__ == '__main__':
main()
helloworld-python ['10.56.217.27']
helloworld-python-access-pod ['10.56.217.29']
testpod1 ['10.56.217.28']
Set Label "isext=true" on a Pod
$ kubectl label pods testpod1 isext=true
Python Script to get pod label of "isext"
#!/usr/bin/python
from kubernetes import client, config
import json
def get_label_isext(my_metadata):
try:
labels = my_metadata.labels
for k, v in labels.items():
if k == 'isext' and v == 'true':
return True
return False
except Exception as e:
return False
def main():
config.load_kube_config()
v1 = client.CoreV1Api()
ret = v1.list_namespaced_pod('')
for i in ret.items:
is_ext = get_label_isext(i.metadata)
print("pod:", i.metadata.name, "isExt:", is_ext)
if __name__ == '__main__':
main()
pod: helloworld-python isExt: False
pod: helloworld-python-access-pod isExt: False
pod: nfd-master-tk2g9 isExt: False
pod: nfd-worker-wdlmw isExt: False
pod: testpod1 isExt: True
