diff --git a/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf b/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf index c01c9afc79..f2974e1950 100644 --- a/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf +++ b/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf @@ -245,12 +245,12 @@ log_config : }; fhi_72 = { - dpdk_devices = ("0000:c3:11.0", "0000:c3:11.1"); + dpdk_devices = ("0000:c3:11.0"); dpdk_iova_mode = "VA"; system_core = 0; io_core = 1; worker_cores = (2); - ru_addr = ("98:ae:71:04:83:e3", "98:ae:71:04:83:e3"); + ru_addr = ("98:ae:71:04:83:e3"); mtu = 9216; fh_config = ({ T1a_cp_dl = (285, 470); diff --git a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml index d8548077fd..7dc7e49b3c 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml @@ -27,9 +27,9 @@ Custom_Script - Setup sriov and network interfaces VVDN + Configure system for low-latency RT performance cacofonix - + diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml index 2e6c044fcb..8c47dd6683 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml @@ -1,4 +1,16 @@ services: + dpdk-init: + image: docker.io/library/oai-dpdk-init:latest + network_mode: host + container_name: dpdk-init + entrypoint: "/tmp/setup_sriov.sh" + privileged: true + devices: + - /dev/vfio:/dev/vfio/ + volumes: + - ./setup_sriov_vvdn.sh:/tmp/setup_sriov.sh + - /lib/modules:/lib/modules + - /usr/lib/modules:/usr/lib/modules oai-gnb: image: ${REGISTRY-oaisoftwarealliance/}oai-gnb-fhi72:${TAG:-develop} cap_add: @@ -16,10 +28,13 @@ services: - ../../conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf:/opt/oai-gnb/etc/gnb.conf - /dev/hugepages:/dev/hugepages # Please change these values based on your system - cpuset: "0,1,2,3,4,5,6,7,8,9,10,11,12" + cpuset: "0-12" networks: oai-net: ipv4_address: 172.21.18.20 + depends_on: + dpdk-init: + condition: service_completed_successfully healthcheck: test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" start_period: 15s diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh old mode 100644 new mode 100755 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_config.sh b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_config.sh new file mode 100755 index 0000000000..8850720d5c --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_config.sh @@ -0,0 +1,4 @@ +set -e +sudo cpupower idle-set -D 0 > /dev/null +sudo sysctl kernel.sched_rt_runtime_us=-1 +sudo sysctl kernel.timer_migration=0 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh old mode 100644 new mode 100755 index 2b7a512f83..c600550518 --- a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh @@ -1,21 +1,37 @@ -set -e -sudo cpupower idle-set -D 0 > /dev/null -sudo sysctl kernel.sched_rt_runtime_us=-1 -sudo sysctl kernel.timer_migration=0 -sudo ethtool -G ens7f1 rx 8160 -sudo ethtool -G ens7f1 tx 8160 -sudo sh -c 'echo 0 > /sys/class/net/ens7f1/device/sriov_numvfs' -sudo sh -c 'echo 2 > /sys/class/net/ens7f1/device/sriov_numvfs' -sudo modprobe -r iavf -sudo modprobe iavf +#!/bin/sh +set -eu + +pci_addr() +{ + PF_IF=$1 + VF_INDEX=$2 + SYSFS_PATH="/sys/class/net/${PF_IF}/device/virtfn${VF_INDEX}" + + if [ ! -e "$SYSFS_PATH" ]; then + echo "VF $VF_INDEX not found for interface $PF_IF" + exit 1 + fi + PCI_ADDR=$(basename "$(readlink "$SYSFS_PATH")") + echo "$PCI_ADDR" +} +IF_NAME=ens7f1 +## O-DU C Plane MAC ADDR and VLAN +C_U_PLANE_MAC_ADD=76:76:64:6e:00:01 +C_U_PLANE_VLAN=4 +MTU=9000 +DPDK_DEVBIND_PREFIX=/usr/local/bin +NUM_VFs=1 +ethtool -G $IF_NAME rx 8160 tx 8160 +sh -c "echo 0 > /sys/class/net/$IF_NAME/device/sriov_numvfs" +sh -c "echo $NUM_VFs > /sys/class/net/$IF_NAME/device/sriov_numvfs" +C_U_PLANE_PCI=$(pci_addr $IF_NAME 0) # this next 2 lines is for C/U planes -sudo ip link set ens7f1 vf 0 mac 76:76:64:6e:00:01 vlan 4 spoofchk off mtu 9216 -sudo ip link set ens7f1 vf 1 mac 76:76:64:6e:00:00 vlan 4 spoofchk off mtu 9216 +ip link set $IF_NAME vf 0 mac $C_U_PLANE_MAC_ADD vlan $C_U_PLANE_VLAN spoofchk off mtu $MTU sleep 1 -# These are the DPDK bindings for C/U-planes on vlan 4 -sudo /usr/local/bin/dpdk-devbind.py --unbind c3:11.0 -sudo /usr/local/bin/dpdk-devbind.py --unbind c3:11.1 -sudo modprobe vfio-pci -sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci c3:11.0 -sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci c3:11.1 +modprobe iavf +${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --unbind $C_U_PLANE_PCI +modprobe vfio-pci +${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --bind vfio-pci $C_U_PLANE_PCI +echo "Successfully configured C-PLANE and U-PLANE: + - C-PLANE MAC: $C_U_PLANE_MAC_ADD, VLAN: $C_U_PLANE_VLAN, PCI: $C_U_PLANE_PCI" exit 0