From 7dffdcd797b114a5c4906c6a17aa8d6a5865e91a Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Fri, 2 Aug 2024 14:28:41 +0200 Subject: [PATCH] CI: Add YAML files and deployment scripts for FHI7.2 pipeline Create directories for gNB deployment with VVDN and Benetel550 RUs. Each directory includes scripts for setting up and cleaning up the host machine running the gNB. --- .../sa_fhi_7.2_benetel550_gnb/README.md | 15 ++++ .../docker-compose.yml | 40 +++++++++ .../setup_cleanup.sh | 5 ++ .../setup_sriov_benetel.sh | 22 +++++ .../yaml_files/sa_fhi_7.2_vvdn_gnb/README.md | 87 +++++++++++++++++++ .../sa_fhi_7.2_vvdn_gnb/docker-compose.yml | 40 +++++++++ .../sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh | 5 ++ .../sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh | 21 +++++ 8 files changed, 235 insertions(+) create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/README.md create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_cleanup.sh create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_sriov_benetel.sh create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/README.md create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh create mode 100644 ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/README.md b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/README.md new file mode 100644 index 0000000000..c9cf235bb2 --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/README.md @@ -0,0 +1,15 @@ + + + + + +
+ + + + + + OAI O-RAN 7.2 Front-haul Docker Compose +
+ +For instructions on OAI gNB Docker deployment with FH 7.2 RU, please refer to the [README](../sa_fhi_7.2_vvdn_gnb/README.md). diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml new file mode 100644 index 0000000000..5783964bbe --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml @@ -0,0 +1,40 @@ +services: + oai-gnb: + image: oai-gnb-fhi72:latest + cap_add: + - SYS_ADMIN + - IPC_LOCK + - SYS_NICE + cap_drop: + - ALL + container_name: oai-gnb + environment: + TZ: Europe/Paris + USE_ADDITIONAL_OPTIONS: --sa --thread-pool 6,7,8,9 + devices: + - /dev/vfio:/dev/vfio/ + volumes: + - ../../conf_files/gnb.sa.band78.273prb.fhi72.4x4-benetel550.conf:/opt/oai-gnb/etc/gnb.conf + - /dev/hugepages:/dev/hugepages + # Please change these values based on your system + cpuset: "0,1,2,3,4,5,6,7,8,9" + networks: + oai-net: + ipv4_address: 172.21.18.20 + healthcheck: + test: /bin/bash -c "pgrep nr-softmodem" + interval: 10s + timeout: 5s + retries: 5 +networks: + oai-net: + driver: macvlan + name: oai-net + ipam: + config: + - subnet: "172.21.16.0/22" + ip_range: "172.21.18.20/32" + gateway: "172.21.19.254" + driver_opts: + com.docker.network.bridge.name: "oai-net" + parent: ens7f0 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_cleanup.sh b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_cleanup.sh new file mode 100644 index 0000000000..9a06336acc --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_cleanup.sh @@ -0,0 +1,5 @@ +set -e +sudo cpupower idle-set -E > /dev/null +sudo sysctl kernel.sched_rt_runtime_us=950000 +sudo sysctl kernel.timer_migration=1 +exit 0 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_sriov_benetel.sh b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_sriov_benetel.sh new file mode 100644 index 0000000000..1bcf941ae0 --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/setup_sriov_benetel.sh @@ -0,0 +1,22 @@ +set -e +sudo cpupower idle-set -D 0 > /dev/null +sudo sysctl kernel.sched_rt_runtime_us=-1 +sudo sysctl kernel.timer_migration=0 +sudo ethtool -G ens7f1 rx 8160 +sudo ethtool -G ens7f1 tx 8160 +sudo ifconfig ens7f1 mtu 9216 +sudo sh -c 'echo 0 > /sys/class/net/ens7f1/device/sriov_numvfs' +sudo sh -c 'echo 2 > /sys/class/net/ens7f1/device/sriov_numvfs' +sudo modprobe -r iavf +sudo modprobe iavf +# this next 2 lines is for C/U planes +sudo ip link set ens7f1 vf 0 mac 00:11:22:33:44:66 vlan 3 qos 0 spoofchk off mtu 9216 +sudo ip link set ens7f1 vf 1 mac 00:11:22:33:44:67 vlan 3 qos 0 spoofchk off mtu 9216 +sleep 1 +# These are the DPDK bindings for C/U-planes on vlan 3 +sudo /usr/local/bin/dpdk-devbind.py --unbind c3:11.0 +sudo /usr/local/bin/dpdk-devbind.py --unbind c3:11.1 +sudo modprobe vfio-pci +sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci c3:11.0 +sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci c3:11.1 +exit 0 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/README.md b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/README.md new file mode 100644 index 0000000000..2fcf39ad2c --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/README.md @@ -0,0 +1,87 @@ + + + + + +
+ + + + + + OAI O-RAN 7.2 Front-haul Docker Compose +
+ +![Docker deploy 7.2](../../../doc/images/docker-deploy-oai-7-2.png) + +This docker-compose is designed to use `OAI-gNB` with a 7.2 compatible Radio Unit. Before using this docker compose you have to configure +the host machine as per the [ORAN_FHI7.2_Tutorial](../../../doc/ORAN_FHI7.2_Tutorial.md). The container image used by the docker compose file is tested only on `Ubuntu 22.04` and `RHEL 9.4` docker host. + +## Build Image (Optional) + +Refer to [OAI Docker/Podman Build and Usage Procedures](../../../docker/README.md) + +## Configure Networking + +### SR-IOV Virtual Functions (VFs) + +In docker-compose environment there is no automated method +to configure the VFs on the fly. The user will have to manually configure +C/U plane VFs before starting the container `OAI-gNB`. + +You can follow the step +[configure-network-interfaces-and-dpdk-vfs](../../../doc/ORAN_FHI7.2_Tutorial.md#configure-network-interfaces-and-dpdk-vfs). + +### Interface towards AMF (N2) + +For `N2` interface we are using `macvlan` driver of docker. + +You can use the `bridge` driver, in situation + +- When the core network is running on the same machine +- or different machine but you have configured +needed `ip route` and forwarding to access the core network from RAN host. + +To configure docker `macvlan` network +you need to choose `ipam.config` and `driver_opts.parent` are per your environment + +``` + oai-net: + driver: macvlan + name: oai-net + ipam: + config: + - subnet: "172.21.16.0/22" + ip_range: "172.21.18.20/32" + gateway: "172.21.19.254" + driver_opts: + com.docker.network.bridge.name: "oai-net" + parent: enp193s0f0 +``` + +To configure `bridge` network you need to choose `ipam.config.subnet` as per your environment. + +``` + oai-net: + driver: bridge + name: oai-net + ipam: + config: + - subnet: 192.168.72.128/26 + driver_opts: + com.docker.network.bridge.name: "oai-net" +``` + +## Deploy OAI-gNB Container + +The [configuration file](../../conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf) used by docker compose is configured for VVDN RU. + +```bash +docker-compose up -d +``` + +To check the logs + +```bash +docker logs oai-gnb -f +``` diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml new file mode 100644 index 0000000000..2798a8e8fc --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml @@ -0,0 +1,40 @@ +services: + oai-gnb: + image: oai-gnb-fhi72:latest + cap_add: + - SYS_ADMIN + - IPC_LOCK + - SYS_NICE + cap_drop: + - ALL + container_name: oai-gnb + environment: + TZ: Europe/Paris + USE_ADDITIONAL_OPTIONS: --sa --thread-pool 6,7,8,9 + devices: + - /dev/vfio:/dev/vfio/ + volumes: + - ../../conf_files/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf:/opt/oai-gnb/etc/gnb.conf + - /dev/hugepages:/dev/hugepages + # Please change these values based on your system + cpuset: "0,1,2,3,4,5,6,7,8,9" + networks: + oai-net: + ipv4_address: 172.21.18.20 + healthcheck: + test: /bin/bash -c "pgrep nr-softmodem" + interval: 10s + timeout: 5s + retries: 5 +networks: + oai-net: + driver: macvlan + name: oai-net + ipam: + config: + - subnet: "172.21.16.0/22" + ip_range: "172.21.18.20/32" + gateway: "172.21.19.254" + driver_opts: + com.docker.network.bridge.name: "oai-net" + parent: ens7f0 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh new file mode 100644 index 0000000000..9a06336acc --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_cleanup.sh @@ -0,0 +1,5 @@ +set -e +sudo cpupower idle-set -E > /dev/null +sudo sysctl kernel.sched_rt_runtime_us=950000 +sudo sysctl kernel.timer_migration=1 +exit 0 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh new file mode 100644 index 0000000000..2b7a512f83 --- /dev/null +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/setup_sriov_vvdn.sh @@ -0,0 +1,21 @@ +set -e +sudo cpupower idle-set -D 0 > /dev/null +sudo sysctl kernel.sched_rt_runtime_us=-1 +sudo sysctl kernel.timer_migration=0 +sudo ethtool -G ens7f1 rx 8160 +sudo ethtool -G ens7f1 tx 8160 +sudo sh -c 'echo 0 > /sys/class/net/ens7f1/device/sriov_numvfs' +sudo sh -c 'echo 2 > /sys/class/net/ens7f1/device/sriov_numvfs' +sudo modprobe -r iavf +sudo modprobe iavf +# this next 2 lines is for C/U planes +sudo ip link set ens7f1 vf 0 mac 76:76:64:6e:00:01 vlan 4 spoofchk off mtu 9216 +sudo ip link set ens7f1 vf 1 mac 76:76:64:6e:00:00 vlan 4 spoofchk off mtu 9216 +sleep 1 +# These are the DPDK bindings for C/U-planes on vlan 4 +sudo /usr/local/bin/dpdk-devbind.py --unbind c3:11.0 +sudo /usr/local/bin/dpdk-devbind.py --unbind c3:11.1 +sudo modprobe vfio-pci +sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci c3:11.0 +sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci c3:11.1 +exit 0