From a22335a89ff32905474127b5a615385110a53cb7 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 8 Feb 2024 09:16:23 +0100 Subject: [PATCH 01/18] FH 7.2: ignore packets with no payload instead of asserting --- radio/fhi_72/oaioran.c | 110 +++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 58 deletions(-) diff --git a/radio/fhi_72/oaioran.c b/radio/fhi_72/oaioran.c index fc756e1018..aa04e16c38 100644 --- a/radio/fhi_72/oaioran.c +++ b/radio/fhi_72/oaioran.c @@ -325,76 +325,70 @@ int xran_fh_rx_read_slot(ru_info_t *ru, int *frame, int *slot) pData = p_sec_desc->pData; ptr = pData; pos = (int32_t *)(start_ptr + (4 * sym_idx * 4096)); + if (ptr == NULL || pos == NULL) + continue; - uint8_t *u8dptr; struct xran_prb_map *pRbMap = pPrbMap; - AssertFatal(ptr != NULL, "ptr NULL\n"); - AssertFatal(pos != NULL, "pos NULL\n"); - if (1) { - uint32_t idxElm = 0; - u8dptr = (uint8_t *)ptr; - int16_t payload_len = 0; - uint8_t *src = (uint8_t *)u8dptr; + uint32_t idxElm = 0; + uint8_t *src = (uint8_t *)ptr; + int16_t payload_len = 0; - LOG_D(PHY, "pRbMap->nPrbElm %d\n", pRbMap->nPrbElm); - for (idxElm = 0; idxElm < pRbMap->nPrbElm; idxElm++) { - LOG_D(PHY, - "prbMap[%d] : PRBstart %d nPRBs %d\n", - idxElm, - pRbMap->prbMap[idxElm].nRBStart, - pRbMap->prbMap[idxElm].nRBSize); - pRbElm = &pRbMap->prbMap[idxElm]; - int pos_len = 0; - int neg_len = 0; - if (pRbElm->nRBStart < (nPRBs >> 1)) // there are PRBs left of DC - neg_len = min((nPRBs * 6) - (pRbElm->nRBStart * 12), pRbElm->nRBSize * N_SC_PER_PRB); - pos_len = (pRbElm->nRBSize * N_SC_PER_PRB) - neg_len; + LOG_D(PHY, "pRbMap->nPrbElm %d\n", pRbMap->nPrbElm); + for (idxElm = 0; idxElm < pRbMap->nPrbElm; idxElm++) { + LOG_D(PHY, + "prbMap[%d] : PRBstart %d nPRBs %d\n", + idxElm, + pRbMap->prbMap[idxElm].nRBStart, + pRbMap->prbMap[idxElm].nRBSize); + pRbElm = &pRbMap->prbMap[idxElm]; + int pos_len = 0; + int neg_len = 0; - src = pData; - // Calculation of the pointer for the section in the buffer. - // positive half - uint8_t *dst1 = (uint8_t *)(pos + (neg_len == 0 ? ((pRbElm->nRBStart * N_SC_PER_PRB) - (nPRBs * 6)) : 0)); - // negative half - uint8_t *dst2 = (uint8_t *)(pos + (pRbElm->nRBStart * N_SC_PER_PRB) + fftsize - (nPRBs * 6)); - int32_t local_dst[pRbElm->nRBSize * N_SC_PER_PRB] __attribute__((aligned(64))); - if (pRbElm->compMethod == XRAN_COMPMETHOD_NONE) { - // NOTE: gcc 11 knows how to generate AVX2 for this! - for (idx = 0; idx < pRbElm->nRBSize * N_SC_PER_PRB * 2; idx++) - ((int16_t *)local_dst)[idx] = ((int16_t)ntohs(((uint16_t *)src)[idx])) >> 2; - memcpy((void *)dst2, (void *)local_dst, neg_len * 4); - memcpy((void *)dst1, (void *)&local_dst[neg_len], pos_len * 4); - } else if (pRbElm->compMethod == XRAN_COMPMETHOD_BLKFLOAT) { - struct xranlib_decompress_request bfp_decom_req; - struct xranlib_decompress_response bfp_decom_rsp; + if (pRbElm->nRBStart < (nPRBs >> 1)) // there are PRBs left of DC + neg_len = min((nPRBs * 6) - (pRbElm->nRBStart * 12), pRbElm->nRBSize * N_SC_PER_PRB); + pos_len = (pRbElm->nRBSize * N_SC_PER_PRB) - neg_len; - payload_len = (3 * pRbElm->iqWidth + 1) * pRbElm->nRBSize; + src = pData; + // Calculation of the pointer for the section in the buffer. + // positive half + uint8_t *dst1 = (uint8_t *)(pos + (neg_len == 0 ? ((pRbElm->nRBStart * N_SC_PER_PRB) - (nPRBs * 6)) : 0)); + // negative half + uint8_t *dst2 = (uint8_t *)(pos + (pRbElm->nRBStart * N_SC_PER_PRB) + fftsize - (nPRBs * 6)); + int32_t local_dst[pRbElm->nRBSize * N_SC_PER_PRB] __attribute__((aligned(64))); + if (pRbElm->compMethod == XRAN_COMPMETHOD_NONE) { + // NOTE: gcc 11 knows how to generate AVX2 for this! + for (idx = 0; idx < pRbElm->nRBSize * N_SC_PER_PRB * 2; idx++) + ((int16_t *)local_dst)[idx] = ((int16_t)ntohs(((uint16_t *)src)[idx])) >> 2; + memcpy((void *)dst2, (void *)local_dst, neg_len * 4); + memcpy((void *)dst1, (void *)&local_dst[neg_len], pos_len * 4); + } else if (pRbElm->compMethod == XRAN_COMPMETHOD_BLKFLOAT) { + struct xranlib_decompress_request bfp_decom_req; + struct xranlib_decompress_response bfp_decom_rsp; - memset(&bfp_decom_req, 0, sizeof(struct xranlib_decompress_request)); - memset(&bfp_decom_rsp, 0, sizeof(struct xranlib_decompress_response)); + payload_len = (3 * pRbElm->iqWidth + 1) * pRbElm->nRBSize; - bfp_decom_req.data_in = (int8_t *)src; - bfp_decom_req.numRBs = pRbElm->nRBSize; - bfp_decom_req.len = payload_len; - bfp_decom_req.compMethod = pRbElm->compMethod; - bfp_decom_req.iqWidth = pRbElm->iqWidth; + memset(&bfp_decom_req, 0, sizeof(struct xranlib_decompress_request)); + memset(&bfp_decom_rsp, 0, sizeof(struct xranlib_decompress_response)); - bfp_decom_rsp.data_out = (int16_t *)local_dst; - bfp_decom_rsp.len = 0; + bfp_decom_req.data_in = (int8_t *)src; + bfp_decom_req.numRBs = pRbElm->nRBSize; + bfp_decom_req.len = payload_len; + bfp_decom_req.compMethod = pRbElm->compMethod; + bfp_decom_req.iqWidth = pRbElm->iqWidth; - xranlib_decompress_avx512(&bfp_decom_req, &bfp_decom_rsp); - memcpy((void *)dst2, (void *)local_dst, neg_len * 4); - memcpy((void *)dst1, (void *)&local_dst[neg_len], pos_len * 4); - outcnt++; - } else { - printf("pRbElm->compMethod == %d is not supported\n", pRbElm->compMethod); - exit(-1); - } + bfp_decom_rsp.data_out = (int16_t *)local_dst; + bfp_decom_rsp.len = 0; + + xranlib_decompress_avx512(&bfp_decom_req, &bfp_decom_rsp); + memcpy((void *)dst2, (void *)local_dst, neg_len * 4); + memcpy((void *)dst1, (void *)&local_dst[neg_len], pos_len * 4); + outcnt++; + } else { + printf("pRbElm->compMethod == %d is not supported\n", pRbElm->compMethod); + exit(-1); } - - } else { - return 0; } } // sym_ind } // ant_ind From 970e7855c4cb26b10d765525463a5c70f5a4536b Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 8 Feb 2024 09:32:25 +0100 Subject: [PATCH 02/18] O-RAN 7.2 doc: describe how to interpret counters --- doc/ORAN_FHI7.2_Tutorial.md | 47 +++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index ccd0c868a2..86696cb3c9 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -624,7 +624,7 @@ sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci 31:06.1 ``` -# Start OAI gNB +# Start and Operation of OAI gNB ```bash cd ran_build/build @@ -632,5 +632,48 @@ cd ran_build/build sudo ./nr-softmodem -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/oran.fh.band78.fr1.273PRB.conf --sa --reorder-thread-disable 1 --thread-pool ``` -For example if you have two numa nodes (for example 18 CPU per socket) in your system and odd cores are non isolated then you can put the thread-pool on `1,3,5,7,9,11,13,15`. Else if you have 1 numa node either you can use isolated cores or non isolated. Just make sure that isolated cores are not the ones defined earlier. +For example if you have two NUMA nodes in your system (for example 18 CPUs per +socket) and odd cores are non-isolated, then you can put the thread-pool on +`1,3,5,7,9,11,13,15`. On the other hand, if you have one NUMA node, you can use +either isolated cores or non isolated cores, but make sure that isolated cores +are not the ones defined earlier for DPDK/xran. +
+Once the gNB runs, you should see counters for PDSCH/PUSCH/PRACH per +antenna port, as follows (4x2 configuration): + +``` +[NR_PHY] [o-du 0][rx 24604 pps 24520 kbps 455611][tx 126652 pps 126092 kbps 2250645][Total Msgs_Rcvd 24604] +[NR_PHY] [o_du0][pusch0 10766 prach0 1536] +[NR_PHY] [o_du0][pusch1 10766 prach1 1536] +``` + +
+ +The first line show RX/TX packet counters, i.e., packets received from the RU +(RX), and sent to the RU (TX). In the second and third line, it shows the +counters for the PUSCH and PRACH ports (2 receive antennas, so two counters +each). These numbers should be equal, otherwise it indicates that you don't +receive enough packets on either port. + +
+If you see many zeroes, then it means that OAI does not receive +packets on the fronthaul from the RU (RX is almost 0, all PUSCH/PRACH counters +are 0). + +``` +[NR_PHY] [o-du 0][rx 2 pps 0 kbps 0][tx 1020100 pps 127488 kbps 4717971][Total Msgs_Rcvd 2] +[NR_PHY] [o_du0][pusch0 0 prach0 0] +[NR_PHY] [o_du0][pusch1 0 prach1 0] +[NR_PHY] [o_du0][pusch2 0 prach2 0] +[NR_PHY] [o_du0][pusch3 0 prach3 0] +``` + +
+ +In this case, please make sure that the O-RU has been configured with the right +ethernet address of the gNB, and has been activated. You might enable port +mirroring at your switch to capture the fronthaul packets: check that you see +(1) packets at all (2) they have the right ethernet address (3) the right VLAN +tag. Although we did not test this, you might make use of the [DPDK packet +capture feature](https://doc.dpdk.org/guides/howto/packet_capture_framework.html) From fd3c50b6c8e38c823598ced3765e7eed6c296624 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 8 Feb 2024 09:36:33 +0100 Subject: [PATCH 03/18] O-RAN 7.2 doc: Correct NUMA node capitalization --- doc/ORAN_FHI7.2_Tutorial.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 86696cb3c9..2ce88924b5 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -74,8 +74,8 @@ Tested libxran releases: Your server could be: -* One NUMA Node (See [one NUMA node example](#111-one-numa-node)): all the processors are sharing a single memory system. -* Two NUMA Node (see [two NUMA node example](#112-two-numa-node)): processors are grouped in 2 memory systems. +* One NUMA node (See [one NUMA node example](#111-one-numa-node)): all the processors are sharing a single memory system. +* Two NUMA nodes (see [two NUMA nodes example](#112-two-numa-node)): processors are grouped in 2 memory systems. - Usually the even (ie `0,2,4,...`) CPUs are on the 1st socket - And the odd (ie (`1,3,5,...`) CPUs are on the 2nd socket @@ -95,13 +95,13 @@ Let summarize for example on a `32-CPU` single NUMA node system, regardless of t |OAI `nr-softmodem` |1,3,5,7,9,11,13,15| |kernel |16-31 | -In below example we have shown the output of `/proc/cmdline` for two different servers, each of them have different number of numa nodes. Be careful in isolating the CPUs in your environment. Apart from CPU allocation there are additional parameters which are important to be present in your boot command. +In below example we have shown the output of `/proc/cmdline` for two different servers, each of them have different number of NUMA nodes. Be careful in isolating the CPUs in your environment. Apart from CPU allocation there are additional parameters which are important to be present in your boot command. Modifying the `linux` command line usually requires to edit the `/etc/default/grub`, run a `grub` command and reboot the server. -### One NUMA NODE +### One NUMA Node -Below is the output of `/proc/cmdline` of a single numa node server, +Below is the output of `/proc/cmdline` of a single NUMA node server, ```bash NUMA: @@ -115,9 +115,9 @@ isolcpus=0-15 nohz_full=0-15 rcu_nocbs=0-15 kthread_cpus=16-31 rcu_nocb_poll nos Example taken for AMD EPYC 9374F 32-Core Processor -### Two numa nodes +### Two NUMA Nodes -Below is the output of `/proc/cmdline` of a two numa node server, +Below is the output of `/proc/cmdline` of a two NUMA node server, ``` NUMA: From 4085544f2883f4a6243e66444c09fa67eb485c40 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 8 Feb 2024 09:38:47 +0100 Subject: [PATCH 04/18] O-RAN 7.2 patch: don't ignore user's PKG_CONFIG_PATH --- .../oran_fhi_integration_patches/E/oaioran_E.patch | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch b/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch index ef44581240..105c301855 100644 --- a/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch +++ b/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch @@ -12,7 +12,7 @@ index 7508117..a93c085 100644 #define APP_O_DU 0 #define APP_O_RU 1 diff --git a/fhi_lib/lib/Makefile b/fhi_lib/lib/Makefile -index de141bf..5bcde3d 100644 +index de141bf..c12b912 100644 --- a/fhi_lib/lib/Makefile +++ b/fhi_lib/lib/Makefile @@ -23,11 +23,11 @@ MYCUSTOMSPACE1='------------------------------------------------------------' @@ -30,6 +30,15 @@ index de141bf..5bcde3d 100644 OBJDUMP := objdump ifeq ($(SHELL),cmd.exe) +@@ -57,7 +57,7 @@ endif + + RTE_TARGET ?= x86_64-native-linux-icc + +-RTE_INC := $(shell PKG_CONFIG_PATH=/usr/lib64/pkgconfig:$(RTE_SDK)/build/meson-uninstalled pkgconf --cflags-only-I libdpdk) ++RTE_INC := $(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH):/usr/lib64/pkgconfig:$(RTE_SDK)/build/meson-uninstalled pkgconf --cflags-only-I libdpdk) + API_DIR := $(PROJECT_DIR)/api + SRC_DIR := $(PROJECT_DIR)/src + ETH_DIR := $(PROJECT_DIR)/ethernet @@ -95,8 +95,7 @@ CPP_SRC = $(SRC_DIR)/xran_compression.cpp \ $(SRC_DIR)/xran_bfp_cplane32.cpp \ $(SRC_DIR)/xran_bfp_cplane64.cpp \ From b612e5f02172892cd9baa4c7e9b0bd9ab5a907f3 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 9 Feb 2024 11:42:45 +0100 Subject: [PATCH 05/18] O-RAN 7.2 doc: make section Configuration, DPDK before gNB config The gNB configuration depends on DPDK (e.g., PCI addresses). Therefore, put DPDK config before, to keep it logically ordered. --- doc/ORAN_FHI7.2_Tutorial.md | 113 ++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 2ce88924b5..6a810379fe 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -431,7 +431,9 @@ cmake .. -GNinja -DOAI_FHI72=ON -Dxran_LOCATION=$HOME/phy/fhi_lib/lib ninja nr-softmodem oran_fhlib_5g params_libconfig ``` -# Configure the RU +# Configuration + +## Configure the RU Contact the RU vendor to get the configuration manual, and configure the RU appropriately. You can orient on the OAI configuration files mentioned further @@ -439,61 +441,6 @@ below. We are evaluating if we can share RU configuration steps. -# Configure OAI gNB - -Sample configuration files for OAI gNB, specific to the manufacturer of the radio unit, are available at: -1. LITE-ON RU: [`gnb.sa.band78.273prb.fhi72.4x4-liteon.conf`](../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf) (band n78, 273 PRBs, 3.5GHz center freq, 4x4 antenna configuration with 9 bit I/Q samples (compressed) for PUSCH/PDSCH/PRACH, 2-layer DL MIMO, UL SISO) -2. Benetel 650 RU: [`gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf`](../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf) (band n78, 273 PRBs, 3.5GHz center freq, 4x2 antenna configuration with 9 bit I/Q samples (compressed) for PUSCH/PDSCH/PRACH, 2-layer DL MIMO, UL SISO) -3. VVDN RU: [`gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf`](../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf) (band n77, 273 PRBs, 4.0GHz center freq, 4x4 antenna configuration with 9 bit I/Q samples (compressed) for PUSCH/PDSCH/PRACH, 2-layer DL MIMO, UL SISO) - -## Adapt the OAI-gNB configuration file to your system/workspace - -Edit the sample OAI gNB configuration file and check following parameters: - -* `gNBs` section - * The PLMN section shall match the one defined in the AMF - * `amf_ip_address` shall be the correct AMF IP address in your system - * `GNB_INTERFACE_NAME_FOR_NG_AMF` and `GNB_IPV4_ADDRESS_FOR_NG_AMF` shall match your DU N2 interface name and IP address - * `GNB_INTERFACE_NAME_FOR_NGU` and `GNB_IPV4_ADDRESS_FOR_NGU` shall match your DU N3 interface name and IP address - * `prach_ConfigurationIndex` - * `prach_msg1_FrequencyStart` - * Adjust the frequency, bandwidth and SSB position - -* `L1s` section - * Set an isolated core for L1 thread `L1_rx_thread_core`, in our environment we are using CPU 8 - * `phase_compensation` should be set to 0 to disable when it is performed in the RU and set to 1 when it should be performed on the DU side - -* `RUs` section - * Set an isolated core for RU thread `ru_thread_core`, in our environment we are using CPU 6 - -* `fhi_72` (FrontHaul Interface) section: this config follows the structure - that is employed by the xRAN library (`xran_fh_init` and `xran_fh_config` - structs in the code): - * `dpdk_devices`: PCI addresses of NIC VFs binded to the DPDK - * `io_core`: absolute CPU core ID for XRAN library - * `worker_cores`: array of absolute CPU core IDs for XRAN library - * `du_addr`: DU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`, - hexadecimal numbers) - * `ru_addr`: RU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`, - hexadecimal numbers) - * `mtu`: Maximum Transmission Unit for the RU, specified by RU vendor - * `fh_config`: parameters that need to match RU parameters - * timing parameters (starting with `T`) depend on the RU: `Tadv_cp_dl` is a - single number, the rest pairs of numbers `(x, y)` specifying minimum and - maximum delays - * `ru_config`: RU-specific configuration: - * `iq_width`: Width of DL/UL IQ samples: if 16, no compression, if <16, applies - compression - * `iq_width_prach`: Width of PRACH IQ samples: if 16, no compression, if <16, applies - compression - * `fft_size`: size of FFT performed by RU, set to 12 by default - * `prach_config`: PRACH-specific configuration - * `eAxC_offset`: PRACH antenna offset - * `kbar`: the PRACH guard interval, provided in RU - -xRAN SRS reception is not supported. The eAxC offsets for DL/UL are determined -from the antenna configuration. - ## Configure Network Interfaces and DPDK VFs The 7.2 fronthaul uses the xran library, which requires DPDK. In this step, we @@ -624,6 +571,60 @@ sudo /usr/local/bin/dpdk-devbind.py --bind vfio-pci 31:06.1 ``` + +## Configure OAI gNB + +Sample configuration files for OAI gNB, specific to the manufacturer of the radio unit, are available at: +1. LITE-ON RU: [`gnb.sa.band78.273prb.fhi72.4x4-liteon.conf`](../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf) (band n78, 273 PRBs, 3.5GHz center freq, 4x4 antenna configuration with 9 bit I/Q samples (compressed) for PUSCH/PDSCH/PRACH, 2-layer DL MIMO, UL SISO) +2. Benetel 650 RU: [`gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf`](../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf) (band n78, 273 PRBs, 3.5GHz center freq, 4x2 antenna configuration with 9 bit I/Q samples (compressed) for PUSCH/PDSCH/PRACH, 2-layer DL MIMO, UL SISO) +3. VVDN RU: [`gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf`](../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf) (band n77, 273 PRBs, 4.0GHz center freq, 4x4 antenna configuration with 9 bit I/Q samples (compressed) for PUSCH/PDSCH/PRACH, 2-layer DL MIMO, UL SISO) + +Edit the sample OAI gNB configuration file and check following parameters: + +* `gNBs` section + * The PLMN section shall match the one defined in the AMF + * `amf_ip_address` shall be the correct AMF IP address in your system + * `GNB_INTERFACE_NAME_FOR_NG_AMF` and `GNB_IPV4_ADDRESS_FOR_NG_AMF` shall match your DU N2 interface name and IP address + * `GNB_INTERFACE_NAME_FOR_NGU` and `GNB_IPV4_ADDRESS_FOR_NGU` shall match your DU N3 interface name and IP address + * `prach_ConfigurationIndex` + * `prach_msg1_FrequencyStart` + * Adjust the frequency, bandwidth and SSB position + +* `L1s` section + * Set an isolated core for L1 thread `L1_rx_thread_core`, in our environment we are using CPU 8 + * `phase_compensation` should be set to 0 to disable when it is performed in the RU and set to 1 when it should be performed on the DU side + +* `RUs` section + * Set an isolated core for RU thread `ru_thread_core`, in our environment we are using CPU 6 + +* `fhi_72` (FrontHaul Interface) section: this config follows the structure + that is employed by the xRAN library (`xran_fh_init` and `xran_fh_config` + structs in the code): + * `dpdk_devices`: PCI addresses of NIC VFs binded to the DPDK + * `io_core`: absolute CPU core ID for XRAN library + * `worker_cores`: array of absolute CPU core IDs for XRAN library + * `du_addr`: DU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`, + hexadecimal numbers) + * `ru_addr`: RU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`, + hexadecimal numbers) + * `mtu`: Maximum Transmission Unit for the RU, specified by RU vendor + * `fh_config`: parameters that need to match RU parameters + * timing parameters (starting with `T`) depend on the RU: `Tadv_cp_dl` is a + single number, the rest pairs of numbers `(x, y)` specifying minimum and + maximum delays + * `ru_config`: RU-specific configuration: + * `iq_width`: Width of DL/UL IQ samples: if 16, no compression, if <16, applies + compression + * `iq_width_prach`: Width of PRACH IQ samples: if 16, no compression, if <16, applies + compression + * `fft_size`: size of FFT performed by RU, set to 12 by default + * `prach_config`: PRACH-specific configuration + * `eAxC_offset`: PRACH antenna offset + * `kbar`: the PRACH guard interval, provided in RU + +xRAN SRS reception is not supported. The eAxC offsets for DL/UL are determined +from the antenna configuration. + # Start and Operation of OAI gNB ```bash From 702c608ca1f0bafde7f6e432be36565bde9d2628 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 9 Feb 2024 11:49:47 +0100 Subject: [PATCH 06/18] O-RAN 7.2 doc: explain layer mapping, streamline start gNB section --- doc/ORAN_FHI7.2_Tutorial.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 6a810379fe..77e3ae69f3 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -622,17 +622,25 @@ Edit the sample OAI gNB configuration file and check following parameters: * `eAxC_offset`: PRACH antenna offset * `kbar`: the PRACH guard interval, provided in RU -xRAN SRS reception is not supported. The eAxC offsets for DL/UL are determined -from the antenna configuration. +Layer mapping (eAxC offsets) happens as follows: +- For PUSCH/PDSCH, the layers are mapped to `[0,1,...,N-1]` where `N` is the + respective RX/TX number of antennas. +- For PRACH, the layers are mapped to `[No,No+1,...No+N-1]` where No is the + `fhi_72.fh_config.[0].prach_config.eAxC_offset` and `N` the number of receive + antennas. + +xRAN SRS reception is not supported. # Start and Operation of OAI gNB +Run the `nr-softmodem` from the build directory: ```bash -cd ran_build/build - +cd ~/openairinterface5g/ran_build/build sudo ./nr-softmodem -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/oran.fh.band78.fr1.273PRB.conf --sa --reorder-thread-disable 1 --thread-pool ``` +You have to set the thread pool option to non-isolated CPUs, since the thread +pool is used for L1 processing which should not interfere with DPDK threads. For example if you have two NUMA nodes in your system (for example 18 CPUs per socket) and odd cores are non-isolated, then you can put the thread-pool on `1,3,5,7,9,11,13,15`. On the other hand, if you have one NUMA node, you can use From 4f7464d0108504f99a71f4de13272cb1a6161598 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 9 Feb 2024 12:14:34 +0100 Subject: [PATCH 07/18] Print additional information if TBS is too small for SIB1 --- openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_bch.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_bch.c b/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_bch.c index 624ee6d243..6f40b0ba52 100644 --- a/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_bch.c +++ b/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_bch.c @@ -358,8 +358,20 @@ static uint32_t schedule_control_sib1(module_id_t module_id, rbSize, tda_info->nrOfSymbols, N_PRB_DMRS * dmrs_length,0, 0,1) >> 3; } while (TBS < gNB_mac->sched_ctrlCommon->num_total_bytes); - AssertFatal(TBS>=gNB_mac->sched_ctrlCommon->num_total_bytes,"Couldn't allocate enough resources for %d bytes in SIB1 PDSCH\n", - gNB_mac->sched_ctrlCommon->num_total_bytes); + if (TBS < gNB_mac->sched_ctrlCommon->num_total_bytes) { + for (int rb = 0; rb < bwpSize; rb++) + LOG_I(NR_MAC, "vrb_map[%d] %x\n", rbStart + rb, vrb_map[rbStart + rb]); + } + AssertFatal( + TBS >= gNB_mac->sched_ctrlCommon->num_total_bytes, + "Couldn't allocate enough resources for %d bytes in SIB1 PDSCH (rbStart %d, rbSize %d, bwpSize %d SLmask %x - [%d,%d])\n", + gNB_mac->sched_ctrlCommon->num_total_bytes, + rbStart, + rbSize, + bwpSize, + SL_to_bitmap(tda_info->startSymbolIndex, tda_info->nrOfSymbols), + tda_info->startSymbolIndex, + tda_info->nrOfSymbols); pdsch->rbSize = rbSize; pdsch->rbStart = 0; From 8dd12ba2bfd6f42f081755ed610cf51c552b11dc Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 9 Feb 2024 12:16:25 +0100 Subject: [PATCH 08/18] O-RAN 7.2: use configured values instead of hardcoded Use configured - FFT size - numerology - slot number Necessary for FR2 operation and different FFT sizes (as opposed to FR1/4096). --- radio/fhi_72/oaioran.c | 31 +++++++++++++++++++------------ radio/fhi_72/oran_isolate.c | 5 +++-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/radio/fhi_72/oaioran.c b/radio/fhi_72/oaioran.c index aa04e16c38..247293c18f 100644 --- a/radio/fhi_72/oaioran.c +++ b/radio/fhi_72/oaioran.c @@ -81,7 +81,11 @@ void oai_xran_fh_rx_callback(void *pCallbackTag, xran_status_t status) struct xran_device_ctx *xran_ctx = xran_dev_get_ctx(); const struct xran_fh_init *fh_init = &xran_ctx->fh_init; int num_ports = fh_init->xran_ports; - static int rx_RU[XRAN_PORTS_NUM][20] = {0}; + + const struct xran_fh_config *fh_config = &xran_ctx->fh_cfg; + const int slots_per_subframe = 1 << fh_config->frame_conf.nNumerology; + + static int rx_RU[XRAN_PORTS_NUM][160] = {0}; uint32_t rx_tti = callback_tag->slotiId; tti = xran_get_slot_idx_from_tti(rx_tti, &frame, &subframe, &slot, &second); @@ -95,7 +99,7 @@ void oai_xran_fh_rx_callback(void *pCallbackTag, xran_status_t status) } first_rx_set = 1; if (first_read_set == 1) { - slot2 = slot + (subframe << 1); + slot2 = slot + (subframe * slots_per_subframe); rx_RU[ru_id][slot2] = 1; if (last_frame > 0 && frame > 0 && ((slot2 > 0 && last_frame != frame) || (slot2 == 0 && last_frame != ((1024 + frame - 1) & 1023)))) @@ -164,14 +168,17 @@ int oai_physide_ul_full_slot_call_back(void *param) int read_prach_data(ru_info_t *ru, int frame, int slot) { /* calculate tti and subframe_id from frame, slot num */ - int tti = 20 * (frame) + (slot); - uint32_t subframe = XranGetSubFrameNum(tti, 2, 10); - uint32_t is_prach_slot = xran_is_prach_slot(0, subframe, (slot % 2)); int sym_idx = 0; struct xran_device_ctx *xran_ctx = xran_dev_get_ctx(); struct xran_prach_cp_config *pPrachCPConfig = &(xran_ctx->PrachCPConfig); struct xran_ru_config *ru_conf = &(xran_ctx->fh_cfg.ru_conf); + int slots_per_frame = 10 << xran_ctx->fh_cfg.frame_conf.nNumerology; + int slots_per_subframe = 1 << xran_ctx->fh_cfg.frame_conf.nNumerology; + + int tti = slots_per_frame * (frame) + (slot); + uint32_t subframe = slot / slots_per_subframe; + uint32_t is_prach_slot = xran_is_prach_slot(0, subframe, (slot % slots_per_subframe)); int nb_rx_per_ru = ru->nb_rx / xran_ctx->fh_init.xran_ports; /* If it is PRACH slot, copy prach IQ from XRAN PRACH buffer to OAI PRACH buffer */ @@ -275,17 +282,19 @@ int xran_fh_rx_read_slot(ru_info_t *ru, int *frame, int *slot) #endif // return(0); - int tti = (*frame * 20) + *slot; + struct xran_device_ctx *xran_ctx = xran_dev_get_ctx(); + int slots_per_frame = 10 << xran_ctx->fh_cfg.frame_conf.nNumerology; + + int tti = slots_per_frame * (*frame) + (*slot); read_prach_data(ru, *frame, *slot); - struct xran_device_ctx *xran_ctx = xran_dev_get_ctx(); const struct xran_fh_init *fh_init = &xran_ctx->fh_init; int nPRBs = xran_ctx->fh_cfg.nULRBs; int fftsize = 1 << xran_ctx->fh_cfg.ru_conf.fftSize; int slot_offset_rxdata = 3 & (*slot); - uint32_t slot_size = 4 * 14 * 4096; + uint32_t slot_size = 4 * 14 * fftsize; uint8_t *rx_data = (uint8_t *)ru->rxdataF[0]; uint8_t *start_ptr = NULL; int nb_rx_per_ru = ru->nb_rx / fh_init->xran_ports; @@ -324,7 +333,7 @@ int xran_fh_rx_read_slot(ru_info_t *ru, int *frame, int *slot) else pData = p_sec_desc->pData; ptr = pData; - pos = (int32_t *)(start_ptr + (4 * sym_idx * 4096)); + pos = (int32_t *)(start_ptr + (4 * sym_idx * fftsize)); if (ptr == NULL || pos == NULL) continue; @@ -451,9 +460,7 @@ int xran_fh_tx_send_slot(ru_info_t *ru, int frame, int slot, uint64_t timestamp) .sBufferList.pBuffers->pData; struct xran_prb_map *pPrbMap = (struct xran_prb_map *)pPrbMapData; ptr = pData; - pos = - &ru->txdataF_BF[ant_id][sym_idx * 4096 /*fp->ofdm_symbol_size*/]; // We had to use a different ru structure than benetel - // so the access to the buffer is not the same. + pos = &ru->txdataF_BF[ant_id][sym_idx * fftsize]; uint8_t *u8dptr; struct xran_prb_map *pRbMap = pPrbMap; diff --git a/radio/fhi_72/oran_isolate.c b/radio/fhi_72/oran_isolate.c index 9624217c5d..bdc786f482 100644 --- a/radio/fhi_72/oran_isolate.c +++ b/radio/fhi_72/oran_isolate.c @@ -209,10 +209,11 @@ void oran_fh_if4p5_south_in(RU_t *ru, int *frame, int *slot) printf("ORAN: %d.%d ORAN_fh_if4p5_south_in ERROR in RX function \n", f, sl); } + int slots_per_frame = 10 << (ru->openair0_cfg.nr_scs_for_raster); proc->tti_rx = sl; proc->frame_rx = f; - proc->tti_tx = (sl + sl_ahead) % 20; - proc->frame_tx = (sl > (19 - sl_ahead)) ? (f + 1) & 1023 : f; + proc->tti_tx = (sl + sl_ahead) % slots_per_frame; + proc->frame_tx = (sl > (slots_per_frame - 1 - sl_ahead)) ? (f + 1) & 1023 : f; if (proc->first_rx == 0) { if (proc->tti_rx != *slot) { From 0fb0a9a48ed2848a43980437061cd5b4e7e76941 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 9 Feb 2024 12:20:37 +0100 Subject: [PATCH 09/18] O-RAN 7.2: Set PRACH filter index correctly for FR2 --- radio/fhi_72/oran-config.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radio/fhi_72/oran-config.c b/radio/fhi_72/oran-config.c index 3c201732c9..50a44c221c 100644 --- a/radio/fhi_72/oran-config.c +++ b/radio/fhi_72/oran-config.c @@ -684,9 +684,6 @@ static bool set_fh_prach_config(const openair0_config_t *oai0, { const split7_config_t *s7cfg = &oai0->split7; - // for FR2, need at least to update nPrachFilterIdx - AssertFatal(oai0->nr_band < 100, "can only handle FR1!\n"); - prach_config->nPrachConfIdx = s7cfg->prach_index; prach_config->nPrachSubcSpacing = oai0->nr_scs_for_raster; prach_config->nPrachZeroCorrConf = 0; @@ -694,7 +691,10 @@ static bool set_fh_prach_config(const openair0_config_t *oai0, prach_config->nPrachRootSeqIdx = 0; prach_config->nPrachFreqStart = s7cfg->prach_freq_start; prach_config->nPrachFreqOffset = (s7cfg->prach_freq_start * 12 - oai0->num_rb_dl * 6) * 2; - prach_config->nPrachFilterIdx = get_prach_filterindex_fr1(oai0->duplex_mode, s7cfg->prach_index); + if (oai0->nr_band < 100) + prach_config->nPrachFilterIdx = get_prach_filterindex_fr1(oai0->duplex_mode, s7cfg->prach_index); + else + prach_config->nPrachFilterIdx = XRAN_FILTERINDEX_PRACH_ABC; prach_config->startSymId = 0; prach_config->lastSymId = 0; prach_config->startPrbc = 0; From b32b9e836f9ecbbfc6e3dc892433f7e975dd0b7b Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 9 Feb 2024 12:42:19 +0100 Subject: [PATCH 10/18] O-RAN 7.2: make DPDK system core configurable system_core is the core reserved for DPDK control threads. It was hardcoded to 0, but it seems that one some systems, CPU 0 cannot be reasonably isolated from kernel threads. Make it therefore configurable (default: 0) Closes: #732 (suggestion 1) Suggested-by: Andrew Sergeev --- doc/ORAN_FHI7.2_Tutorial.md | 2 ++ radio/fhi_72/oran-config.c | 2 +- radio/fhi_72/oran-params.h | 2 ++ .../CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf | 1 + .../CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf | 1 + .../CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 77e3ae69f3..175123ea62 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -601,6 +601,8 @@ Edit the sample OAI gNB configuration file and check following parameters: that is employed by the xRAN library (`xran_fh_init` and `xran_fh_config` structs in the code): * `dpdk_devices`: PCI addresses of NIC VFs binded to the DPDK + * `system_core`: absolute CPU core ID for DPDK control threads + (`rte_mp_handle`, `eal-intr-thread`, `iavf-event-thread`) * `io_core`: absolute CPU core ID for XRAN library * `worker_cores`: array of absolute CPU core IDs for XRAN library * `du_addr`: DU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`, diff --git a/radio/fhi_72/oran-config.c b/radio/fhi_72/oran-config.c index 50a44c221c..ec7eabc459 100644 --- a/radio/fhi_72/oran-config.c +++ b/radio/fhi_72/oran-config.c @@ -500,7 +500,7 @@ static bool set_fh_io_cfg(struct xran_io_cfg *io_cfg, const paramdef_t *fhip, in io_cfg->dpdkIoVaMode = 0; /* IOVA mode */ io_cfg->dpdkMemorySize = 0; /* DPDK memory size */ io_cfg->core = *gpd(fhip, nump, ORAN_CONFIG_IO_CORE)->iptr; - io_cfg->system_core = 0; /* TODO how called in sample app? */ + io_cfg->system_core = *gpd(fhip, nump, ORAN_CONFIG_SYSTEM_CORE)->iptr; io_cfg->pkt_proc_core = get_u64_mask(gpd(fhip, nump, ORAN_CONFIG_WORKER_CORES)); io_cfg->pkt_proc_core_64_127 = 0x0; // bitmap 0 -> no core io_cfg->pkt_aux_core = 0; /* sapmle app says 0 = "do not start" */ diff --git a/radio/fhi_72/oran-params.h b/radio/fhi_72/oran-params.h index 55157ec94e..a3fb22267b 100644 --- a/radio/fhi_72/oran-params.h +++ b/radio/fhi_72/oran-params.h @@ -28,6 +28,7 @@ #define CONFIG_STRING_ORAN "fhi_72" #define ORAN_CONFIG_DPDK_DEVICES "dpdk_devices" +#define ORAN_CONFIG_SYSTEM_CORE "system_core" #define ORAN_CONFIG_IO_CORE "io_core" #define ORAN_CONFIG_WORKER_CORES "worker_cores" #define ORAN_CONFIG_DU_ADDR "du_addr" @@ -42,6 +43,7 @@ // TODO: ethernet addr check #define ORAN_GLOBALPARAMS_DESC { \ {ORAN_CONFIG_DPDK_DEVICES, "PCI addr of devices for DPDK\n", PARAMFLAG_MANDATORY, .strlistptr=NULL, .defstrlistval=NULL, TYPE_STRINGLIST, 0}, \ + {ORAN_CONFIG_SYSTEM_CORE, "DPDK control threads core\n", PARAMFLAG_MANDATORY, .iptr=NULL, .defintval=0, TYPE_INT, 0}, \ {ORAN_CONFIG_IO_CORE, "DPDK Core used for IO\n", PARAMFLAG_MANDATORY, .iptr=NULL, .defintval=4, TYPE_INT, 0}, \ {ORAN_CONFIG_WORKER_CORES, "CPU Cores to use for workers\n", PARAMFLAG_MANDATORY, .uptr=NULL, .defintarrayval=NULL,TYPE_UINTARRAY, 0}, \ {ORAN_CONFIG_DU_ADDR, "Ether addr of DU\n", PARAMFLAG_MANDATORY, .strlistptr=NULL, .defstrlistval=NULL, TYPE_STRINGLIST, 0}, \ diff --git a/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf b/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf index 1e377e5664..567637fb17 100644 --- a/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf +++ b/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band77.273prb.fhi72.4x4-vvdn.conf @@ -284,6 +284,7 @@ log_config : { fhi_72 = { dpdk_devices = ("0000:31:06.0", "0000:31:06.1"); + system_core = 0; io_core = 4; worker_cores = (2); du_addr = ("76:76:64:6e:00:01", "76:76:64:6e:00:01"); diff --git a/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf b/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf index c362f05ac1..69625e0c40 100644 --- a/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf +++ b/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x2-benetel650.conf @@ -272,6 +272,7 @@ log_config : fhi_72 = { dpdk_devices = ("0000:31:06.0", "0000:31:06.1"); + system_core = 0; io_core = 4; worker_cores = (2); du_addr = ("00:11:22:33:44:66", "00:11:22:33:44:67"); diff --git a/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf b/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf index 5de3d3a4b5..59d85536a7 100644 --- a/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf +++ b/targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.273prb.fhi72.4x4-liteon.conf @@ -275,6 +275,7 @@ log_config : fhi_72 = { dpdk_devices = ("0000:31:06.0", "0000:31:06.1"); + system_core = 0; io_core = 4; worker_cores = (2); du_addr = ("00:11:22:33:44:99", "00:11:22:33:44:99"); From b5ba3a69f1dbf6acf4ce5d7947e235d2bfd031f3 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 9 Feb 2024 13:02:50 +0100 Subject: [PATCH 11/18] O-RAN 7.2: set CPU affinity for ru_thread explicitly Force the CPU affinity for ru_thread explicitly to ru_thread_core. It is already done via threadCreate(), but force again as per change request. Delete commented 7.2 code that did the equivalent. We cannot set it in the driver, as we don't have access to variable ru->ru_thread_core. Closes: #732 (suggestion 2) Suggested-by: Andrew Sergeev --- executables/nr-ru.c | 12 ++++++++++++ radio/fhi_72/oran-init.c | 31 ------------------------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/executables/nr-ru.c b/executables/nr-ru.c index c61088dc88..3e4ebec5cc 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -1204,6 +1204,18 @@ void *ru_thread( void *param ) { malloc_IF4p5_buffer(ru); } + int cpu = sched_getcpu(); + if (ru->ru_thread_core > -1 && cpu != ru->ru_thread_core) { + /* we start the ru_thread using threadCreate(), which already sets CPU + * affinity; let's force it here again as per feature request #732 */ + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(ru->ru_thread_core, &cpuset); + int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + AssertFatal(ret == 0, "Error in pthread_getaffinity_np(): ret: %d, errno: %d", ret, errno); + LOG_I(PHY, "RU %d: manually set CPU affinity to CPU %d\n", ru->idx, ru->ru_thread_core); + } + LOG_I(PHY,"Starting IF interface for RU %d, nb_rx %d\n",ru->idx,ru->nb_rx); AssertFatal(ru->nr_start_if(ru,NULL) == 0, "Could not start the IF device\n"); diff --git a/radio/fhi_72/oran-init.c b/radio/fhi_72/oran-init.c index 6586b2cda7..f05ae260c5 100644 --- a/radio/fhi_72/oran-init.c +++ b/radio/fhi_72/oran-init.c @@ -78,32 +78,6 @@ static uint32_t get_nFpgaToSW_FTH_RxBufferLen(int mu) } } -/* is this necessary? -int32_t set_main_core(uint32_t main_core) -{ - struct sched_param sched_param; - cpu_set_t cpuset; - int32_t result = 0; - memset(&sched_param, 0, sizeof(struct sched_param)); - // set main thread affinity mask to CPU2 - sched_param.sched_priority = 99; - CPU_ZERO(&cpuset); - - printf("This system has %d processors configured and %d processors available.\n", get_nprocs_conf(), get_nprocs()); - - if (main_core < get_nprocs_conf()) - CPU_SET(main_core, &cpuset); - else - return -1; - - if ((result = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset))) { - printf("pthread_setaffinity_np failed: coreId = 2, result = %d\n", result); - } - printf("%s [CPU %2d] [PID: %6d]\n", __FUNCTION__, sched_getcpu(), getpid()); - return result; -} -*/ - static struct xran_prb_map get_xran_prb_map_dl(const struct xran_fh_config *f) { struct xran_prb_map prbmap = { @@ -432,11 +406,6 @@ int *oai_oran_initialize(const openair0_config_t *openair0_cfg) print_fh_config(&xran_fh_config[o_xu_id]); } - // if ((xret = set_main_core(init.something)) < 0) { - // printf("set_main_core() failed %d\n", xret); - // exit(-1); - // } - xret = xran_init(0, NULL, &init, NULL, &gxran_handle); if (xret != XRAN_STATUS_SUCCESS) { printf("xran_init failed %d\n", xret); From c5211b0f8c2c9ac061c4178eebf98abadae78115 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 9 Feb 2024 13:08:41 +0100 Subject: [PATCH 12/18] O-RAN 7.2: modify patch to switch of DPDK telemetry, bump version Disable DPDK telemetry to skip creation of one/two threads (depending on DPDK version). To force users to do this, bump the version of the patch checked by cmake. More information on DPDK telemetry: https://doc.dpdk.org/guides/howto/telemetry.html Closes: #732 (suggestion 2) Suggested-by: Andrew Sergeev --- cmake_targets/tools/MODULES/Findxran.cmake | 2 ++ .../E/oaioran_E.patch | 18 +++++++++++++----- radio/fhi_72/CMakeLists.txt | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmake_targets/tools/MODULES/Findxran.cmake b/cmake_targets/tools/MODULES/Findxran.cmake index 1405fd6053..810206c32c 100644 --- a/cmake_targets/tools/MODULES/Findxran.cmake +++ b/cmake_targets/tools/MODULES/Findxran.cmake @@ -107,6 +107,8 @@ if (xran_VERSION_STRING STREQUAL "oran_e_maintenance_release_v1.0") set(xran_VERSION 5.1.0) elseif (xran_VERSION_STRING STREQUAL "oran_e_maintenance_release_v1.1") set(xran_VERSION 5.1.1) +elseif (xran_VERSION_STRING STREQUAL "oran_e_maintenance_release_v1.2") + set(xran_VERSION 5.1.2) endif() unset(xran_VERSION_LINE) unset(xran_VERSION_STRING) diff --git a/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch b/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch index 105c301855..6252623542 100644 --- a/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch +++ b/cmake_targets/tools/oran_fhi_integration_patches/E/oaioran_E.patch @@ -1,5 +1,5 @@ diff --git a/fhi_lib/app/src/common.h b/fhi_lib/app/src/common.h -index 7508117..a93c085 100644 +index 7508117..0dbe0bd 100644 --- a/fhi_lib/app/src/common.h +++ b/fhi_lib/app/src/common.h @@ -28,7 +28,7 @@ @@ -7,7 +7,7 @@ index 7508117..a93c085 100644 #include -#define VERSIONX "oran_e_maintenance_release_v1.0" -+#define VERSIONX "oran_e_maintenance_release_v1.1" ++#define VERSIONX "oran_e_maintenance_release_v1.2" #define APP_O_DU 0 #define APP_O_RU 1 @@ -126,10 +126,18 @@ index 7d3afc5..3e00c5a 100644 uint16_t *num_prbu, uint16_t *start_prbu, diff --git a/fhi_lib/lib/ethernet/ethdi.c b/fhi_lib/lib/ethernet/ethdi.c -index b6ba257..74b3c26 100644 +index b6ba257..b53325f 100644 --- a/fhi_lib/lib/ethernet/ethdi.c +++ b/fhi_lib/lib/ethernet/ethdi.c -@@ -479,11 +479,13 @@ xran_ethdi_init_dpdk_io(char *name, const struct xran_io_cfg *io_cfg, +@@ -335,6 +335,7 @@ xran_ethdi_init_dpdk_io(char *name, const struct xran_io_cfg *io_cfg, + uint16_t count; + + char *argv[] = { name, core_mask, "-n2", iova_mode, socket_mem, socket_limit, "--proc-type=auto", ++ "--no-telemetry", + "--file-prefix", name, "-a0000:00:00.0", bbdev_wdev, bbdev_vdev}; + + +@@ -479,11 +480,13 @@ xran_ethdi_init_dpdk_io(char *name, const struct xran_io_cfg *io_cfg, ctx->tx_ring[i] = rte_ring_create(ring_name, NUM_MBUFS_RING_TRX, rte_lcore_to_socket_id(*lcore_id), RING_F_SC_DEQ); PANIC_ON(ctx->tx_ring[i] == NULL, "failed to allocate rx ring"); @@ -143,7 +151,7 @@ index b6ba257..74b3c26 100644 } } } else { -@@ -553,7 +555,7 @@ xran_ethdi_init_dpdk_io(char *name, const struct xran_io_cfg *io_cfg, +@@ -553,7 +556,7 @@ xran_ethdi_init_dpdk_io(char *name, const struct xran_io_cfg *io_cfg, ctx->up_dl_pkt_gen_ring[i] = rte_ring_create(ring_name, NUM_MBUFS_RING, rte_lcore_to_socket_id(*lcore_id), /*RING_F_SC_DEQ*/0); PANIC_ON(ctx->up_dl_pkt_gen_ring[i] == NULL, "failed to allocate dl gen ring"); diff --git a/radio/fhi_72/CMakeLists.txt b/radio/fhi_72/CMakeLists.txt index 05cc50b486..88552eb46b 100644 --- a/radio/fhi_72/CMakeLists.txt +++ b/radio/fhi_72/CMakeLists.txt @@ -2,7 +2,7 @@ pkg_check_modules(dpdk REQUIRED libdpdk) pkg_check_modules(numa REQUIRED numa) -find_package(xran 5.1.1 EXACT REQUIRED) # E release -> 5 +find_package(xran 5.1.2 EXACT REQUIRED) # E release -> 5 # Ignore xran-specific warning: we don't care/can't change the following warning, so suppress # alignment 1 of 'struct XYZ' is less than 2 From d5f11902b7f3faa7b8101b0a210c1cfb2f3b5275 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 8 Feb 2024 10:46:39 +0100 Subject: [PATCH 13/18] Handle PUCCH frame/slot mismatches gracefully For different reasons, the frame/slot numbers might jump (e.g., gNB and RU machines not properly synced). The scheduler would assert for PUCCH allocations. Handle this gracefully by printing an error message, and resetting the PUCCH structure. --- .../LAYER2/NR_MAC_gNB/gNB_scheduler_uci.c | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_uci.c b/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_uci.c index 9f77dab513..aef5b11340 100644 --- a/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_uci.c +++ b/openair2/LAYER2/NR_MAC_gNB/gNB_scheduler_uci.c @@ -185,7 +185,16 @@ void nr_schedule_pucch(gNB_MAC_INST *nrmac, NR_sched_pucch_t *curr_pucch = &UE->UE_sched_ctrl.sched_pucch[pucch_index]; if (!curr_pucch->active) continue; - DevAssert(frameP == curr_pucch->frame && slotP == curr_pucch->ul_slot); + if (frameP != curr_pucch->frame || slotP != curr_pucch->ul_slot) { + LOG_E(NR_MAC, + "PUCCH frame/slot mismatch: current %4d.%2d vs. request %4d.%2d: not scheduling PUCCH\n", + curr_pucch->frame, + curr_pucch->ul_slot, + frameP, + slotP); + memset(curr_pucch, 0, sizeof(*curr_pucch));; + continue; + } const uint16_t O_ack = curr_pucch->dai_c; const uint16_t O_csi = curr_pucch->csi_bits; @@ -1296,8 +1305,13 @@ int nr_acknack_scheduling(gNB_MAC_INST *mac, return pucch_index; // index of current PUCCH structure } else if (curr_pucch->active) { - AssertFatal(1==0, "This shouldn't happen! curr_pucch frame.slot %d.%d not matching with computed frame.slot %d.%d\n", - curr_pucch->frame, curr_pucch->ul_slot, pucch_frame, pucch_slot); + LOG_E(NR_MAC, + "current PUCCH inactive: curr_pucch frame.slot %d.%d not matching with computed frame.slot %d.%d\n", + curr_pucch->frame, + curr_pucch->ul_slot, + pucch_frame, + pucch_slot); + memset(curr_pucch, 0, sizeof(*curr_pucch)); } else { // unoccupied occasion // checking if in ul_slot the resources potentially to be assigned to this PUCCH are available @@ -1392,8 +1406,13 @@ void nr_sr_reporting(gNB_MAC_INST *nrmac, frame_t SFN, sub_frame_t slot) curr_pucch->resource_indicator == idx) curr_pucch->sr_flag = true; else if (curr_pucch->active) { - AssertFatal(1==0, "This shouldn't happen! curr_pucch frame.slot %d.%d not matching with SR function frame.slot %d.%d\n", - curr_pucch->frame, curr_pucch->ul_slot, SFN, slot); + LOG_E(NR_MAC, + "current PUCCH inactive: curr_pucch frame.slot %d.%d not matching with computed frame.slot %d.%d\n", + curr_pucch->frame, + curr_pucch->ul_slot, + SFN, + slot); + memset(curr_pucch, 0, sizeof(*curr_pucch)); continue; } else { From 435376398483bb379c7abe2ef633ea310ed7d6be Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 12 Feb 2024 18:41:26 +0100 Subject: [PATCH 14/18] ORAN 7.2/doc: update NIC info --- doc/ORAN_FHI7.2_Tutorial.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 175123ea62..313c7644f7 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -30,11 +30,12 @@ The hardware on which we have tried this tutorial: NICs we have tested so far: -|Vendor |Firmware Version | -|------------|------------------------| -|Intel X710 |9.20 0x8000d95e 22.0.9 | -|Intel E810 |4.00 0x8001184e 1.3236.0| -|Intel XXV710|6.02 0x80003888 | +|Vendor |Firmware Version | +|---------------|------------------------| +|Intel X710 |9.20 0x8000d95e 22.0.9 | +|Intel E810-XXV |4.00 0x8001184e 1.3236.0| +|E810-C |4.20 0x8001784e 22.0.9 | +|Intel XXV710 |6.02 0x80003888 | PTP enabled switches and grandmaster clock we have in are lab: From 6fe655c7d41faa59d612aafb16813dc5cae0bdb6 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 12 Feb 2024 18:41:42 +0100 Subject: [PATCH 15/18] ORAN 7.2/doc: update s-plane info --- doc/ORAN_FHI7.2_Tutorial.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 313c7644f7..61b329fed1 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -45,8 +45,17 @@ PTP enabled switches and grandmaster clock we have in are lab: |Fibrolan Falcon-RX/812/G|8.0.25.4 | |Qulsar Qg2 (Grandmaster)|12.1.27 | +**S-Plane synchronization is mandatory.** S-plane support is done via `ptp4l` +and `phc2sys`. -Radio units we are testing/integrating: +| Software | Software Version | +|-----------|------------------| +| `ptp4l` | 3.1.1 | +| `phc2sys` | 3.1.1 | + +We have only verified LLS-C3 configuration in our lab, i.e. using an external +grandmaster, a switch as a boundary clock, and the gNB/DU and RU. We haven't +tested any RU without S-plane. Radio units we are testing/integrating: |Vendor |Software Version | |-----------|-----------------| From 5421a4d1725d96211a3050b5fa9a01a7fff9742f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 12 Feb 2024 18:42:49 +0100 Subject: [PATCH 16/18] O-RAN 7.2/doc: Update info on L1_tx_thread (diff d.txt ../graphs/_dijkstraData.txt ) --- doc/ORAN_FHI7.2_Tutorial.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 61b329fed1..2ef96895f6 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -91,7 +91,7 @@ Your server could be: DPDK, OAI and kernel threads require to be properly allocated to extract maximum real-time performance for your use case. -1. **NOTE**: Currently the default OAI 7.2 configuration file requires isolated **CPUs 0,2,4** for DPDK/libXRAN, **CPU 6** for `ru_thread` and **CPU 8** for `L1_rx_thread`. It is preferrable to have all these threads on the same socket. +1. **NOTE**: Currently the default OAI 7.2 configuration file requires isolated **CPUs 0,2,4** for DPDK/libXRAN, **CPU 6** for `ru_thread`, **CPU 8** for `L1_rx_thread` and **CPU 10** for `L1_tx_thread`. It is preferrable to have all these threads on the same socket. 2. Allocating CPUs to the OAI nr-softmodem is done using the `--thread-pool` option. Allocating 4 CPUs is the minimal configuration but we recommend to allocate at least **8** CPUs. And they can be on a different socket as the DPDK threads. 3. And to avoid kernel preempting these allocated CPUs, it is better to force the kernel to use un-allocated CPUs. @@ -102,6 +102,7 @@ Let summarize for example on a `32-CPU` single NUMA node system, regardless of t |XRAN DPDK usage |0,2,4 | |OAI `ru_thread` |6 | |OAI `L1_rx_thread` |8 | +|OAI `L1_tx_thread` |10 | |OAI `nr-softmodem` |1,3,5,7,9,11,13,15| |kernel |16-31 | @@ -602,6 +603,7 @@ Edit the sample OAI gNB configuration file and check following parameters: * `L1s` section * Set an isolated core for L1 thread `L1_rx_thread_core`, in our environment we are using CPU 8 + * Set an isolated core for L1 thread `L1_tx_thread_core`, in our environment we are using CPU 10 * `phase_compensation` should be set to 0 to disable when it is performed in the RU and set to 1 when it should be performed on the DU side * `RUs` section From a7f56d0d527fc29393e57efed625d1ba5a4e83a7 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 12 Feb 2024 19:17:37 +0100 Subject: [PATCH 17/18] O-RAN 7.2/doc: Update info on frame jumps --- doc/ORAN_FHI7.2_Tutorial.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 2ef96895f6..96df869783 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -700,3 +700,28 @@ mirroring at your switch to capture the fronthaul packets: check that you see (1) packets at all (2) they have the right ethernet address (3) the right VLAN tag. Although we did not test this, you might make use of the [DPDK packet capture feature](https://doc.dpdk.org/guides/howto/packet_capture_framework.html) + +
+If you see messages about `Received time doesn't correspond to the +time we think it is` or `Jump in frame counter`, the S-plane is not working. + +``` +[PHY] Received Time doesn't correspond to the time we think it is (slot mismatch, received 480.5, expected 475.8) +[PHY] Received Time doesn't correspond to the time we think it is (frame mismatch, 480.5 , expected 475.5) +[PHY] Jump in frame counter last_frame 480 => 519, slot 19 +[PHY] Received Time doesn't correspond to the time we think it is (slot mismatch, received 519.19, expected 480.12) +[PHY] Received Time doesn't correspond to the time we think it is (frame mismatch, 519.19 , expected 480.19) +[PHY] Received Time doesn't correspond to the time we think it is (slot mismatch, received 520.1, expected 520.0) +``` + +You can see that the frame numbers jump around, by 5-40 frames (corresponding +to 50-400ms!). This indicates the gNB receives packets on the fronthaul that +don't match its internal time, and the synchronization between gNB and RU is +not working! + +
+ +In this case, you should reverify that `ptp4l` and `phc2sys` are working, e.g., +do not do any jumps (during the last hour). While an occasional jump is not +necessarily problematic for the gNB, many such messages mean that the system is +not working, and UEs might not be able to attach or reach good performance. From 3bb9e6b0f50641e0a9a61196b52513a43b24c7dd Mon Sep 17 00:00:00 2001 From: beurdouc Date: Tue, 13 Feb 2024 10:00:05 +0000 Subject: [PATCH 18/18] ORAN 7.2/doc: indicate CPU cores in our environment --- doc/ORAN_FHI7.2_Tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ORAN_FHI7.2_Tutorial.md b/doc/ORAN_FHI7.2_Tutorial.md index 96df869783..4b8023acea 100644 --- a/doc/ORAN_FHI7.2_Tutorial.md +++ b/doc/ORAN_FHI7.2_Tutorial.md @@ -615,8 +615,8 @@ Edit the sample OAI gNB configuration file and check following parameters: * `dpdk_devices`: PCI addresses of NIC VFs binded to the DPDK * `system_core`: absolute CPU core ID for DPDK control threads (`rte_mp_handle`, `eal-intr-thread`, `iavf-event-thread`) - * `io_core`: absolute CPU core ID for XRAN library - * `worker_cores`: array of absolute CPU core IDs for XRAN library + * `io_core`: absolute CPU core ID for XRAN library, it should be an isolated core, in our environment we are using CPU 4 + * `worker_cores`: array of absolute CPU core IDs for XRAN library, they should be isolated cores, in our environment we are using CPU 2 * `du_addr`: DU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`, hexadecimal numbers) * `ru_addr`: RU C- and U-plane MAC-addresses (format `UU:VV:WW:XX:YY:ZZ`,