diff --git a/CMakeLists.txt b/CMakeLists.txt index 4aefd12b54..82f158bbd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -500,10 +500,10 @@ target_link_libraries(shlib_loader PRIVATE CONFIG_LIB) add_boolean_option(ENABLE_LDPC_T2 OFF "Build support for LDPC Offload to T2 library" OFF) set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/x86_64-linux-gnu/pkgconfig") if (ENABLE_LDPC_T2) - pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.7) - find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS "/opt/dpdk-t2/lib/x86_64-linux-gnu/") + pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.9) + find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS ${LIBDPDK_T2_LIBRARY_DIRS}) if (NOT PMD_T2) - message(FATAL_ERROR "could not find PMD T2") + message(FATAL_ERROR "could not find poll-mode driver for AccelerComm T2 LDPC Offload (rte_baseband_accl_ldpc.so)") endif() message(STATUS "T2 build: use ${PMD_T2}") add_library(ldpc_t2 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c) diff --git a/doc/BUILD.md b/doc/BUILD.md index 6fd6d76e3a..87b6dcca89 100644 --- a/doc/BUILD.md +++ b/doc/BUILD.md @@ -184,6 +184,7 @@ Some libraries have further dependencies and might not build on every system: - `nrqtscope`: Qt5 - `ldpc_cuda`: CUDA - `websrv`: npm and others +- `ldpc_t2`: DPDK with patch # Running `cmake` directly diff --git a/doc/LDPC_T2_OFFLOAD_SETUP.md b/doc/LDPC_T2_OFFLOAD_SETUP.md index 551ddcc1df..485ad4c4a4 100644 --- a/doc/LDPC_T2_OFFLOAD_SETUP.md +++ b/doc/LDPC_T2_OFFLOAD_SETUP.md @@ -21,19 +21,21 @@ This documentation aims to provide a tutorial for AMD Xilinx T2 Telco card integ # Requirements - bitstream image and PMD driver for the T2 card provided by AccelerComm - - DPDK 20.11.7 with patch from Accelercomm (also tested with DPDK 20.11.3) - - tested on RHEL7.9, RHEL9.2, Ubuntu 20.04, Ubuntu 22.04 + - DPDK 20.11.9 with patch from Accelercomm: version ACCL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch + - tested on RHEL7.9, RHEL9.2, Ubuntu 22.04 # DPDK setup ## DPDK installation +*Note: Following instructions are valid for ACCL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch version, which is compatible with DPDK 20.11.9. Installation steps, which should be followed for older versions of the patch file (for example ACL_BBDEV_DPDK20.11.3_BL_1006_build_1105_dev_branch_MCT_optimisations_1106_physical_std.patch) are present in older version of this documentation, under the tag 2023.w48.* + ``` # Get DPDK source code git clone https://github.com/DPDK/dpdk-stable.git ~/dpdk-stable cd ~/dpdk-stable -git checkout v20.11.7 -git apply ~/ACL_BBDEV_DPDK20.11.3_xxx.patch +git checkout v20.11.9 +git apply ~/ACL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch ``` -Replace `~/ACL_BBDEV_DPDK20.11.3_xxx.patch` by patch file provided by +Replace `~/ACL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch` by patch file provided by Accelercomm. ``` cd ~/dpdk-stable @@ -47,16 +49,15 @@ sudo ldconfig ## DPDK configuration - load required kernel module ``` -sudo modprobe igb_uio -sudo insmod ~/dpdk-stable/kernel/linux/igb_uio/igb_uio.ko +sudo modprobe vfio-pci ``` - check presence of the card and its PCI addres on the host machine ``` lspci | grep "Xilinx" ``` - - bind the card with igb_uio driver + - bind the card with vfio-pci driver ``` -sudo python3 ~/dpdk-stable/usertools/dpdk-devbind.py -b igb_uio 41:00.0 +sudo python3 ~/dpdk-stable/usertools/dpdk-devbind.py --bind=vfio-pci 41:00.0 ``` Replace PCI address of the card *41:00.0* by address detected by *lspci | grep "Xilinx"* command - hugepages setup (10 x 1GB hugepages) @@ -68,12 +69,12 @@ sudo python3 ~/dpdk-stable/usertools/dpdk-hugepages.py -p 1G --setup 10G` the host machine* # Modifications in the OAI code -## PMD path specification -Path to the PMD for operating the card is specified in `CMakeLists.txt` file in +## DPDK lib and PMD path specification +Path to the DPDK lib and Accelercomm PMD for operating the card is specified in the `CMakeLists.txt` file, in *LDPC OFFLOAD library* section. Modify following line based on the location of -the PMD on your system. By deafult, path to the PMD is set to `/opt/dpdk-t2/lib/x86_64-linux-gnu/`. +`libdpdk.pc` file associated with the target DPDK library on your system. By default, the path is set to `/usr/local/lib/x86_64-linux-gnu/pkgconfig`. ``` -find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS "/opt/dpdk-t2/lib/x86_64-linux-gnu/") +set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/x86_64-linux-gnu/pkgconfig") ``` ## T2 card DPDK initialization @@ -106,7 +107,7 @@ cd ~/openairinterface5g/cmake_targets cd ~/openairinterface5g source oaienv cd cmake_targets -./build_oai -w USRP --ninja --gNB -P -C --build-lib "ldpc_t2" +./build_oai -w USRP --ninja --gNB -P --build-lib "ldpc_t2" -C ``` Shared object file *libldpc_t2.so* is created during the compilation. This object is conditionally compiled. Selection of the library to compile is done using *--build-lib ldpc_t2*. @@ -131,14 +132,14 @@ sudo ./nr_dlsim -n300 -s30 -R 106 -e 27 -c ``` # OTA test -Offload of the channel encoding and decoding to the AMD Xilinx T2 card is enabled by *--ldpc-offload-enable 1* option. +Offload of the channel encoding and decoding to the AMD Xilinx T2 card is enabled by *--ldpc-offload-enable* option. ## Run OAI gNB with USRP B210 ``` cd ~/openairinterface5g source oaienv cd cmake_targets/ran_build/build -sudo ./nr-softmodem --sa -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.106PRB.usrpb210.conf --ldpc-offload-enable 1 +sudo ./nr-softmodem --sa -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.106PRB.usrpb210.conf --ldpc-offload-enable ``` # Limitations diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index 3738fa3f28..ef38a8ee1f 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -593,7 +593,7 @@ void init_eNB_afterRU(void) { LOG_I(PHY,"RC.nb_nr_CC[inst:%d]:%p\n", inst, RC.gNB[inst]); gNB = RC.gNB[inst]; - gNB->ldpc_offload_flag = ldpc_offload_flag; + gNB->ldpc_offload_flag = get_softmodem_params()->ldpc_offload_flag; gNB->reorder_thread_disable = get_softmodem_params()->reorder_thread_disable; phy_init_nr_gNB(gNB); diff --git a/executables/nr-softmodem-common.h b/executables/nr-softmodem-common.h index da743e4342..19760bdd26 100644 --- a/executables/nr-softmodem-common.h +++ b/executables/nr-softmodem-common.h @@ -98,7 +98,6 @@ #define CONFIG_HLP_WORKER_CMD "two option for worker 'WORKER_DISABLE' or 'WORKER_ENABLE'\n" #define CONFIG_HLP_USRP_THREAD "having extra thead for usrp tx\n" #define CONFIG_HLP_DISABLNBIOT "disable nb-iot, even if defined in config\n" -#define CONFIG_HLP_LDPC_OFFLOAD "enable LDPC offload to AMD Xilinx T2 card\n" #define CONFIG_HLP_USRP_ARGS "set the arguments to identify USRP (same syntax as in UHD)\n" #define CONFIG_HLP_TX_SUBDEV "set the arguments to select tx_subdev (same syntax as in UHD)\n" #define CONFIG_HLP_RX_SUBDEV "set the arguments to select rx_subdev (same syntax as in UHD)\n" diff --git a/executables/nr-softmodem.h b/executables/nr-softmodem.h index d4414291df..e7eed10cff 100644 --- a/executables/nr-softmodem.h +++ b/executables/nr-softmodem.h @@ -29,7 +29,6 @@ {"D" , CONFIG_HLP_DLBM_PHYTEST, 0, .u64ptr=&dlsch_slot_bitmap, .defintval=0, TYPE_UINT64, 0}, \ {"U" , CONFIG_HLP_ULBM_PHYTEST, 0, .u64ptr=&ulsch_slot_bitmap, .defintval=0, TYPE_UINT64, 0}, \ {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD, 0, .iptr=&usrp_tx_thread, .defstrval=0, TYPE_INT, 0}, \ - {"ldpc-offload-enable", CONFIG_HLP_LDPC_OFFLOAD, 0, .iptr=&ldpc_offload_flag, .defstrval=0, TYPE_INT, 0}, \ {"uecap_file", CONFIG_HLP_UECAP_FILE, 0, .strptr=&uecap_file, .defstrval="./uecap_ports1.xml", TYPE_STRING, 0}, \ {"s" , CONFIG_HLP_SNR, 0, .dblptr=&snr_dB, .defdblval=25, TYPE_DOUBLE, 0}, \ } @@ -44,7 +43,6 @@ extern uint32_t target_ul_bw; extern uint64_t dlsch_slot_bitmap; extern uint64_t ulsch_slot_bitmap; extern char *uecap_file; -extern int ldpc_offload_flag; // In nr-gnb.c extern void init_gNB(int single_thread_flag,int wait_for_sync); diff --git a/executables/softmodem-common.c b/executables/softmodem-common.c index f72d201622..316a5b14c9 100644 --- a/executables/softmodem-common.c +++ b/executables/softmodem-common.c @@ -47,7 +47,6 @@ char *parallel_config=NULL; char *worker_config=NULL; int usrp_tx_thread = 0; char *nfapi_str=NULL; -int ldpc_offload_flag=0; uint8_t nfapi_mode=0; static mapping softmodem_funcs[] = MAPPING_SOFTMODEM_FUNCTIONS; diff --git a/executables/softmodem-common.h b/executables/softmodem-common.h index e72e4dd824..b856bf252c 100644 --- a/executables/softmodem-common.h +++ b/executables/softmodem-common.h @@ -109,7 +109,8 @@ extern "C" #define CONFIG_HLP_SYNC_REF "Sync Reference in Sidelink\n" #define CONFIG_HLP_NID1 "Set NID1 value in Sidelink\n" #define CONFIG_HLP_NID2 "Set NID2 value in Sidelink\n" -#define CONFIG_HLP_NOITTI "Do not start itti threads, call queue processing in place, inside the caller thread" +#define CONFIG_HLP_NOITTI "Do not start itti threads, call queue processing in place, inside the caller thread" +#define CONFIG_HLP_LDPC_OFFLOAD "Enable LDPC offload to AMD Xilinx T2 telco card\n" /*-----------------------------------------------------------------------------------------------------------------------------------------------------*/ /* command line parameters common to eNodeB and UE */ @@ -144,6 +145,7 @@ extern "C" #define SYNC_REF softmodem_params.sync_ref #define NID1 softmodem_params.nid1 #define NID2 softmodem_params.nid2 +#define LDPC_OFFLOAD_FLAG softmodem_params.ldpc_offload_flag #define REORDER_THREAD_DISABLE softmodem_params.reorder_thread_disable #define DEFAULT_RFCONFIG_FILE "/usr/local/etc/syriq/ue.band7.tm1.PRB100.NR40.dat"; @@ -191,7 +193,8 @@ extern int usrp_tx_thread; {"disable-stats", CONFIG_HLP_STATS_DISABLE, PARAMFLAG_BOOL, .iptr=&stats_disabled, .defintval=0, TYPE_INT, 0}, \ {"nid1", CONFIG_HLP_NID1, 0, .iptr=&NID1, .defintval=10, TYPE_INT, 0}, \ {"nid2", CONFIG_HLP_NID2, 0, .iptr=&NID2, .defintval=1, TYPE_INT, 0}, \ - {"no-itti-threads", CONFIG_HLP_NOITTI, PARAMFLAG_BOOL, .iptr=&softmodem_params.no_itti, .defintval=0, TYPE_INT, 0}, \ + {"no-itti-threads", CONFIG_HLP_NOITTI, PARAMFLAG_BOOL, .iptr=&softmodem_params.no_itti, .defintval=0, TYPE_INT, 0}, \ + {"ldpc-offload-enable", CONFIG_HLP_LDPC_OFFLOAD, PARAMFLAG_BOOL, .iptr=&LDPC_OFFLOAD_FLAG, .defstrval=0, TYPE_INT, 0}, \ } // clang-format on @@ -241,6 +244,7 @@ extern int usrp_tx_thread; { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ + { .s5 = { NULL } }, \ } // clang-format on @@ -353,6 +357,7 @@ typedef struct { int nid1; int nid2; int no_itti; + int ldpc_offload_flag; } softmodem_params_t; extern uint64_t get_softmodem_optmask(void); @@ -368,7 +373,6 @@ extern int32_t uplink_frequency_offset[MAX_NUM_CCs][4]; extern int usrp_tx_thread; extern uint16_t sl_ahead; extern uint16_t sf_ahead; -extern int ldpc_offload_flag; extern int oai_exit; void tx_func(void *param);