diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cd36d625b..d2c594aad5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -500,10 +500,10 @@ target_link_libraries(shlib_loader PRIVATE CONFIG_LIB) add_boolean_option(ENABLE_LDPC_T2 OFF "Build support for LDPC Offload to T2 library" OFF) set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/x86_64-linux-gnu/pkgconfig") if (ENABLE_LDPC_T2) - pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.7) - find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS "/opt/dpdk-t2/lib/x86_64-linux-gnu/") + pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.9) + find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS ${LIBDPDK_T2_LIBRARY_DIRS}) if (NOT PMD_T2) - message(FATAL_ERROR "could not find PMD T2") + message(FATAL_ERROR "could not find poll-mode driver for AccelerComm T2 LDPC Offload (rte_baseband_accl_ldpc.so)") endif() message(STATUS "T2 build: use ${PMD_T2}") add_library(ldpc_t2 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c) diff --git a/doc/BUILD.md b/doc/BUILD.md index 6fd6d76e3a..87b6dcca89 100644 --- a/doc/BUILD.md +++ b/doc/BUILD.md @@ -184,6 +184,7 @@ Some libraries have further dependencies and might not build on every system: - `nrqtscope`: Qt5 - `ldpc_cuda`: CUDA - `websrv`: npm and others +- `ldpc_t2`: DPDK with patch # Running `cmake` directly diff --git a/doc/LDPC_T2_OFFLOAD_SETUP.md b/doc/LDPC_T2_OFFLOAD_SETUP.md index 551ddcc1df..485ad4c4a4 100644 --- a/doc/LDPC_T2_OFFLOAD_SETUP.md +++ b/doc/LDPC_T2_OFFLOAD_SETUP.md @@ -21,19 +21,21 @@ This documentation aims to provide a tutorial for AMD Xilinx T2 Telco card integ # Requirements - bitstream image and PMD driver for the T2 card provided by AccelerComm - - DPDK 20.11.7 with patch from Accelercomm (also tested with DPDK 20.11.3) - - tested on RHEL7.9, RHEL9.2, Ubuntu 20.04, Ubuntu 22.04 + - DPDK 20.11.9 with patch from Accelercomm: version ACCL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch + - tested on RHEL7.9, RHEL9.2, Ubuntu 22.04 # DPDK setup ## DPDK installation +*Note: Following instructions are valid for ACCL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch version, which is compatible with DPDK 20.11.9. Installation steps, which should be followed for older versions of the patch file (for example ACL_BBDEV_DPDK20.11.3_BL_1006_build_1105_dev_branch_MCT_optimisations_1106_physical_std.patch) are present in older version of this documentation, under the tag 2023.w48.* + ``` # Get DPDK source code git clone https://github.com/DPDK/dpdk-stable.git ~/dpdk-stable cd ~/dpdk-stable -git checkout v20.11.7 -git apply ~/ACL_BBDEV_DPDK20.11.3_xxx.patch +git checkout v20.11.9 +git apply ~/ACL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch ``` -Replace `~/ACL_BBDEV_DPDK20.11.3_xxx.patch` by patch file provided by +Replace `~/ACL_BBDEV_DPDK20.11.3_ldpc_3.1.918.patch` by patch file provided by Accelercomm. ``` cd ~/dpdk-stable @@ -47,16 +49,15 @@ sudo ldconfig ## DPDK configuration - load required kernel module ``` -sudo modprobe igb_uio -sudo insmod ~/dpdk-stable/kernel/linux/igb_uio/igb_uio.ko +sudo modprobe vfio-pci ``` - check presence of the card and its PCI addres on the host machine ``` lspci | grep "Xilinx" ``` - - bind the card with igb_uio driver + - bind the card with vfio-pci driver ``` -sudo python3 ~/dpdk-stable/usertools/dpdk-devbind.py -b igb_uio 41:00.0 +sudo python3 ~/dpdk-stable/usertools/dpdk-devbind.py --bind=vfio-pci 41:00.0 ``` Replace PCI address of the card *41:00.0* by address detected by *lspci | grep "Xilinx"* command - hugepages setup (10 x 1GB hugepages) @@ -68,12 +69,12 @@ sudo python3 ~/dpdk-stable/usertools/dpdk-hugepages.py -p 1G --setup 10G` the host machine* # Modifications in the OAI code -## PMD path specification -Path to the PMD for operating the card is specified in `CMakeLists.txt` file in +## DPDK lib and PMD path specification +Path to the DPDK lib and Accelercomm PMD for operating the card is specified in the `CMakeLists.txt` file, in *LDPC OFFLOAD library* section. Modify following line based on the location of -the PMD on your system. By deafult, path to the PMD is set to `/opt/dpdk-t2/lib/x86_64-linux-gnu/`. +`libdpdk.pc` file associated with the target DPDK library on your system. By default, the path is set to `/usr/local/lib/x86_64-linux-gnu/pkgconfig`. ``` -find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS "/opt/dpdk-t2/lib/x86_64-linux-gnu/") +set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/x86_64-linux-gnu/pkgconfig") ``` ## T2 card DPDK initialization @@ -106,7 +107,7 @@ cd ~/openairinterface5g/cmake_targets cd ~/openairinterface5g source oaienv cd cmake_targets -./build_oai -w USRP --ninja --gNB -P -C --build-lib "ldpc_t2" +./build_oai -w USRP --ninja --gNB -P --build-lib "ldpc_t2" -C ``` Shared object file *libldpc_t2.so* is created during the compilation. This object is conditionally compiled. Selection of the library to compile is done using *--build-lib ldpc_t2*. @@ -131,14 +132,14 @@ sudo ./nr_dlsim -n300 -s30 -R 106 -e 27 -c ``` # OTA test -Offload of the channel encoding and decoding to the AMD Xilinx T2 card is enabled by *--ldpc-offload-enable 1* option. +Offload of the channel encoding and decoding to the AMD Xilinx T2 card is enabled by *--ldpc-offload-enable* option. ## Run OAI gNB with USRP B210 ``` cd ~/openairinterface5g source oaienv cd cmake_targets/ran_build/build -sudo ./nr-softmodem --sa -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.106PRB.usrpb210.conf --ldpc-offload-enable 1 +sudo ./nr-softmodem --sa -O ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.106PRB.usrpb210.conf --ldpc-offload-enable ``` # Limitations diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index 3738fa3f28..ef38a8ee1f 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -593,7 +593,7 @@ void init_eNB_afterRU(void) { LOG_I(PHY,"RC.nb_nr_CC[inst:%d]:%p\n", inst, RC.gNB[inst]); gNB = RC.gNB[inst]; - gNB->ldpc_offload_flag = ldpc_offload_flag; + gNB->ldpc_offload_flag = get_softmodem_params()->ldpc_offload_flag; gNB->reorder_thread_disable = get_softmodem_params()->reorder_thread_disable; phy_init_nr_gNB(gNB); diff --git a/executables/nr-softmodem-common.h b/executables/nr-softmodem-common.h index da743e4342..19760bdd26 100644 --- a/executables/nr-softmodem-common.h +++ b/executables/nr-softmodem-common.h @@ -98,7 +98,6 @@ #define CONFIG_HLP_WORKER_CMD "two option for worker 'WORKER_DISABLE' or 'WORKER_ENABLE'\n" #define CONFIG_HLP_USRP_THREAD "having extra thead for usrp tx\n" #define CONFIG_HLP_DISABLNBIOT "disable nb-iot, even if defined in config\n" -#define CONFIG_HLP_LDPC_OFFLOAD "enable LDPC offload to AMD Xilinx T2 card\n" #define CONFIG_HLP_USRP_ARGS "set the arguments to identify USRP (same syntax as in UHD)\n" #define CONFIG_HLP_TX_SUBDEV "set the arguments to select tx_subdev (same syntax as in UHD)\n" #define CONFIG_HLP_RX_SUBDEV "set the arguments to select rx_subdev (same syntax as in UHD)\n" diff --git a/executables/nr-softmodem.h b/executables/nr-softmodem.h index d4414291df..e7eed10cff 100644 --- a/executables/nr-softmodem.h +++ b/executables/nr-softmodem.h @@ -29,7 +29,6 @@ {"D" , CONFIG_HLP_DLBM_PHYTEST, 0, .u64ptr=&dlsch_slot_bitmap, .defintval=0, TYPE_UINT64, 0}, \ {"U" , CONFIG_HLP_ULBM_PHYTEST, 0, .u64ptr=&ulsch_slot_bitmap, .defintval=0, TYPE_UINT64, 0}, \ {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD, 0, .iptr=&usrp_tx_thread, .defstrval=0, TYPE_INT, 0}, \ - {"ldpc-offload-enable", CONFIG_HLP_LDPC_OFFLOAD, 0, .iptr=&ldpc_offload_flag, .defstrval=0, TYPE_INT, 0}, \ {"uecap_file", CONFIG_HLP_UECAP_FILE, 0, .strptr=&uecap_file, .defstrval="./uecap_ports1.xml", TYPE_STRING, 0}, \ {"s" , CONFIG_HLP_SNR, 0, .dblptr=&snr_dB, .defdblval=25, TYPE_DOUBLE, 0}, \ } @@ -44,7 +43,6 @@ extern uint32_t target_ul_bw; extern uint64_t dlsch_slot_bitmap; extern uint64_t ulsch_slot_bitmap; extern char *uecap_file; -extern int ldpc_offload_flag; // In nr-gnb.c extern void init_gNB(int single_thread_flag,int wait_for_sync); diff --git a/executables/softmodem-common.c b/executables/softmodem-common.c index f72d201622..316a5b14c9 100644 --- a/executables/softmodem-common.c +++ b/executables/softmodem-common.c @@ -47,7 +47,6 @@ char *parallel_config=NULL; char *worker_config=NULL; int usrp_tx_thread = 0; char *nfapi_str=NULL; -int ldpc_offload_flag=0; uint8_t nfapi_mode=0; static mapping softmodem_funcs[] = MAPPING_SOFTMODEM_FUNCTIONS; diff --git a/executables/softmodem-common.h b/executables/softmodem-common.h index e72e4dd824..b856bf252c 100644 --- a/executables/softmodem-common.h +++ b/executables/softmodem-common.h @@ -109,7 +109,8 @@ extern "C" #define CONFIG_HLP_SYNC_REF "Sync Reference in Sidelink\n" #define CONFIG_HLP_NID1 "Set NID1 value in Sidelink\n" #define CONFIG_HLP_NID2 "Set NID2 value in Sidelink\n" -#define CONFIG_HLP_NOITTI "Do not start itti threads, call queue processing in place, inside the caller thread" +#define CONFIG_HLP_NOITTI "Do not start itti threads, call queue processing in place, inside the caller thread" +#define CONFIG_HLP_LDPC_OFFLOAD "Enable LDPC offload to AMD Xilinx T2 telco card\n" /*-----------------------------------------------------------------------------------------------------------------------------------------------------*/ /* command line parameters common to eNodeB and UE */ @@ -144,6 +145,7 @@ extern "C" #define SYNC_REF softmodem_params.sync_ref #define NID1 softmodem_params.nid1 #define NID2 softmodem_params.nid2 +#define LDPC_OFFLOAD_FLAG softmodem_params.ldpc_offload_flag #define REORDER_THREAD_DISABLE softmodem_params.reorder_thread_disable #define DEFAULT_RFCONFIG_FILE "/usr/local/etc/syriq/ue.band7.tm1.PRB100.NR40.dat"; @@ -191,7 +193,8 @@ extern int usrp_tx_thread; {"disable-stats", CONFIG_HLP_STATS_DISABLE, PARAMFLAG_BOOL, .iptr=&stats_disabled, .defintval=0, TYPE_INT, 0}, \ {"nid1", CONFIG_HLP_NID1, 0, .iptr=&NID1, .defintval=10, TYPE_INT, 0}, \ {"nid2", CONFIG_HLP_NID2, 0, .iptr=&NID2, .defintval=1, TYPE_INT, 0}, \ - {"no-itti-threads", CONFIG_HLP_NOITTI, PARAMFLAG_BOOL, .iptr=&softmodem_params.no_itti, .defintval=0, TYPE_INT, 0}, \ + {"no-itti-threads", CONFIG_HLP_NOITTI, PARAMFLAG_BOOL, .iptr=&softmodem_params.no_itti, .defintval=0, TYPE_INT, 0}, \ + {"ldpc-offload-enable", CONFIG_HLP_LDPC_OFFLOAD, PARAMFLAG_BOOL, .iptr=&LDPC_OFFLOAD_FLAG, .defstrval=0, TYPE_INT, 0}, \ } // clang-format on @@ -241,6 +244,7 @@ extern int usrp_tx_thread; { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ + { .s5 = { NULL } }, \ } // clang-format on @@ -353,6 +357,7 @@ typedef struct { int nid1; int nid2; int no_itti; + int ldpc_offload_flag; } softmodem_params_t; extern uint64_t get_softmodem_optmask(void); @@ -368,7 +373,6 @@ extern int32_t uplink_frequency_offset[MAX_NUM_CCs][4]; extern int usrp_tx_thread; extern uint16_t sl_ahead; extern uint16_t sf_ahead; -extern int ldpc_offload_flag; extern int oai_exit; void tx_func(void *param); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c index a77fdfc2f7..1f2163ca5e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c @@ -543,8 +543,8 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, } LOG_D(PHY,"ULSCH %02d HARQPID %02d R %02d COMBIN %d RV %d NCB %05d \n", ulsch_id, harq_pid, r, p_offloadParams->setCombIn, p_offloadParams->rv, p_offloadParams->n_cb); ops[i]->ldpc_dec.code_block_mode = 1; // ldpc_dec->code_block_mode; - ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id * (32 * 1024 * 1024) + harq_pid * (2 * 1024 * 1024) + r * (1024 * 32); - ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id * (32 * 1024 * 1024) + harq_pid * (2 * 1024 * 1024) + r * (1024 * 32); + ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + r * LDPC_MAX_CB_SIZE; + ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + r * LDPC_MAX_CB_SIZE; if (bufs->hard_outputs != NULL) ops[i]->ldpc_dec.hard_output = bufs->hard_outputs[start_idx + i]; if (bufs->inputs != NULL) @@ -968,7 +968,7 @@ int32_t LDPCinit() int socket_id = GET_SOCKET(info.socket_id); int out_max_sz = 8448; // max code block size (for BG1), 22 * 384 - int in_max_sz = 25344; // max number of encoded bits (for BG1), 66 * 384 + int in_max_sz = LDPC_MAX_CB_SIZE; // max number of encoded bits (for BG2 and MCS0) int num_ops = 1; int f_ret = create_mempools(ad, socket_id, num_ops, out_max_sz, in_max_sz); if (f_ret != TEST_SUCCESS) { diff --git a/openair1/PHY/CODING/nrLDPC_extern.h b/openair1/PHY/CODING/nrLDPC_extern.h index dbfdae21b2..f93e7eb9aa 100644 --- a/openair1/PHY/CODING/nrLDPC_extern.h +++ b/openair1/PHY/CODING/nrLDPC_extern.h @@ -21,7 +21,8 @@ #ifndef _NRLDPC_EXTERN_H__ #define _NRLDPC_EXTERN_H__ #include "openair1/PHY/CODING/nrLDPC_defs.h" - +/* LDPC maximum code block size - maximum E */ +#define LDPC_MAX_CB_SIZE 32768 /* ldpc coder/decoder API*/ typedef struct ldpc_interface_s { LDPC_initfunc_t *LDPCinit; diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c index a3041838b9..e04d8fc41a 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c @@ -232,8 +232,8 @@ int decode_offload(PHY_VARS_gNB *phy_vars_gNB, { NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id]; NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process; - int16_t z_ol[68 * 384 + 16] __attribute__((aligned(16))); - int8_t l_ol[68 * 384 + 16] __attribute__((aligned(16))); + int16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); + int8_t l_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); uint8_t Qm = pusch_pdu->qam_mod_order; uint8_t n_layers = pusch_pdu->nrOfLayers; const int Kr = harq_process->K;