From 79f8d16da67b44605343ccd765cb25530bed5b83 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Tue, 14 Feb 2023 14:07:42 +0100 Subject: [PATCH 1/6] Rename T1 offload lib and conditional compilation --- CMakeLists.txt | 43 ++++++++----------- .../nrLDPC_decoder/nrLDPC_decoder_offload.c | 2 +- openair1/PHY/CODING/nrLDPC_load.c | 2 +- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 116b8a55d2..2cf58436fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -618,26 +618,19 @@ target_link_libraries(benetel_5g PRIVATE asn1_nr_rrc asn1_lte_rrc) # LDPC offload library ########################################################## -# there is no generic way to test for T1 Offload, it simply comes with the -# shared object -if (EXISTS "/usr/local/lib64/librte_pmd_hpac_sdfec_pmd.so") - set(T1_OFFLOAD_FOUND TRUE) -else() - set(T1_OFFLOAD_FOUND FALSE) +add_boolean_option(ENABLE_LDPC_T1 OFF "Build support for LDPC Offload to T1 library") +if (ENABLE_LDPC_T1) + pkg_check_modules(LIBDPDK_T1 REQUIRED libdpdk=20.05.0) + find_library(T1 NAMES rte_pmd_hpac_sdfec_pmd REQUIRED) + if (NOT T1) + message(FATAL_ERROR "Library rte_pmd_hpac_sdfec_pmd for T1 offload not found") + endif() + add_library(ldpc_t1 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c) + set_target_properties(ldpc_t1 PROPERTIES COMPILE_FLAGS "-DALLOW_EXPERIMENTAL_API") + target_compile_options(ldpc_t1 PRIVATE ${LIBDPDK_T1_CFLAGS}) + target_link_libraries(ldpc_t1 ${LIBDPDK_T1_LDFLAGS} ${T1}) endif() -if(T1_OFFLOAD_FOUND) - message(STATUS "T1 Offload support detected") - set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib64/pkgconfig/") - pkg_search_module(LIBDPDK REQUIRED libdpdk=20.05.0) - add_library(ldpc_offload MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c) - set_target_properties(ldpc_offload PROPERTIES COMPILE_FLAGS "-DALLOW_EXPERIMENTAL_API") - target_compile_options(ldpc_offload PRIVATE ${LIBDPDK_CFLAGS}) - target_link_libraries(ldpc_offload ${LIBDPDK_LDFLAGS} rte_pmd_hpac_sdfec_pmd "-Wl,-rpath /usr/local/lib64") -else() - message(STATUS "No T1 Offload support detected") -endif () - ########################################################## include_directories ("${OPENAIR_DIR}/radio/COMMON") @@ -2384,11 +2377,11 @@ target_link_libraries(nr-softmodem PRIVATE ${LIB_LMS_LIBRARIES}) target_link_libraries(nr-softmodem PRIVATE ${T_LIB}) target_link_libraries(nr-softmodem PRIVATE asn1_nr_rrc asn1_lte_rrc) -add_dependencies( nr-softmodem ldpc_orig ldpc_optim ldpc_optim8seg ldpc) +add_dependencies(nr-softmodem ldpc_orig ldpc_optim ldpc_optim8seg ldpc) -if(T1_OFFLOAD_FOUND) -add_dependencies( nr-softmodem ldpc_offload) -endif () +if (ENABLE_LDPC_T1) + add_dependencies(nr-softmodem ldpc_t1) +endif() # force the generation of ASN.1 so that we don't need to wait during the build target_link_libraries(nr-softmodem PRIVATE @@ -2634,9 +2627,9 @@ add_executable(nr_ulsim ${SHLIB_LOADER_SOURCES} ) -if(T1_OFFLOAD_FOUND) -add_dependencies( nr_ulsim ldpc_offload) -endif () +if (ENABLE_LDPC_T1) + add_dependencies(nr_ulsim ldpc_t1) +endif() target_link_libraries(nr_ulsim PRIVATE -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB SCHED_NR_UE_LIB MAC_NR MAC_UE_NR MAC_NR_COMMON lte_rrc nr_rrc CONFIG_LIB L2_LTE_NR L2_NR HASHTABLE x2ap SECU_CN ngap NFAPI_COMMON_LIB NFAPI_LIB NFAPI_PNF_LIB NFAPI_USER_LIB -lz -Wl,--end-group diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c index a523e960e2..33c0026314 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c @@ -1434,7 +1434,7 @@ int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid, argv_re[1] = "-l"; argv_re[2] = "31"; argv_re[3] = "-w"; - argv_re[4] = "81:00.0"; + argv_re[4] = "b0:00.0"; argv_re[5] = "--file-prefix=b6"; argv_re[6] = "--"; diff --git a/openair1/PHY/CODING/nrLDPC_load.c b/openair1/PHY/CODING/nrLDPC_load.c index 49ca9bf0de..400fb8d836 100644 --- a/openair1/PHY/CODING/nrLDPC_load.c +++ b/openair1/PHY/CODING/nrLDPC_load.c @@ -75,7 +75,7 @@ int load_nrLDPClib_offload(void) { loader_shlibfunc_t shlib_decoffload_fdesc; shlib_decoffload_fdesc.fname = "nrLDPC_decod_offload"; - int ret=load_module_shlib("ldpc_offload",&shlib_decoffload_fdesc,1,NULL); + int ret=load_module_shlib("ldpc_t1",&shlib_decoffload_fdesc,1,NULL); AssertFatal( (ret >= 0),"Error loading ldpc decoder offload"); nrLDPC_decoder_offload = (nrLDPC_decoffloadfunc_t)shlib_decoffload_fdesc.fptr; From 52e160a08a882ed3c8344f6c0fa996df57943a3a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Sat, 18 Feb 2023 11:54:21 +0100 Subject: [PATCH 2/6] Conditional compilation for CUDA LDPC Offload --- CMakeLists.txt | 57 +++++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cf58436fd..7da5bbc5e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,29 +140,6 @@ set (OPENAIR_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}) project (OpenAirInterface) -############################################## -# Base CUDA setting -############################################## -find_package(CUDA) -if (CUDA_FOUND) - message ("cuda include ${CUDA_INCLUDE_DIRS}") - message ("cuda library ${CUDA_LIBRARY_DIRS}") - - add_definitions("-L/usr/local/cuda/lib64") - - SET(CUDA_NVCC_FLAGS - "${CUDA_NVCC_FLAGS};-arch=sm_60;") - - # Disable warnings for CUDA - SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-lpthread;-w;-O3;--default-stream;per-thread;-I/usr/local/cuda/inc;-L/usr/local/cuda/lib -lcutil;-rdc=true;-lcudadevrt") - SET(CUDA_VERBOSE_BUILD ON) - - SET(CUDA_SEPARABLE_COMPILATION ON) - -else (CUDA_FOUND) - message ("No CUDA tool installed") -endif () - #################################################### # compilation flags ############################################# @@ -1026,16 +1003,26 @@ add_library(ldpc_cl MODULE ${PHY_LDPC_CL_SRC} ) target_link_libraries(ldpc_cl OpenCL) add_dependencies(ldpc_cl nrLDPC_decoder_kernels_CL) -if (CUDA_FOUND) - cuda_add_library(ldpc_cuda MODULE ${PHY_LDPC_CUDA_SRC} ) + +############################################## +# Base CUDA setting +############################################## + +add_boolean_option(BUILD_CUDA False "Build support for CUDA") +if (ENABLE_LDPC_CUDA) + find_package(CUDA REQUIRED) + SET(CUDA_NVCC_FLAG "${CUDA_NVCC_FLAGS};-arch=sm_60;") + SET(CUDA_VERBOSE_BUILD ON) + cuda_add_library(ldpc_cuda MODULE ${PHY_LDPC_CUDA_SRC}) set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON) -# CUDA_ADD_CUFFT_TO_TARGET(ldpc_cuda) -endif (CUDA_FOUND) + if (NOT CUDA_FOUND) + message(FATAL_ERROR "no CUDA found") + endif() +endif() add_library(ldpc MODULE ${PHY_LDPC_OPTIM8SEGMULTI_SRC} ) target_link_libraries(ldpc PRIVATE ldpc_gen_HEADERS) - add_library(coding MODULE ${PHY_TURBOSRC} ) target_link_libraries(coding PRIVATE asn1_nr_rrc asn1_lte_rrc) @@ -2444,10 +2431,10 @@ target_link_libraries(nr-uesoftmodem PRIVATE ${T_LIB}) target_link_libraries(nr-uesoftmodem PRIVATE asn1_nr_rrc asn1_lte_rrc) add_dependencies( nr-uesoftmodem ldpc_orig ldpc_optim ldpc_optim8seg ldpc ) -if (CUDA_FOUND) - add_dependencies( nr-uesoftmodem ldpc_cuda) - add_dependencies( nr-softmodem ldpc_cuda) -endif (CUDA_FOUND) +if (ENABLE_LDPC_CUDA) + add_dependencies(nr-uesoftmodem ldpc_cuda) + add_dependencies(nr-softmodem ldpc_cuda) +endif() # force the generation of ASN.1 so that we don't need to wait during the build target_link_libraries(nr-uesoftmodem PRIVATE @@ -2516,9 +2503,9 @@ add_executable(ldpctest ) add_dependencies( ldpctest ldpc_orig ldpc_optim ldpc_optim8seg ldpc ) -if (CUDA_FOUND) - add_dependencies( ldpctest ldpc_cuda) -endif (CUDA_FOUND) +if (ENABLE_LDPC_CUDA) + add_dependencies(ldpctest ldpc_cuda) +endif() target_link_libraries(ldpctest PRIVATE -Wl,--start-group UTIL SIMU PHY_NR PHY_COMMON PHY_NR_COMMON CONFIG_LIB -Wl,--end-group m pthread ${ATLAS_LIBRARIES} dl From b89b0d50744ea90d3023b645d566d0c8baec4f70 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Sat, 18 Feb 2023 11:54:29 +0100 Subject: [PATCH 3/6] build_oai: include ldpc_cuda and ldpc_t1 as optional libraries --- cmake_targets/build_oai | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmake_targets/build_oai b/cmake_targets/build_oai index f9166e0555..f27a0cb315 100755 --- a/cmake_targets/build_oai +++ b/cmake_targets/build_oai @@ -55,7 +55,7 @@ CMAKE_BUILD_TYPE="RelWithDebInfo" CMAKE_CMD="$CMAKE" BUILD_ECLIPSE=0 NR="False" -OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope" +OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_cuda ldpc_t1" ulfiusdep=$(basename ./`find /usr/lib* -name libulfius.so`) jssondep=$(basename ./`find /usr/lib* -name libjansson.so`) if [ "$ulfiusdep" == "libulfius.so" -a "$jssondep" == "libjansson.so" ] ; then @@ -392,13 +392,16 @@ function main() { if [ "$alib" = "$oklib" ] ; then BUILD_OPTLIB="$BUILD_OPTLIB $alib" echo_info "Enabling build of lib${alib}.so" - fi - done - done - if [ "${BUILD_OPTLIB## }" != "$2" ] ; then - echo_fatal "Unknown optional library in $2, valid libraries are $OPTIONAL_LIBRARIES" + fi + done + done + if [ "${BUILD_OPTLIB## }" != "$2" ] ; then + echo_fatal "Unknown optional library in $2, valid libraries are $OPTIONAL_LIBRARIES" fi fi + for oklib in $BUILD_OPTLIB ; do + CMAKE_CMD="$CMAKE_CMD -DENABLE_${oklib^^}=ON" + done shift 2;; --noavx512) CMAKE_CMD="$CMAKE_CMD -DAVX512=OFF" From 4f097b425a7ee3de15a2165d2d47f41d8e1831a3 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Sat, 18 Feb 2023 11:59:11 +0100 Subject: [PATCH 4/6] Build CI: only build selected optional libraries --- docker/Dockerfile.build.rhel8.2 | 2 +- docker/Dockerfile.build.ubuntu18 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.build.rhel8.2 b/docker/Dockerfile.build.rhel8.2 index 6fc2598276..4978c0c50d 100644 --- a/docker/Dockerfile.build.rhel8.2 +++ b/docker/Dockerfile.build.rhel8.2 @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib all -w USRP --verbose-ci --noavx512 -c + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP --verbose-ci --noavx512 -c diff --git a/docker/Dockerfile.build.ubuntu18 b/docker/Dockerfile.build.ubuntu18 index 8c1ea3556d..62c3b84c10 100644 --- a/docker/Dockerfile.build.ubuntu18 +++ b/docker/Dockerfile.build.ubuntu18 @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib all -w USRP --verbose-ci --noavx512 -c + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP --verbose-ci --noavx512 -c From 8f7a4bf353f7b315e24737fc7a2e6afe83af7c58 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Sat, 18 Feb 2023 12:04:41 +0100 Subject: [PATCH 5/6] All-in-one tutorial: only build selected libraries --- doc/NR_SA_CN5G_gNB_USRP_COTS_UE_Tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/NR_SA_CN5G_gNB_USRP_COTS_UE_Tutorial.md b/doc/NR_SA_CN5G_gNB_USRP_COTS_UE_Tutorial.md index 8d82b1964f..13624042ee 100644 --- a/doc/NR_SA_CN5G_gNB_USRP_COTS_UE_Tutorial.md +++ b/doc/NR_SA_CN5G_gNB_USRP_COTS_UE_Tutorial.md @@ -144,7 +144,7 @@ cd cmake_targets cd ~/openairinterface5g source oaienv cd cmake_targets -./build_oai -w USRP --ninja --nrUE --gNB --build-lib all -c +./build_oai -w USRP --ninja --nrUE --gNB --build-lib "telnetsrv nrscope" -c ``` # 4. Run OAI CN5G and OAI gNB From 972cbd5cb31cff3730a684b8f62a4469bb0806d9 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Sat, 18 Feb 2023 15:06:47 +0100 Subject: [PATCH 6/6] Build CUDA lib in caracal test --- ci-scripts/xml_files/gnb_usrp_build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/xml_files/gnb_usrp_build.xml b/ci-scripts/xml_files/gnb_usrp_build.xml index 3fc1b1c955..45adb6a2c1 100644 --- a/ci-scripts/xml_files/gnb_usrp_build.xml +++ b/ci-scripts/xml_files/gnb_usrp_build.xml @@ -34,7 +34,7 @@ TesteNB Build_eNB Build gNB (USRP) - --gNB -w USRP --ninja -c -P + --gNB -w USRP --ninja -c -P --build-lib "ldpc_cuda" True