diff --git a/CMakeLists.txt b/CMakeLists.txt index 95667e42e9..8101f9f6e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,17 +55,17 @@ if (${RF_BOARD} STREQUAL "OAI_USRP") find_package(Boost REQUIRED) elseif (${RF_BOARD} STREQUAL "OAI_IRIS") - include_directories("${OPENAIR_DIR}/sdr/IRIS/USERSPACE/LIB/") + include_directories("${OPENAIR_DIR}/radio/IRIS/USERSPACE/LIB/") set(HW_SOURCE ${HW_SOURCE} - ${OPENAIR_DIR}/sdr/IRIS/USERSPACE/LIB/iris_lib.cpp) + ${OPENAIR_DIR}/radio/IRIS/USERSPACE/LIB/iris_lib.cpp) LINK_DIRECTORIES("/usr/local/lib") set(option_HW_lib "-lSoapySDR -rdynamic -ldl") elseif (${RF_BOARD} STREQUAL "OAI_AW2SORI") - include_directories("${OPENAIR_DIR}/sdr/AW2SORI") + include_directories("${OPENAIR_DIR}/radio/AW2SORI") set(HW_SOURCE ${HW_SOURCE} - ${OPENAIR_DIR}/sdr/AW2SORI/ARCH/AW2SORI/oaiori.c) + ${OPENAIR_DIR}/radio/AW2SORI/ARCH/AW2SORI/oaiori.c) LINK_DIRECTORIES("/usr/local/lib") set(openair_HW_lib "-shared -fPIC -msse4 -g -ggdb -lori") @@ -193,43 +193,56 @@ endif () if (CMAKE_BUILD_TYPE STREQUAL "") set(CMAKE_BUILD_TYPE "RelWithDebInfo") endif() -message("CMAKE_BUILD_TYPE is ${CMAKE_BUILD_TYPE}") +message(STATUS "CMAKE_BUILD_TYPE is ${CMAKE_BUILD_TYPE}") add_list_string_option(CMAKE_BUILD_TYPE "RelWithDebInfo" "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel." Debug Release RelWithDebInfo MinSizeRel) -Message("Architecture is ${CMAKE_SYSTEM_PROCESSOR}") # in case /proc/cpuinfo exists we want to inspect available Intrinsics # -so not to go always through SIMDE emulation # -so to avoid AVX512 instructions generation by gcc -if(EXISTS "/proc/cpuinfo") - file(STRINGS "/proc/cpuinfo" CPUINFO REGEX flags LIMIT_COUNT 1) - message("AVX512 is ${AVX512}") - message("AVX2 is ${AVX2}") - if ("${AVX512}" STREQUAL "False") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -mno-avx512f -march=native -DSIMDE_X86_AVX2_NATIVE -DSIMDE_X86_AVX2_NATIVE -DSIMDE_X86_AVX512BW_NATIVE -DSIMDE_X86_AVX512F_NATIVE -DSIMDE_X86_AVX512VL_NATIVE -DSIMDE_X86_AVX_NATIVE -DSIMDE_X86_AVX_NATIVE -DSIMDE_X86_F16C_NATIVE -DSIMDE_X86_FMA_NATIVE -DSIMDE_X86_GFNI_NATIVE -DSIMDE_X86_MMX_NATIVE -DSIMDE_X86_PCLMUL_NATIVE -DSIMDE_X86_SSE2_NATIVE -DSIMDE_X86_SSE3_NATIVE -DSIMDE_X86_SSE4_1_NATIVE -DSIMDE_X86_SSE4_2_NATIVE -DSIMDE_X86_SSE_NATIVE -DSIMDE_X86_SSSE3_NATIVE -DSIMDE_X86_VPCLMULQDQ_NATIVE -DSIMDE_X86_XOP_HAVE_COM_ -DSIMDE_X86_XOP_NATIVE") - else() - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -mavx512bw -march=skylake-avx512 -mtune=skylake-avx512") - endif() - if (CPUINFO MATCHES "avx2" AND "${AVX2}" STREQUAL "True") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX2_NATIVE -DSIMDE_X86_VPCLMULQDQ_NATIVE") - endif() - if (CPUINFO MATCHES "sse4_1") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSE4_1_NATIVE") - endif() - if (CPUINFO MATCHES "sse4_2") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSE4_2_NATIVE") - endif() - if (CPUINFO MATCHES "ssse3") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSSE3_NATIVE") - endif() +execute_process(COMMAND uname -m OUTPUT_VARIABLE CPUARCH OUTPUT_STRIP_TRAILING_WHITESPACE) +message(STATUS "CPUARCH ${CPUARCH}") +if(EXISTS "/proc/cpuinfo") + file(STRINGS "/proc/cpuinfo" CPUFLAGS REGEX flags LIMIT_COUNT 1) +else() + message(WARNING "did not find /proc/cpuinfo -- not setting any x86-specific compilation variables") +endif() + +eval_boolean(AUTODETECT_AVX512 DEFINED CPUFLAGS AND CPUFLAGS MATCHES "avx512") +add_boolean_option(AVX512 ${AUTODETECT_AVX512} "Whether AVX512 intrinsics is available on the host processor") + +eval_boolean(AUTODETECT_AVX2 DEFINED CPUFLAGS AND CPUFLAGS MATCHES "avx2") +add_boolean_option(AVX2 ${AUTODETECT_AVX2} "Whether AVX2 intrinsics is available on the host processor") + +if(${CPUARCH} STREQUAL "x86_64" AND DEFINED CPUFLAGS) + # The following intrinsics are assumed to be available on any x86 system + # (avx, f16c, fma, gnfi, mmx, pclmul, sse, sse2, sse3, xop) + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX_NATIVE -DSIMDE_X86_AVX_NATIVE -DSIMDE_X86_F16C_NATIVE -DSIMDE_X86_FMA_NATIVE -DSIMDE_X86_GFNI_NATIVE -DSIMDE_X86_MMX_NATIVE -DSIMDE_X86_PCLMUL_NATIVE -DSIMDE_X86_SSE2_NATIVE -DSIMDE_X86_SSE3_NATIVE -DSIMDE_X86_SSE_NATIVE -DSIMDE_X86_XOP_HAVE_COM_ -DSIMDE_X86_XOP_NATIVE") + message(STATUS "AVX512 intrinsics are ${AVX512}") + if(${AVX512}) + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX512BW_NATIVE -DSIMDE_X86_AVX512F_NATIVE -DSIMDE_X86_AVX512VL_NATIVE -mavx512bw -march=skylake-avx512 -mtune=skylake-avx512") + else() + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -mno-avx512f -march=native") + endif() + message(STATUS "AVX2 intrinsics are ${AVX2}") + if(${AVX2}) + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX2_NATIVE -DSIMDE_X86_VPCLMULQDQ_NATIVE") + endif() + if (CPUINFO MATCHES "sse4_1") + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSE4_1_NATIVE") + endif() + if(CPUINFO MATCHES "sse4_2") + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSE4_2_NATIVE") + endif() + if(CPUINFO MATCHES "ssse3") + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSSE3_NATIVE") + endif() +elseif(${CPUARCH} NOT STREQUAL "x86_64") + message(FATAL_ERROR "Cannot compile for CPU architecture ${CPUARCH}") endif() set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -fno-var-tracking-assignments -march=native") -Message("C_FLAGS_PROCESSOR is ${C_FLAGS_PROCESSOR}") - -# # add autotools definitions that were maybe used! - add_definitions("-DSTDC_HEADERS=1 -DHAVE_SYS_TYPES_H=1 -DHAVE_SYS_STAT_H=1 -DHAVE_STDLIB_H=1 -DHAVE_STRING_H=1 -DHAVE_MEMORY_H=1 -DHAVE_STRINGS_H=1 -DHAVE_INTTYPES_H=1 -DHAVE_STDINT_H=1 -DHAVE_UNISTD_H=1 -DHAVE_FCNTL_H=1 -DHAVE_ARPA_INET_H=1 -DHAVE_SYS_TIME_H=1 -DHAVE_SYS_SOCKET_H=1 -DHAVE_STRERROR=1 -DHAVE_SOCKET=1 -DHAVE_MEMSET=1 -DHAVE_GETTIMEOFDAY=1 -DHAVE_STDLIB_H=1 -DHAVE_MALLOC=1 -DHAVE_LIBSCTP") set(commonOpts "-pipe -Wno-packed-bitfield-compat -fPIC -Wall -fno-strict-aliasing -rdynamic") @@ -276,7 +289,7 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ggdb2 -Wl,-rpath -Wl,${C # set a flag for changes in the source code # these changes are related to hardcoded path to include .h files set(debugOpt "-ggdb2 -DMALLOC_CHECK_=3 -fno-delete-null-pointer-checks") -set(CMAKE_C_FLAGS_DEBUG "${debugOpt} -O0") +set(CMAKE_C_FLAGS_DEBUG "${debugOpt} -Og") set(CMAKE_C_FLAGS_RELWITHDEBINFO "${debugOpt} -O2") set(CMAKE_C_FLAGS_RELEASE "-O3") @@ -290,7 +303,6 @@ set(GIT_COMMIT_DATE "UNKNOWN") find_package(Git) if(GIT_FOUND) - message("git found: ${GIT_EXECUTABLE}") # Get the current working branch execute_process( COMMAND git rev-parse --abbrev-ref HEAD @@ -752,53 +764,53 @@ set (SHLIB_LOADER_SOURCES ###################################################################### set(HWLIB_USRP_SOURCE - ${OPENAIR_DIR}/sdr/USRP/USERSPACE/LIB/usrp_lib.cpp + ${OPENAIR_DIR}/radio/USRP/USERSPACE/LIB/usrp_lib.cpp ) add_library(oai_usrpdevif MODULE ${HWLIB_USRP_SOURCE} ) target_include_directories(oai_usrpdevif PRIVATE - "${OPENAIR_DIR}/sdr/USRP/USERSPACE/LIB/" + "${OPENAIR_DIR}/radio/USRP/USERSPACE/LIB/" ${Boost_INCLUDE_DIR} ) target_link_libraries(oai_usrpdevif uhd) -include_directories("${OPENAIR_DIR}/sdr/BLADERF/USERSPACE/LIB/") +include_directories("${OPENAIR_DIR}/radio/BLADERF/USERSPACE/LIB/") set(HWLIB_BLADERF_SOURCE - ${OPENAIR_DIR}/sdr/BLADERF/USERSPACE/LIB/bladerf_lib.c + ${OPENAIR_DIR}/radio/BLADERF/USERSPACE/LIB/bladerf_lib.c ) add_library(oai_bladerfdevif MODULE ${HWLIB_BLADERF_SOURCE} ) target_link_libraries(oai_bladerfdevif bladeRF) -include_directories("${OPENAIR_DIR}/sdr/LMSSDR/USERSPACE/LIB/") +include_directories("${OPENAIR_DIR}/radio/LMSSDR/USERSPACE/LIB/") set(HWLIB_LMSSDR_SOURCE - ${OPENAIR_DIR}/sdr/LMSSDR/USERSPACE/LIB/lms_lib.cpp + ${OPENAIR_DIR}/radio/LMSSDR/USERSPACE/LIB/lms_lib.cpp ) add_library(oai_lmssdrdevif MODULE ${HWLIB_LMSSDR_SOURCE} ) target_include_directories(oai_lmssdrdevif PRIVATE /usr/local/include/lime) target_link_libraries(oai_lmssdrdevif LimeSuite ) -include_directories("${OPENAIR_DIR}/sdr/ETHERNET/USERSPACE/LIB/") +include_directories("${OPENAIR_DIR}/radio/ETHERNET/USERSPACE/LIB/") set(TPLIB_ETHERNET_SOURCE - ${OPENAIR_DIR}/sdr/ETHERNET/USERSPACE/LIB/ethernet_lib.c - ${OPENAIR_DIR}/sdr/ETHERNET/USERSPACE/LIB/eth_udp.c - ${OPENAIR_DIR}/sdr/ETHERNET/USERSPACE/LIB/eth_raw.c + ${OPENAIR_DIR}/radio/ETHERNET/USERSPACE/LIB/ethernet_lib.c + ${OPENAIR_DIR}/radio/ETHERNET/USERSPACE/LIB/eth_udp.c + ${OPENAIR_DIR}/radio/ETHERNET/USERSPACE/LIB/eth_raw.c ) add_library(oai_eth_transpro MODULE ${TPLIB_ETHERNET_SOURCE} ) -include_directories("${OPENAIR_DIR}/sdr/AW2SORI/") +include_directories("${OPENAIR_DIR}/radio/AW2SORI/") link_directories("/usr/local/lib") set(HWLIB_AW2SORI_SOURCE - ${OPENAIR_DIR}/sdr/AW2SORI/oaiori.c + ${OPENAIR_DIR}/radio/AW2SORI/oaiori.c ) add_library(aw2sori_transpro MODULE ${HWLIB_AW2SORI_SOURCE}) target_compile_options(aw2sori_transpro PRIVATE -shared -fPIC -msse4 -g -ggdb -DLITE_COMPILATION) target_link_libraries(aw2sori_transpro libori.so) -include_directories("${OPENAIR_DIR}/sdr/IRIS/USERSPACE/LIB/") +include_directories("${OPENAIR_DIR}/radio/IRIS/USERSPACE/LIB/") set(option_HWIRISLIB_lib "-l SoapySDR") set(HWLIB_IRIS_SOURCE - ${OPENAIR_DIR}/sdr/IRIS/USERSPACE/LIB/iris_lib.cpp + ${OPENAIR_DIR}/radio/IRIS/USERSPACE/LIB/iris_lib.cpp ) add_library(oai_irisdevif MODULE ${HWLIB_IRIS_SOURCE}) target_include_directories(oai_irisdevif PRIVATE /usr/local/lib/SoapySDR/modules0.7/) @@ -811,11 +823,11 @@ target_link_libraries(oai_irisdevif SoapySDR) include_directories ("/usr/include/dpdk") set(HWLIB_BENETEL_4G_SOURCE - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/4g/benetel.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/4g/shared_buffers.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/4g/low.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/4g/low_dpdk.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/4g/dpdk_driver.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/4g/benetel.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/4g/shared_buffers.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/4g/low.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/4g/low_dpdk.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/4g/dpdk_driver.c ) add_library(benetel_4g MODULE ${HWLIB_BENETEL_4G_SOURCE} ) @@ -829,11 +841,11 @@ TARGET_LINK_LIBRARIES(benetel_4g pthread dl rt m numa) ###################################################################### set(HWLIB_BENETEL_5G_SOURCE - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/5g/benetel.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/5g/shared_buffers.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/5g/low.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/5g/low_dpdk.c - ${OPENAIR_DIR}/sdr/ETHERNET/benetel/5g/dpdk_driver.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/5g/benetel.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/5g/shared_buffers.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/5g/low.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/5g/low_dpdk.c + ${OPENAIR_DIR}/radio/ETHERNET/benetel/5g/dpdk_driver.c ) add_library(benetel_5g MODULE ${HWLIB_BENETEL_5G_SOURCE} ) @@ -870,7 +882,7 @@ endif () ########################################################## -include_directories ("${OPENAIR_DIR}/sdr/COMMON") +include_directories ("${OPENAIR_DIR}/radio/COMMON") ############################################################## # ???!!! TO BE DOCUMENTED OPTIONS !!!??? @@ -1035,7 +1047,7 @@ include_directories("${OPENAIR3_DIR}/ocp-gtpu") include_directories("${OPENAIR3_DIR}/M3AP") include_directories("${OPENAIR3_DIR}/MME_APP") include_directories("${OPENAIR_DIR}/targets/COMMON") -include_directories("${OPENAIR_DIR}/sdr/COMMON") +include_directories("${OPENAIR_DIR}/radio/COMMON") include_directories("${OPENAIR2_DIR}/ENB_APP/CONTROL_MODULES/PHY") include_directories("${OPENAIR2_DIR}/ENB_APP/CONTROL_MODULES/MAC") include_directories("${OPENAIR2_DIR}/ENB_APP/CONTROL_MODULES/RRC") @@ -2112,12 +2124,6 @@ include_directories("${OPENAIR1_DIR}/SCHED_NR_UE") # CN libs ########################## -add_library(CN_UTILS - ${OPENAIR3_DIR}/UTILS/conversions.c - ${OPENAIR3_DIR}/UTILS/enum_string.c - ${OPENAIR3_DIR}/UTILS/mcc_mnc_itu.c - ) - add_library (GTPV1U ${NR_RRC_DIR}/rrc_gNB_GTPV1U.c ${RRC_DIR}/rrc_eNB_GTPV1U.c @@ -2539,15 +2545,15 @@ target_link_libraries(nrscope ${XFORMS_LIBRARIES}) add_library(rfsimulator MODULE - ${OPENAIR_DIR}/sdr/rfsimulator/simulator.c - ${OPENAIR_DIR}/sdr/rfsimulator/apply_channelmod.c - ${OPENAIR_DIR}/sdr/rfsimulator/new_channel_sim.c + ${OPENAIR_DIR}/radio/rfsimulator/simulator.c + ${OPENAIR_DIR}/radio/rfsimulator/apply_channelmod.c + ${OPENAIR_DIR}/radio/rfsimulator/new_channel_sim.c ${OPENAIR1_DIR}/PHY/TOOLS/signal_energy.c ) target_link_libraries(rfsimulator SIMU_COMMON ${ATLAS_LIBRARIES}) add_library(oai_iqplayer MODULE - ${OPENAIR_DIR}/sdr/iqplayer/iqplayer_lib.c + ${OPENAIR_DIR}/radio/iqplayer/iqplayer_lib.c ) set(CMAKE_MODULE_PATH "${OPENAIR_DIR}/cmake_targets/tools/MODULES" "${CMAKE_MODULE_PATH}") @@ -2608,7 +2614,7 @@ add_executable(nfapi_test ) add_executable(replay_node - ${OPENAIR_DIR}/sdr/rfsimulator/stored_node.c + ${OPENAIR_DIR}/radio/rfsimulator/stored_node.c ) target_link_libraries (replay_node minimal_lib) @@ -2633,10 +2639,10 @@ add_executable(lte-softmodem ${OPENAIR_DIR}/executables/softmodem-common.c ${OPENAIR2_DIR}/ENB_APP/NB_IoT_interface.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c - ${OPENAIR_TARGETS}/COMMON/create_tasks.c - ${OPENAIR_TARGETS}/COMMON/create_tasks_mbms.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c - ${OPENAIR_DIR}/sdr/COMMON/record_player.c + ${OPENAIR_DIR}/executables/create_tasks.c + ${OPENAIR_DIR}/executables/create_tasks_mbms.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/record_player.c ${OPENAIR2_DIR}/RRC/NAS/nas_config.c ${OPENAIR2_DIR}/RRC/NAS/rb_config.c ${OPENAIR2_DIR}/F1AP/dummy_enb.c @@ -2673,10 +2679,10 @@ add_executable(ocp-enb ${OPENAIR_DIR}/executables/transport_split.c ${OPENAIR2_DIR}/ENB_APP/NB_IoT_interface.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c - ${OPENAIR_TARGETS}/COMMON/create_tasks.c - ${OPENAIR_TARGETS}/COMMON/create_tasks_mbms.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c - ${OPENAIR_DIR}/sdr/COMMON/record_player.c + ${OPENAIR_DIR}/executables/create_tasks.c + ${OPENAIR_DIR}/executables/create_tasks_mbms.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/record_player.c ${OPENAIR2_DIR}/RRC/NAS/nas_config.c ${OPENAIR2_DIR}/RRC/NAS/rb_config.c ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c @@ -2704,8 +2710,8 @@ target_link_libraries (ocp-enb ${LIBXML2_LIBRARIES} pthread m CONFIG_LIB rt cryp add_executable(oairu ${OPENAIR_TARGETS}/RT/USER/lte-ru.c ${OPENAIR_TARGETS}/RT/USER/ru_control.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c - ${OPENAIR_DIR}/sdr/COMMON/record_player.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/record_player.c ${OPENAIR_DIR}/executables/softmodem-common.c ${OPENAIR_DIR}/openair1/SCHED/phy_procedures_lte_common.c ${OPENAIR_DIR}/executables/main_ru.c @@ -2729,9 +2735,9 @@ add_executable(lte-uesoftmodem ${OPENAIR_TARGETS}/RT/USER/lte-ue.c ${OPENAIR_TARGETS}/RT/USER/lte-uesoftmodem.c ${OPENAIR_DIR}/executables/softmodem-common.c - ${OPENAIR_TARGETS}/COMMON/create_tasks_ue.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c - ${OPENAIR_DIR}/sdr/COMMON/record_player.c + ${OPENAIR_DIR}/executables/create_tasks_ue.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/record_player.c ${OPENAIR2_DIR}/RRC/NAS/nas_config.c ${OPENAIR2_DIR}/RRC/NAS/rb_config.c ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c @@ -2776,8 +2782,8 @@ add_executable(nr-softmodem ${OPENAIR_DIR}/executables/nr-softmodem.c ${OPENAIR_DIR}/executables/softmodem-common.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c - ${OPENAIR_DIR}/sdr/COMMON/record_player.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/record_player.c ${OPENAIR2_DIR}/RRC/NAS/nas_config.c ${OPENAIR2_DIR}/RRC/NAS/rb_config.c ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c @@ -2821,8 +2827,8 @@ add_executable(nr-uesoftmodem ${OPENAIR_DIR}/executables/nr-ue.c ${OPENAIR_DIR}/executables/softmodem-common.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c - ${OPENAIR_DIR}/sdr/COMMON/record_player.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/record_player.c ${OPENAIR2_DIR}/RRC/NAS/nas_config.c ${OPENAIR2_DIR}/LAYER2/NR_MAC_COMMON/nr_mac_common.c ${OPENAIR2_DIR}/RRC/NAS/rb_config.c @@ -2875,7 +2881,7 @@ target_link_libraries (dlsim_tm4 add_executable(rftest ${OPENAIR_DIR}/openair1/PHY/TOOLS/calibration_test.c ${OPENAIR_DIR}/openair1/PHY/TOOLS/calibration_scope.c - ${OPENAIR_DIR}/sdr/COMMON/common_lib.c + ${OPENAIR_DIR}/radio/COMMON/common_lib.c ${OPENAIR_DIR}/executables/softmodem-common.c ${SHLIB_LOADER_SOURCES} ) diff --git a/ci-scripts/cls_amarisoft_ue.py b/ci-scripts/cls_amarisoft_ue.py index 367bfaac3f..9f4976792d 100644 --- a/ci-scripts/cls_amarisoft_ue.py +++ b/ci-scripts/cls_amarisoft_ue.py @@ -61,7 +61,7 @@ class AS_UE: def KillASUE(self): mySSH = sshconnection.SSHConnection() mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) - mySSH.command('killall --signal SIGKILL lteue-avx2', '#', 5) + mySSH.command('killall --signal SIGKILL lteue-avx2', '\$', 5) mySSH.close() def RunScenario(self): @@ -70,10 +70,10 @@ class AS_UE: logging.debug("Deleting old artifacts :") cmd='rm -rf ' + self.Ping + ' ' + self.UELog - mySSH.command(cmd,'#',5) + mySSH.command(cmd, '\$',5) logging.debug("Running scenario :") cmd='echo $USER; nohup '+self.Cmd + ' ' + self.Config + ' &' - mySSH.command(cmd,'#',5) + mySSH.command(cmd, '\$',5) mySSH.close() diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 5cf6b76b38..e3893b8179 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -81,7 +81,7 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge if ranTargetBranch == '': ranTargetBranch = 'develop' logging.debug(f'Merging with the target branch: {ranTargetBranch}') - sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 5) + sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30) def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw): sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5) @@ -208,6 +208,7 @@ class Containerize(): self.imageToCopy = '' self.registrySvrId = '' self.testSvrId = '' + self.imageToPull = [] #checkers from xml self.ran_checkers={} @@ -547,72 +548,194 @@ class Containerize(): HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) - def Copy_Image_to_Test_Server(self, HTML): - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' + def ImageTagToUse(self, imageName): + shortCommit = self.ranCommitID[0:8] + if self.ranAllowMerge: + tagToUse = f'{self.ranBranch}-{shortCommit}' + else: + tagToUse = f'develop-{shortCommit}' + fullTag = f'porcepix.sboai.cs.eurecom.fr/{imageName}:{tagToUse}' + return fullTag - lSsh = SSH.SSHConnection() - # Going to the Docker Registry server + def Push_Image_to_Local_Registry(self, HTML): if self.registrySvrId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath elif self.registrySvrId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath elif self.registrySvrId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password - lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - ret = lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') - if ret != 0: - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('cannot remove'): - HTML.CreateHtmlTestRow('file not created by docker save', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.close() + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Pushing images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) - # Going to the Test Server + mySSH.command('docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Login Succeeded', mySSH.getBefore()) is None: + msg = 'Could not log into local registry' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + + orgTag = 'develop' + if self.ranAllowMerge: + orgTag = 'ci-temp' + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + mySSH.command(f'docker image tag {image}:{orgTag} {tagToUse}', '\$', 5) + mySSH.command(f'docker push {tagToUse}', '\$', 120) + if re.search(': digest:', mySSH.getBefore()) is None: + logging.debug(mySSH.getBefore()) + msg = f'Could not push {image} to local registry : {tagToUse}' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + mySSH.command(f'docker rmi {tagToUse} {image}:{orgTag}', '\$', 30) + + mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Removing login credentials', mySSH.getBefore()) is None: + msg = 'Could not log off from local registry' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + + mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + return True + + def Pull_Image_from_Local_Registry(self, HTML): + # This method can be called either onto a remote server (different from python executor) + # or directly on the python executor (ie lIpAddr == 'none') if self.testSvrId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath elif self.testSvrId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath elif self.testSvrId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password - lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~') - # copyout has no return code and will quit if something fails - lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10) - lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('o such file') or lSsh.getBefore().count('invalid tar header'): - logging.debug(lSsh.getBefore()) - HTML.CreateHtmlTestRow('problem during docker load', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('cannot remove'): - HTML.CreateHtmlTestRow('file not copied during scp?', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.close() + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + if lIpAddr != 'none': + logging.debug('Pulling images onto server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + else: + logging.debug('Pulling images locally') - if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'): - os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz') + cmd = 'docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True, stderr=subprocess.STDOUT) + if re.search('Login Succeeded', response) is None: + msg = 'Could not log into local registry' + logging.error(msg) + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + for image in self.imageToPull: + tagToUse = self.ImageTagToUse(image) + cmd = f'docker pull {tagToUse}' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 120) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True) + if re.search('Status: Downloaded newer image for |Status: Image is up to date for', response) is None: + logging.debug(response) + msg = f'Could not pull {image} from local registry : {tagToUse}' + logging.error(msg) + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK) + return False + + cmd = 'docker logout porcepix.sboai.cs.eurecom.fr' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True) + if re.search('Removing login credentials', response) is None: + msg = 'Could not log off from local registry' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + return True + + def Clean_Test_Server_Images(self, HTML): + # This method can be called either onto a remote server (different from python executor) + # or directly on the python executor (ie lIpAddr == 'none') + if self.testSvrId == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.testSvrId == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.testSvrId == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + if lIpAddr != 'none': + logging.debug('Removing test images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + else: + logging.debug('Removing test images locally') + + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + cmd = f'docker rmi {self.ImageTagToUse(image)} || true' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + else: + logging.info(cmd) + subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + if lIpAddr != 'none': + mySSH.close() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True @@ -639,16 +762,15 @@ class Containerize(): mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) - + CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5) - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' - mySSH.command('sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) - mySSH.command('sed -i -e "s/image: oai-gnb:latest/image: oai-gnb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) + imagesList = ['oai-enb', 'oai-gnb'] + for image in imagesList: + imageToUse = self.ImageTagToUse(image) + mySSH.command(f'sed -i -e "s#image: {image}:latest#image: {imageToUse}#" ci-docker-compose.yml', '\$', 2) localMmeIpAddr = EPC.MmeIPAddress mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2) # if self.flexranCtrlDeployed: @@ -672,6 +794,8 @@ class Containerize(): healthyNb = 0 startingNb = 0 containerName = '' + usedImage = '' + imageInfo = '' if result is not None: containerName = result.group('container_name') time.sleep(5) @@ -686,6 +810,20 @@ class Containerize(): else: time.sleep(10) cnt += 1 + mySSH.command('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName, '\$', 5) + for stdoutLine in mySSH.getBefore().split('\n'): + if stdoutLine.count('ImageUsed: porcepix'): + usedImage = stdoutLine.replace('ImageUsed: porcepix', 'porcepix').strip() + logging.debug('Used image is ' + usedImage) + if usedImage != '': + mySSH.command('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage, '\$', 5, silent=True) + for stdoutLine in mySSH.getBefore().split('\n'): + if re.search('Size = [0-9]', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' + if re.search('Creation = [0-9]', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' + if re.search('Id = sha256', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') @@ -715,11 +853,24 @@ class Containerize(): mySSH.copyin(lIpAddr, lUserName, lPassWord, logfilename, '.') mySSH.close() + html_queue = SimpleQueue() + html_cell = '
\n'
+		if usedImage != '':
+			html_cell += f'Used Image = {usedImage} :\n'
+			html_cell += imageInfo
+		else:
+			html_cell += 'Could not retrieve used image info!\n'
 		if status:
-			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+			html_cell += '\nHealthy deployment!\n'
+		else:
+			html_cell += '\nUnhealthy deployment! -- Check logs for reason!\n'
+		html_cell += '
' + html_queue.put(html_cell) + if status: + HTML.CreateHtmlTestRowQueue('N/A', 'OK', CONST.ENB_PROCESS_OK, html_queue) else: self.exitStatus = 1 - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRowQueue('N/A', 'KO', CONST.ENB_PROCESS_OK, html_queue) def UndeployObject(self, HTML, RAN): @@ -800,33 +951,40 @@ class Containerize(): def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0 - logging.info('\u001B[1m Checking Services to deploy\u001B[0m') + logging.debug('\u001B[1m Checking Services to deploy\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services' - logging.debug(cmd) + logging.info(cmd) try: listServices = subprocess.check_output(cmd, shell=True, universal_newlines=True) except Exception as e: self.exitStatus = 1 HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) return + displayUsedTag = False for reqSvc in self.services[0].split(' '): res = re.search(reqSvc, listServices) if res is None: logging.error(reqSvc + ' not found in specified docker-compose') self.exitStatus = 1 + res = re.search('oai-gnb|oai-nr-ue|oai-cu|oai-du|oai_enb|oai_ue', reqSvc) + if res is not None: + displayUsedTag = True if (self.exitStatus == 1): HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) return - if (self.ranAllowMerge): - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@ci-temp@" docker-compose.y*ml > docker-compose-ci.yml' - else: - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@develop@" docker-compose.y*ml > docker-compose-ci.yml' - logging.debug(cmd) - subprocess.run(cmd, shell=True) + cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml' + subprocess.run(cmd, shell=True) + if displayUsedTag: + tagToUse = self.ImageTagToUse('oai-xxx') + logging.info(f'\u001B[1m Using Image Tag: {tagToUse}\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml up -d ' + self.services[0] - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -835,7 +993,7 @@ class Containerize(): HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK) return - logging.info('\u001B[1m Checking if all deployed healthy\u001B[0m') + logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a' count = 0 healthy = 0 @@ -930,7 +1088,7 @@ class Containerize(): cmd += ' -w /tmp/capture_' ymlPath = self.yamlPath[0].split('/') cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &' - logging.debug(cmd) + logging.info(cmd) networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) self.tsharkStarted = True @@ -939,22 +1097,23 @@ class Containerize(): ymlPath = self.yamlPath[0].split('/') logPath = '../cmake_targets/log/' + ymlPath[1] - if (self.ranAllowMerge): - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@ci-temp@" docker-compose.y*ml > docker-compose-ci.yml' - else: - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@develop@" docker-compose.y*ml > docker-compose-ci.yml' - logging.debug(cmd) + cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' subprocess.run(cmd, shell=True) + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml' + subprocess.run(cmd, shell=True) # check which containers are running for log recovery later cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' - logging.debug(cmd) + logging.info(cmd) deployStatusLogs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) # Stop the containers to shut down objects logging.debug('\u001B[1m Stopping containers \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml stop' - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -976,16 +1135,16 @@ class Containerize(): anyLogs = True cName = res.group('container_name') cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1' - logging.debug(cmd) + logging.info(cmd) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) if re.search('magma-mme', cName) is not None: cmd = 'cd ' + self.yamlPath[0] + ' && docker cp -L ' + cName + ':/var/log/mme.log ' + cName + '-full.log' - logging.debug(cmd) + logging.info(cmd) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) fullStatus = True if anyLogs: cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) # Analyzing log file(s)! @@ -1035,25 +1194,25 @@ class Containerize(): HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK) cmd = 'rm ' + self.yamlPath[0] + '/*.log' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) if self.tsharkStarted: self.tsharkStarted = True ymlPath = self.yamlPath[0].split('/') cmd = 'sudo chmod 666 /tmp/capture_' + ymlPath[1] + '.pcap' - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) cmd = 'cp /tmp/capture_' + ymlPath[1] + '.pcap ' + logPath - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) cmd = 'sudo rm /tmp/capture_' + ymlPath[1] + '.pcap' - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) self.tsharkStarted = False logging.debug('\u001B[1m Undeploying \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down' - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -1066,7 +1225,7 @@ class Containerize(): self.deployedContainers = [] # Cleaning any created tmp volume cmd = 'docker volume prune --force || true' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) if fullStatus: @@ -1074,7 +1233,7 @@ class Containerize(): logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m') else: HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK) - logging.info('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') + logging.error('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') def StatsFromGenObject(self, HTML): self.exitStatus = 0 @@ -1083,7 +1242,7 @@ class Containerize(): # if the containers are running, recover the logs! cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) cmd = 'docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" ' anyLogs = False @@ -1099,7 +1258,7 @@ class Containerize(): cmd += res.group('container_name') + ' ' message = '' if anyLogs: - logging.debug(cmd) + logging.info(cmd) stats = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) for statLine in stats.split('\n'): logging.debug(statLine) @@ -1119,7 +1278,7 @@ class Containerize(): cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ' + logPath + '/ping_' + HTML.testCase_id + '.log || true' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', deployStatus) @@ -1176,9 +1335,9 @@ class Containerize(): logging.error('\u001B[1;37;41m ping test FAIL -- ' + message + ' \u001B[0m') HTML.CreateHtmlTestRowQueue(self.pingOptions, 'KO', 1, html_queue) # Automatic undeployment - logging.debug('----------------------------------------') - logging.debug('\u001B[1m Starting Automatic undeployment \u001B[0m') - logging.debug('----------------------------------------') + logging.warning('----------------------------------------') + logging.warning('\u001B[1m Starting Automatic undeployment \u001B[0m') + logging.warning('----------------------------------------') HTML.testCase_id = 'AUTO-UNDEPLOY' HTML.desc = 'Automatic Un-Deployment' self.UndeployGenObject(HTML, RAN, UE) @@ -1194,23 +1353,23 @@ class Containerize(): # Start the server process cmd = 'docker exec -d ' + self.svrContName + ' /bin/bash -c "nohup iperf ' + self.svrOptions + ' > /tmp/iperf_server.log 2>&1" || true' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) time.sleep(5) # Start the client process cmd = 'docker exec ' + self.cliContName + ' /bin/bash -c "iperf ' + self.cliOptions + '" 2>&1 | tee '+ logPath + '/iperf_client_' + HTML.testCase_id + '.log || true' - logging.debug(cmd) + logging.info(cmd) clientStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) # Stop the server process cmd = 'docker exec ' + self.svrContName + ' /bin/bash -c "pkill iperf" || true' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) time.sleep(5) cmd = 'docker cp ' + self.svrContName + ':/tmp/iperf_server.log '+ logPath + '/iperf_server_' + HTML.testCase_id + '.log' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) # Analyze client output diff --git a/ci-scripts/cls_log_mgt.py b/ci-scripts/cls_log_mgt.py index f1dbf6b8ec..a78eee5a33 100644 --- a/ci-scripts/cls_log_mgt.py +++ b/ci-scripts/cls_log_mgt.py @@ -35,11 +35,6 @@ import re import subprocess import sshconnection -logging.basicConfig( - level=logging.DEBUG, - format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" -) - class Log_Mgt: def __init__(self,Username, IPAddress,Password,Path): diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 536f2ff49e..48123f52ad 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -33,6 +33,7 @@ #----------------------------------------------------------- import sys # arg import re # reg +import fileinput import logging import os import time @@ -102,9 +103,9 @@ class HTMLManagement(): def CreateHtmlHeader(self, ADBIPAddress): if (not self.htmlHeaderCreated): - logging.debug('\u001B[1m----------------------------------------\u001B[0m') - logging.debug('\u001B[1m Creating HTML header \u001B[0m') - logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m Creating HTML header \u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') self.htmlFile = open('test_results.html', 'w') self.htmlFile.write('\n') self.htmlFile.write('\n') @@ -195,7 +196,10 @@ class HTMLManagement(): self.htmlFile.write('