diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 9570610cd8..b8dc8b32fb 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -579,9 +579,17 @@ class Containerize(): lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') + lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + ret = lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') + if ret != 0: + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + if lSsh.getBefore().count('cannot remove'): + HTML.CreateHtmlTestRow('file not created by docker save', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.close() # Going to the Test Server @@ -599,15 +607,26 @@ class Containerize(): lPassWord = self.eNB2Password lSsh.open(lIpAddr, lUserName, lPassWord) lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~') + # copyout has no return code and will quit if something fails lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10) lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + if lSsh.getBefore().count('o such file') or lSsh.getBefore().count('invalid tar header'): + logging.debug(lSsh.getBefore()) + HTML.CreateHtmlTestRow('problem during docker load', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + if lSsh.getBefore().count('cannot remove'): + HTML.CreateHtmlTestRow('file not copied during scp?', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.close() if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'): os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz') HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + return True def DeployObject(self, HTML, EPC): if self.eNB_serverId[self.eNB_instance] == '0': @@ -683,6 +702,9 @@ class Containerize(): logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') + self.testCase_id = HTML.testCase_id + self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log' + status = False if healthyNb == 1: cnt = 0 @@ -697,11 +719,12 @@ class Containerize(): status = True logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m') time.sleep(10) + else: + # containers are unhealthy, so we won't start. However, logs are stored at the end + # in UndeployObject so we here store the logs of the unhealthy container to report it + mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) mySSH.close() - self.testCase_id = HTML.testCase_id - self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log' - if status: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) else: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index baed6a6011..8a4b59ede6 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -921,7 +921,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'Build_Proxy': CONTAINERS.BuildProxy(HTML) elif action == 'Copy_Image_to_Test': - CONTAINERS.Copy_Image_to_Test_Server(HTML) + success = CONTAINERS.Copy_Image_to_Test_Server(HTML) + if not success: + RAN.prematureExit = True elif action == 'Deploy_Object': CONTAINERS.DeployObject(HTML, EPC) if CONTAINERS.exitStatus==1: diff --git a/cmake_targets/tools/build_helper b/cmake_targets/tools/build_helper index d35f90fcce..608ace0486 100755 --- a/cmake_targets/tools/build_helper +++ b/cmake_targets/tools/build_helper @@ -237,8 +237,9 @@ compilations() { echo_error "$2 compilation failed" exit 1 fi - if [ -s $3 ] ; then - cp $3 $4 + if [ -s "$3" ] ; then + rm -f "$4" + ln -s "$PWD/$3" "$4" echo_success "$2 compiled" check_warnings "$dlog/$2.$REL.txt" else diff --git a/docker/Dockerfile.eNB.rhel8.2 b/docker/Dockerfile.eNB.rhel8.2 index 5296ac2172..23afb5f38d 100644 --- a/docker/Dockerfile.eNB.rhel8.2 +++ b/docker/Dockerfile.eNB.rhel8.2 @@ -43,6 +43,7 @@ RUN yum update -y && \ tzdata \ procps-ng \ atlas \ + gdb \ python3 \ python3-pip \ net-tools \ diff --git a/docker/Dockerfile.eNB.ubuntu18 b/docker/Dockerfile.eNB.ubuntu18 index f95a03398a..2b50dcadfb 100644 --- a/docker/Dockerfile.eNB.ubuntu18 +++ b/docker/Dockerfile.eNB.ubuntu18 @@ -53,6 +53,7 @@ RUN apt-get update && \ iputils-ping \ iproute2 \ iperf \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/Dockerfile.gNB.rhel8.2 b/docker/Dockerfile.gNB.rhel8.2 index 9ec7ee7556..f782384f58 100644 --- a/docker/Dockerfile.gNB.rhel8.2 +++ b/docker/Dockerfile.gNB.rhel8.2 @@ -46,6 +46,7 @@ RUN yum repolist --disablerepo=* && \ lksctp-tools \ nettle \ tzdata \ + gdb \ python3 \ python3-pip \ net-tools \ diff --git a/docker/Dockerfile.gNB.ubuntu18 b/docker/Dockerfile.gNB.ubuntu18 index 6a98c8d3ce..11669c2af4 100644 --- a/docker/Dockerfile.gNB.ubuntu18 +++ b/docker/Dockerfile.gNB.ubuntu18 @@ -52,6 +52,7 @@ RUN apt-get update && \ net-tools \ iproute2 \ iputils-ping \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/Dockerfile.lteRU.rhel8.2 b/docker/Dockerfile.lteRU.rhel8.2 index b41aee36dc..f20910ff51 100644 --- a/docker/Dockerfile.lteRU.rhel8.2 +++ b/docker/Dockerfile.lteRU.rhel8.2 @@ -41,6 +41,7 @@ RUN yum update -y && \ tzdata \ procps-ng \ atlas \ + gdb \ python3 \ python3-pip \ net-tools \ diff --git a/docker/Dockerfile.lteRU.ubuntu18 b/docker/Dockerfile.lteRU.ubuntu18 index 1810afb621..157a5a615a 100644 --- a/docker/Dockerfile.lteRU.ubuntu18 +++ b/docker/Dockerfile.lteRU.ubuntu18 @@ -47,6 +47,7 @@ RUN apt-get update && \ libatlas3-base \ libconfig9 \ net-tools \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/Dockerfile.lteUE.rhel8.2 b/docker/Dockerfile.lteUE.rhel8.2 index ab7d224939..303394ea27 100644 --- a/docker/Dockerfile.lteUE.rhel8.2 +++ b/docker/Dockerfile.lteUE.rhel8.2 @@ -44,6 +44,7 @@ RUN yum update -y && \ nettle \ tzdata \ atlas \ + gdb \ python3 \ python3-pip \ iproute \ diff --git a/docker/Dockerfile.nrUE.rhel8.2 b/docker/Dockerfile.nrUE.rhel8.2 index dfb28cfd1d..3ee93ed5d2 100644 --- a/docker/Dockerfile.nrUE.rhel8.2 +++ b/docker/Dockerfile.nrUE.rhel8.2 @@ -46,6 +46,7 @@ RUN yum update -y && \ iputils \ iproute \ atlas \ + gdb \ python3 \ python3-pip \ libXpm \ diff --git a/docker/Dockerfile.nrUE.ubuntu18 b/docker/Dockerfile.nrUE.ubuntu18 index 9f68b947cd..0997783b8b 100644 --- a/docker/Dockerfile.nrUE.ubuntu18 +++ b/docker/Dockerfile.nrUE.ubuntu18 @@ -50,6 +50,7 @@ RUN apt-get update && \ libconfig9 \ openssl \ net-tools \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/scripts/enb_entrypoint.sh b/docker/scripts/enb_entrypoint.sh index 825dfd73c1..2d49a0ed20 100755 --- a/docker/scripts/enb_entrypoint.sh +++ b/docker/scripts/enb_entrypoint.sh @@ -51,9 +51,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then @@ -62,6 +68,9 @@ elif [[ -v USE_N3XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t n3xx fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting eNB soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/gnb_entrypoint.sh b/docker/scripts/gnb_entrypoint.sh index 6deff8da10..4e8983599e 100755 --- a/docker/scripts/gnb_entrypoint.sh +++ b/docker/scripts/gnb_entrypoint.sh @@ -51,9 +51,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then @@ -62,6 +68,9 @@ elif [[ -v USE_N3XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t n3xx fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting gNB soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/lte_ru_entrypoint.sh b/docker/scripts/lte_ru_entrypoint.sh index 297e437ac0..746efb30b3 100755 --- a/docker/scripts/lte_ru_entrypoint.sh +++ b/docker/scripts/lte_ru_entrypoint.sh @@ -31,9 +31,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then @@ -42,6 +48,9 @@ elif [[ -v USE_N3XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t n3xx fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting eNB soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/lte_ue_entrypoint.sh b/docker/scripts/lte_ue_entrypoint.sh index 82e96dd07f..2bdcb873c9 100755 --- a/docker/scripts/lte_ue_entrypoint.sh +++ b/docker/scripts/lte_ue_entrypoint.sh @@ -30,6 +30,10 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done #now generate USIM files @@ -38,6 +42,8 @@ cd $PREFIX $PREFIX/bin/conf2uedata -c $PREFIX/etc/ue_usim.conf -o $PREFIX # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then @@ -57,6 +63,9 @@ if [[ -v USE_NFAPI ]]; then new_args+=("$PREFIX/etc/ue.conf") fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting LTE UE soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/nr_ue_entrypoint.sh b/docker/scripts/nr_ue_entrypoint.sh index 38b080a41b..616d71b555 100755 --- a/docker/scripts/nr_ue_entrypoint.sh +++ b/docker/scripts/nr_ue_entrypoint.sh @@ -40,9 +40,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then @@ -58,6 +64,9 @@ while [[ $# -gt 0 ]]; do shift done +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting NR UE soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then