From 3a0a14cf7d4c59996956ed670a66fe9e476679cc Mon Sep 17 00:00:00 2001 From: laurent Date: Thu, 18 Aug 2022 14:59:16 +0200 Subject: [PATCH 1/5] gzip optim, links for size --- ci-scripts/cls_containerize.py | 2 +- cmake_targets/tools/build_helper | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 9570610cd8..24e2f9b187 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -579,7 +579,7 @@ class Containerize(): lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) lSsh.close() diff --git a/cmake_targets/tools/build_helper b/cmake_targets/tools/build_helper index d35f90fcce..608ace0486 100755 --- a/cmake_targets/tools/build_helper +++ b/cmake_targets/tools/build_helper @@ -237,8 +237,9 @@ compilations() { echo_error "$2 compilation failed" exit 1 fi - if [ -s $3 ] ; then - cp $3 $4 + if [ -s "$3" ] ; then + rm -f "$4" + ln -s "$PWD/$3" "$4" echo_success "$2 compiled" check_warnings "$dlog/$2.$REL.txt" else From c1b448cd1ac97ab9faea20115c32a0951054fe55 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 19 Aug 2022 10:01:55 +0200 Subject: [PATCH 2/5] Handle errors in Copy_Image_to_Test_Server() --- ci-scripts/cls_containerize.py | 21 ++++++++++++++++++++- ci-scripts/main.py | 4 +++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 24e2f9b187..81714c18c7 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -580,8 +580,16 @@ class Containerize(): lPassWord = self.eNB2Password lSsh.open(lIpAddr, lUserName, lPassWord) lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') + ret = lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') + if ret != 0: + HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + if lSsh.getBefore().count('cannot remove'): + HTML.CreateHtmlTestRow('file not created by docker save', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.close() # Going to the Test Server @@ -599,15 +607,26 @@ class Containerize(): lPassWord = self.eNB2Password lSsh.open(lIpAddr, lUserName, lPassWord) lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~') + # copyout has no return code and will quit if something fails lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10) lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + if lSsh.getBefore().count('o such file') or lSsh.getBefore().count('invalid tar header'): + logging.debug(lSsh.getBefore()) + HTML.CreateHtmlTestRow('problem during docker load', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) + if lSsh.getBefore().count('cannot remove'): + HTML.CreateHtmlTestRow('file not copied during scp?', 'KO', CONST.ALL_PROCESSES_OK) + self.exitStatus = 1 + return False lSsh.close() if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'): os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz') HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + return True def DeployObject(self, HTML, EPC): if self.eNB_serverId[self.eNB_instance] == '0': diff --git a/ci-scripts/main.py b/ci-scripts/main.py index baed6a6011..8a4b59ede6 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -921,7 +921,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'Build_Proxy': CONTAINERS.BuildProxy(HTML) elif action == 'Copy_Image_to_Test': - CONTAINERS.Copy_Image_to_Test_Server(HTML) + success = CONTAINERS.Copy_Image_to_Test_Server(HTML) + if not success: + RAN.prematureExit = True elif action == 'Deploy_Object': CONTAINERS.DeployObject(HTML, EPC) if CONTAINERS.exitStatus==1: From 23cf8059aa87bf2633b1112aeeeb8b656d46ab4f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 19 Aug 2022 10:09:34 +0200 Subject: [PATCH 3/5] Save logs of unhealthy containers during deployment --- ci-scripts/cls_containerize.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 81714c18c7..b8dc8b32fb 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -702,6 +702,9 @@ class Containerize(): logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') + self.testCase_id = HTML.testCase_id + self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log' + status = False if healthyNb == 1: cnt = 0 @@ -716,11 +719,12 @@ class Containerize(): status = True logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m') time.sleep(10) + else: + # containers are unhealthy, so we won't start. However, logs are stored at the end + # in UndeployObject so we here store the logs of the unhealthy container to report it + mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) mySSH.close() - self.testCase_id = HTML.testCase_id - self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log' - if status: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) else: From 4dc26bf575770a6b210b14397f843fe479f4d3da Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 11 Aug 2022 16:07:40 +0200 Subject: [PATCH 4/5] docker entrypoints: print config, log USRP loader --- docker/scripts/enb_entrypoint.sh | 6 ++++++ docker/scripts/gnb_entrypoint.sh | 6 ++++++ docker/scripts/lte_ru_entrypoint.sh | 6 ++++++ docker/scripts/lte_ue_entrypoint.sh | 6 ++++++ docker/scripts/nr_ue_entrypoint.sh | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/docker/scripts/enb_entrypoint.sh b/docker/scripts/enb_entrypoint.sh index 825dfd73c1..7e787abaa6 100755 --- a/docker/scripts/enb_entrypoint.sh +++ b/docker/scripts/enb_entrypoint.sh @@ -51,9 +51,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then diff --git a/docker/scripts/gnb_entrypoint.sh b/docker/scripts/gnb_entrypoint.sh index 6deff8da10..146e9e8ecf 100755 --- a/docker/scripts/gnb_entrypoint.sh +++ b/docker/scripts/gnb_entrypoint.sh @@ -51,9 +51,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then diff --git a/docker/scripts/lte_ru_entrypoint.sh b/docker/scripts/lte_ru_entrypoint.sh index 297e437ac0..395ea6b109 100755 --- a/docker/scripts/lte_ru_entrypoint.sh +++ b/docker/scripts/lte_ru_entrypoint.sh @@ -31,9 +31,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then diff --git a/docker/scripts/lte_ue_entrypoint.sh b/docker/scripts/lte_ue_entrypoint.sh index 82e96dd07f..6de61f23e4 100755 --- a/docker/scripts/lte_ue_entrypoint.sh +++ b/docker/scripts/lte_ue_entrypoint.sh @@ -30,6 +30,10 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done #now generate USIM files @@ -38,6 +42,8 @@ cd $PREFIX $PREFIX/bin/conf2uedata -c $PREFIX/etc/ue_usim.conf -o $PREFIX # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then diff --git a/docker/scripts/nr_ue_entrypoint.sh b/docker/scripts/nr_ue_entrypoint.sh index 38b080a41b..888c4da792 100755 --- a/docker/scripts/nr_ue_entrypoint.sh +++ b/docker/scripts/nr_ue_entrypoint.sh @@ -40,9 +40,15 @@ for c in ${CONFIG_FILES}; do # render template and inline replace config file sed -i "${EXPRESSIONS}" ${c} + + echo "==================================" + echo "== Configuration file: ${c}" + cat ${c} done # Load the USRP binaries +echo "==================================" +echo "== Load USRP binaries" if [[ -v USE_B2XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t b2xx elif [[ -v USE_X3XX ]]; then From a58b3989d177dd743678ae85c675d1a3eab4901e Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 19 Aug 2022 10:59:16 +0200 Subject: [PATCH 5/5] Install gdb in RAN images to have stack traces on asserts --- docker/Dockerfile.eNB.rhel8.2 | 1 + docker/Dockerfile.eNB.ubuntu18 | 1 + docker/Dockerfile.gNB.rhel8.2 | 1 + docker/Dockerfile.gNB.ubuntu18 | 1 + docker/Dockerfile.lteRU.rhel8.2 | 1 + docker/Dockerfile.lteRU.ubuntu18 | 1 + docker/Dockerfile.lteUE.rhel8.2 | 1 + docker/Dockerfile.nrUE.rhel8.2 | 1 + docker/Dockerfile.nrUE.ubuntu18 | 1 + docker/scripts/enb_entrypoint.sh | 3 +++ docker/scripts/gnb_entrypoint.sh | 3 +++ docker/scripts/lte_ru_entrypoint.sh | 3 +++ docker/scripts/lte_ue_entrypoint.sh | 3 +++ docker/scripts/nr_ue_entrypoint.sh | 3 +++ 14 files changed, 24 insertions(+) diff --git a/docker/Dockerfile.eNB.rhel8.2 b/docker/Dockerfile.eNB.rhel8.2 index 5296ac2172..23afb5f38d 100644 --- a/docker/Dockerfile.eNB.rhel8.2 +++ b/docker/Dockerfile.eNB.rhel8.2 @@ -43,6 +43,7 @@ RUN yum update -y && \ tzdata \ procps-ng \ atlas \ + gdb \ python3 \ python3-pip \ net-tools \ diff --git a/docker/Dockerfile.eNB.ubuntu18 b/docker/Dockerfile.eNB.ubuntu18 index f95a03398a..2b50dcadfb 100644 --- a/docker/Dockerfile.eNB.ubuntu18 +++ b/docker/Dockerfile.eNB.ubuntu18 @@ -53,6 +53,7 @@ RUN apt-get update && \ iputils-ping \ iproute2 \ iperf \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/Dockerfile.gNB.rhel8.2 b/docker/Dockerfile.gNB.rhel8.2 index 9ec7ee7556..f782384f58 100644 --- a/docker/Dockerfile.gNB.rhel8.2 +++ b/docker/Dockerfile.gNB.rhel8.2 @@ -46,6 +46,7 @@ RUN yum repolist --disablerepo=* && \ lksctp-tools \ nettle \ tzdata \ + gdb \ python3 \ python3-pip \ net-tools \ diff --git a/docker/Dockerfile.gNB.ubuntu18 b/docker/Dockerfile.gNB.ubuntu18 index 6a98c8d3ce..11669c2af4 100644 --- a/docker/Dockerfile.gNB.ubuntu18 +++ b/docker/Dockerfile.gNB.ubuntu18 @@ -52,6 +52,7 @@ RUN apt-get update && \ net-tools \ iproute2 \ iputils-ping \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/Dockerfile.lteRU.rhel8.2 b/docker/Dockerfile.lteRU.rhel8.2 index b41aee36dc..f20910ff51 100644 --- a/docker/Dockerfile.lteRU.rhel8.2 +++ b/docker/Dockerfile.lteRU.rhel8.2 @@ -41,6 +41,7 @@ RUN yum update -y && \ tzdata \ procps-ng \ atlas \ + gdb \ python3 \ python3-pip \ net-tools \ diff --git a/docker/Dockerfile.lteRU.ubuntu18 b/docker/Dockerfile.lteRU.ubuntu18 index 1810afb621..157a5a615a 100644 --- a/docker/Dockerfile.lteRU.ubuntu18 +++ b/docker/Dockerfile.lteRU.ubuntu18 @@ -47,6 +47,7 @@ RUN apt-get update && \ libatlas3-base \ libconfig9 \ net-tools \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/Dockerfile.lteUE.rhel8.2 b/docker/Dockerfile.lteUE.rhel8.2 index ab7d224939..303394ea27 100644 --- a/docker/Dockerfile.lteUE.rhel8.2 +++ b/docker/Dockerfile.lteUE.rhel8.2 @@ -44,6 +44,7 @@ RUN yum update -y && \ nettle \ tzdata \ atlas \ + gdb \ python3 \ python3-pip \ iproute \ diff --git a/docker/Dockerfile.nrUE.rhel8.2 b/docker/Dockerfile.nrUE.rhel8.2 index dfb28cfd1d..3ee93ed5d2 100644 --- a/docker/Dockerfile.nrUE.rhel8.2 +++ b/docker/Dockerfile.nrUE.rhel8.2 @@ -46,6 +46,7 @@ RUN yum update -y && \ iputils \ iproute \ atlas \ + gdb \ python3 \ python3-pip \ libXpm \ diff --git a/docker/Dockerfile.nrUE.ubuntu18 b/docker/Dockerfile.nrUE.ubuntu18 index 9f68b947cd..0997783b8b 100644 --- a/docker/Dockerfile.nrUE.ubuntu18 +++ b/docker/Dockerfile.nrUE.ubuntu18 @@ -50,6 +50,7 @@ RUN apt-get update && \ libconfig9 \ openssl \ net-tools \ + gdb \ python \ python3 \ python3-six \ diff --git a/docker/scripts/enb_entrypoint.sh b/docker/scripts/enb_entrypoint.sh index 7e787abaa6..2d49a0ed20 100755 --- a/docker/scripts/enb_entrypoint.sh +++ b/docker/scripts/enb_entrypoint.sh @@ -68,6 +68,9 @@ elif [[ -v USE_N3XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t n3xx fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting eNB soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/gnb_entrypoint.sh b/docker/scripts/gnb_entrypoint.sh index 146e9e8ecf..4e8983599e 100755 --- a/docker/scripts/gnb_entrypoint.sh +++ b/docker/scripts/gnb_entrypoint.sh @@ -68,6 +68,9 @@ elif [[ -v USE_N3XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t n3xx fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting gNB soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/lte_ru_entrypoint.sh b/docker/scripts/lte_ru_entrypoint.sh index 395ea6b109..746efb30b3 100755 --- a/docker/scripts/lte_ru_entrypoint.sh +++ b/docker/scripts/lte_ru_entrypoint.sh @@ -48,6 +48,9 @@ elif [[ -v USE_N3XX ]]; then $PREFIX/bin/uhd_images_downloader.py -t n3xx fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting eNB soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/lte_ue_entrypoint.sh b/docker/scripts/lte_ue_entrypoint.sh index 6de61f23e4..2bdcb873c9 100755 --- a/docker/scripts/lte_ue_entrypoint.sh +++ b/docker/scripts/lte_ue_entrypoint.sh @@ -63,6 +63,9 @@ if [[ -v USE_NFAPI ]]; then new_args+=("$PREFIX/etc/ue.conf") fi +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting LTE UE soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then diff --git a/docker/scripts/nr_ue_entrypoint.sh b/docker/scripts/nr_ue_entrypoint.sh index 888c4da792..616d71b555 100755 --- a/docker/scripts/nr_ue_entrypoint.sh +++ b/docker/scripts/nr_ue_entrypoint.sh @@ -64,6 +64,9 @@ while [[ $# -gt 0 ]]; do shift done +# enable printing of stack traces on assert +export gdbStacks=1 + echo "==================================" echo "== Starting NR UE soft modem" if [[ -v USE_ADDITIONAL_OPTIONS ]]; then