Merge remote-tracking branch 'origin/optimize-ci-time' into integration_2022_wk35

ci-scripts/cls_containerize.py

@@ -579,9 +579,17 @@ class Containerize():
 			lUserName = self.eNB2UserName
 			lPassWord = self.eNB2Password
 		lSsh.open(lIpAddr, lUserName, lPassWord)
-		lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
-		lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.')
+		ret = lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.')
+		if ret != 0:
+			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
+			self.exitStatus = 1
+			return False
 		lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		if lSsh.getBefore().count('cannot remove'):
+			HTML.CreateHtmlTestRow('file not created by docker save', 'KO', CONST.ALL_PROCESSES_OK)
+			self.exitStatus = 1
+			return False
 		lSsh.close()
 
 		# Going to the Test Server
@@ -599,15 +607,26 @@ class Containerize():
 			lPassWord = self.eNB2Password
 		lSsh.open(lIpAddr, lUserName, lPassWord)
 		lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~')
+		# copyout has no return code and will quit if something fails
 		lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10)
 		lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		if lSsh.getBefore().count('o such file') or lSsh.getBefore().count('invalid tar header'):
+			logging.debug(lSsh.getBefore())
+			HTML.CreateHtmlTestRow('problem during docker load', 'KO', CONST.ALL_PROCESSES_OK)
+			self.exitStatus = 1
+			return False
 		lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		if lSsh.getBefore().count('cannot remove'):
+			HTML.CreateHtmlTestRow('file not copied during scp?', 'KO', CONST.ALL_PROCESSES_OK)
+			self.exitStatus = 1
+			return False
 		lSsh.close()
 
 		if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'):
 			os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz')
 
 		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+		return True
 
 	def DeployObject(self, HTML, EPC):
 		if self.eNB_serverId[self.eNB_instance] == '0':
@@ -683,6 +702,9 @@ class Containerize():
 		logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)')
 		logging.debug(' -- ' + str(startingNb) + ' still starting container(s)')
 
+		self.testCase_id = HTML.testCase_id
+		self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log'
+
 		status = False
 		if healthyNb == 1:
 			cnt = 0
@@ -697,11 +719,12 @@ class Containerize():
 					status = True
 					logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m')
 					time.sleep(10)
+		else:
+			# containers are unhealthy, so we won't start. However, logs are stored at the end
+			# in UndeployObject so we here store the logs of the unhealthy container to report it
+			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
 		mySSH.close()
 
-		self.testCase_id = HTML.testCase_id
-		self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log'
-
 		if status:
 			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
 		else:

ci-scripts/main.py

@@ -921,7 +921,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 				elif action == 'Build_Proxy':
 					CONTAINERS.BuildProxy(HTML)
 				elif action == 'Copy_Image_to_Test':
-					CONTAINERS.Copy_Image_to_Test_Server(HTML)
+					success = CONTAINERS.Copy_Image_to_Test_Server(HTML)
+					if not success:
+						RAN.prematureExit = True
 				elif action == 'Deploy_Object':
 					CONTAINERS.DeployObject(HTML, EPC)
 					if CONTAINERS.exitStatus==1:

cmake_targets/tools/build_helper

@@ -237,8 +237,9 @@ compilations() {
      echo_error "$2 compilation failed"
      exit 1
   fi
-  if [ -s $3 ] ; then
+  if [ -s "$3" ] ; then
-     cp $3 $4
+     rm -f "$4"
+     ln -s "$PWD/$3" "$4"
      echo_success "$2 compiled"
      check_warnings "$dlog/$2.$REL.txt"
   else

docker/Dockerfile.eNB.rhel8.2

@@ -43,6 +43,7 @@ RUN yum update -y && \
         tzdata \
         procps-ng \
         atlas \
+        gdb \
         python3 \
         python3-pip \
         net-tools \

docker/Dockerfile.eNB.ubuntu18

@@ -53,6 +53,7 @@ RUN apt-get update && \
         iputils-ping \
         iproute2 \
         iperf \
+        gdb \
         python \
         python3 \
         python3-six \

docker/Dockerfile.gNB.rhel8.2

@@ -46,6 +46,7 @@ RUN yum repolist --disablerepo=* && \
         lksctp-tools \
         nettle \
         tzdata \
+        gdb \
         python3 \
         python3-pip \
         net-tools \

docker/Dockerfile.gNB.ubuntu18

@@ -52,6 +52,7 @@ RUN apt-get update && \
         net-tools \
         iproute2 \
         iputils-ping \
+        gdb \
         python \
         python3 \
         python3-six \

docker/Dockerfile.lteRU.rhel8.2

@@ -41,6 +41,7 @@ RUN yum update -y && \
         tzdata \
         procps-ng \
         atlas \
+        gdb \
         python3 \
         python3-pip \
         net-tools \

docker/Dockerfile.lteRU.ubuntu18

@@ -47,6 +47,7 @@ RUN apt-get update && \
         libatlas3-base \
         libconfig9 \
         net-tools \
+        gdb \
         python \
         python3 \
         python3-six \

docker/Dockerfile.lteUE.rhel8.2

@@ -44,6 +44,7 @@ RUN yum update -y && \
         nettle \
         tzdata \
         atlas \
+        gdb \
         python3 \
         python3-pip \
         iproute \

docker/Dockerfile.nrUE.rhel8.2

@@ -46,6 +46,7 @@ RUN yum update -y && \
         iputils \
         iproute \
         atlas \
+        gdb \
         python3 \
         python3-pip \
         libXpm \

docker/Dockerfile.nrUE.ubuntu18

@@ -50,6 +50,7 @@ RUN apt-get update && \
         libconfig9 \
         openssl \
         net-tools \
+        gdb \
         python \
         python3 \
         python3-six \

docker/scripts/enb_entrypoint.sh

@@ -51,9 +51,15 @@ for c in ${CONFIG_FILES}; do
 
     # render template and inline replace config file
     sed -i "${EXPRESSIONS}" ${c}
+
+    echo "=================================="
+    echo "== Configuration file: ${c}"
+    cat ${c}
 done
 
 # Load the USRP binaries
+echo "=================================="
+echo "== Load USRP binaries"
 if [[ -v USE_B2XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t b2xx
 elif [[ -v USE_X3XX ]]; then
@@ -62,6 +68,9 @@ elif [[ -v USE_N3XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t n3xx
 fi
 
+# enable printing of stack traces on assert
+export gdbStacks=1
+
 echo "=================================="
 echo "== Starting eNB soft modem"
 if [[ -v USE_ADDITIONAL_OPTIONS ]]; then

docker/scripts/gnb_entrypoint.sh

@@ -51,9 +51,15 @@ for c in ${CONFIG_FILES}; do
 
     # render template and inline replace config file
     sed -i "${EXPRESSIONS}" ${c}
+
+    echo "=================================="
+    echo "== Configuration file: ${c}"
+    cat ${c}
 done
 
 # Load the USRP binaries
+echo "=================================="
+echo "== Load USRP binaries"
 if [[ -v USE_B2XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t b2xx
 elif [[ -v USE_X3XX ]]; then
@@ -62,6 +68,9 @@ elif [[ -v USE_N3XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t n3xx
 fi
 
+# enable printing of stack traces on assert
+export gdbStacks=1
+
 echo "=================================="
 echo "== Starting gNB soft modem"
 if [[ -v USE_ADDITIONAL_OPTIONS ]]; then

docker/scripts/lte_ru_entrypoint.sh

@@ -31,9 +31,15 @@ for c in ${CONFIG_FILES}; do
 
     # render template and inline replace config file
     sed -i "${EXPRESSIONS}" ${c}
+
+    echo "=================================="
+    echo "== Configuration file: ${c}"
+    cat ${c}
 done
 
 # Load the USRP binaries
+echo "=================================="
+echo "== Load USRP binaries"
 if [[ -v USE_B2XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t b2xx
 elif [[ -v USE_X3XX ]]; then
@@ -42,6 +48,9 @@ elif [[ -v USE_N3XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t n3xx
 fi
 
+# enable printing of stack traces on assert
+export gdbStacks=1
+
 echo "=================================="
 echo "== Starting eNB soft modem"
 if [[ -v USE_ADDITIONAL_OPTIONS ]]; then

docker/scripts/lte_ue_entrypoint.sh

@@ -30,6 +30,10 @@ for c in ${CONFIG_FILES}; do
 
     # render template and inline replace config file
     sed -i "${EXPRESSIONS}" ${c}
+
+    echo "=================================="
+    echo "== Configuration file: ${c}"
+    cat ${c}
 done
 
 #now generate USIM files
@@ -38,6 +42,8 @@ cd $PREFIX
 $PREFIX/bin/conf2uedata -c $PREFIX/etc/ue_usim.conf -o $PREFIX
 
 # Load the USRP binaries
+echo "=================================="
+echo "== Load USRP binaries"
 if [[ -v USE_B2XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t b2xx
 elif [[ -v USE_X3XX ]]; then
@@ -57,6 +63,9 @@ if [[ -v USE_NFAPI ]]; then
   new_args+=("$PREFIX/etc/ue.conf")
 fi
 
+# enable printing of stack traces on assert
+export gdbStacks=1
+
 echo "=================================="
 echo "== Starting LTE UE soft modem"
 if [[ -v USE_ADDITIONAL_OPTIONS ]]; then

docker/scripts/nr_ue_entrypoint.sh

@@ -40,9 +40,15 @@ for c in ${CONFIG_FILES}; do
 
     # render template and inline replace config file
     sed -i "${EXPRESSIONS}" ${c}
+
+    echo "=================================="
+    echo "== Configuration file: ${c}"
+    cat ${c}
 done
 
 # Load the USRP binaries
+echo "=================================="
+echo "== Load USRP binaries"
 if [[ -v USE_B2XX ]]; then
     $PREFIX/bin/uhd_images_downloader.py -t b2xx
 elif [[ -v USE_X3XX ]]; then
@@ -58,6 +64,9 @@ while [[ $# -gt 0 ]]; do
   shift
 done
 
+# enable printing of stack traces on assert
+export gdbStacks=1
+
 echo "=================================="
 echo "== Starting NR UE soft modem"
 if [[ -v USE_ADDITIONAL_OPTIONS ]]; then