Merge remote-tracking branch 'origin/ci-collect-ue-logs' into integration_2025_w32 (!3560)

CI: Refactor log collection, collect also (COTS) UE logs

Refactor log handling in CI: collect according to currently used XML files. This
makes it homogeneous in the pipelines, and avoids accidental overwriting (as
happens in phytest-timing), as well as harmonizing the log placement
(e.g., no YAML lookup in Ping/Iperf).

Allow to trace and archive UE logs (e.g., Quectel via QLog, Amarisoft UE logs).
By default, no logs will be collected as of this MR, as many files are big.

Various cleanup in Jenkinsfiles, some bugfixes, add unit tests.

ci-scripts/Jenkinsfile-tmp-full-ran

@@ -216,24 +216,6 @@ pipeline {
         }
         stage('Log Collection') {
             parallel {
-                stage('Log Collection (eNB - Build)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
                 stage('Log Collection (eNB - Run)') {
                     steps {
                         withCredentials([

ci-scripts/Jenkinsfile-tmp-multi-enb

@@ -222,24 +222,6 @@ pipeline {
         }
         stage('Log Collection') {
             parallel {
-                stage('Log Collection (eNB - Build)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
                 stage('Log Collection (eNB - Run)') {
                     steps {
                         withCredentials([

ci-scripts/Jenkinsfile-tmp-multi-enb-nsa

@@ -210,24 +210,6 @@ pipeline {
     }
     stage('Log Collection') {
       parallel {
-        stage('Log Collection (eNB - Build)') {
-          steps {
-            withCredentials([
-               [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-            ]) {
-              echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-              sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-              echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-              sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-            }
-            script {
-              if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-              }
-            }
-          }
-        }
         stage('Log Collection (eNB - Run)') {
           steps {
             withCredentials([

ci-scripts/Jenkinsfile-tmp-ran

@@ -227,24 +227,6 @@ pipeline {
         }
         stage('Log Collection') {
             parallel {
-                stage('Log Collection (eNB - Build)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
                 stage('Log Collection (eNB - Run)') {
                     steps {
                         withCredentials([

ci-scripts/ci_infra.yaml

@@ -5,10 +5,12 @@ idefix:
   AttachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 wup
   DetachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 detach
   NetworkScript: ip a show dev wwan0
+  #Tracing:
+  #  Start: 'sudo rm -f ci_qlog/*; nohup sudo -b QLog/QLog -s ci_qlog -f NR5G.cfg &'
+  #  Stop: 'sudo killall QLog'
+  #  Collect: 'sudo mv ci_qlog/* %%log_dir%%/'
   IF: wwan0
   MTU: 1500
-  Trace: True
-  LogStore: /media/usb-drive/ci_qlogs
 
 up2:
   Host: up2
@@ -46,7 +48,6 @@ adb_ue_1:
   NetworkScript: adb -s R3CM40LZPHT shell "ip address show | grep rmnet_data0"
   CmdPrefix: adb -s R3CM40LZPHT shell
   MTU: 1500
-  LogStore: /media/usb-drive/ci_adb_1-logs
 adb_ue_2:
   Host: nano
   InitScript: /home/oaicicd/ci_ctl_adb.sh initialize 5200c00db4413517
@@ -59,7 +60,6 @@ adb_ue_2:
   NetworkScript: adb -s 5200c00db4413517 shell "ip address show | grep rmnet"
   CmdPrefix: adb -s 5200c00db4413517 shell
   MTU: 1500
-  LogStore: /media/usb-drive/ci_adb_2-logs
 
 oc-cn5g:
   Host: avra
@@ -150,6 +150,10 @@ amarisoft_ue:
   InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz/aw2s-multi-00102-20.cfg &
   TermScript: /root/lteue-linux-2025-03-15/ws.js -t 10 127.0.0.1:9002 '{"message":"quit"}' || killall -KILL lteue-avx2
   NetworkScript: ip netns exec ue1 ip a show dev pdn0
+  #Tracing:
+  #  Start: '' # nothing to be done
+  #  Stop: '' # nothing to be done
+  #  Collect: 'cp /tmp/ue.log %%log_dir%%'
 amarisoft_ue_2x2:
   Host: amariue
   InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz_2x2/aw2s-multi-00102-2x2-v2.cfg &

ci-scripts/cls_analysis.py

@@ -84,7 +84,7 @@ class Analysis():
 
 		return success,msg
 
-	def analyze_oc_physim(result_junit, details_json):
+	def analyze_oc_physim(result_junit, details_json, logPath):
 		try:
 			tree = ET.parse(result_junit)
 			root = tree.getroot()
@@ -124,7 +124,7 @@ class Analysis():
 			output = test.findtext("system-out")
 			output_check = "exceeds the threshold" not in output
 			# collect logs
-			log_dir = f'../cmake_targets/log/{test_exec}'
+			log_dir = f'{logPath}/{test_exec}'
 			os.makedirs(log_dir, exist_ok=True)
 			with open(f'{log_dir}/{test_name}.log', 'w') as f:
 				f.write(output)

ci-scripts/cls_ci_helper.py

@@ -0,0 +1,44 @@
+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+
+import os
+from typing import NamedTuple
+import logging
+
+class TestCaseCtx(NamedTuple):
+    count: int
+    test_id: int
+    logPath: str
+
+    def Default(logPath):
+        return TestCaseCtx(123, 112233, logPath)
+    def baseFilename(self):
+        # typically, the test ID is of form 001234 (6 digits)
+        return f"{self.logPath}/{self.count}-{self.test_id:06d}"
+
+def archiveArtifact(cmd, ctx, remote_path):
+    base = os.path.basename(remote_path)
+    local = f"{ctx.baseFilename()}-{base}"
+    logging.info(f"Archive artifact '{local}'")
+    success = cmd.copyin(remote_path, local)
+    if success:
+        cmd.run(f'rm {remote_path}', silent=True)
+    return local if success else None

ci-scripts/cls_cluster.py

@@ -39,6 +39,7 @@ import constants as CONST
 import helpreadme as HELP
 import cls_containerize
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc"
 NAMESPACE = "oaicicd-ran"
@@ -207,7 +208,12 @@ class Cluster:
 			HTML.CreateHtmlTestRowQueue(param, 'KO', [msg])
 		return success
 
-	def BuildClusterImage(self, HTML):
+	def _retrieveOCLog(self, ctx, job, lSourcePath, image):
+		fn = f'{lSourcePath}/cmake_targets/log/{image}.log'
+		self.cmd.run(f'oc logs {job} &> {fn}')
+		return (image, archiveArtifact(self.cmd, ctx, fn))
+
+	def BuildClusterImage(self, ctx, HTML):
 		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
 			HELP.GenericHelp(CONST.Version)
 			raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}')
@@ -280,22 +286,27 @@ class Cluster:
 		self._recreate_entitlements()
 
 		status = True # flag to abandon compiling if any image fails
-		attemptedImages = []
+		log_files = []
+		build_metrics = f"{lSourcePath}/cmake_targets/log/build-metrics.log"
 		if forceBaseImageBuild:
 			self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml')
 			self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml')
 			ranbase_job = self._start_build('ran-base')
-			attemptedImages += ['ran-base']
 			status = ranbase_job is not None and self._wait_build_end([ranbase_job], 1000)
 			if not status: logging.error('failure during build of ran-base')
-			self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect
+			log_files.append(self._retrieveOCLog(ctx, ranbase_job, lSourcePath, 'ran-base'))
+
 			# recover logs by mounting image
 			self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml')
 			pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml')
 			if pod is not None:
-				self.cmd.run(f'mkdir -p cmake_targets/log/ran-base')
+				logdir = f'{lSourcePath}/cmake_targets/log/ran-base'
-				self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base')
+				self.cmd.run(f'mkdir -p {logdir}')
+				self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ {logdir}')
 				self._undeploy_pod('openshift/ran-base-log-retrieval.yaml')
+				ret = self.cmd.run(f'ls {logdir}')
+				for f in ret.stdout.splitlines():
+					archiveArtifact(self.cmd, ctx, f"{logdir}/{f}")
 			else:
 				status = False
 
@@ -304,27 +315,24 @@ class Cluster:
 			self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel9')
 			physim_job = self._start_build('oai-physim')
-			attemptedImages += ['oai-physim']
 
 			self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml')
 			self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel9')
 			ranbuild_job = self._start_build('ran-build')
-			attemptedImages += ['ran-build']
 
 			self._recreate_is_tag('oai-clang', imageTag, 'openshift/oai-clang-is.yaml')
 			self._recreate_bc('oai-clang', imageTag, 'openshift/oai-clang-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.clang.rhel9')
 			clang_job = self._start_build('oai-clang')
-			attemptedImages += ['oai-clang']
 
 			wait = ranbuild_job is not None and physim_job is not None and clang_job is not None and self._wait_build_end([ranbuild_job, physim_job, clang_job], 1200)
 			if not wait: logging.error('error during build of ranbuild_job or physim_job or clang_job')
 			status = status and wait
-			self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log')
+			log_files.append(self._retrieveOCLog(ctx, ranbuild_job, lSourcePath, 'ran-build'))
-			self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log')
+			log_files.append(self._retrieveOCLog(ctx, physim_job, lSourcePath, 'oai-physim'))
-			self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log')
+			log_files.append(self._retrieveOCLog(ctx, clang_job, lSourcePath, 'oai-clang'))
-			self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
+			self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
 
 		if status:
 			self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
@@ -332,72 +340,66 @@ class Cluster:
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel9')
 			self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel9')
 			enb_job = self._start_build('oai-enb')
-			attemptedImages += ['oai-enb']
 
 			self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml')
 			self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel9')
 			self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel9')
 			gnb_job = self._start_build('oai-gnb')
-			attemptedImages += ['oai-gnb']
 
 			self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml')
 			self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel9')
 			self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel9')
 			gnb_aw2s_job = self._start_build('oai-gnb-aw2s')
-			attemptedImages += ['oai-gnb-aw2s']
 
 			wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 800)
 			if not wait: logging.error('error during build of eNB/gNB')
 			status = status and wait
 			# recover logs
-			self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log')
+			log_files.append(self._retrieveOCLog(ctx, enb_job, lSourcePath, 'oai-enb'))
-			self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log')
+			log_files.append(self._retrieveOCLog(ctx, gnb_job, lSourcePath, 'oai-gnb'))
-			self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log')
+			log_files.append(self._retrieveOCLog(ctx, gnb_aw2s_job, lSourcePath, 'oai-gnb-aw2s'))
+			self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
 
 			self._recreate_is_tag('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-is.yaml')
 			self._recreate_bc('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nr-cuup.rhel9')
 			self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nr-cuup.rhel9')
 			nr_cuup_job = self._start_build('oai-nr-cuup')
-			attemptedImages += ['oai-nr-cuup']
 
 			self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml')
 			self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel9')
 			self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel9')
 			lteue_job = self._start_build('oai-lte-ue')
-			attemptedImages += ['oai-lte-ue']
 
 			self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml')
 			self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel9')
 			self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel9')
 			nrue_job = self._start_build('oai-nr-ue')
-			attemptedImages += ['oai-nr-ue']
 
 			wait = nr_cuup_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end([nr_cuup_job, lteue_job, nrue_job], 800)
 			if not wait: logging.error('error during build of nr-cuup/lteUE/nrUE')
 			status = status and wait
 			# recover logs
-			self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log')
+			log_files.append(self._retrieveOCLog(ctx, nr_cuup_job, lSourcePath, 'oai-nr-cuup'))
-			self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log')
+			log_files.append(self._retrieveOCLog(ctx, lteue_job, lSourcePath, 'oai-lte-ue'))
-			self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log')
+			log_files.append(self._retrieveOCLog(ctx, nrue_job, lSourcePath, 'oai-nr-ue'))
-			self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
+			self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
 
 		if status:
 			self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml')
 			self._recreate_bc('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-bc.yaml')
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.fhi72.rhel9')
 			ranbuildfhi72_job = self._start_build('ran-build-fhi72')
-			attemptedImages += ['ran-build-fhi72']
 
 			wait = ranbuildfhi72_job is not None and self._wait_build_end([ranbuildfhi72_job], 1200)
 			if not wait: logging.error('error during build of ranbuildfhi72_job')
 			status = status and wait
-			self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log')
+			log_files.append(self._retrieveOCLog(ctx, ranbuildfhi72_job, lSourcePath, 'ran-build-fhi72'))
-			self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
+			self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
 
 		if status:
 			self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml')
@@ -405,19 +407,17 @@ class Cluster:
 			self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.fhi72.rhel9')
 			self._retag_image_statement('ran-build-fhi72', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build-fhi72', imageTag, 'docker/Dockerfile.gNB.fhi72.rhel9')
 			gnb_fhi72_job = self._start_build('oai-gnb-fhi72')
-			attemptedImages += ['oai-gnb-fhi72']
 
 			wait = gnb_fhi72_job is not None and self._wait_build_end([gnb_fhi72_job], 600)
 			if not wait: logging.error('error during build of gNB-fhi72')
 			status = status and wait
 			# recover logs
-			self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log')
+			log_files.append(self._retrieveOCLog(ctx, gnb_fhi72_job, lSourcePath, 'oai-gnb-fhi72'))
-			self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
+			self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
 
 		# split and analyze logs
 		imageSize = {}
-		for image in attemptedImages:
+		for image, _ in log_files:
-			self.cmd.run(f'mkdir -p cmake_targets/log/{image}')
 			tag = imageTag if image != 'ran-base' else baseTag
 			size = self._get_image_size(image, tag)
 			if size <= 0:
@@ -428,12 +428,14 @@ class Cluster:
 				imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)'
 			logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
 
-		grep_exp = r"\|".join(attemptedImages)
+		archiveArtifact(self.cmd, ctx, build_metrics)
-		self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log');
+		logfile = f'{lSourcePath}/cmake_targets/log/image_registry.log'
-		self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done')
+		grep_exp = r"\|".join([i for i,f in log_files])
-
+		self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> {logfile}');
-		build_log_name = f'build_log_{self.testCase_id}'
+		archiveArtifact(self.cmd, ctx, logfile)
-		cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name)
+		logfile = f'{lSourcePath}/cmake_targets/log/build_pod_summary.log'
+		self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> {logfile}; done')
+		archiveArtifact(self.cmd, ctx, logfile)
 
 		self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done')
 
@@ -442,7 +444,7 @@ class Cluster:
 		self.cmd.close()
 
 		# Analyze the logs
-		collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status)
+		collectInfo = cls_containerize.AnalyzeBuildLogs(log_files, status)
 		for img in collectInfo:
 			for f in collectInfo[img]:
 				status = status and collectInfo[img][f]['status']
@@ -458,9 +460,16 @@ class Cluster:
 
 		HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
 
+		# TODO fix groovy script, remove the following.
+		# the groovy scripts expects all logs in
+		# <jenkins-workspace>/<pipeline>/ci-scripts, so copy it there
+		with cls_cmd.LocalCmd() as c:
+			c.run(f'mkdir -p {os.getcwd()}/test_log_{ctx.test_id}/')
+			c.run(f'cp -r {ctx.logPath} {os.getcwd()}/test_log_{ctx.test_id}/')
+
 		return status
 
-	def deploy_oc_physim(self, HTML, oc_release, svr_id):
+	def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id):
 		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
 			HELP.GenericHelp(CONST.Version)
 			raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}')
@@ -470,17 +479,13 @@ class Cluster:
 		options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}"
 		ret = cls_cmd.runScript(svr_id, script, 600, options)
 		logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
-		log_dir = f'{os.getcwd()}/../cmake_targets/log'
-		os.makedirs(log_dir, exist_ok=True)
-		result_junit = f'{oc_release}-run.xml'
-		details_json = f'{oc_release}-tests.json'
 		with cls_cmd.getConnection(svr_id) as ssh:
-			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}')
+			details_json = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-tests.json')
-			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}')
+			result_junit = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-run.xml')
-			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt')
+			archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt')
-			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt')
+			archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt')
-			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log')
+			archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log')
-		test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}')
+		test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(result_junit, details_json, ctx.logPath)
 		if test_summary:
 			if test_status:
 				HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)

ci-scripts/cls_containerize.py

@@ -46,6 +46,7 @@ import cls_cmd
 import helpreadme as HELP
 import constants as CONST
 import cls_oaicitest
+from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
 # Helper functions used here and in other classes
@@ -81,67 +82,26 @@ def CreateTag(ranCommitID, ranBranch, ranAllowMerge):
 		tagToUse = f'develop-{shortCommit}'
 	return tagToUse
 
-def CopyLogsToExecutor(cmd, sourcePath, log_name):
+def AnalyzeBuildLogs(logfiles, globalStatus):
-	cmd.cd(f'{sourcePath}/cmake_targets')
-	cmd.run(f'rm -f {log_name}.zip')
-	cmd.run(f'mkdir -p {log_name}')
-	cmd.run(f'mv log/* {log_name}')
-	cmd.run(f'zip -r -qq {log_name}.zip {log_name}')
-
-	# copy zip to executor for analysis
-	if (os.path.isfile(f'./{log_name}.zip')):
-		os.remove(f'./{log_name}.zip')
-	if (os.path.isdir(f'./{log_name}')):
-		shutil.rmtree(f'./{log_name}')
-	cmd.copyin(src=f'{sourcePath}/cmake_targets/{log_name}.zip', tgt=f'{os.getcwd()}/{log_name}.zip')
-	cmd.run(f'rm -f {log_name}.zip')
-	ZipFile(f'{log_name}.zip').extractall('.')
-
-def AnalyzeBuildLogs(buildRoot, images, globalStatus):
 	collectInfo = {}
-	for image in images:
+	for image, lf in logfiles:
 		files = {}
-		file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')]
+		errorandwarnings = {}
-		# Analyze the "sub-logs" of every target image
+		committed = False
-		for fil in file_list:
+		tagged = False
-			errorandwarnings = {}
+		with open(lf, mode='r') as inputfile:
-			warningsNo = 0
+			for line in inputfile:
-			errorsNo = 0
+				lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
-			with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile:
+				lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
-				for line in inputfile:
+				tagged = tagged or lineHasTag or lineHasTag2
-					result = re.search(' ERROR ', str(line))
+				# the OpenShift Cluster builder prepends image registry URL
-					if result is not None:
+				lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None
-						errorsNo += 1
+				committed = committed or lineHasCommit
-					result = re.search(' error:', str(line))
+		errorandwarnings['errors'] = 0 if committed or tagged else 1
-					if result is not None:
+		errorandwarnings['warnings'] = 0
-						errorsNo += 1
+		errorandwarnings['status'] = committed or tagged
-					result = re.search(' WARNING ', str(line))
+		logging.info(f"Analyzing {image}, file {lf}: {errorandwarnings}")
-					if result is not None:
+		files['Target Image Creation'] = errorandwarnings
-						warningsNo += 1
-					result = re.search(' warning:', str(line))
-					if result is not None:
-						warningsNo += 1
-				errorandwarnings['errors'] = errorsNo
-				errorandwarnings['warnings'] = warningsNo
-				errorandwarnings['status'] = globalStatus
-			files[fil] = errorandwarnings
-		# Analyze the target image
-		if os.path.isfile(f'{buildRoot}/{image}.log'):
-			errorandwarnings = {}
-			committed = False
-			tagged = False
-			with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
-				for line in inputfile:
-					lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
-					lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
-					tagged = tagged or lineHasTag or lineHasTag2
-					# the OpenShift Cluster builder prepends image registry URL
-					lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
-					committed = committed or lineHasCommit
-			errorandwarnings['errors'] = 0 if committed or tagged else 1
-			errorandwarnings['warnings'] = 0
-			errorandwarnings['status'] = committed or tagged
-			files['Target Image Creation'] = errorandwarnings
 		collectInfo[image] = files
 	return collectInfo
 
@@ -202,12 +162,10 @@ def GetServices(ssh, requested, file):
     else:
         return requested
 
-def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename):
+def CopyinServiceLog(ssh, lSourcePath, svcName, wd_yaml, ctx):
-	remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}"
+	remote_filename = f"{lSourcePath}/cmake_targets/log/{svcName}.logs"
 	ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}')
-	local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}"
+	return archiveArtifact(ssh, ctx, remote_filename)
-	local_filename = f"{local_dir}/{filename}"
-	return ssh.copyin(remote_filename, local_filename)
 
 def GetRunningServices(ssh, file):
 	ret = ssh.run(f'docker compose -f {file} config --services')
@@ -229,14 +187,13 @@ def GetRunningServices(ssh, file):
 	logging.info(f'stopping services: {running_services}')
 	return running_services
 
-def CheckLogs(self, yaml, service_name, HTML, RAN):
+def CheckLogs(self, filename, HTML, RAN):
-	logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}'
-	filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log'
 	success = True
-	if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
+	name = os.path.basename(filename)
+	if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
 		logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
 		logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN)
-		opt = f"UE log analysis for service {service_name}"
+		opt = f"UE log analysis ({name})"
 		# usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files
 		# abuse HTML to store their reports, and we here want to put custom options,
 		# which is not possible with CreateHtmlTestRow
@@ -247,13 +204,13 @@ def CheckLogs(self, yaml, service_name, HTML, RAN):
 		else:
 			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg])
 		HTML.htmlUEFailureMsg = ""
-	elif service_name == 'nv-cubb':
+	elif 'nv-cubb' in name:
 		msg = 'Undeploy PNF/Nvidia CUBB'
 		HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK)
-	elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])):
+	elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb'])):
 		logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m')
 		logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
-		opt = f"xNB log analysis for service {service_name}"
+		opt = f"xNB log analysis ({name})"
 		if (logStatus < 0):
 			HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg])
 			success = False
@@ -261,9 +218,9 @@ def CheckLogs(self, yaml, service_name, HTML, RAN):
 			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg])
 		HTML.htmleNBFailureMsg = ""
 	else:
-		logging.info(f'Skipping to analyze log for service name {service_name}')
+		logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE")
-		HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"])
+		HTML.CreateHtmlTestRowQueue(f"file {name}", 'OK', ["no analysis function"])
-	logging.debug(f"log check: service {service_name} passed analysis {success}")
+	logging.debug(f"log check: file {filename} passed analysis {success}")
 	return success
 
 #-----------------------------------------------------------
@@ -299,8 +256,6 @@ class Containerize():
 		self.services = ['', '', '']
 		self.deploymentTag = ''
 
-		self.testCase_id = ''
-
 		self.cli = ''
 		self.cliBuildOptions = ''
 		self.dockerfileprefix = ''
@@ -330,11 +285,12 @@ class Containerize():
 			raise ValueError(f'Insufficient Parameter: IP/node {ip}, path {path}')
 		return (ip, path)
 
-	def BuildImage(self, HTML):
+	def BuildImage(self, ctx, HTML):
 		svr = self.eNB_serverId[self.eNB_instance]
 		lIpAddr, lSourcePath = self.GetCredentials(svr)
 		logging.debug('Building on server: ' + lIpAddr)
 		cmd = cls_cmd.getConnection(lIpAddr)
+		log_files = []
 	
 		# Checking the hostname to get adapted on cli and dockerfileprefixes
 		cmd.run('hostnamectl')
@@ -390,7 +346,6 @@ class Containerize():
 			imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
 			imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', ''))
 		
-		self.testCase_id = HTML.testCase_id
 		cmd.cd(lSourcePath)
 		# if asterix, copy the entitlement and subscription manager configurations
 		if self.host == 'Red Hat':
@@ -420,13 +375,24 @@ class Containerize():
 		# Let's remove any previous run artifacts if still there
 		cmd.run(f"{self.cli} image prune --force")
 		for image,pattern,name,option in imageNames:
-			cmd.run(f"{self.cli} image rm {name}:{imageTag}")
+			cmd.run(f"{self.cli} image rm {name}:{imageTag}", reportNonZero=False)
 
 		# Build the base image only on Push Events (not on Merge Requests)
 		# On when the base image docker file is being modified.
 		if forceBaseImageBuild:
 			cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
-			cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600)
+			logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log'
+			cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> {logfile}", timeout=1600)
+			t = ("ran-base", archiveArtifact(cmd, ctx, logfile))
+			log_files.append(t)
+
+			# Recover build logs, for the moment only possible when build is successful
+			cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
+			cmd.run("mkdir -p cmake_targets/log/ran-base")
+			logfile = f'{lSourcePath}/cmake_targets/log/ran-base.log'
+			cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/all.txt {logfile}")
+			cmd.run(f"{self.cli} rm -f test")
+			archiveArtifact(cmd, ctx, logfile)
 		# First verify if the base image was properly created.
 		ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
 		allImagesSize = {}
@@ -441,7 +407,7 @@ class Containerize():
 			HTML.CreateHtmlTabFooter(False)
 			return False
 		else:
-			result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
+			result = re.search(r'Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
 			if result is not None:
 				size = float(result.group("size")) / 1000000
 				imageSizeStr = f'{size:.1f}'
@@ -450,17 +416,9 @@ class Containerize():
 			else:
 				logging.debug('ran-base size is unknown')
 
-		# Recover build logs, for the moment only possible when build is successful
-		cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
-		cmd.run("mkdir -p cmake_targets/log/ran-base")
-		cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base")
-		cmd.run(f"{self.cli} rm -f test")
-
 		# Build the target image(s)
 		status = True
-		attemptedImages = ['ran-base']
 		for image,pattern,name,option in imageNames:
-			attemptedImages += [name]
 			# the archived Dockerfiles have "ran-base:latest" as base image
 			# we need to update them with proper tag
 			cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}')
@@ -474,15 +432,18 @@ class Containerize():
 				cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
 			if image == 'oai-gnb-aerial':
 				cmd.run('cp -f /opt/nvidia-ipc/nvipc.src.2025.05.20.tar.gz .')
-			ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200)
+			logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log'
+			ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > {logfile} 2>&1', timeout=1200)
+			t = (name, archiveArtifact(cmd, ctx, logfile))
+			log_files.append(t)
 			if image == 'oai-gnb-aerial':
 				cmd.run('rm -f nvipc.src.2025.05.20.tar.gz')
-			if image == 'ran-build' and ret.returncode == 0:
+			if image == 'ran-build' or image == 'ran-build-asan' or image == 'ran-build-fhi72':
 				cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true")
-				cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/")
+				logfile = f'{lSourcePath}/{image}.ninja.log'
+				cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/all.txt {logfile}")
 				cmd.run(f"docker rm -f test-log")
-			else:
+				archiveArtifact(cmd, ctx, logfile)
-				cmd.run(f"mkdir -p cmake_targets/log/{name}")
 			# check the status of the build
 			ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}")
 			if ret.returncode != 0:
@@ -493,7 +454,7 @@ class Containerize():
 				allImagesSize[name] = 'N/A -- Build Failed'
 				break
 			else:
-				result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
+				result = re.search(r'Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
 				if result is not None:
 					size = float(result.group("size")) / 1000000 # convert to MB
 					imageSizeStr = f'{size:.1f}'
@@ -516,13 +477,10 @@ class Containerize():
 		logging.debug(cmd.run("df -h").stdout)
 		logging.debug(cmd.run("docker system df").stdout)
 
-		# create a zip with all logs
-		build_log_name = f'build_log_{self.testCase_id}'
-		CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
 		cmd.close()
 
 		# Analyze the logs
-		collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
+		collectInfo = AnalyzeBuildLogs(log_files, status)
 		
 		if status:
 			logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
@@ -536,13 +494,12 @@ class Containerize():
 			HTML.CreateHtmlTabFooter(False)
 			return False
 
-	def BuildProxy(self, HTML):
+	def BuildProxy(self, ctx, HTML):
 		svr = self.eNB_serverId[self.eNB_instance]
 		lIpAddr, lSourcePath = self.GetCredentials(svr)
 		logging.debug('Building on server: ' + lIpAddr)
 		ssh = cls_cmd.getConnection(lIpAddr)
 
-		self.testCase_id = HTML.testCase_id
 		oldRanCommidID = self.ranCommitID
 		oldRanRepository = self.ranRepository
 		oldRanAllowMerge = self.ranAllowMerge
@@ -568,15 +525,10 @@ class Containerize():
 			if not success:
 				raise Exception("could not clone proxy repository")
 
-			filename = f'build_log_{self.testCase_id}'
+			fullpath = f'{lSourcePath}/proxy_build.log'
-			fullpath = f'{lSourcePath}/{filename}'
 
 			ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1')
-			ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}')
+			archiveArtifact(ssh, ctx, fullpath)
-			local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip"
-			ssh.copyin(f'{fullpath}.zip', local_file)
-			# don't delete such that we might recover the zips
-			#ssh.run(f'rm -f {fullpath}.zip')
 
 			ssh.run('docker image prune --force')
 			ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
@@ -616,7 +568,7 @@ class Containerize():
 		collectInfo = {}
 		collectInfo['proxy'] = files
 		ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
-		result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
+		result = re.search(r'Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
 		# Cleaning any created tmp volume
 		ssh.run('docker volume prune --force')
 		ssh.close()
@@ -638,7 +590,7 @@ class Containerize():
 			HTML.CreateHtmlTabFooter(False)
 			return False
 
-	def BuildRunTests(self, HTML):
+	def BuildRunTests(self, ctx, HTML):
 		svr = self.eNB_serverId[self.eNB_instance]
 		lIpAddr, lSourcePath = self.GetCredentials(svr)
 		logging.debug('Building on server: ' + lIpAddr)
@@ -655,7 +607,6 @@ class Containerize():
 
 		if self.forcedWorkspaceCleanup:
 			cmd.run(f'sudo -S rm -Rf {lSourcePath}')
-		self.testCase_id = HTML.testCase_id
 	
 		# check that ran-base image exists as we expect it
 		baseImage = 'ran-base'
@@ -675,10 +626,10 @@ class Containerize():
 
 		# build ran-unittests image
 		dockerfile = "ci-scripts/docker/Dockerfile.unittest.ubuntu22"
-		ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {lSourcePath}/cmake_targets/log/unittest-build.log')
+		logfile = f'{lSourcePath}/cmake_targets/log/unittest-build.log'
+		ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {logfile}')
+		archiveArtifact(cmd, ctx, logfile)
 		if ret.returncode != 0:
-			build_log_name = f'build_log_{self.testCase_id}'
-			CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
 			logging.error(f'Cannot build unit tests')
 			HTML.CreateHtmlTestRow("Unit test build failed", 'KO', [dockerfile])
 			HTML.CreateHtmlTabFooter(False)
@@ -687,10 +638,15 @@ class Containerize():
 		HTML.CreateHtmlTestRowQueue("Build unit tests", 'OK', [dockerfile])
 
 		# it worked, build and execute tests, and close connection
-		ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --rm ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)')
+		# I would like to run it with --rm and mount the ctest result directory to avoid 'docker cp'
-		cmd.run(f'docker rmi ran-unittests:{baseTag}')
+		# below, but then permissions are messed up and we can't remove the directory without sudo
-		build_log_name = f'build_log_{self.testCase_id}'
+		# making the next pipeline fail
-		CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
+		ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --name ran-unittests ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)')
+		cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTest.log .')
+		archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTest.log')
+		cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTestsFailed.log .')
+		archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTestsFailed.log')
+		cmd.run('docker rm ran-unittests')
 		cmd.close()
 
 		if ret.returncode == 0:
@@ -823,18 +779,14 @@ class Containerize():
 			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"])
 		return success
 
-	def DeployObject(self, HTML):
+	def DeployObject(self, ctx, HTML):
 		svr = self.eNB_serverId[self.eNB_instance]
 		num_attempts = self.num_attempts
 		lIpAddr, lSourcePath = self.GetCredentials(svr)
 		logging.debug(f'Deploying OAI Object on server: {lIpAddr}')
 		yaml = self.yamlPath[self.eNB_instance].strip('/')
-		# creating the log folder by default
-		local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}"
-		os.system(f'mkdir -p {local_dir}')
 		wd = f'{lSourcePath}/{yaml}'
 		wd_yaml = f'{wd}/docker-compose.y*ml'
-		yaml_dir = yaml.split('/')[-1]
 		with cls_cmd.getConnection(lIpAddr) as ssh:
 			services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml)
 			if services == [] or services == ' ' or services == None:
@@ -861,7 +813,7 @@ class Containerize():
 					logging.warning(warning_msg)
 					HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [warning_msg])
 					for svc in services.split():
-						CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log')
+						CopyinServiceLog(ssh, lSourcePath, svc, wd_yaml, ctx)
 					ssh.run(f'docker compose -f {wd_yaml} down -- {services}')
 		imagesInfo = info.stdout.splitlines()[1:]
 		logging.debug(f'{info.stdout.splitlines()[1:]}')
@@ -871,13 +823,12 @@ class Containerize():
 			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
 		return deployed
 
-	def UndeployObject(self, HTML, RAN):
+	def UndeployObject(self, ctx, HTML, RAN):
 		svr = self.eNB_serverId[self.eNB_instance]
 		lIpAddr, lSourcePath = self.GetCredentials(svr)
 		logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
 		yaml = self.yamlPath[self.eNB_instance].strip('/')
 		wd = f'{lSourcePath}/{yaml}'
-		yaml_dir = yaml.split('/')[-1]
 		with cls_cmd.getConnection(lIpAddr) as ssh:
 			ExistEnvFilePrint(ssh, wd)
 			services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml")
@@ -885,7 +836,7 @@ class Containerize():
 			if services is not None:
 				all_serv = " ".join([s for s, _ in services])
 				ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
-				copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services)
+				copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, f"{wd}/docker-compose.y*ml", ctx) for s, _ in services]
 			else:
 				logging.warning('could not identify services to stop => no log file')
 			ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
@@ -894,7 +845,7 @@ class Containerize():
 			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)'])
 			return False
 		else:
-			log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services]
+			log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res]
 			success = all(log_results)
 		if success:
 			logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')

ci-scripts/cls_corenetwork.py

@@ -29,6 +29,7 @@ import yaml
 import re
 
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 def listify(s):
 	if s is None:
@@ -114,8 +115,7 @@ class CoreNetwork:
 		logging.info(f'deployed core network {self}, pingable IP address {ip}')
 		return True, output
 
-	def _collect_logs(self, log_dir):
+	def _collect_logs(self, ctx):
-		logging.info(f'collecting logs into (local) {log_dir}')
 		remote_dir = "/tmp/cn-undeploy-logs"
 		with cls_cmd.getConnection(self._host) as c:
 			# create a directory for log collection
@@ -134,18 +134,17 @@ class CoreNetwork:
 				logging.error("cannot enumerate log files")
 				return []
 			log_files = []
-			# copy them to the executor one by one, and store in log_dir
+			# copy them to the executor one by one
 			for f in ret.stdout.split("\n"):
-				l = f.replace(remote_dir, log_dir)
+				name = archiveArtifact(c, ctx, f)
-				c.copyin(f, l)
+				log_files.append(name)
-				log_files.append(l)
 			c.run(f'rm -rf {remote_dir}')
 			return log_files
 
-	def undeploy(self, log_dir=None):
+	def undeploy(self, ctx=None):
 		log_files = []
-		if log_dir is not None:
+		if ctx is not None:
-			log_files = self._collect_logs(log_dir)
+			log_files = self._collect_logs(ctx)
 		else:
 			logging.warning("no directory for log collection specified, cannot retrieve core network logs")
 		logging.info(f'undeploy core network {self}')

ci-scripts/cls_module.py

@@ -32,6 +32,7 @@ import re
 import yaml
 
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 class Module_UE:
 
@@ -59,26 +60,31 @@ class Module_UE:
 			}
 			self.interface = m.get('IF')
 			self.MTU = m.get('MTU')
-			self.trace = m.get('trace') == True
-			self.logStore = m.get('LogStore')
 			self.cmd_prefix = m.get('CmdPrefix')
 			logging.info(f'initialized UE {self} from {filename}')
 
+			t = m.get('Tracing')
+			self.trace = t is not None
+			if self.trace:
+				if t.get('Start') is None or t.get('Stop') is None or t.get('Collect')is None :
+					raise ValueError("need to have Start/Stop/Collect for tracing")
+				self.cmd_dict["traceStart"] = t.get('Start')
+				self.cmd_dict["traceStop"] = t.get('Stop')
+				self._logCollect = t.get('Collect')
+				if "%%log_dir%%" not in self._logCollect:
+					raise ValueError(f"(At least one) LogCollect expression for {module_name} must contain \"%%log_dir%%\"")
+
 	def __str__(self):
-		return f"{self.module_name}@{self.host} [IP: {self.getIP()}]"
+		return f"{self.module_name}@{self.host}"
 
 	def __repr__(self):
 		return self.__str__()
 
-	def _command(self, cmd, silent = False):
+	def _command(self, cmd, silent=False, reportNonZero=True):
 		if cmd is None:
 			raise Exception("no command provided")
-		if self.host == "" or self.host == "localhost":
+		with cls_cmd.getConnection(self.host) as c:
-			c = cls_cmd.LocalCmd()
+			response = c.run(cmd, silent=silent, reportNonZero=reportNonZero)
-		else:
-			c = cls_cmd.RemoteCmd(self.host)
-		response = c.run(cmd, silent=silent)
-		c.close()
 		return response
 
 #-----------------$
@@ -86,25 +92,23 @@ class Module_UE:
 #-----------------$
 
 	def initialize(self):
-		if self.trace:
-			raise Exception("UE tracing not implemented yet")
-			self._enableTrace()
 		# we first terminate to make sure the UE has been stopped
 		if self.cmd_dict["detach"]:
 			self._command(self.cmd_dict["detach"], silent=True)
 		self._command(self.cmd_dict["terminate"], silent=True)
 		ret = self._command(self.cmd_dict["initialize"])
 		logging.info(f'For command: {ret.args} | return output: {ret.stdout} | Code: {ret.returncode}')
+		if self.trace:
+			self._enableTrace()
 		# Here each UE returns differently for the successful initialization, requires check based on UE
 		return ret.returncode == 0
 
 
-	def terminate(self):
+	def terminate(self, ctx=None):
 		self._command(self.cmd_dict["terminate"])
-		if self.trace:
+		if self.trace and ctx is not None:
-			raise Exception("UE tracing not implemented yet")
 			self._disableTrace()
-			return self._logCollect()
+			return self._collectTrace(ctx)
 		return None
 
 	def attach(self, attach_tries = 4, attach_timeout = 60):
@@ -113,10 +117,11 @@ class Module_UE:
 			self._command(self.cmd_dict["attach"])
 			timeout = attach_timeout
 			logging.debug("Waiting for IP address to be assigned")
+			ip = self.getIP(silent=False, reportNonZero=True)
 			while timeout > 0 and not ip:
-				time.sleep(5)
+				time.sleep(1)
-				timeout -= 5
+				timeout -= 1
-				ip = self.getIP()
+				ip = self.getIP(silent=True, reportNonZero=False)
 			if ip:
 				break
 			logging.warning(f"UE did not receive IP address after {attach_timeout} s, detaching")
@@ -160,8 +165,8 @@ class Module_UE:
 			logging.error(message)
 			return False
 
-	def getIP(self):
+	def getIP(self, silent=True, reportNonZero=True):
-		output = self._command(self.cmd_dict["getNetwork"], silent=True)
+		output = self._command(self.cmd_dict["getNetwork"], silent=silent, reportNonZero=reportNonZero)
 		result = re.search(r'inet (?P<ip>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', output.stdout)
 		if result and result.group('ip'):
 			ip = result.group('ip')
@@ -191,10 +196,33 @@ class Module_UE:
 		return self.cmd_prefix if self.cmd_prefix else ""
 
 	def _enableTrace(self):
-		raise Exception("not implemented")
+		logging.info(f'UE {self}: start UE tracing')
+		self._command(self.cmd_dict["traceStart"])
 
 	def _disableTrace(self):
-		raise Exception("not implemented")
+		logging.info(f'UE {self}: stop UE tracing')
+		self._command(self.cmd_dict["traceStop"])
 
-	def _logCollect(self):
+	def _collectTrace(self, ctx):
-		raise Exception("not implemented")
+		remote_dir = "/tmp/ue-trace-logs"
+		with cls_cmd.getConnection(self.host) as c:
+			# create a directory for log collection
+			c.run(f'rm -rf {remote_dir}')
+			ret = c.run(f'mkdir {remote_dir}')
+			if ret.returncode != 0:
+				logging.error("cannot create directory for log collection")
+				return []
+			log_cmd = self._logCollect.replace('%%log_dir%%', remote_dir)
+			self._command(log_cmd)
+			# enumerate collected files
+			ret = c.run(f'ls {remote_dir}/*')
+			if ret.returncode != 0:
+				logging.error("cannot enumerate log files")
+				return []
+			log_files = []
+			# copy them to the executor one by one, and store in log_dir
+			for f in ret.stdout.split("\n"):
+				name = archiveArtifact(c, ctx, f)
+				log_files.append(name)
+			c.run(f'rm -rf {remote_dir}')
+			return log_files

ci-scripts/cls_native.py

@@ -28,27 +28,24 @@ import cls_cmd
 import cls_oai_html
 import cls_analysis
 import constants as CONST
+from cls_ci_helper import archiveArtifact
 
-LOG_PATH_PHYSIM = 'phy_sim_logs'
 DPDK_PATH = '/opt/dpdk-t2-22.11.0'
 
 class Native():
 
-	def Build(test_case, HTML, host, directory, options):
+	def Build(ctx, test_case, HTML, host, directory, options):
 		logging.debug(f'Building on server: {host}')
 		HTML.testCase_id = test_case
 
 		with cls_cmd.getConnection(host) as ssh:
 			base = f"{directory}/cmake_targets"
-			ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/log/build_oai.log", timeout=900)
+			ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/build_oai.log", timeout=900)
 			success = ret.returncode == 0
-			logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True)
+			logs = ssh.run(f"cat {base}/build_oai.log", silent=True)
 			logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}")
 
-			# create log directory, and copy build logs
+			archiveArtifact(ssh, ctx, f'{base}/build_oai.log')
-			target = f"{base}/build_log_{test_case}/"
-			ssh.run(f"mkdir -p {target}")
-			ssh.run(f"mv {base}/log/* {target}")
 
 			# check if build artifacts are there
 			# NOTE: build_oai should fail with exit code if it could not build, but it does not
@@ -68,15 +65,14 @@ class Native():
 			HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK)
 		return success
 
-	def Run_Physim(HTML, host, directory, options, physim_test, threshold):
+	def Run_Physim(ctx, HTML, host, directory, options, physim_test, threshold):
 		logging.debug(f'Runnin {physim_test} on server: {host}')
 		workSpacePath = f'{directory}/cmake_targets'
-		os.system(f'mkdir -p ./{LOG_PATH_PHYSIM}')
+		runLogFile=f'{workSpacePath}/physim.log'
-		runLogFile=f'physim_{HTML.testCase_id}.log'
 		with cls_cmd.getConnection(host) as cmd:
-			cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {workSpacePath}/{runLogFile}')
+			cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {runLogFile}')
-			cmd.copyin(src=f'{workSpacePath}/{runLogFile}', tgt=f'{LOG_PATH_PHYSIM}/{runLogFile}')
+			physim_file = archiveArtifact(cmd, ctx, runLogFile)
-		success, msg = cls_analysis.Analysis.analyze_physim(f'{LOG_PATH_PHYSIM}/{runLogFile}', physim_test, options, threshold)
+		success, msg = cls_analysis.Analysis.analyze_physim(physim_file, physim_test, options, threshold)
 		if success:
 			HTML.CreateHtmlTestRowQueue(options, 'OK', [msg])
 		else:

ci-scripts/cls_oaicitest.py

@@ -47,6 +47,7 @@ import constants as CONST
 import cls_module
 import cls_corenetwork
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
 # Helper functions used here and in other classes
@@ -259,9 +260,7 @@ class OaiCiTest():
 		self.ranAllowMerge = False
 		self.ranTargetBranch = ''
 
-		self.testCase_id = ''
 		self.testXMLfiles = []
-		self.desc = ''
 		self.ping_args = ''
 		self.ping_packetloss_threshold = ''
 		self.ping_rttavg_threshold =''
@@ -355,7 +354,7 @@ class OaiCiTest():
 		HTML.CreateHtmlTestRowQueue('NA', 'OK', messages)
 		return True
 
-	def Ping_common(self, cn, ue, logPath):
+	def Ping_common(self, ctx, cn, ue):
 		ping_status = 0
 		ueIP = ue.getIP()
 		if not ueIP:
@@ -363,13 +362,12 @@ class OaiCiTest():
 		svrIP = cn.getIP()
 		if not svrIP:
 			return (False, f"CN {cn.getName()} has no IP address")
-		ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
+		ping_log_file = f'/tmp/ping_{ue.getName()}.log'
 		ping_time = re.findall(r"-c *(\d+)",str(self.ping_args))
-		local_ping_log_file = f'{logPath}/{ping_log_file}'
 		if re.search('%cn_ip%', self.ping_args) or re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+', self.ping_args):
 			raise Exception(f"ping_args should not have IP address: {self.ping_args}")
 		interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
-		ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee /tmp/{ping_log_file}'
+		ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee {ping_log_file}'
 		cmd = cls_cmd.getConnection(ue.getHost())
 		response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
 		ue_header = f'UE {ue.getName()} ({ueIP})'
@@ -377,8 +375,7 @@ class OaiCiTest():
 			message = ue_header + ': ping crashed: TIMEOUT?'
 			return (False, message)
 
-		#copy the ping log file to have it locally for analysis (ping stats)
+		local_ping_log_file = archiveArtifact(cmd, ctx, ping_log_file)
-		cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=local_ping_log_file)
 		cmd.close()
 
 		with open(local_ping_log_file, 'r') as f:
@@ -418,18 +415,13 @@ class OaiCiTest():
 
 		return (True, message)
 
-	def Ping(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"):
+	def Ping(self, ctx, HTML, infra_file="ci_infra.yaml"):
 		if self.ue_ids == [] or self.svr_id == None:
 			raise Exception("no module names in self.ue_ids or/and self.svr_id provided")
-		# Creating destination log folder if needed on the python executor workspace
-		with cls_cmd.getConnection('localhost') as local:
-			ymlPath = CONTAINERS.yamlPath[0].split('/')
-			logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}'
-			local.run(f'mkdir -p {logPath}', silent=True)
 		ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)]
 		cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file)
 		with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:
-			futures = [executor.submit(self.Ping_common, cn, ue, logPath) for ue in ues]
+			futures = [executor.submit(self.Ping_common, ctx, cn, ue) for ue in ues]
 			results = [f.result() for f in futures]
 			# each result in results is a tuple, first member goes to successes, second to messages
 			successes, messages = map(list, zip(*results))
@@ -450,7 +442,7 @@ class OaiCiTest():
 			HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages)
 		return success
 
-	def Iperf_Module(self, cn, ue, idx, ue_num, logPath):
+	def Iperf_Module(self, ctx, cn, ue, idx, ue_num):
 		ueIP = ue.getIP()
 		if not ueIP:
 			return (False, f"UE {ue.getName()} has no IP address")
@@ -463,7 +455,7 @@ class OaiCiTest():
 		serverReport = ""
 		udpIperf = re.search('-u', iperf_opt) is not None
 		bidirIperf = re.search('--bidir', iperf_opt) is not None
-		client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log'
+		client_filename = f'/tmp/iperf_client_{ue.getName()}.log'
 		if udpIperf:
 			target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(idx, ue_num, self.iperf_profile, self.iperf_args)
 			# note: for UDP testing we don't want to use json report - reports 0 Mbps received bitrate
@@ -478,14 +470,11 @@ class OaiCiTest():
 			port = 5002 + idx
 			# note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False`
 			t = iperf_time * 2.5
-			cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True)
+			cmd_ue.run(f'rm {client_filename}', reportNonZero=False, silent=True)
 			if cn.runIperf3Server():
 				cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} >> /dev/null &', timeout=t)
-			cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=t)
+			cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> {client_filename}', timeout=t)
-			# note: copy iperf3 log to the current directory for log analysis and log collection
+			dest_filename = archiveArtifact(cmd_ue, ctx, client_filename)
-			dest_filename = f'{logPath}/{client_filename}'
-			cmd_ue.copyin(f'/tmp/{client_filename}', dest_filename)
-			cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True)
 		if udpIperf:
 			status, msg = Iperf_analyzeV3UDP(dest_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
 		elif bidirIperf:
@@ -495,20 +484,15 @@ class OaiCiTest():
 
 		return (status, f'{ue_header}\n{msg}')
 
-	def Iperf(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"):
+	def Iperf(self, ctx, HTML, infra_file="ci_infra.yaml"):
 		logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"')
 
 		if self.ue_ids == [] or self.svr_id == None:
 			raise Exception("no module names in self.ue_ids or/and self.svr_id provided")
-		# create log directory on executor node
-		with cls_cmd.getConnection('localhost') as local:
-			ymlPath = CONTAINERS.yamlPath[0].split('/')
-			logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}'
-			local.run(f'mkdir -p {logPath}', silent=True)
 		ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)]
 		cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file)
 		with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:
-			futures = [executor.submit(self.Iperf_Module, cn, ue, i, len(ues), logPath) for i, ue in enumerate(ues)]
+			futures = [executor.submit(self.Iperf_Module, ctx, cn, ue, i, len(ues)) for i, ue in enumerate(ues)]
 			results = [f.result() for f in futures]
 			# each result in results is a tuple, first member goes to successes, second to messages
 			successes, messages = map(list, zip(*results))
@@ -529,7 +513,7 @@ class OaiCiTest():
 			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', messages)
 		return success
 
-	def Iperf2_Unidir(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"):
+	def Iperf2_Unidir(self, ctx, HTML, infra_file="ci_infra.yaml"):
 		if self.ue_ids == [] or self.svr_id == None or len(self.ue_ids) != 1:
 			raise Exception("no module names in self.ue_ids or/and self.svr_id provided, multi UE scenario not supported")
 		ue = cls_module.Module_UE(self.ue_ids[0].strip(),self.nodes[0].strip(), infra_file)
@@ -540,23 +524,17 @@ class OaiCiTest():
 		svrIP = cn.getIP()
 		if not svrIP:
 			return False
-		server_filename = f'iperf_server_{self.testCase_id}_{ue.getName()}.log'
+		server_filename = f'/tmp/iperf_server_{ue.getName()}.log'
-		ymlPath = CONTAINERS.yamlPath[0].split('/')
-		logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}'
 		iperf_time = Iperf_ComputeTime(self.iperf_args)
 		target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(0, 1, self.iperf_profile, self.iperf_args)
 		t = iperf_time*2.5
-		with cls_cmd.getConnection('localhost') as local:
-			local.run(f'mkdir -p {logPath}')
 		with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(cn.getHost()) as cmd_svr:
-			cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False)
+			cmd_ue.run(f'rm {server_filename}', reportNonZero=False)
-			cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> /tmp/{server_filename} &', timeout=t)
+			cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> {server_filename} &', timeout=t)
 			cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf -c {ueIP} -B {svrIP} {iperf_opt} -i1 >> /dev/null', timeout=t)
 			localPath = f'{os.getcwd()}'
-			# note: copy iperf2 log to the directory for log collection
+			local = archiveArtifact(cmd_ue, ctx, server_filename)
-			cmd_ue.copyin(f'/tmp/{server_filename}', f'{logPath}/{server_filename}')
+		success, msg = Iperf_analyzeV2UDP(local, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
-			cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False)
-		success, msg = Iperf_analyzeV2UDP(f'{logPath}/{server_filename}', self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
 		ue_header = f'UE {ue.getName()} ({ueIP})'
 		logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m')
 		for l in msg.split('\n'):
@@ -838,17 +816,17 @@ class OaiCiTest():
 				global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC
 		return global_status
 
-	def TerminateUE(self, HTML):
+	def TerminateUE(self, ctx, HTML):
 		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
 		with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:
-			futures = [executor.submit(ue.terminate) for ue in ues]
+			futures = [executor.submit(ue.terminate, ctx) for ue in ues]
 			archives = [f.result() for f in futures]
 		archive_info = [f'Log at: {a}' if a else 'No log available' for a in archives]
 		messages = [f"UE {ue.getName()}: {log}" for (ue, log) in zip(ues, archive_info)]
 		HTML.CreateHtmlTestRowQueue(f'N/A', 'OK', messages)
 		return True
 
-	def DeployCoreNetwork(cn_id, HTML):
+	def DeployCoreNetwork(cn_id, ctx, HTML):
 		core_name = cn_id.strip()
 		cn = cls_corenetwork.CoreNetwork(core_name)
 		success, output = cn.deploy()
@@ -862,41 +840,11 @@ class OaiCiTest():
 			HTML.CreateHtmlTestRowQueue(core_name, 'KO', [msg])
 		return success
 
-	def UndeployCoreNetwork(cn_id, HTML):
+	def UndeployCoreNetwork(cn_id, ctx, HTML):
-		# Ping, Iperf, DeployObject put logs into a path based on YAML. We
-		# can't do this here (because there is no yaml), so hardcode a path for
-		# "cn_logs" for the moment
-		logPath = f'{os.getcwd()}/../cmake_targets/log/cn_logs'
-		with cls_cmd.getConnection('localhost') as local:
-			local.run(f'mkdir -p {logPath}', silent=True)
 		core_name = cn_id.strip()
 		cn = cls_corenetwork.CoreNetwork(core_name)
-		logs, output = cn.undeploy(log_dir=logPath)
+		logs, output = cn.undeploy(ctx=ctx)
 		logging.info(f"undeployed core network {core_name}, logs {logs}, output:\n{output}")
-		message = "Log files: " + ", ".join([os.path.basename(l) for l in logs])
+		message = "Log files:\n" + "\n".join([os.path.basename(l) for l in logs])
 		HTML.CreateHtmlTestRowQueue(core_name, 'OK', [message])
 		return True
-
-	def LogCollectBuild(self,RAN):
-		# Some pipelines are using "none" IP / Credentials
-		# In that case, just forget about it
-		if RAN.eNBIPAddress == 'none':
-			sys.exit(0)
-
-		if (RAN.eNBIPAddress != '' and RAN.eNBUserName != '' and RAN.eNBPassword != ''):
-			IPAddress = RAN.eNBIPAddress
-			UserName = RAN.eNBUserName
-			Password = RAN.eNBPassword
-			SourceCodePath = RAN.eNBSourceCodePath
-		else:
-			sys.exit('Insufficient Parameter')
-		with cls_cmd.getConnection(IPAddress) as cmd:
-			d = f'{SourceCodePath}/cmake_targets'
-			cmd.run(f'rm -f {d}/build.log.zip')
-			cmd.run(f'cd {d} && zip -r build.log.zip build_log_*/*')
-
-	def ShowTestID(self):
-		logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
-		logging.info(f'\u001B[1m Test ID: {self.testCase_id} \u001B[0m')
-		logging.info(f'\u001B[1m {self.desc} \u001B[0m')
-		logging.info(f'\u001B[1m----------------------------------------\u001B[0m')

ci-scripts/cls_static_code_analysis.py

@@ -43,6 +43,7 @@ from pathlib import Path
 import helpreadme as HELP
 import constants as CONST
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
 # Class Declaration
@@ -80,7 +81,7 @@ class StaticCodeAnalysis():
 		self.eNBPassword = ''
 		self.eNBSourceCodePath = ''
 
-	def CppCheckAnalysis(self, HTML):
+	def CppCheckAnalysis(self, ctx, HTML):
 		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -103,7 +104,7 @@ class StaticCodeAnalysis():
 			full_ran_repo_name = self.ranRepository + '.git'
 
 		cmd.cd(lSourcePath)
-		logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}'
+		logDir = f'{lSourcePath}/cmake_targets/log'
 		cmd.run(f'mkdir -p {logDir}')
 		cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal')
 		cmd.run(f'sed -e "s@xenial@bionic@" {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.xenial > {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.bionic')
@@ -112,31 +113,21 @@ class StaticCodeAnalysis():
 		cmd.run(f'docker build --tag oai-cppcheck:focal --file {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.focal . > {logDir}/cppcheck-focal.txt 2>&1')
 		cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal')
 
-		# Analyzing the logs
+		bionic = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-bionic.txt')
-		cmd.copyin(f'{logDir}/cppcheck-bionic.txt', 'cppcheck-bionic.txt')
+		focal = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-focal.txt')
-		cmd.copyin(f'{logDir}/cppcheck-focal.txt', 'cppcheck-focal.txt')
 		cmd.close()
 
 		CCR = CppCheckResults()
 		CCR_ref = CppCheckResults()
 		vId = 0
 		for variant in CCR.variants:
-			refAvailable = False
+			filename = ctx.baseFilename() + '-cppcheck-'+ variant + '.txt'
-			if self.ranAllowMerge:
+			logging.info(f"will check file '{filename}'")
-				refFolder = str(Path.home()) + '/cppcheck-references'
+			if not os.path.isfile(filename):
-				if (os.path.isfile(refFolder + '/cppcheck-'+ variant + '.txt')):
+				raise FileNotFoundError(f"{filename} is not a file")
-					refAvailable = True
+			else:
-					with open(refFolder + '/cppcheck-'+ variant + '.txt', 'r') as refFile:
-						for line in refFile:
-							ret = re.search(' (?P<nb_errors>[0-9\.]+) errors', str(line))
-							if ret is not None:
-								CCR_ref.nbErrors[vId] = int(ret.group('nb_errors'))
-							ret = re.search(' (?P<nb_warnings>[0-9\.]+) warnings', str(line))
-							if ret is not None:
-								CCR_ref.nbWarnings[vId] = int(ret.group('nb_warnings'))
-			if (os.path.isfile('./cppcheck-'+ variant + '.txt')):
 				xmlStart = False
-				with open('./cppcheck-'+ variant + '.txt', 'r') as logfile:
+				with open(filename, 'r') as logfile:
 					for line in logfile:
 						ret = re.search('cppcheck version="(?P<version>[0-9\.]+)"', str(line))
 						if ret is not None:
@@ -185,26 +176,6 @@ class StaticCodeAnalysis():
 			vMsg += '   Wrong Scanf Nb Args:             ' + str(CCR.nbWrongScanfArg[vId]) + '\n'
 			for vLine in vMsg.split('\n'):
 				logging.debug(vLine)
-			if self.ranAllowMerge and refAvailable:
-				if CCR_ref.nbErrors[vId] == CCR.nbErrors[vId]:
-					logging.debug('   No change in number of errors')
-				elif CCR_ref.nbErrors[vId] > CCR.nbErrors[vId]:
-					logging.debug('   Good! Decrease in number of errors')
-				else:
-					logging.debug('   Bad! increase in number of errors')
-				if CCR_ref.nbWarnings[vId] == CCR.nbWarnings[vId]:
-					logging.debug('   No change in number of warnings')
-				elif CCR_ref.nbWarnings[vId] > CCR.nbWarnings[vId]:
-					logging.debug('   Good! Decrease in number of warnings')
-				else:
-					logging.debug('   Bad! increase in number of warnings')
-			# Create new reference file
-			if not self.ranAllowMerge:
-				refFolder = str(Path.home()) + '/cppcheck-references'
-				if not os.path.isdir(refFolder):
-					os.mkdir(refFolder)
-				with open(refFolder + '/cppcheck-'+ variant + '.txt', 'w') as refFile:
-					refFile.write(vMsg)
 			vId += 1
 
 		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
@@ -213,7 +184,7 @@ class StaticCodeAnalysis():
 
 		return True
 
-	def LicenceAndFormattingCheck(self, HTML):
+	def LicenceAndFormattingCheck(self, ctx, HTML):
 		# Workspace is no longer recreated from scratch.
 		# It implies that this method shall be called last within a build pipeline
 		# where workspace is already created
@@ -238,7 +209,7 @@ class StaticCodeAnalysis():
 			else:
 				check_options += f' --build-arg TARGET_BRANCH={self.ranTargetBranch}'
 
-		logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}'
+		logDir = f'{lSourcePath}/cmake_targets/log/'
 		cmd.run(f'mkdir -p {logDir}')
 		cmd.run('docker image rm oai-formatting-check:latest')
 		cmd.run(f'docker build --target oai-formatting-check --tag oai-formatting-check:latest {check_options} --file {lSourcePath}/ci-scripts/docker/Dockerfile.formatting.bionic {lSourcePath} > {logDir}/oai-formatting-check.txt 2>&1')
@@ -247,12 +218,11 @@ class StaticCodeAnalysis():
 		cmd.run('docker image prune --force')
 		cmd.run('docker volume prune --force')
 
-		# Analyzing the logs
+		file = archiveArtifact(cmd, ctx, f'{logDir}/oai-formatting-check.txt')
-		cmd.copyin(f'{logDir}/oai-formatting-check.txt', 'oai-formatting-check.txt')
 		cmd.close()
 
 		finalStatus = 0
-		if (os.path.isfile('./oai-formatting-check.txt')):
+		if (os.path.isfile(file)):
 			analyzed = False
 			nbFilesNotFormatted = 0
 			listFiles = False
@@ -263,7 +233,7 @@ class StaticCodeAnalysis():
 			gnuGplLicenceFiles = []
 			suspectLicence = False
 			suspectLicenceFiles = []
-			with open('./oai-formatting-check.txt', 'r') as logfile:
+			with open(file, 'r') as logfile:
 				for line in logfile:
 					ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line))
 					if ret is not None:

ci-scripts/main.py

@@ -43,8 +43,10 @@ import cls_containerize	 #class Containerize for all container-based operations
 import cls_static_code_analysis  #class for static code analysis
 import cls_cluster		 # class for building/deploying on cluster
 import cls_native        # class for all native/source-based operations
+from cls_ci_helper import TestCaseCtx
 
 import ran
+import cls_cmd
 import cls_oai_html
 
 
@@ -91,7 +93,7 @@ def AssignParams(params_dict):
 		setattr(RAN, key, value)
 		setattr(HTML, key, value)
 
-def ExecuteActionWithParam(action):
+def ExecuteActionWithParam(action, ctx):
 	global RAN
 	global HTML
 	global CONTAINERS
@@ -134,15 +136,15 @@ def ExecuteActionWithParam(action):
 		if proxy_commit is not None:
 			CONTAINERS.proxyCommit = proxy_commit
 		if action == 'Build_eNB':
-			success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
+			success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
 		elif action == 'Build_Image':
-			success = CONTAINERS.BuildImage(HTML)
+			success = CONTAINERS.BuildImage(ctx, HTML)
 		elif action == 'Build_Proxy':
-			success = CONTAINERS.BuildProxy(HTML)
+			success = CONTAINERS.BuildProxy(ctx, HTML)
 		elif action == 'Build_Cluster_Image':
-			success = CLUSTER.BuildClusterImage(HTML)
+			success = CLUSTER.BuildClusterImage(ctx, HTML)
 		elif action == 'Build_Run_Tests':
-			success = CONTAINERS.BuildRunTests(HTML)
+			success = CONTAINERS.BuildRunTests(ctx, HTML)
 
 	elif action == 'Initialize_eNB':
 		datalog_rt_stats_file=test.findtext('rt_stats_cfg')
@@ -204,7 +206,7 @@ def ExecuteActionWithParam(action):
 			RAN.air_interface[RAN.eNB_instance] = 'lte-softmodem'
 		else:
 			RAN.air_interface[RAN.eNB_instance] = air_interface.lower() +'-softmodem'
-		success = RAN.TerminateeNB(HTML)
+		success = RAN.TerminateeNB(ctx, HTML)
 
 	elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE':
 		CiTestObj.ue_ids = test.findtext('id').split(' ')
@@ -226,7 +228,7 @@ def ExecuteActionWithParam(action):
 		elif action == 'Detach_UE':
 			success = CiTestObj.DetachUE(HTML)
 		elif action == 'Terminate_UE':
-			success = CiTestObj.TerminateUE(HTML)
+			success = CiTestObj.TerminateUE(ctx, HTML)
 		elif action == 'CheckStatusUE':
 			success = CiTestObj.CheckStatusUE(HTML)
 		elif action == 'DataEnable_UE':
@@ -253,7 +255,7 @@ def ExecuteActionWithParam(action):
 			else:
 				CiTestObj.nodes = [None] * len(CiTestObj.ue_ids)
 		ping_rttavg_threshold = test.findtext('ping_rttavg_threshold') or ''
-		success = CiTestObj.Ping(HTML, CONTAINERS)
+		success = CiTestObj.Ping(ctx, HTML)
 
 	elif action == 'Iperf' or action == 'Iperf2_Unidir':
 		CiTestObj.iperf_args = test.findtext('iperf_args')
@@ -284,9 +286,9 @@ def ExecuteActionWithParam(action):
 			logging.error('test-case has wrong option ' + CiTestObj.iperf_options)
 			CiTestObj.iperf_options = 'check'
 		if action == 'Iperf':
-			success = CiTestObj.Iperf(HTML, CONTAINERS)
+			success = CiTestObj.Iperf(ctx, HTML)
 		elif action == 'Iperf2_Unidir':
-			success = CiTestObj.Iperf2_Unidir(HTML, CONTAINERS)
+			success = CiTestObj.Iperf2_Unidir(ctx, HTML)
 
 	elif action == 'IdleSleep':
 		st = test.findtext('idle_sleep_time_in_sec') or "5"
@@ -295,12 +297,12 @@ def ExecuteActionWithParam(action):
 	elif action == 'Deploy_Run_PhySim':
 		oc_release = test.findtext('oc_release')
 		svr_id = test.findtext('svr_id') or None
-		success = CLUSTER.deploy_oc_physim(HTML, oc_release, svr_id)
+		success = CLUSTER.deploy_oc_physim(ctx, HTML, oc_release, svr_id)
 
 	elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork':
 		cn_id = test.findtext('cn_id')
 		core_op = getattr(cls_oaicitest.OaiCiTest, action)
-		success = core_op(cn_id, HTML)
+		success = core_op(cn_id, ctx, HTML)
 
 	elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace":
 		eNB_instance=test.findtext('eNB_instance')
@@ -328,9 +330,9 @@ def ExecuteActionWithParam(action):
 		CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
 		CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
 		if action == 'Deploy_Object':
-			success = CONTAINERS.DeployObject(HTML)
+			success = CONTAINERS.DeployObject(ctx, HTML)
 		elif action == 'Undeploy_Object':
-			success = CONTAINERS.UndeployObject(HTML, RAN)
+			success = CONTAINERS.UndeployObject(ctx, HTML, RAN)
 		elif action == 'Create_Workspace':
 			if force_local:
 				# Do not create a working directory when running locally. Current repo directory will be used
@@ -341,13 +343,13 @@ def ExecuteActionWithParam(action):
 		physim_options = test.findtext('physim_run_args')
 		physim_test = test.findtext('physim_test')
 		physim_threshold = test.findtext('physim_time_threshold') or 'inf'
-		success = cls_native.Native.Run_Physim(HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold)
+		success = cls_native.Native.Run_Physim(ctx, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold)
 
 	elif action == 'LicenceAndFormattingCheck':
-		success = SCA.LicenceAndFormattingCheck(HTML)
+		success = SCA.LicenceAndFormattingCheck(ctx, HTML)
 
 	elif action == 'Cppcheck_Analysis':
-		success = SCA.CppCheckAnalysis(HTML)
+		success = SCA.CppCheckAnalysis(ctx, HTML)
 
 	elif action == 'Push_Local_Registry':
 		svr_id = test.findtext('svr_id')
@@ -407,10 +409,11 @@ def test_in_list(test, list):
 def receive_signal(signum, frame):
 	sys.exit(1)
 
-
+def ShowTestID(ctx, desc):
-
+    logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
-
+    logging.info(f'\u001B[1m Test ID: {ctx.test_id} (#{ctx.count}) \u001B[0m')
-
+    logging.info(f'\u001B[1m {desc} \u001B[0m')
+    logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
 
 #-----------------------------------------------------------
 # MAIN PART
@@ -466,15 +469,7 @@ if py_param_file_present == True:
 cwd = os.getcwd()
 
 if re.match('^TerminateeNB$', mode, re.IGNORECASE):
-	if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '':
+	logging.warning("Option TerminateeNB ignored")
-		HELP.GenericHelp(CONST.Version)
-		sys.exit('Insufficient Parameter')
-	if RAN.eNBIPAddress == 'none':
-		sys.exit(0)
-	RAN.eNB_instance=0
-	RAN.eNB_serverId[0]='0'
-	RAN.eNBSourceCodePath='/tmp/'
-	RAN.TerminateeNB(HTML)
 elif re.match('^TerminateHSS$', mode, re.IGNORECASE):
 	logging.warning("Option TerminateHSS ignored")
 elif re.match('^TerminateMME$', mode, re.IGNORECASE):
@@ -482,12 +477,7 @@ elif re.match('^TerminateMME$', mode, re.IGNORECASE):
 elif re.match('^TerminateSPGW$', mode, re.IGNORECASE):
 	logging.warning("Option TerminateSPGW ignored")
 elif re.match('^LogCollectBuild$', mode, re.IGNORECASE):
-	if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '':
+	logging.warning("Option LogCollectBuild ignored")
-		logging.warning("nothing to collect (eNBIPAddress/eNBUserName/eNBPassword/eNBSourceCodePath is '')")
-		sys.exit(0)
-	if RAN.eNBIPAddress == 'none':
-		sys.exit(0)
-	CiTestObj.LogCollectBuild(RAN)
 elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
 	if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '':
 		HELP.GenericHelp(CONST.Version)
@@ -565,6 +555,16 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	else:
 		xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[0]
 
+	# directory where all log artifacts will be placed
+	logPath = f"{cwd}/../cmake_targets/log/{CiTestObj.testXMLfiles[0].split('/')[-1]}.d"
+	# we run from within ci-scripts, but the logPath is absolute, so replace
+	# the ci-scripts/..; if it does not exist, nothing will happen
+	logPath = logPath.replace(r'/ci-scripts/..', '')
+	logging.info(f"placing all artifacts for this run in {logPath}/")
+	with cls_cmd.LocalCmd() as c:
+		c.run(f"rm -rf {logPath}")
+		c.run(f"mkdir -p {logPath}")
+
 	xmlTree = ET.parse(xml_test_file)
 	xmlRoot = xmlTree.getroot()
 
@@ -615,29 +615,32 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	task_set_succeeded = True
 	HTML.startTime=int(round(time.time() * 1000))
 
+	i = 0
 	for test_case_id in todo_tests:
 		for test in all_tests:
 			id = test.get('id')
 			if test_case_id != id:
 				continue
+			i += 1
 			CiTestObj.testCase_id = id
+			ctx = TestCaseCtx(i, int(id), logPath)
 			HTML.testCase_id=CiTestObj.testCase_id
-			CiTestObj.desc = test.findtext('desc')
+			desc = test.findtext('desc')
 			always_exec = test.findtext('always_exec') in ['True', 'true', 'Yes', 'yes']
 			may_fail = test.findtext('may_fail') in ['True', 'true', 'Yes', 'yes']
-			HTML.desc=CiTestObj.desc
+			HTML.desc = desc
 			action = test.findtext('class')
 			if (CheckClassValidity(xml_class_list, action, id) == False):
 				task_set_succeeded = False
 				continue
-			CiTestObj.ShowTestID()
+			ShowTestID(ctx, desc)
 			if not task_set_succeeded and not always_exec:
 				msg = f"skipping test due to prior error"
 				logging.warning(msg)
 				HTML.CreateHtmlTestRowQueue(msg, "SKIP", [])
 				break
 			try:
-				test_succeeded = ExecuteActionWithParam(action)
+				test_succeeded = ExecuteActionWithParam(action, ctx)
 				if not test_succeeded and may_fail:
 					logging.warning(f"test ID {test_case_id} action {action} may or may not fail, proceeding despite error")
 				elif not test_succeeded:

ci-scripts/ran.py

@@ -46,6 +46,7 @@ import cls_cmd
 import sshconnection as SSH
 import helpreadme as HELP
 import constants as CONST
+from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
 # Class Declaration
@@ -124,7 +125,6 @@ class RANManagement():
 
 		self.testCase_id = HTML.testCase_id
 		mySSH = SSH.SSHConnection()
-		cwd = os.getcwd()
 		
 		mySSH.open(lIpAddr, lUserName, lPassWord)
 		mySSH.command('cd ' + lSourcePath, '\$', 5)
@@ -249,42 +249,7 @@ class RANManagement():
 		logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m')
 		return enbDidSync
 
-	def CheckeNBProcess(self, status_queue):
+	def TerminateeNB(self, ctx, HTML):
-		# At least the instance 0 SHALL be on!
-		if self.eNBstatuses[0] == 0:
-			lIpAddr = self.eNBIPAddress
-			lUserName = self.eNBUserName
-			lPassWord = self.eNBPassword
-		elif self.eNBstatuses[0] == 1:
-			lIpAddr = self.eNB1IPAddress
-			lUserName = self.eNB1UserName
-			lPassWord = self.eNB1Password
-		elif self.eNBstatuses[0] == 2:
-			lIpAddr = self.eNB2IPAddress
-			lUserName = self.eNB2UserName
-			lPassWord = self.eNB2Password
-		else:
-			lIpAddr = self.eNBIPAddress
-			lUserName = self.eNBUserName
-			lPassWord = self.eNBPassword
-		mySSH = SSH.SSHConnection()
-		mySSH.open(lIpAddr, lUserName, lPassWord)
-		if self.air_interface[self.eNB_instance] == '':
-			pattern = 'softmodem'
-		else:
-			pattern = self.air_interface[self.eNB_instance]
-		mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5)
-		result = re.search(pattern, mySSH.getBefore())
-		success = result is not None
-		if not success:
-			logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
-			status_queue.put(CONST.ENB_PROCESS_FAILED)
-		else:
-			status_queue.put(CONST.ENB_PROCESS_OK)
-		mySSH.close()
-		return success
-
-	def TerminateeNB(self, HTML):
 		if self.eNB_serverId[self.eNB_instance] == '0':
 			lIpAddr = self.eNBIPAddress
 			lUserName = self.eNBUserName
@@ -356,13 +321,11 @@ class RANManagement():
 				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
 				self.eNBLogFiles[int(self.eNB_instance)] = ''
 			if analyzeFile:
-				#*stats.log files + pickle + png
+				with cls_cmd.getConnection(lIpAddr) as cmd:
-				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.')
+					file = archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/' + fileToAnalyze)
-				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.')
+					archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrL1_stats.log')
-				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.')
+					archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrMAC_stats.log')
-				#
+				if file is None:
-				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
-				if (copyin_res == -1):
 					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
 					HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
 					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
@@ -375,8 +338,8 @@ class RANManagement():
 					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
 					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
 					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
-				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
+				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + file)
-				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML, self.ran_checkers)
+				logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers)
 				if (logStatus < 0):
 					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
 					#display rt stats for gNB only
@@ -734,11 +697,16 @@ class RANManagement():
 			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
 		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  
 
-		if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'):
+		# nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file
+		# currently the link is only implicit as below based on pattern matching
+		# I will rework this to give the file explicitly
+		l1_stats_fn = re.sub(r'enb_[0-9]+', 'nrL1_stats', eNBlogFile)
+		mac_stats_fn = re.sub(r'enb_[0-9]+', 'nrMAC_stats', eNBlogFile)
+		if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn):
 			# don't use CI-nrL1_stats.log, as this will increase the processing time for
 			# no reason, we just need the last occurence
-			nrL1_stats = open('./nrL1_stats.log', 'r')
+			nrL1_stats = open(l1_stats_fn, 'r')
-			nrMAC_stats = open('./nrMAC_stats.log', 'r')
+			nrMAC_stats = open(mac_stats_fn, 'r')
 			for line in nrL1_stats.readlines():
 				for k in rt_keys:
 					result = re.search(k, line)     
@@ -758,7 +726,7 @@ class RANManagement():
 			nrL1_stats.close()
 			nrMAC_stats.close()
 		else:
-			logging.debug("NR Stats files for RT analysis not found")
+			logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}")
 
 		#stdout log file and stat log files analysis completed
 		logging.debug('   File analysis (stdout, stats) completed')

ci-scripts/tests/config/test_module_infra.yaml

@@ -0,0 +1,34 @@
+test:
+  Host: localhost
+  InitScript: ""
+  TermScript: ""
+  AttachScript: "true"
+  DetachScript: "true"
+  NetworkScript: ip a show dev lo
+  IF: lo
+  MTU: 65536
+
+test-fail:
+  Host: localhost
+  InitScript: ""
+  TermScript: ""
+  AttachScript: "false"
+  DetachScript: "true"
+  NetworkScript: ip a show dev TESTDEVICEDOESNOTEXIST
+  IF: TESTDEVICEDOESNOTEXIST
+  MTU: 123
+
+test-trace:
+  Host: localhost
+  InitScript: ""
+  TermScript: ""
+  AttachScript: "true"
+  DetachScript: "true"
+  Tracing:
+    # tttf: temp-trace-test-file
+    Start: "rm -f /tmp/tttf; for i in {1..8}; do echo $i >> /tmp/tttf; sleep 1; done &"
+    Stop: "while [[ $(wc -l /tmp/tttf) < 4 ]]; do sleep 1; done"
+    Collect: "cp /tmp/tttf %%log_dir%%"
+  NetworkScript: ip a show dev lo
+  IF: lo
+  MTU: 65536

ci-scripts/tests/corenetwork.py

@@ -6,14 +6,22 @@ logging.basicConfig(
 	format="[%(asctime)s] %(levelname)8s: %(message)s"
 )
 import os
+import tempfile
 
 import unittest
 
 sys.path.append('./') # to find OAI imports below
 import cls_corenetwork
+from cls_ci_helper import TestCaseCtx
 import cls_cmd
 
 class TestCoreNetwork(unittest.TestCase):
+    def setUp(self):
+        self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+    def tearDown(self):
+        with cls_cmd.LocalCmd() as c:
+            c.run(f'rm -rf {self.ctx.logPath}')
+
     def test_simple_core(self):
         c = cls_corenetwork.CoreNetwork("test", filename="tests/config/test_core_infra.yaml")
         success, output = c.deploy()
@@ -22,14 +30,14 @@ class TestCoreNetwork(unittest.TestCase):
         # should not have it set in config
         self.assertTrue(c.runIperf3Server())
         self.assertEqual(c.getIP(), "127.0.0.1")
-        log_files, output = c.undeploy(log_dir="/tmp")
+        log_files, output = c.undeploy(self.ctx)
         self.assertEqual(output, "undeploy")
         with cls_cmd.LocalCmd() as cmd:
             # there must be one log file for this (test) core
             self.assertEqual(len(log_files), 1)
-            # test core writes to %%log_dir%%/logs, which expands to /tmp/logs
+            # undeploy uses archiveArtifact(), which writes to {prefix}-logs
             l = log_files[0]
-            self.assertEqual(l, "/tmp/logs")
+            self.assertEqual(l, f"{self.ctx.baseFilename()}-logs")
             ret = cmd.run(f"cat {l}")
             self.assertEqual(ret.returncode, 0) # command must succeed
             self.assertEqual(ret.stdout, "logs") # output should be "logs"
@@ -39,14 +47,14 @@ class TestCoreNetwork(unittest.TestCase):
         success, _ = c.deploy()
         self.assertTrue(success)
         self.assertFalse(c.runIperf3Server())
-        c.undeploy(log_dir="/tmp")
+        c.undeploy(None)
 
     def test_core_fail(self):
         c = cls_corenetwork.CoreNetwork("test_fail", filename="tests/config/test_core_infra.yaml")
         success, _ = c.deploy()
         self.assertFalse(success)
         # undeployment should still work
-        c.undeploy(log_dir="/tmp")
+        c.undeploy(None)
 
     def test_core_script(self):
         c = cls_corenetwork.CoreNetwork("test_script", filename="tests/config/test_core_infra.yaml")

ci-scripts/tests/deployment.py

@@ -6,6 +6,7 @@ logging.basicConfig(
 	format="[%(asctime)s] %(levelname)8s: %(message)s"
 )
 import os
+import tempfile
 os.system(f'rm -rf cmake_targets')
 os.system(f'mkdir -p cmake_targets/log')
 import unittest
@@ -14,6 +15,7 @@ sys.path.append('./') # to find OAI imports below
 import cls_oai_html
 import cls_oaicitest
 import cls_containerize
+from cls_ci_helper import TestCaseCtx
 import ran
 import cls_cmd
 
@@ -48,12 +50,16 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.eNBPassword = None
 		self.cont.eNBSourceCodePath = os.getcwd()
 		self.cont.num_attempts = 3
+		self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+	def tearDown(self):
+		with cls_cmd.LocalCmd() as c:
+			c.run(f'rm -rf {self.ctx.logPath}')
 
 	def test_deploy(self):
 		self.cont.yamlPath[0] = 'tests/simple-dep/'
 		self.cont.deploymentTag = "noble"
-		deploy = self.cont.DeployObject(self.html)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -61,8 +67,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		# fails reliably
 		old = self.cont.yamlPath
 		self.cont.yamlPath[0] = 'tests/simple-fail/'
-		deploy = self.cont.DeployObject(self.html)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
-		self.cont.UndeployObject(self.html, self.ran)
+		self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertFalse(deploy)
 		self.cont.yamlPath = old
 
@@ -70,8 +76,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		# fails reliably
 		old = self.cont.yamlPath
 		self.cont.yamlPath[0] = 'tests/simple-fail-2svc/'
-		deploy = self.cont.DeployObject(self.html)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
-		self.cont.UndeployObject(self.html, self.ran)
+		self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertFalse(deploy)
 		self.cont.yamlPath = old
 
@@ -79,8 +85,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
 		self.cont.services[0] = "oai-gnb"
 		self.cont.deploymentTag = 'develop-12345678'
-		deploy = self.cont.DeployObject(self.html)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -88,8 +94,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
 		self.cont.services[0] = "oai-gnb oai-nr-ue"
 		self.cont.deploymentTag = 'develop-12345678'
-		deploy = self.cont.DeployObject(self.html)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -97,10 +103,10 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
 		self.cont.services[0] = "oai-gnb"
 		self.cont.deploymentTag = 'develop-12345678'
-		deploy1 = self.cont.DeployObject(self.html)
+		deploy1 = self.cont.DeployObject(self.ctx, self.html)
 		self.cont.services[0] = "oai-nr-ue"
-		deploy2 = self.cont.DeployObject(self.html)
+		deploy2 = self.cont.DeployObject(self.ctx, self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy1)
 		self.assertTrue(deploy2)
 		self.assertTrue(undeploy)

ci-scripts/tests/module.py

@@ -0,0 +1,64 @@
+import sys
+import logging
+import tempfile
+logging.basicConfig(
+	level=logging.DEBUG,
+	stream=sys.stdout,
+	format="[%(asctime)s] %(levelname)8s: %(message)s"
+)
+import os
+
+import unittest
+
+sys.path.append('./') # to find OAI imports below
+import cls_module
+from cls_ci_helper import TestCaseCtx
+import cls_cmd
+
+class TestModule(unittest.TestCase):
+    def setUp(self):
+        self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+    def tearDown(self):
+        with cls_cmd.LocalCmd() as c:
+            c.run(f'rm -rf {self.ctx.logPath}')
+
+    def test_simple_module(self):
+        c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml")
+        self.assertFalse(c.trace)
+        success = c.initialize()
+        self.assertTrue(success)
+        ip = c.attach()
+        self.assertEqual(ip, "127.0.0.1")
+        self.assertTrue(c.checkMTU())
+        c.detach()
+        logs = c.terminate()
+        self.assertEqual(logs, None) # no tracing
+
+    @unittest.skip("this test takes long: it verifies the UE cannot attach")
+    def test_simple_fail(self):
+        c = cls_module.Module_UE("test-fail", filename="tests/config/test_module_infra.yaml")
+        success = c.initialize()
+        self.assertTrue(success)
+        ip = c.attach()
+        self.assertEqual(ip, None)
+        self.assertFalse(c.checkMTU())
+        c.detach()
+        logs = c.terminate()
+        self.assertEqual(logs, None) # no tracing
+
+    def test_simple_trace(self):
+        c = cls_module.Module_UE("test-trace", filename="tests/config/test_module_infra.yaml")
+        self.assertTrue(c.trace)
+        success = c.initialize()
+        self.assertTrue(success)
+        ip = c.attach()
+        self.assertEqual(ip, "127.0.0.1")
+        self.assertTrue(c.checkMTU())
+        c.detach()
+        tmp = tempfile.mkdtemp()
+        log_file = c.terminate(self.ctx)
+        # undeploy uses archiveArtifact(), which writes to {prefix}-logs
+        self.assertEqual(log_file, [f"{self.ctx.baseFilename()}-tttf"]) # matches test-trace UE collection file
+
+if __name__ == '__main__':
+    unittest.main()

ci-scripts/tests/ping-iperf.py

@@ -9,11 +9,13 @@ import os
 os.system(f'rm -rf cmake_targets')
 os.system(f'mkdir -p cmake_targets/log')
 import unittest
+import tempfile
 
 sys.path.append('./') # to find OAI imports below
 import cls_oai_html
+from cls_ci_helper import TestCaseCtx
 import cls_oaicitest
-import cls_containerize
+import cls_cmd
 
 class TestPingIperf(unittest.TestCase):
 	def setUp(self):
@@ -22,7 +24,10 @@ class TestPingIperf(unittest.TestCase):
 		self.ci = cls_oaicitest.OaiCiTest()
 		self.ci.ue_ids = ["test"]
 		self.ci.nodes = ["localhost"]
-		self.cont = cls_containerize.Containerize()
+		self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+	def tearDown(self):
+		with cls_cmd.LocalCmd() as c:
+			c.run(f'rm -rf {self.ctx.logPath}')
 
 	def test_ping(self):
 		self.ci.ping_args = "-c3"
@@ -30,7 +35,7 @@ class TestPingIperf(unittest.TestCase):
 		self.ci.svr_id = "test"
 		infra_file = "tests/config/infra_ping_iperf.yaml"
 		# TODO Should need nothing but options and UE(s) to use
-		success = self.ci.Ping(self.html, self.cont, infra_file=infra_file)
+		success = self.ci.Ping(self.ctx, self.html, infra_file=infra_file)
 		self.assertTrue(success)
 
 	def test_iperf(self):
@@ -45,7 +50,7 @@ class TestPingIperf(unittest.TestCase):
 		self.ci.iperf_profile = "balanced"
 		infra_file = "tests/config/infra_ping_iperf.yaml"
 		# TODO Should need nothing but options and UE(s) to use
-		success = self.ci.Iperf(self.html, self.cont, infra_file=infra_file)
+		success = self.ci.Iperf(self.ctx, self.html, infra_file=infra_file)
 		self.assertTrue(success)
 
 	def test_iperf2_unidir(self):
@@ -57,7 +62,7 @@ class TestPingIperf(unittest.TestCase):
 		self.ci.iperf_profile = "balanced"
 		infra_file = "tests/config/infra_ping_iperf.yaml"
 		# TODO Should need nothing but options and UE(s) to use
-		success = self.ci.Iperf2_Unidir(self.html, self.cont, infra_file=infra_file)
+		success = self.ci.Iperf2_Unidir(self.ctx, self.html, infra_file=infra_file)
 		self.assertTrue(success)
 
 	def test_iperf_highrate(self):

ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml

@@ -26,7 +26,6 @@
         <htmlTabIcon>wrench</htmlTabIcon>
         <TestCaseRequestedList>
  111111
- 100001
  800813
  000001
  000002

ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml

@@ -26,7 +26,6 @@
         <htmlTabIcon>wrench</htmlTabIcon>
         <TestCaseRequestedList>
  111111
- 100001
  800813
  000001
  000002

ci-scripts/xml_files/gnb_usrp_build.xml

@@ -47,6 +47,7 @@
 	<testCase id="100000">
 		<class>Custom_Command</class>
 		<desc>Reboot USRP</desc>
+		<may_fail>true</may_fail>
 		<node>caracal</node>
 		<command>ssh root@192.168.20.2 reboot</command>
         </testCase>

docker/Dockerfile.build.fhi72.native_arm.ubuntu22

@@ -78,7 +78,7 @@ RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
     ./build_oai \
-      --gNB \
+      --ninja --gNB \
       --build-lib "telnetsrv enbscope uescope nrscope" \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
       --build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \

docker/Dockerfile.build.fhi72.rhel9

@@ -58,7 +58,7 @@ RUN /bin/sh oaienv && \
     mkdir -p log && \
     scl enable gcc-toolset-13 "\
     ./build_oai \
-      --gNB \
+      --ninja --gNB \
       --build-lib 'telnetsrv enbscope uescope nrscope' \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
       --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION" &&\

docker/Dockerfile.build.fhi72.ubuntu22

@@ -68,7 +68,7 @@ RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
     ./build_oai \
-      --gNB \
+      --ninja --gNB \
       --build-lib "telnetsrv enbscope uescope nrscope" \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
       --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \