From af3302d2d6d41f7b8a4b2da6f790f086c3537c30 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 1 Aug 2025 11:32:49 +0200 Subject: [PATCH 01/38] Remove unused statement --- ci-scripts/ran.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index f0f790ba2c..46711b1b73 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -124,7 +124,6 @@ class RANManagement(): self.testCase_id = HTML.testCase_id mySSH = SSH.SSHConnection() - cwd = os.getcwd() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath, '\$', 5) From be60dbd38c280064c18afe79eaf259e859e38d00 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 18:28:46 +0200 Subject: [PATCH 02/38] CI: shorten UE names (IP might not be available) --- ci-scripts/cls_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index 3ec6ffc9e5..65edbfdb56 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -65,7 +65,7 @@ class Module_UE: logging.info(f'initialized UE {self} from {filename}') def __str__(self): - return f"{self.module_name}@{self.host} [IP: {self.getIP()}]" + return f"{self.module_name}@{self.host}" def __repr__(self): return self.__str__() From 442c29cfcd51b9f00db1433ef262638c450b0069 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 18:29:00 +0200 Subject: [PATCH 03/38] CI: use getConnection() to simplify code --- ci-scripts/cls_module.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index 65edbfdb56..3fed4f3dd6 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -73,12 +73,8 @@ class Module_UE: def _command(self, cmd, silent = False): if cmd is None: raise Exception("no command provided") - if self.host == "" or self.host == "localhost": - c = cls_cmd.LocalCmd() - else: - c = cls_cmd.RemoteCmd(self.host) - response = c.run(cmd, silent=silent) - c.close() + with cls_cmd.getConnection(self.host) as c: + response = c.run(cmd, silent=silent) return response #-----------------$ From a1c354bd6eb203aaa81b342e830a3aaaaa0fc381 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 18:30:23 +0200 Subject: [PATCH 04/38] CI: Remove old tracing information It will be replaced with a new mechanism in a later commit. --- ci-scripts/ci_infra.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci-scripts/ci_infra.yaml b/ci-scripts/ci_infra.yaml index 804852c937..fd2d414a03 100644 --- a/ci-scripts/ci_infra.yaml +++ b/ci-scripts/ci_infra.yaml @@ -7,8 +7,6 @@ idefix: NetworkScript: ip a show dev wwan0 IF: wwan0 MTU: 1500 - Trace: True - LogStore: /media/usb-drive/ci_qlogs up2: Host: up2 @@ -46,7 +44,6 @@ adb_ue_1: NetworkScript: adb -s R3CM40LZPHT shell "ip address show | grep rmnet_data0" CmdPrefix: adb -s R3CM40LZPHT shell MTU: 1500 - LogStore: /media/usb-drive/ci_adb_1-logs adb_ue_2: Host: nano InitScript: /home/oaicicd/ci_ctl_adb.sh initialize 5200c00db4413517 @@ -59,7 +56,6 @@ adb_ue_2: NetworkScript: adb -s 5200c00db4413517 shell "ip address show | grep rmnet" CmdPrefix: adb -s 5200c00db4413517 shell MTU: 1500 - LogStore: /media/usb-drive/ci_adb_2-logs oc-cn5g: Host: avra From cc1b2315af6f354afb4bde5d057a5ab282a241ec Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 16:32:23 +0200 Subject: [PATCH 05/38] CI: add simple unit test for UE handling --- .../tests/config/test_module_infra.yaml | 19 +++++++++ ci-scripts/tests/module.py | 41 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 ci-scripts/tests/config/test_module_infra.yaml create mode 100644 ci-scripts/tests/module.py diff --git a/ci-scripts/tests/config/test_module_infra.yaml b/ci-scripts/tests/config/test_module_infra.yaml new file mode 100644 index 0000000000..fadc721232 --- /dev/null +++ b/ci-scripts/tests/config/test_module_infra.yaml @@ -0,0 +1,19 @@ +test: + Host: localhost + InitScript: "" + TermScript: "" + AttachScript: "true" + DetachScript: "true" + NetworkScript: ip a show dev lo + IF: lo + MTU: 65536 + +test-fail: + Host: localhost + InitScript: "" + TermScript: "" + AttachScript: "false" + DetachScript: "true" + NetworkScript: ip a show dev TESTDEVICEDOESNOTEXIST + IF: TESTDEVICEDOESNOTEXIST + MTU: 123 diff --git a/ci-scripts/tests/module.py b/ci-scripts/tests/module.py new file mode 100644 index 0000000000..b6898405d4 --- /dev/null +++ b/ci-scripts/tests/module.py @@ -0,0 +1,41 @@ +import sys +import logging +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import os + +import unittest + +sys.path.append('./') # to find OAI imports below +import cls_module +import cls_cmd + +class TestModule(unittest.TestCase): + def test_simple_module(self): + c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml") + success = c.initialize() + self.assertTrue(success) + ip = c.attach() + self.assertEqual(ip, "127.0.0.1") + self.assertTrue(c.checkMTU()) + c.detach() + logs = c.terminate() + self.assertEqual(logs, None) # no tracing + + @unittest.skip("this test takes long: it verifies the UE cannot attach") + def test_simple_fail(self): + c = cls_module.Module_UE("test-fail", filename="tests/config/test_module_infra.yaml") + success = c.initialize() + self.assertTrue(success) + ip = c.attach() + self.assertEqual(ip, None) + self.assertFalse(c.checkMTU()) + c.detach() + logs = c.terminate() + self.assertEqual(logs, None) # no tracing + +if __name__ == '__main__': + unittest.main() From de2d32be092774aec283b9be4465b2c27feca769 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 18:29:19 +0200 Subject: [PATCH 06/38] CI UE Attach: add more frequent checks, improve output check for IP once before sleeping in case UE IP address exists, which should speed up the CI. With this, Only print the actual attach command once. Print errors only at the beginning. This should simplify attach command output logs. --- ci-scripts/cls_module.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index 3fed4f3dd6..d387783bf7 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -70,11 +70,11 @@ class Module_UE: def __repr__(self): return self.__str__() - def _command(self, cmd, silent = False): + def _command(self, cmd, silent=False, reportNonZero=True): if cmd is None: raise Exception("no command provided") with cls_cmd.getConnection(self.host) as c: - response = c.run(cmd, silent=silent) + response = c.run(cmd, silent=silent, reportNonZero=reportNonZero) return response #-----------------$ @@ -109,10 +109,11 @@ class Module_UE: self._command(self.cmd_dict["attach"]) timeout = attach_timeout logging.debug("Waiting for IP address to be assigned") + ip = self.getIP(silent=False, reportNonZero=True) while timeout > 0 and not ip: - time.sleep(5) - timeout -= 5 - ip = self.getIP() + time.sleep(1) + timeout -= 1 + ip = self.getIP(silent=True, reportNonZero=False) if ip: break logging.warning(f"UE did not receive IP address after {attach_timeout} s, detaching") @@ -156,8 +157,8 @@ class Module_UE: logging.error(message) return False - def getIP(self): - output = self._command(self.cmd_dict["getNetwork"], silent=True) + def getIP(self, silent=True, reportNonZero=True): + output = self._command(self.cmd_dict["getNetwork"], silent=silent, reportNonZero=reportNonZero) result = re.search(r'inet (?P[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', output.stdout) if result and result.group('ip'): ip = result.group('ip') From a3d46ed0e6c07343f8085d829a9cb9dc9b00043a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 01:03:06 +0200 Subject: [PATCH 07/38] 5g rfsim NTN LEO/GEO: remove leading undeploy step It is not necessary (the final undeploys at the end are "always_exec"), and makes the CI potentially fail since there is nothing to copy at this point. --- ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml | 1 - ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml | 1 - 2 files changed, 2 deletions(-) diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml index 2cefc32722..88173b3659 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml @@ -26,7 +26,6 @@ wrench 111111 - 100001 800813 000001 000002 diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml index 61bf4d5e48..0d2634192d 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml @@ -26,7 +26,6 @@ wrench 111111 - 100001 800813 000001 000002 From ce9f2c9a45ff339c96254e81ac59cc192ed4c735 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 18:28:16 +0200 Subject: [PATCH 08/38] Add possibility to trace UEs --- ci-scripts/cls_module.py | 58 ++++++++++++++----- .../tests/config/test_module_infra.yaml | 15 +++++ ci-scripts/tests/module.py | 17 ++++++ 3 files changed, 77 insertions(+), 13 deletions(-) diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index d387783bf7..5493511bd8 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -59,11 +59,20 @@ class Module_UE: } self.interface = m.get('IF') self.MTU = m.get('MTU') - self.trace = m.get('trace') == True - self.logStore = m.get('LogStore') self.cmd_prefix = m.get('CmdPrefix') logging.info(f'initialized UE {self} from {filename}') + t = m.get('Tracing') + self.trace = t is not None + if self.trace: + if t.get('Start') is None or t.get('Stop') is None or t.get('Collect')is None : + raise ValueError("need to have Start/Stop/Collect for tracing") + self.cmd_dict["traceStart"] = t.get('Start') + self.cmd_dict["traceStop"] = t.get('Stop') + self._logCollect = t.get('Collect') + if "%%log_dir%%" not in self._logCollect: + raise ValueError(f"(At least one) LogCollect expression for {module_name} must contain \"%%log_dir%%\"") + def __str__(self): return f"{self.module_name}@{self.host}" @@ -82,25 +91,23 @@ class Module_UE: #-----------------$ def initialize(self): - if self.trace: - raise Exception("UE tracing not implemented yet") - self._enableTrace() # we first terminate to make sure the UE has been stopped if self.cmd_dict["detach"]: self._command(self.cmd_dict["detach"], silent=True) self._command(self.cmd_dict["terminate"], silent=True) ret = self._command(self.cmd_dict["initialize"]) logging.info(f'For command: {ret.args} | return output: {ret.stdout} | Code: {ret.returncode}') + if self.trace: + self._enableTrace() # Here each UE returns differently for the successful initialization, requires check based on UE return ret.returncode == 0 - def terminate(self): + def terminate(self, log_dir=None): self._command(self.cmd_dict["terminate"]) - if self.trace: - raise Exception("UE tracing not implemented yet") + if self.trace and log_dir is not None: self._disableTrace() - return self._logCollect() + return self._collectTrace(log_dir) return None def attach(self, attach_tries = 4, attach_timeout = 60): @@ -188,10 +195,35 @@ class Module_UE: return self.cmd_prefix if self.cmd_prefix else "" def _enableTrace(self): - raise Exception("not implemented") + logging.info(f'UE {self}: start UE tracing') + self._command(self.cmd_dict["traceStart"]) def _disableTrace(self): - raise Exception("not implemented") + logging.info(f'UE {self}: stop UE tracing') + self._command(self.cmd_dict["traceStop"]) - def _logCollect(self): - raise Exception("not implemented") + def _collectTrace(self, log_dir): + logging.info(f'collecting logs into (local) {log_dir}') + remote_dir = "/tmp/ue-trace-logs" + with cls_cmd.getConnection(self.host) as c: + # create a directory for log collection + c.run(f'rm -rf {remote_dir}') + ret = c.run(f'mkdir {remote_dir}') + if ret.returncode != 0: + logging.error("cannot create directory for log collection") + return [] + log_cmd = self._logCollect.replace('%%log_dir%%', remote_dir) + self._command(log_cmd) + # enumerate collected files + ret = c.run(f'ls {remote_dir}/*') + if ret.returncode != 0: + logging.error("cannot enumerate log files") + return [] + log_files = [] + # copy them to the executor one by one, and store in log_dir + for f in ret.stdout.split("\n"): + l = f.replace(remote_dir, log_dir) + c.copyin(f, l) + log_files.append(l) + c.run(f'rm -rf {remote_dir}') + return log_files diff --git a/ci-scripts/tests/config/test_module_infra.yaml b/ci-scripts/tests/config/test_module_infra.yaml index fadc721232..7bfdfc1d9d 100644 --- a/ci-scripts/tests/config/test_module_infra.yaml +++ b/ci-scripts/tests/config/test_module_infra.yaml @@ -17,3 +17,18 @@ test-fail: NetworkScript: ip a show dev TESTDEVICEDOESNOTEXIST IF: TESTDEVICEDOESNOTEXIST MTU: 123 + +test-trace: + Host: localhost + InitScript: "" + TermScript: "" + AttachScript: "true" + DetachScript: "true" + Tracing: + # tttf: temp-trace-test-file + Start: "rm -f /tmp/tttf; for i in {1..8}; do echo $i >> /tmp/tttf; sleep 1; done &" + Stop: "while [[ $(wc -l /tmp/tttf) < 4 ]]; do sleep 1; done" + Collect: "cp /tmp/tttf %%log_dir%%" + NetworkScript: ip a show dev lo + IF: lo + MTU: 65536 diff --git a/ci-scripts/tests/module.py b/ci-scripts/tests/module.py index b6898405d4..366593873c 100644 --- a/ci-scripts/tests/module.py +++ b/ci-scripts/tests/module.py @@ -1,5 +1,6 @@ import sys import logging +import tempfile logging.basicConfig( level=logging.DEBUG, stream=sys.stdout, @@ -16,6 +17,7 @@ import cls_cmd class TestModule(unittest.TestCase): def test_simple_module(self): c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml") + self.assertFalse(c.trace) success = c.initialize() self.assertTrue(success) ip = c.attach() @@ -37,5 +39,20 @@ class TestModule(unittest.TestCase): logs = c.terminate() self.assertEqual(logs, None) # no tracing + def test_simple_trace(self): + c = cls_module.Module_UE("test-trace", filename="tests/config/test_module_infra.yaml") + self.assertTrue(c.trace) + success = c.initialize() + self.assertTrue(success) + ip = c.attach() + self.assertEqual(ip, "127.0.0.1") + self.assertTrue(c.checkMTU()) + c.detach() + tmp = tempfile.mkdtemp() + log_file = c.terminate(log_dir=tmp) + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {tmp}') + self.assertEqual(log_file, [f"{tmp}/tttf"]) # matches test-trace UE collection file + if __name__ == '__main__': unittest.main() From fe2cda1309155fb2bdb8c12cdb8e4ab859b1cd3b Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 19:44:50 +0200 Subject: [PATCH 09/38] CI: log UE files in artifacts --- ci-scripts/cls_oaicitest.py | 4 ++-- ci-scripts/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 53bdacaacc..0a550016d4 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -835,10 +835,10 @@ class OaiCiTest(): global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC return global_status - def TerminateUE(self, HTML): + def TerminateUE(self, HTML, logPath): ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids] with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(ue.terminate) for ue in ues] + futures = [executor.submit(ue.terminate, logPath) for ue in ues] archives = [f.result() for f in futures] archive_info = [f'Log at: {a}' if a else 'No log available' for a in archives] messages = [f"UE {ue.getName()}: {log}" for (ue, log) in zip(ues, archive_info)] diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 18956ae28f..fd794de22c 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -226,7 +226,7 @@ def ExecuteActionWithParam(action): elif action == 'Detach_UE': success = CiTestObj.DetachUE(HTML) elif action == 'Terminate_UE': - success = CiTestObj.TerminateUE(HTML) + success = CiTestObj.TerminateUE(HTML, logPath) elif action == 'CheckStatusUE': success = CiTestObj.CheckStatusUE(HTML) elif action == 'DataEnable_UE': From af580f3a2f012682a3b24ed0614e61cb32694271 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Jul 2025 18:33:42 +0200 Subject: [PATCH 10/38] Add example tracing for Amarisoft UE sim Commented, as the resulting files are very big. --- ci-scripts/ci_infra.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci-scripts/ci_infra.yaml b/ci-scripts/ci_infra.yaml index fd2d414a03..2f91576a60 100644 --- a/ci-scripts/ci_infra.yaml +++ b/ci-scripts/ci_infra.yaml @@ -146,6 +146,10 @@ amarisoft_ue: InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz/aw2s-multi-00102-20.cfg & TermScript: /root/lteue-linux-2025-03-15/ws.js -t 10 127.0.0.1:9002 '{"message":"quit"}' || killall -KILL lteue-avx2 NetworkScript: ip netns exec ue1 ip a show dev pdn0 + #Tracing: + # Start: '' # nothing to be done + # Stop: '' # nothing to be done + # Collect: 'cp /tmp/ue.log %%log_dir%%' amarisoft_ue_2x2: Host: amariue InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz_2x2/aw2s-multi-00102-2x2-v2.cfg & From 5492de0e0b92819bb0f2c58175235bf85e0630aa Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 1 Aug 2025 07:51:55 +0200 Subject: [PATCH 11/38] Example tracing for QLog Note: this does not actually work. In the pipelines, we see that for some reason, quectel-CM cannot establish a connection. Manually, it works though(?) --- ci-scripts/ci_infra.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci-scripts/ci_infra.yaml b/ci-scripts/ci_infra.yaml index 2f91576a60..e5888ef6fe 100644 --- a/ci-scripts/ci_infra.yaml +++ b/ci-scripts/ci_infra.yaml @@ -5,6 +5,10 @@ idefix: AttachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 wup DetachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 detach NetworkScript: ip a show dev wwan0 + #Tracing: + # Start: 'sudo rm -f ci_qlog/*; nohup sudo -b QLog/QLog -s ci_qlog -f NR5G.cfg &' + # Stop: 'sudo killall QLog' + # Collect: 'sudo mv ci_qlog/* %%log_dir%%/' IF: wwan0 MTU: 1500 From 5c16fc6189b3669c8b409d3c4a99ec42d8bd671c Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 4 Aug 2025 09:25:11 +0200 Subject: [PATCH 12/38] Fix Python "invalid escape sequence" warnings cls_containerize.py:139: SyntaxWarning: invalid escape sequence '\.' lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None cls_containerize.py:441: SyntaxWarning: invalid escape sequence '\-' result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) cls_containerize.py:493: SyntaxWarning: invalid escape sequence '\-' result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) cls_containerize.py:616: SyntaxWarning: invalid escape sequence '\-' result = re.search('Size *= *(?P[0-9\-]+) *bytes', ret.stdout) --- ci-scripts/cls_containerize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index b878e82caa..3d63de0834 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -136,7 +136,7 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus): lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None tagged = tagged or lineHasTag or lineHasTag2 # the OpenShift Cluster builder prepends image registry URL - lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None + lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None committed = committed or lineHasCommit errorandwarnings['errors'] = 0 if committed or tagged else 1 errorandwarnings['warnings'] = 0 @@ -441,7 +441,7 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) return False else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) + result = re.search(r'Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 imageSizeStr = f'{size:.1f}' @@ -493,7 +493,7 @@ class Containerize(): allImagesSize[name] = 'N/A -- Build Failed' break else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) + result = re.search(r'Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 # convert to MB imageSizeStr = f'{size:.1f}' @@ -616,7 +616,7 @@ class Containerize(): collectInfo = {} collectInfo['proxy'] = files ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') - result = re.search('Size *= *(?P[0-9\-]+) *bytes', ret.stdout) + result = re.search(r'Size *= *(?P[0-9\-]+) *bytes', ret.stdout) # Cleaning any created tmp volume ssh.run('docker volume prune --force') ssh.close() From f766b21b5776ab94755292c4101e83cb5d29c83c Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 08:16:48 +0200 Subject: [PATCH 13/38] Add CI helper class This class provides a generic "context" for individual tasks about the current task step, and where to log results. It's used by archiveArtifact() that would automatically log files in the right directory, based on a directory in the context, and in a way that order of files is preserved for long-running pipelines. --- ci-scripts/cls_ci_helper.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 ci-scripts/cls_ci_helper.py diff --git a/ci-scripts/cls_ci_helper.py b/ci-scripts/cls_ci_helper.py new file mode 100644 index 0000000000..eb47a32892 --- /dev/null +++ b/ci-scripts/cls_ci_helper.py @@ -0,0 +1,44 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ + +import os +from typing import NamedTuple +import logging + +class TestCaseCtx(NamedTuple): + count: int + test_id: int + logPath: str + + def Default(logPath): + return TestCaseCtx(123, 112233, logPath) + def baseFilename(self): + # typically, the test ID is of form 001234 (6 digits) + return f"{self.logPath}/{self.count}-{self.test_id:06d}" + +def archiveArtifact(cmd, ctx, remote_path): + base = os.path.basename(remote_path) + local = f"{ctx.baseFilename()}-{base}" + logging.info(f"Archive artifact '{local}'") + success = cmd.copyin(remote_path, local) + if success: + cmd.run(f'rm {remote_path}', silent=True) + return local if success else None From 28ce0dc444fa206b21cbfe3655639ab7d2954d25 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:42:06 +0200 Subject: [PATCH 14/38] Main: pass in ctx Set the context for each task step. Set the artifact result path based on the XML, so it is harmonized; currently, most pipelines use the "YAML path" of docker-compose, but that does not always make sense. For instance, Ping/Iperf() needs to look up this path, but does not inherently belong to one specific YAML path (there might even be more). --- ci-scripts/main.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index fd794de22c..3cc550eba5 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -43,8 +43,10 @@ import cls_containerize #class Containerize for all container-based operations import cls_static_code_analysis #class for static code analysis import cls_cluster # class for building/deploying on cluster import cls_native # class for all native/source-based operations +from cls_ci_helper import TestCaseCtx import ran +import cls_cmd import cls_oai_html @@ -91,7 +93,7 @@ def AssignParams(params_dict): setattr(RAN, key, value) setattr(HTML, key, value) -def ExecuteActionWithParam(action): +def ExecuteActionWithParam(action, ctx): global RAN global HTML global CONTAINERS @@ -565,6 +567,16 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re else: xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[0] + # directory where all log artifacts will be placed + logPath = f"{cwd}/../cmake_targets/log/{CiTestObj.testXMLfiles[0].split('/')[-1]}.d" + # we run from within ci-scripts, but the logPath is absolute, so replace + # the ci-scripts/..; if it does not exist, nothing will happen + logPath = logPath.replace(r'/ci-scripts/..', '') + logging.info(f"placing all artifacts for this run in {logPath}/") + with cls_cmd.LocalCmd() as c: + c.run(f"rm -rf {logPath}") + c.run(f"mkdir -p {logPath}") + xmlTree = ET.parse(xml_test_file) xmlRoot = xmlTree.getroot() @@ -615,12 +627,15 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re task_set_succeeded = True HTML.startTime=int(round(time.time() * 1000)) + i = 0 for test_case_id in todo_tests: for test in all_tests: id = test.get('id') if test_case_id != id: continue + i += 1 CiTestObj.testCase_id = id + ctx = TestCaseCtx(i, int(id), logPath) HTML.testCase_id=CiTestObj.testCase_id CiTestObj.desc = test.findtext('desc') always_exec = test.findtext('always_exec') in ['True', 'true', 'Yes', 'yes'] @@ -637,7 +652,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML.CreateHtmlTestRowQueue(msg, "SKIP", []) break try: - test_succeeded = ExecuteActionWithParam(action) + test_succeeded = ExecuteActionWithParam(action, ctx) if not test_succeeded and may_fail: logging.warning(f"test ID {test_case_id} action {action} may or may not fail, proceeding despite error") elif not test_succeeded: From dcbb3369e19d8cf6f20e6ac8e67ea1a01a923b84 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:58:06 +0200 Subject: [PATCH 15/38] Simplify ShowTestID() make it a non-class function, stateless, more information on the current CI step (the number). --- ci-scripts/cls_oaicitest.py | 7 ------- ci-scripts/main.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 0a550016d4..b2972b667d 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -258,7 +258,6 @@ class OaiCiTest(): self.testCase_id = '' self.testXMLfiles = [] - self.desc = '' self.ping_args = '' self.ping_packetloss_threshold = '' self.ping_rttavg_threshold ='' @@ -891,9 +890,3 @@ class OaiCiTest(): d = f'{SourceCodePath}/cmake_targets' cmd.run(f'rm -f {d}/build.log.zip') cmd.run(f'cd {d} && zip -r build.log.zip build_log_*/*') - - def ShowTestID(self): - logging.info(f'\u001B[1m----------------------------------------\u001B[0m') - logging.info(f'\u001B[1m Test ID: {self.testCase_id} \u001B[0m') - logging.info(f'\u001B[1m {self.desc} \u001B[0m') - logging.info(f'\u001B[1m----------------------------------------\u001B[0m') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 3cc550eba5..275686abec 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -409,10 +409,11 @@ def test_in_list(test, list): def receive_signal(signum, frame): sys.exit(1) - - - - +def ShowTestID(ctx, desc): + logging.info(f'\u001B[1m----------------------------------------\u001B[0m') + logging.info(f'\u001B[1m Test ID: {ctx.test_id} (#{ctx.count}) \u001B[0m') + logging.info(f'\u001B[1m {desc} \u001B[0m') + logging.info(f'\u001B[1m----------------------------------------\u001B[0m') #----------------------------------------------------------- # MAIN PART @@ -637,15 +638,15 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CiTestObj.testCase_id = id ctx = TestCaseCtx(i, int(id), logPath) HTML.testCase_id=CiTestObj.testCase_id - CiTestObj.desc = test.findtext('desc') + desc = test.findtext('desc') always_exec = test.findtext('always_exec') in ['True', 'true', 'Yes', 'yes'] may_fail = test.findtext('may_fail') in ['True', 'true', 'Yes', 'yes'] - HTML.desc=CiTestObj.desc + HTML.desc = desc action = test.findtext('class') if (CheckClassValidity(xml_class_list, action, id) == False): task_set_succeeded = False continue - CiTestObj.ShowTestID() + ShowTestID(ctx, desc) if not task_set_succeeded and not always_exec: msg = f"skipping test due to prior error" logging.warning(msg) From 2b4e0e5244601a7b895b4cc4d1a735813750141f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:41:40 +0200 Subject: [PATCH 16/38] Use archiveArtifact() for CN undeploy --- ci-scripts/cls_corenetwork.py | 17 ++++++++--------- ci-scripts/cls_oaicitest.py | 14 ++++---------- ci-scripts/main.py | 2 +- ci-scripts/tests/corenetwork.py | 18 +++++++++++++----- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/ci-scripts/cls_corenetwork.py b/ci-scripts/cls_corenetwork.py index 579bad585b..ad83eae622 100644 --- a/ci-scripts/cls_corenetwork.py +++ b/ci-scripts/cls_corenetwork.py @@ -29,6 +29,7 @@ import yaml import re import cls_cmd +from cls_ci_helper import archiveArtifact def listify(s): if s is None: @@ -114,8 +115,7 @@ class CoreNetwork: logging.info(f'deployed core network {self}, pingable IP address {ip}') return True, output - def _collect_logs(self, log_dir): - logging.info(f'collecting logs into (local) {log_dir}') + def _collect_logs(self, ctx): remote_dir = "/tmp/cn-undeploy-logs" with cls_cmd.getConnection(self._host) as c: # create a directory for log collection @@ -134,18 +134,17 @@ class CoreNetwork: logging.error("cannot enumerate log files") return [] log_files = [] - # copy them to the executor one by one, and store in log_dir + # copy them to the executor one by one for f in ret.stdout.split("\n"): - l = f.replace(remote_dir, log_dir) - c.copyin(f, l) - log_files.append(l) + name = archiveArtifact(c, ctx, f) + log_files.append(name) c.run(f'rm -rf {remote_dir}') return log_files - def undeploy(self, log_dir=None): + def undeploy(self, ctx=None): log_files = [] - if log_dir is not None: - log_files = self._collect_logs(log_dir) + if ctx is not None: + log_files = self._collect_logs(ctx) else: logging.warning("no directory for log collection specified, cannot retrieve core network logs") logging.info(f'undeploy core network {self}') diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index b2972b667d..290ad38598 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -844,7 +844,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(f'N/A', 'OK', messages) return True - def DeployCoreNetwork(cn_id, HTML): + def DeployCoreNetwork(cn_id, ctx, HTML): core_name = cn_id.strip() cn = cls_corenetwork.CoreNetwork(core_name) success, output = cn.deploy() @@ -858,18 +858,12 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(core_name, 'KO', [msg]) return success - def UndeployCoreNetwork(cn_id, HTML): - # Ping, Iperf, DeployObject put logs into a path based on YAML. We - # can't do this here (because there is no yaml), so hardcode a path for - # "cn_logs" for the moment - logPath = f'{os.getcwd()}/../cmake_targets/log/cn_logs' - with cls_cmd.getConnection('localhost') as local: - local.run(f'mkdir -p {logPath}', silent=True) + def UndeployCoreNetwork(cn_id, ctx, HTML): core_name = cn_id.strip() cn = cls_corenetwork.CoreNetwork(core_name) - logs, output = cn.undeploy(log_dir=logPath) + logs, output = cn.undeploy(ctx=ctx) logging.info(f"undeployed core network {core_name}, logs {logs}, output:\n{output}") - message = "Log files: " + ", ".join([os.path.basename(l) for l in logs]) + message = "Log files:\n" + "\n".join([os.path.basename(l) for l in logs]) HTML.CreateHtmlTestRowQueue(core_name, 'OK', [message]) return True diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 275686abec..a6da7b399d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -302,7 +302,7 @@ def ExecuteActionWithParam(action, ctx): elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork': cn_id = test.findtext('cn_id') core_op = getattr(cls_oaicitest.OaiCiTest, action) - success = core_op(cn_id, HTML) + success = core_op(cn_id, ctx, HTML) elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace": eNB_instance=test.findtext('eNB_instance') diff --git a/ci-scripts/tests/corenetwork.py b/ci-scripts/tests/corenetwork.py index 77e5c7629e..7bc1603305 100644 --- a/ci-scripts/tests/corenetwork.py +++ b/ci-scripts/tests/corenetwork.py @@ -6,14 +6,22 @@ logging.basicConfig( format="[%(asctime)s] %(levelname)8s: %(message)s" ) import os +import tempfile import unittest sys.path.append('./') # to find OAI imports below import cls_corenetwork +from cls_ci_helper import TestCaseCtx import cls_cmd class TestCoreNetwork(unittest.TestCase): + def setUp(self): + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') + def test_simple_core(self): c = cls_corenetwork.CoreNetwork("test", filename="tests/config/test_core_infra.yaml") success, output = c.deploy() @@ -22,14 +30,14 @@ class TestCoreNetwork(unittest.TestCase): # should not have it set in config self.assertTrue(c.runIperf3Server()) self.assertEqual(c.getIP(), "127.0.0.1") - log_files, output = c.undeploy(log_dir="/tmp") + log_files, output = c.undeploy(self.ctx) self.assertEqual(output, "undeploy") with cls_cmd.LocalCmd() as cmd: # there must be one log file for this (test) core self.assertEqual(len(log_files), 1) - # test core writes to %%log_dir%%/logs, which expands to /tmp/logs + # undeploy uses archiveArtifact(), which writes to {prefix}-logs l = log_files[0] - self.assertEqual(l, "/tmp/logs") + self.assertEqual(l, f"{self.ctx.baseFilename()}-logs") ret = cmd.run(f"cat {l}") self.assertEqual(ret.returncode, 0) # command must succeed self.assertEqual(ret.stdout, "logs") # output should be "logs" @@ -39,14 +47,14 @@ class TestCoreNetwork(unittest.TestCase): success, _ = c.deploy() self.assertTrue(success) self.assertFalse(c.runIperf3Server()) - c.undeploy(log_dir="/tmp") + c.undeploy(None) def test_core_fail(self): c = cls_corenetwork.CoreNetwork("test_fail", filename="tests/config/test_core_infra.yaml") success, _ = c.deploy() self.assertFalse(success) # undeployment should still work - c.undeploy(log_dir="/tmp") + c.undeploy(None) def test_core_script(self): c = cls_corenetwork.CoreNetwork("test_script", filename="tests/config/test_core_infra.yaml") From fa5c91e03f4c73455cfb7217f4e000ae7f313e39 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:40:28 +0200 Subject: [PATCH 17/38] Use archiveArtifact() for (Un-)DeployObject() --- ci-scripts/cls_containerize.py | 45 ++++++++++++++-------------------- ci-scripts/main.py | 4 +-- ci-scripts/tests/deployment.py | 32 ++++++++++++++---------- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 3d63de0834..591d6d413d 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -46,6 +46,7 @@ import cls_cmd import helpreadme as HELP import constants as CONST import cls_oaicitest +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Helper functions used here and in other classes @@ -202,12 +203,10 @@ def GetServices(ssh, requested, file): else: return requested -def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename): - remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}" +def CopyinServiceLog(ssh, lSourcePath, svcName, wd_yaml, ctx): + remote_filename = f"{lSourcePath}/cmake_targets/log/{svcName}.logs" ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}') - local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}" - local_filename = f"{local_dir}/{filename}" - return ssh.copyin(remote_filename, local_filename) + return archiveArtifact(ssh, ctx, remote_filename) def GetRunningServices(ssh, file): ret = ssh.run(f'docker compose -f {file} config --services') @@ -229,14 +228,13 @@ def GetRunningServices(ssh, file): logging.info(f'stopping services: {running_services}') return running_services -def CheckLogs(self, yaml, service_name, HTML, RAN): - logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}' - filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log' +def CheckLogs(self, filename, HTML, RAN): success = True - if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): + name = os.path.basename(filename) + if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m') logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN) - opt = f"UE log analysis for service {service_name}" + opt = f"UE log analysis ({name})" # usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files # abuse HTML to store their reports, and we here want to put custom options, # which is not possible with CreateHtmlTestRow @@ -247,13 +245,13 @@ def CheckLogs(self, yaml, service_name, HTML, RAN): else: HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg]) HTML.htmlUEFailureMsg = "" - elif service_name == 'nv-cubb': + elif 'nv-cubb' in name: msg = 'Undeploy PNF/Nvidia CUBB' HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK) - elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])): + elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb'])): logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m') logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) - opt = f"xNB log analysis for service {service_name}" + opt = f"xNB log analysis ({name})" if (logStatus < 0): HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg]) success = False @@ -261,9 +259,9 @@ def CheckLogs(self, yaml, service_name, HTML, RAN): HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg]) HTML.htmleNBFailureMsg = "" else: - logging.info(f'Skipping to analyze log for service name {service_name}') - HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"]) - logging.debug(f"log check: service {service_name} passed analysis {success}") + logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE") + HTML.CreateHtmlTestRowQueue(f"file {name}", 'OK', ["no analysis function"]) + logging.debug(f"log check: file {filename} passed analysis {success}") return success #----------------------------------------------------------- @@ -823,18 +821,14 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"]) return success - def DeployObject(self, HTML): + def DeployObject(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] num_attempts = self.num_attempts lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug(f'Deploying OAI Object on server: {lIpAddr}') yaml = self.yamlPath[self.eNB_instance].strip('/') - # creating the log folder by default - local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}" - os.system(f'mkdir -p {local_dir}') wd = f'{lSourcePath}/{yaml}' wd_yaml = f'{wd}/docker-compose.y*ml' - yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml) if services == [] or services == ' ' or services == None: @@ -861,7 +855,7 @@ class Containerize(): logging.warning(warning_msg) HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [warning_msg]) for svc in services.split(): - CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') + CopyinServiceLog(ssh, lSourcePath, svc, wd_yaml, ctx) ssh.run(f'docker compose -f {wd_yaml} down -- {services}') imagesInfo = info.stdout.splitlines()[1:] logging.debug(f'{info.stdout.splitlines()[1:]}') @@ -871,13 +865,12 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)]) return deployed - def UndeployObject(self, HTML, RAN): + def UndeployObject(self, ctx, HTML, RAN): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m') yaml = self.yamlPath[self.eNB_instance].strip('/') wd = f'{lSourcePath}/{yaml}' - yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: ExistEnvFilePrint(ssh, wd) services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml") @@ -885,7 +878,7 @@ class Containerize(): if services is not None: all_serv = " ".join([s for s, _ in services]) ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}') - copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services) + copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, f"{wd}/docker-compose.y*ml", ctx) for s, _ in services] else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v') @@ -894,7 +887,7 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)']) return False else: - log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services] + log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res] success = all(log_results) if success: logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index a6da7b399d..7f5efefc34 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -330,9 +330,9 @@ def ExecuteActionWithParam(action, ctx): CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) if action == 'Deploy_Object': - success = CONTAINERS.DeployObject(HTML) + success = CONTAINERS.DeployObject(ctx, HTML) elif action == 'Undeploy_Object': - success = CONTAINERS.UndeployObject(HTML, RAN) + success = CONTAINERS.UndeployObject(ctx, HTML, RAN) elif action == 'Create_Workspace': if force_local: # Do not create a working directory when running locally. Current repo directory will be used diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index a7337f6e52..d2fb0cda57 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -6,6 +6,7 @@ logging.basicConfig( format="[%(asctime)s] %(levelname)8s: %(message)s" ) import os +import tempfile os.system(f'rm -rf cmake_targets') os.system(f'mkdir -p cmake_targets/log') import unittest @@ -14,6 +15,7 @@ sys.path.append('./') # to find OAI imports below import cls_oai_html import cls_oaicitest import cls_containerize +from cls_ci_helper import TestCaseCtx import ran import cls_cmd @@ -48,12 +50,16 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.eNBPassword = None self.cont.eNBSourceCodePath = os.getcwd() self.cont.num_attempts = 3 + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') def test_deploy(self): self.cont.yamlPath[0] = 'tests/simple-dep/' self.cont.deploymentTag = "noble" - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -61,8 +67,8 @@ class TestDeploymentMethods(unittest.TestCase): # fails reliably old = self.cont.yamlPath self.cont.yamlPath[0] = 'tests/simple-fail/' - deploy = self.cont.DeployObject(self.html) - self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertFalse(deploy) self.cont.yamlPath = old @@ -70,8 +76,8 @@ class TestDeploymentMethods(unittest.TestCase): # fails reliably old = self.cont.yamlPath self.cont.yamlPath[0] = 'tests/simple-fail-2svc/' - deploy = self.cont.DeployObject(self.html) - self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertFalse(deploy) self.cont.yamlPath = old @@ -79,8 +85,8 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -88,8 +94,8 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb oai-nr-ue" self.cont.deploymentTag = 'develop-12345678' - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -97,10 +103,10 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' - deploy1 = self.cont.DeployObject(self.html) + deploy1 = self.cont.DeployObject(self.ctx, self.html) self.cont.services[0] = "oai-nr-ue" - deploy2 = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy2 = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy1) self.assertTrue(deploy2) self.assertTrue(undeploy) From 62ae72925b9b14a9effe6a88a84482548b31b07a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:27:18 +0200 Subject: [PATCH 18/38] Use archiveArtifact() for module_UE --- ci-scripts/cls_module.py | 15 +++++++-------- ci-scripts/cls_oaicitest.py | 4 ++-- ci-scripts/main.py | 2 +- ci-scripts/tests/module.py | 14 ++++++++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index 5493511bd8..cfdd96477f 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -32,6 +32,7 @@ import re import yaml import cls_cmd +from cls_ci_helper import archiveArtifact class Module_UE: @@ -103,11 +104,11 @@ class Module_UE: return ret.returncode == 0 - def terminate(self, log_dir=None): + def terminate(self, ctx=None): self._command(self.cmd_dict["terminate"]) - if self.trace and log_dir is not None: + if self.trace and ctx is not None: self._disableTrace() - return self._collectTrace(log_dir) + return self._collectTrace(ctx) return None def attach(self, attach_tries = 4, attach_timeout = 60): @@ -202,8 +203,7 @@ class Module_UE: logging.info(f'UE {self}: stop UE tracing') self._command(self.cmd_dict["traceStop"]) - def _collectTrace(self, log_dir): - logging.info(f'collecting logs into (local) {log_dir}') + def _collectTrace(self, ctx): remote_dir = "/tmp/ue-trace-logs" with cls_cmd.getConnection(self.host) as c: # create a directory for log collection @@ -222,8 +222,7 @@ class Module_UE: log_files = [] # copy them to the executor one by one, and store in log_dir for f in ret.stdout.split("\n"): - l = f.replace(remote_dir, log_dir) - c.copyin(f, l) - log_files.append(l) + name = archiveArtifact(c, ctx, f) + log_files.append(name) c.run(f'rm -rf {remote_dir}') return log_files diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 290ad38598..52cea6a3c4 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -834,10 +834,10 @@ class OaiCiTest(): global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC return global_status - def TerminateUE(self, HTML, logPath): + def TerminateUE(self, ctx, HTML): ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids] with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(ue.terminate, logPath) for ue in ues] + futures = [executor.submit(ue.terminate, ctx) for ue in ues] archives = [f.result() for f in futures] archive_info = [f'Log at: {a}' if a else 'No log available' for a in archives] messages = [f"UE {ue.getName()}: {log}" for (ue, log) in zip(ues, archive_info)] diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 7f5efefc34..509cd19109 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -228,7 +228,7 @@ def ExecuteActionWithParam(action, ctx): elif action == 'Detach_UE': success = CiTestObj.DetachUE(HTML) elif action == 'Terminate_UE': - success = CiTestObj.TerminateUE(HTML, logPath) + success = CiTestObj.TerminateUE(ctx, HTML) elif action == 'CheckStatusUE': success = CiTestObj.CheckStatusUE(HTML) elif action == 'DataEnable_UE': diff --git a/ci-scripts/tests/module.py b/ci-scripts/tests/module.py index 366593873c..47760cfa27 100644 --- a/ci-scripts/tests/module.py +++ b/ci-scripts/tests/module.py @@ -12,9 +12,16 @@ import unittest sys.path.append('./') # to find OAI imports below import cls_module +from cls_ci_helper import TestCaseCtx import cls_cmd class TestModule(unittest.TestCase): + def setUp(self): + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') + def test_simple_module(self): c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml") self.assertFalse(c.trace) @@ -49,10 +56,9 @@ class TestModule(unittest.TestCase): self.assertTrue(c.checkMTU()) c.detach() tmp = tempfile.mkdtemp() - log_file = c.terminate(log_dir=tmp) - with cls_cmd.LocalCmd() as c: - c.run(f'rm -rf {tmp}') - self.assertEqual(log_file, [f"{tmp}/tttf"]) # matches test-trace UE collection file + log_file = c.terminate(self.ctx) + # undeploy uses archiveArtifact(), which writes to {prefix}-logs + self.assertEqual(log_file, [f"{self.ctx.baseFilename()}-tttf"]) # matches test-trace UE collection file if __name__ == '__main__': unittest.main() From da9cec0bfffcbf899f35ca8e992a498a23b37019 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:41:31 +0200 Subject: [PATCH 19/38] Use archiveArtifact() for Ping+IPerf --- ci-scripts/cls_oaicitest.py | 61 +++++++++++----------------------- ci-scripts/main.py | 6 ++-- ci-scripts/tests/ping-iperf.py | 15 ++++++--- 3 files changed, 33 insertions(+), 49 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 52cea6a3c4..ba322ca608 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -47,6 +47,7 @@ import constants as CONST import cls_module import cls_corenetwork import cls_cmd +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Helper functions used here and in other classes @@ -256,7 +257,6 @@ class OaiCiTest(): self.ranAllowMerge = False self.ranTargetBranch = '' - self.testCase_id = '' self.testXMLfiles = [] self.ping_args = '' self.ping_packetloss_threshold = '' @@ -351,7 +351,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue('NA', 'OK', messages) return True - def Ping_common(self, cn, ue, logPath): + def Ping_common(self, ctx, cn, ue): ping_status = 0 ueIP = ue.getIP() if not ueIP: @@ -359,13 +359,12 @@ class OaiCiTest(): svrIP = cn.getIP() if not svrIP: return (False, f"CN {cn.getName()} has no IP address") - ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log' + ping_log_file = f'/tmp/ping_{ue.getName()}.log' ping_time = re.findall(r"-c *(\d+)",str(self.ping_args)) - local_ping_log_file = f'{logPath}/{ping_log_file}' if re.search('%cn_ip%', self.ping_args) or re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+', self.ping_args): raise Exception(f"ping_args should not have IP address: {self.ping_args}") interface = f'-I {ue.getIFName()}' if ue.getIFName() else '' - ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee /tmp/{ping_log_file}' + ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee {ping_log_file}' cmd = cls_cmd.getConnection(ue.getHost()) response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5) ue_header = f'UE {ue.getName()} ({ueIP})' @@ -373,8 +372,7 @@ class OaiCiTest(): message = ue_header + ': ping crashed: TIMEOUT?' return (False, message) - #copy the ping log file to have it locally for analysis (ping stats) - cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=local_ping_log_file) + local_ping_log_file = archiveArtifact(cmd, ctx, ping_log_file) cmd.close() with open(local_ping_log_file, 'r') as f: @@ -414,18 +412,13 @@ class OaiCiTest(): return (True, message) - def Ping(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"): + def Ping(self, ctx, HTML, infra_file="ci_infra.yaml"): if self.ue_ids == [] or self.svr_id == None: raise Exception("no module names in self.ue_ids or/and self.svr_id provided") - # Creating destination log folder if needed on the python executor workspace - with cls_cmd.getConnection('localhost') as local: - ymlPath = CONTAINERS.yamlPath[0].split('/') - logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}' - local.run(f'mkdir -p {logPath}', silent=True) ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)] cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file) with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(self.Ping_common, cn, ue, logPath) for ue in ues] + futures = [executor.submit(self.Ping_common, ctx, cn, ue) for ue in ues] results = [f.result() for f in futures] # each result in results is a tuple, first member goes to successes, second to messages successes, messages = map(list, zip(*results)) @@ -446,7 +439,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages) return success - def Iperf_Module(self, cn, ue, idx, ue_num, logPath): + def Iperf_Module(self, ctx, cn, ue, idx, ue_num): ueIP = ue.getIP() if not ueIP: return (False, f"UE {ue.getName()} has no IP address") @@ -459,7 +452,7 @@ class OaiCiTest(): serverReport = "" udpIperf = re.search('-u', iperf_opt) is not None bidirIperf = re.search('--bidir', iperf_opt) is not None - client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log' + client_filename = f'/tmp/iperf_client_{ue.getName()}.log' if udpIperf: target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(idx, ue_num, self.iperf_profile, self.iperf_args) # note: for UDP testing we don't want to use json report - reports 0 Mbps received bitrate @@ -474,14 +467,11 @@ class OaiCiTest(): port = 5002 + idx # note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False` t = iperf_time * 2.5 - cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True) + cmd_ue.run(f'rm {client_filename}', reportNonZero=False, silent=True) if cn.runIperf3Server(): cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} >> /dev/null &', timeout=t) - cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=t) - # note: copy iperf3 log to the current directory for log analysis and log collection - dest_filename = f'{logPath}/{client_filename}' - cmd_ue.copyin(f'/tmp/{client_filename}', dest_filename) - cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True) + cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> {client_filename}', timeout=t) + dest_filename = archiveArtifact(cmd_ue, ctx, client_filename) if udpIperf: status, msg = Iperf_analyzeV3UDP(dest_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate) elif bidirIperf: @@ -491,20 +481,15 @@ class OaiCiTest(): return (status, f'{ue_header}\n{msg}') - def Iperf(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"): + def Iperf(self, ctx, HTML, infra_file="ci_infra.yaml"): logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"') if self.ue_ids == [] or self.svr_id == None: raise Exception("no module names in self.ue_ids or/and self.svr_id provided") - # create log directory on executor node - with cls_cmd.getConnection('localhost') as local: - ymlPath = CONTAINERS.yamlPath[0].split('/') - logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}' - local.run(f'mkdir -p {logPath}', silent=True) ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)] cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file) with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(self.Iperf_Module, cn, ue, i, len(ues), logPath) for i, ue in enumerate(ues)] + futures = [executor.submit(self.Iperf_Module, ctx, cn, ue, i, len(ues)) for i, ue in enumerate(ues)] results = [f.result() for f in futures] # each result in results is a tuple, first member goes to successes, second to messages successes, messages = map(list, zip(*results)) @@ -525,7 +510,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', messages) return success - def Iperf2_Unidir(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"): + def Iperf2_Unidir(self, ctx, HTML, infra_file="ci_infra.yaml"): if self.ue_ids == [] or self.svr_id == None or len(self.ue_ids) != 1: raise Exception("no module names in self.ue_ids or/and self.svr_id provided, multi UE scenario not supported") ue = cls_module.Module_UE(self.ue_ids[0].strip(),self.nodes[0].strip(), infra_file) @@ -536,23 +521,17 @@ class OaiCiTest(): svrIP = cn.getIP() if not svrIP: return False - server_filename = f'iperf_server_{self.testCase_id}_{ue.getName()}.log' - ymlPath = CONTAINERS.yamlPath[0].split('/') - logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}' + server_filename = f'/tmp/iperf_server_{ue.getName()}.log' iperf_time = Iperf_ComputeTime(self.iperf_args) target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(0, 1, self.iperf_profile, self.iperf_args) t = iperf_time*2.5 - with cls_cmd.getConnection('localhost') as local: - local.run(f'mkdir -p {logPath}') with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(cn.getHost()) as cmd_svr: - cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False) - cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> /tmp/{server_filename} &', timeout=t) + cmd_ue.run(f'rm {server_filename}', reportNonZero=False) + cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> {server_filename} &', timeout=t) cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf -c {ueIP} -B {svrIP} {iperf_opt} -i1 >> /dev/null', timeout=t) localPath = f'{os.getcwd()}' - # note: copy iperf2 log to the directory for log collection - cmd_ue.copyin(f'/tmp/{server_filename}', f'{logPath}/{server_filename}') - cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False) - success, msg = Iperf_analyzeV2UDP(f'{logPath}/{server_filename}', self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate) + local = archiveArtifact(cmd_ue, ctx, server_filename) + success, msg = Iperf_analyzeV2UDP(local, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate) ue_header = f'UE {ue.getName()} ({ueIP})' logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m') for l in msg.split('\n'): diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 509cd19109..7ec30aa477 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -255,7 +255,7 @@ def ExecuteActionWithParam(action, ctx): else: CiTestObj.nodes = [None] * len(CiTestObj.ue_ids) ping_rttavg_threshold = test.findtext('ping_rttavg_threshold') or '' - success = CiTestObj.Ping(HTML, CONTAINERS) + success = CiTestObj.Ping(ctx, HTML) elif action == 'Iperf' or action == 'Iperf2_Unidir': CiTestObj.iperf_args = test.findtext('iperf_args') @@ -286,9 +286,9 @@ def ExecuteActionWithParam(action, ctx): logging.error('test-case has wrong option ' + CiTestObj.iperf_options) CiTestObj.iperf_options = 'check' if action == 'Iperf': - success = CiTestObj.Iperf(HTML, CONTAINERS) + success = CiTestObj.Iperf(ctx, HTML) elif action == 'Iperf2_Unidir': - success = CiTestObj.Iperf2_Unidir(HTML, CONTAINERS) + success = CiTestObj.Iperf2_Unidir(ctx, HTML) elif action == 'IdleSleep': st = test.findtext('idle_sleep_time_in_sec') or "5" diff --git a/ci-scripts/tests/ping-iperf.py b/ci-scripts/tests/ping-iperf.py index 225fe4e868..9d50567a07 100644 --- a/ci-scripts/tests/ping-iperf.py +++ b/ci-scripts/tests/ping-iperf.py @@ -9,11 +9,13 @@ import os os.system(f'rm -rf cmake_targets') os.system(f'mkdir -p cmake_targets/log') import unittest +import tempfile sys.path.append('./') # to find OAI imports below import cls_oai_html +from cls_ci_helper import TestCaseCtx import cls_oaicitest -import cls_containerize +import cls_cmd class TestPingIperf(unittest.TestCase): def setUp(self): @@ -22,7 +24,10 @@ class TestPingIperf(unittest.TestCase): self.ci = cls_oaicitest.OaiCiTest() self.ci.ue_ids = ["test"] self.ci.nodes = ["localhost"] - self.cont = cls_containerize.Containerize() + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') def test_ping(self): self.ci.ping_args = "-c3" @@ -30,7 +35,7 @@ class TestPingIperf(unittest.TestCase): self.ci.svr_id = "test" infra_file = "tests/config/infra_ping_iperf.yaml" # TODO Should need nothing but options and UE(s) to use - success = self.ci.Ping(self.html, self.cont, infra_file=infra_file) + success = self.ci.Ping(self.ctx, self.html, infra_file=infra_file) self.assertTrue(success) def test_iperf(self): @@ -45,7 +50,7 @@ class TestPingIperf(unittest.TestCase): self.ci.iperf_profile = "balanced" infra_file = "tests/config/infra_ping_iperf.yaml" # TODO Should need nothing but options and UE(s) to use - success = self.ci.Iperf(self.html, self.cont, infra_file=infra_file) + success = self.ci.Iperf(self.ctx, self.html, infra_file=infra_file) self.assertTrue(success) def test_iperf2_unidir(self): @@ -57,7 +62,7 @@ class TestPingIperf(unittest.TestCase): self.ci.iperf_profile = "balanced" infra_file = "tests/config/infra_ping_iperf.yaml" # TODO Should need nothing but options and UE(s) to use - success = self.ci.Iperf2_Unidir(self.html, self.cont, infra_file=infra_file) + success = self.ci.Iperf2_Unidir(self.ctx, self.html, infra_file=infra_file) self.assertTrue(success) if __name__ == '__main__': From 1d3e795c0cbe5e1fca282a5b6a05028fd22a0c92 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:40:42 +0200 Subject: [PATCH 20/38] Use archiveArtifact() for deploy_oc_physim() --- ci-scripts/cls_analysis.py | 4 ++-- ci-scripts/cls_cluster.py | 18 +++++++----------- ci-scripts/main.py | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/ci-scripts/cls_analysis.py b/ci-scripts/cls_analysis.py index ac591b52fb..236a4743a1 100644 --- a/ci-scripts/cls_analysis.py +++ b/ci-scripts/cls_analysis.py @@ -84,7 +84,7 @@ class Analysis(): return success,msg - def analyze_oc_physim(result_junit, details_json): + def analyze_oc_physim(result_junit, details_json, logPath): try: tree = ET.parse(result_junit) root = tree.getroot() @@ -124,7 +124,7 @@ class Analysis(): output = test.findtext("system-out") output_check = "exceeds the threshold" not in output # collect logs - log_dir = f'../cmake_targets/log/{test_exec}' + log_dir = f'{logPath}/{test_exec}' os.makedirs(log_dir, exist_ok=True) with open(f'{log_dir}/{test_name}.log', 'w') as f: f.write(output) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 7e88c6b07b..7d41e9d9cc 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -460,7 +460,7 @@ class Cluster: return status - def deploy_oc_physim(self, HTML, oc_release, svr_id): + def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}') @@ -470,17 +470,13 @@ class Cluster: options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}" ret = cls_cmd.runScript(svr_id, script, 600, options) logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}') - log_dir = f'{os.getcwd()}/../cmake_targets/log' - os.makedirs(log_dir, exist_ok=True) - result_junit = f'{oc_release}-run.xml' - details_json = f'{oc_release}-tests.json' with cls_cmd.getConnection(svr_id) as ssh: - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log') - test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}') + details_json = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-tests.json') + result_junit = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-run.xml') + archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt') + archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt') + archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log') + test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(result_junit, details_json, ctx.logPath) if test_summary: if test_status: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 7ec30aa477..035437ac8a 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -297,7 +297,7 @@ def ExecuteActionWithParam(action, ctx): elif action == 'Deploy_Run_PhySim': oc_release = test.findtext('oc_release') svr_id = test.findtext('svr_id') or None - success = CLUSTER.deploy_oc_physim(HTML, oc_release, svr_id) + success = CLUSTER.deploy_oc_physim(ctx, HTML, oc_release, svr_id) elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork': cn_id = test.findtext('cn_id') From 7ef8829167a342466342bc5a67a09830e310e989 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:41:01 +0200 Subject: [PATCH 21/38] Use archiveArtifact() for cls_native.Build() --- ci-scripts/cls_native.py | 13 +++++-------- ci-scripts/main.py | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ci-scripts/cls_native.py b/ci-scripts/cls_native.py index fe508b84e3..86268a7dc1 100644 --- a/ci-scripts/cls_native.py +++ b/ci-scripts/cls_native.py @@ -28,27 +28,24 @@ import cls_cmd import cls_oai_html import cls_analysis import constants as CONST +from cls_ci_helper import archiveArtifact -LOG_PATH_PHYSIM = 'phy_sim_logs' DPDK_PATH = '/opt/dpdk-t2-22.11.0' class Native(): - def Build(test_case, HTML, host, directory, options): + def Build(ctx, test_case, HTML, host, directory, options): logging.debug(f'Building on server: {host}') HTML.testCase_id = test_case with cls_cmd.getConnection(host) as ssh: base = f"{directory}/cmake_targets" - ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/log/build_oai.log", timeout=900) + ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/build_oai.log", timeout=900) success = ret.returncode == 0 - logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True) + logs = ssh.run(f"cat {base}/build_oai.log", silent=True) logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}") - # create log directory, and copy build logs - target = f"{base}/build_log_{test_case}/" - ssh.run(f"mkdir -p {target}") - ssh.run(f"mv {base}/log/* {target}") + archiveArtifact(ssh, ctx, f'{base}/build_oai.log') # check if build artifacts are there # NOTE: build_oai should fail with exit code if it could not build, but it does not diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 035437ac8a..317b32b0f9 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -136,7 +136,7 @@ def ExecuteActionWithParam(action, ctx): if proxy_commit is not None: CONTAINERS.proxyCommit = proxy_commit if action == 'Build_eNB': - success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) + success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) elif action == 'Build_Image': success = CONTAINERS.BuildImage(HTML) elif action == 'Build_Proxy': From 83d037057bbfb0828cc4d5b870dfeab3af1d2b83 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 09:41:08 +0200 Subject: [PATCH 22/38] Use archiveArtifact() for cls_native.run_physim() --- ci-scripts/cls_native.py | 11 +++++------ ci-scripts/main.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ci-scripts/cls_native.py b/ci-scripts/cls_native.py index 86268a7dc1..3dde286c3a 100644 --- a/ci-scripts/cls_native.py +++ b/ci-scripts/cls_native.py @@ -65,15 +65,14 @@ class Native(): HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK) return success - def Run_Physim(HTML, host, directory, options, physim_test, threshold): + def Run_Physim(ctx, HTML, host, directory, options, physim_test, threshold): logging.debug(f'Runnin {physim_test} on server: {host}') workSpacePath = f'{directory}/cmake_targets' - os.system(f'mkdir -p ./{LOG_PATH_PHYSIM}') - runLogFile=f'physim_{HTML.testCase_id}.log' + runLogFile=f'{workSpacePath}/physim.log' with cls_cmd.getConnection(host) as cmd: - cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {workSpacePath}/{runLogFile}') - cmd.copyin(src=f'{workSpacePath}/{runLogFile}', tgt=f'{LOG_PATH_PHYSIM}/{runLogFile}') - success, msg = cls_analysis.Analysis.analyze_physim(f'{LOG_PATH_PHYSIM}/{runLogFile}', physim_test, options, threshold) + cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {runLogFile}') + physim_file = archiveArtifact(cmd, ctx, runLogFile) + success, msg = cls_analysis.Analysis.analyze_physim(physim_file, physim_test, options, threshold) if success: HTML.CreateHtmlTestRowQueue(options, 'OK', [msg]) else: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 317b32b0f9..34f0946de0 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -343,7 +343,7 @@ def ExecuteActionWithParam(action, ctx): physim_options = test.findtext('physim_run_args') physim_test = test.findtext('physim_test') physim_threshold = test.findtext('physim_time_threshold') or 'inf' - success = cls_native.Native.Run_Physim(HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold) + success = cls_native.Native.Run_Physim(ctx, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold) elif action == 'LicenceAndFormattingCheck': success = SCA.LicenceAndFormattingCheck(HTML) From 1ffbba525de7c3c77fd0451659752d3268bf20d8 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 17:41:56 +0200 Subject: [PATCH 23/38] Use archiveArtifact() for BuildRunTests() Collect also unit test logs, so that developers could see the reason for failed logs. --- ci-scripts/cls_containerize.py | 22 +++++++++++++--------- ci-scripts/main.py | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 591d6d413d..4b84b9ab86 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -636,7 +636,7 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) return False - def BuildRunTests(self, HTML): + def BuildRunTests(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) @@ -653,7 +653,6 @@ class Containerize(): if self.forcedWorkspaceCleanup: cmd.run(f'sudo -S rm -Rf {lSourcePath}') - self.testCase_id = HTML.testCase_id # check that ran-base image exists as we expect it baseImage = 'ran-base' @@ -673,10 +672,10 @@ class Containerize(): # build ran-unittests image dockerfile = "ci-scripts/docker/Dockerfile.unittest.ubuntu22" - ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {lSourcePath}/cmake_targets/log/unittest-build.log') + logfile = f'{lSourcePath}/cmake_targets/log/unittest-build.log' + ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {logfile}') + archiveArtifact(cmd, ctx, logfile) if ret.returncode != 0: - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) logging.error(f'Cannot build unit tests') HTML.CreateHtmlTestRow("Unit test build failed", 'KO', [dockerfile]) HTML.CreateHtmlTabFooter(False) @@ -685,10 +684,15 @@ class Containerize(): HTML.CreateHtmlTestRowQueue("Build unit tests", 'OK', [dockerfile]) # it worked, build and execute tests, and close connection - ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --rm ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)') - cmd.run(f'docker rmi ran-unittests:{baseTag}') - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) + # I would like to run it with --rm and mount the ctest result directory to avoid 'docker cp' + # below, but then permissions are messed up and we can't remove the directory without sudo + # making the next pipeline fail + ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --name ran-unittests ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)') + cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTest.log .') + archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTest.log') + cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTestsFailed.log .') + archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTestsFailed.log') + cmd.run('docker rm ran-unittests') cmd.close() if ret.returncode == 0: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 34f0946de0..4a8e63f8cb 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -144,7 +144,7 @@ def ExecuteActionWithParam(action, ctx): elif action == 'Build_Cluster_Image': success = CLUSTER.BuildClusterImage(HTML) elif action == 'Build_Run_Tests': - success = CONTAINERS.BuildRunTests(HTML) + success = CONTAINERS.BuildRunTests(ctx, HTML) elif action == 'Initialize_eNB': datalog_rt_stats_file=test.findtext('rt_stats_cfg') From 63f6368ca7c0c363f8dcd0329bdf16fa33908c67 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 17:41:40 +0200 Subject: [PATCH 24/38] use archiveArtifact() for BuildProxy() --- ci-scripts/cls_containerize.py | 11 +++-------- ci-scripts/main.py | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 4b84b9ab86..381e955127 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -534,7 +534,7 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) return False - def BuildProxy(self, HTML): + def BuildProxy(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) @@ -566,15 +566,10 @@ class Containerize(): if not success: raise Exception("could not clone proxy repository") - filename = f'build_log_{self.testCase_id}' - fullpath = f'{lSourcePath}/{filename}' + fullpath = f'{lSourcePath}/proxy_build.log' ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1') - ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}') - local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip" - ssh.copyin(f'{fullpath}.zip', local_file) - # don't delete such that we might recover the zips - #ssh.run(f'rm -f {fullpath}.zip') + archiveArtifact(ssh, ctx, fullpath) ssh.run('docker image prune --force') ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 4a8e63f8cb..f201bdbcc6 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -140,7 +140,7 @@ def ExecuteActionWithParam(action, ctx): elif action == 'Build_Image': success = CONTAINERS.BuildImage(HTML) elif action == 'Build_Proxy': - success = CONTAINERS.BuildProxy(HTML) + success = CONTAINERS.BuildProxy(ctx, HTML) elif action == 'Build_Cluster_Image': success = CLUSTER.BuildClusterImage(HTML) elif action == 'Build_Run_Tests': From 5ffa4315141d8dc689b7ceddcdc09d81909f09c2 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 17:41:34 +0200 Subject: [PATCH 25/38] Use archiveArtifact() for Build()/BuildCluster() I have to make both together, since they both depend on AnalyzeBuildLogs(). - remove mkdir for legacy sub-logfiles in build_oai, that does not exist anymore; also, remove corresponding log analysis, I think we don't use it and it is overkill (either build works or not) - manually try to copy "all.txt" - explicitly list log files to copy, then analyze - AnalyzeBuildLogs(): pass log files to inspect explicitly --- ci-scripts/cls_cluster.py | 91 ++++++++++++++------------ ci-scripts/cls_containerize.py | 116 +++++++++++---------------------- ci-scripts/main.py | 4 +- 3 files changed, 91 insertions(+), 120 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 7d41e9d9cc..185e90a13e 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -39,6 +39,7 @@ import constants as CONST import helpreadme as HELP import cls_containerize import cls_cmd +from cls_ci_helper import archiveArtifact IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc" NAMESPACE = "oaicicd-ran" @@ -207,7 +208,12 @@ class Cluster: HTML.CreateHtmlTestRowQueue(param, 'KO', [msg]) return success - def BuildClusterImage(self, HTML): + def _retrieveOCLog(self, ctx, job, lSourcePath, image): + fn = f'{lSourcePath}/cmake_targets/log/{image}.log' + self.cmd.run(f'oc logs {job} &> {fn}') + return (image, archiveArtifact(self.cmd, ctx, fn)) + + def BuildClusterImage(self, ctx, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}') @@ -280,22 +286,27 @@ class Cluster: self._recreate_entitlements() status = True # flag to abandon compiling if any image fails - attemptedImages = [] + log_files = [] + build_metrics = f"{lSourcePath}/cmake_targets/log/build-metrics.log" if forceBaseImageBuild: self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml') self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml') ranbase_job = self._start_build('ran-base') - attemptedImages += ['ran-base'] status = ranbase_job is not None and self._wait_build_end([ranbase_job], 1000) if not status: logging.error('failure during build of ran-base') - self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect + log_files.append(self._retrieveOCLog(ctx, ranbase_job, lSourcePath, 'ran-base')) + # recover logs by mounting image self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml') pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml') if pod is not None: - self.cmd.run(f'mkdir -p cmake_targets/log/ran-base') - self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base') + logdir = f'{lSourcePath}/cmake_targets/log/ran-base' + self.cmd.run(f'mkdir -p {logdir}') + self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ {logdir}') self._undeploy_pod('openshift/ran-base-log-retrieval.yaml') + ret = self.cmd.run(f'ls {logdir}') + for f in ret.stdout.splitlines(): + archiveArtifact(self.cmd, ctx, f"{logdir}/{f}") else: status = False @@ -304,27 +315,24 @@ class Cluster: self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel9') physim_job = self._start_build('oai-physim') - attemptedImages += ['oai-physim'] self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml') self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel9') ranbuild_job = self._start_build('ran-build') - attemptedImages += ['ran-build'] self._recreate_is_tag('oai-clang', imageTag, 'openshift/oai-clang-is.yaml') self._recreate_bc('oai-clang', imageTag, 'openshift/oai-clang-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.clang.rhel9') clang_job = self._start_build('oai-clang') - attemptedImages += ['oai-clang'] wait = ranbuild_job is not None and physim_job is not None and clang_job is not None and self._wait_build_end([ranbuild_job, physim_job, clang_job], 1200) if not wait: logging.error('error during build of ranbuild_job or physim_job or clang_job') status = status and wait - self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log') - self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log') - self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, ranbuild_job, lSourcePath, 'ran-build')) + log_files.append(self._retrieveOCLog(ctx, physim_job, lSourcePath, 'oai-physim')) + log_files.append(self._retrieveOCLog(ctx, clang_job, lSourcePath, 'oai-clang')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml') @@ -332,72 +340,66 @@ class Cluster: self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel9') enb_job = self._start_build('oai-enb') - attemptedImages += ['oai-enb'] self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml') self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel9') gnb_job = self._start_build('oai-gnb') - attemptedImages += ['oai-gnb'] self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml') self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel9') gnb_aw2s_job = self._start_build('oai-gnb-aw2s') - attemptedImages += ['oai-gnb-aw2s'] wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 800) if not wait: logging.error('error during build of eNB/gNB') status = status and wait # recover logs - self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log') - self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log') - self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log') + log_files.append(self._retrieveOCLog(ctx, enb_job, lSourcePath, 'oai-enb')) + log_files.append(self._retrieveOCLog(ctx, gnb_job, lSourcePath, 'oai-gnb')) + log_files.append(self._retrieveOCLog(ctx, gnb_aw2s_job, lSourcePath, 'oai-gnb-aw2s')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') self._recreate_is_tag('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-is.yaml') self._recreate_bc('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nr-cuup.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nr-cuup.rhel9') nr_cuup_job = self._start_build('oai-nr-cuup') - attemptedImages += ['oai-nr-cuup'] self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml') self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel9') lteue_job = self._start_build('oai-lte-ue') - attemptedImages += ['oai-lte-ue'] self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml') self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel9') nrue_job = self._start_build('oai-nr-ue') - attemptedImages += ['oai-nr-ue'] wait = nr_cuup_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end([nr_cuup_job, lteue_job, nrue_job], 800) if not wait: logging.error('error during build of nr-cuup/lteUE/nrUE') status = status and wait # recover logs - self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log') - self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log') - self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, nr_cuup_job, lSourcePath, 'oai-nr-cuup')) + log_files.append(self._retrieveOCLog(ctx, lteue_job, lSourcePath, 'oai-lte-ue')) + log_files.append(self._retrieveOCLog(ctx, nrue_job, lSourcePath, 'oai-nr-ue')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml') self._recreate_bc('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.fhi72.rhel9') ranbuildfhi72_job = self._start_build('ran-build-fhi72') - attemptedImages += ['ran-build-fhi72'] wait = ranbuildfhi72_job is not None and self._wait_build_end([ranbuildfhi72_job], 1200) if not wait: logging.error('error during build of ranbuildfhi72_job') status = status and wait - self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, ranbuildfhi72_job, lSourcePath, 'ran-build-fhi72')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml') @@ -405,19 +407,17 @@ class Cluster: self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.fhi72.rhel9') self._retag_image_statement('ran-build-fhi72', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build-fhi72', imageTag, 'docker/Dockerfile.gNB.fhi72.rhel9') gnb_fhi72_job = self._start_build('oai-gnb-fhi72') - attemptedImages += ['oai-gnb-fhi72'] wait = gnb_fhi72_job is not None and self._wait_build_end([gnb_fhi72_job], 600) if not wait: logging.error('error during build of gNB-fhi72') status = status and wait # recover logs - self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, gnb_fhi72_job, lSourcePath, 'oai-gnb-fhi72')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') # split and analyze logs imageSize = {} - for image in attemptedImages: - self.cmd.run(f'mkdir -p cmake_targets/log/{image}') + for image, _ in log_files: tag = imageTag if image != 'ran-base' else baseTag size = self._get_image_size(image, tag) if size <= 0: @@ -428,12 +428,14 @@ class Cluster: imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)' logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m') - grep_exp = r"\|".join(attemptedImages) - self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log'); - self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done') - - build_log_name = f'build_log_{self.testCase_id}' - cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name) + archiveArtifact(self.cmd, ctx, build_metrics) + logfile = f'{lSourcePath}/cmake_targets/log/image_registry.log' + grep_exp = r"\|".join([i for i,f in log_files]) + self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> {logfile}'); + archiveArtifact(self.cmd, ctx, logfile) + logfile = f'{lSourcePath}/cmake_targets/log/build_pod_summary.log' + self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> {logfile}; done') + archiveArtifact(self.cmd, ctx, logfile) self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') @@ -442,7 +444,7 @@ class Cluster: self.cmd.close() # Analyze the logs - collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status) + collectInfo = cls_containerize.AnalyzeBuildLogs(log_files, status) for img in collectInfo: for f in collectInfo[img]: status = status and collectInfo[img][f]['status'] @@ -458,6 +460,13 @@ class Cluster: HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize) + # TODO fix groovy script, remove the following. + # the groovy scripts expects all logs in + # //ci-scripts, so copy it there + with cls_cmd.LocalCmd() as c: + c.run(f'mkdir -p {os.getcwd()}/test_log_{ctx.test_id}/') + c.run(f'cp -r {ctx.logPath} {os.getcwd()}/test_log_{ctx.test_id}/') + return status def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id): diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 381e955127..3af34dca14 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -82,67 +82,25 @@ def CreateTag(ranCommitID, ranBranch, ranAllowMerge): tagToUse = f'develop-{shortCommit}' return tagToUse -def CopyLogsToExecutor(cmd, sourcePath, log_name): - cmd.cd(f'{sourcePath}/cmake_targets') - cmd.run(f'rm -f {log_name}.zip') - cmd.run(f'mkdir -p {log_name}') - cmd.run(f'mv log/* {log_name}') - cmd.run(f'zip -r -qq {log_name}.zip {log_name}') - - # copy zip to executor for analysis - if (os.path.isfile(f'./{log_name}.zip')): - os.remove(f'./{log_name}.zip') - if (os.path.isdir(f'./{log_name}')): - shutil.rmtree(f'./{log_name}') - cmd.copyin(src=f'{sourcePath}/cmake_targets/{log_name}.zip', tgt=f'{os.getcwd()}/{log_name}.zip') - cmd.run(f'rm -f {log_name}.zip') - ZipFile(f'{log_name}.zip').extractall('.') - -def AnalyzeBuildLogs(buildRoot, images, globalStatus): +def AnalyzeBuildLogs(logfiles, globalStatus): collectInfo = {} - for image in images: + for image, lf in logfiles: files = {} - file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')] - # Analyze the "sub-logs" of every target image - for fil in file_list: - errorandwarnings = {} - warningsNo = 0 - errorsNo = 0 - with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile: - for line in inputfile: - result = re.search(' ERROR ', str(line)) - if result is not None: - errorsNo += 1 - result = re.search(' error:', str(line)) - if result is not None: - errorsNo += 1 - result = re.search(' WARNING ', str(line)) - if result is not None: - warningsNo += 1 - result = re.search(' warning:', str(line)) - if result is not None: - warningsNo += 1 - errorandwarnings['errors'] = errorsNo - errorandwarnings['warnings'] = warningsNo - errorandwarnings['status'] = globalStatus - files[fil] = errorandwarnings - # Analyze the target image - if os.path.isfile(f'{buildRoot}/{image}.log'): - errorandwarnings = {} - committed = False - tagged = False - with open(f'{buildRoot}/{image}.log', mode='r') as inputfile: - for line in inputfile: - lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None - lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None - tagged = tagged or lineHasTag or lineHasTag2 - # the OpenShift Cluster builder prepends image registry URL - lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None - committed = committed or lineHasCommit - errorandwarnings['errors'] = 0 if committed or tagged else 1 - errorandwarnings['warnings'] = 0 - errorandwarnings['status'] = committed or tagged - files['Target Image Creation'] = errorandwarnings + errorandwarnings = {} + committed = False + tagged = False + with open(lf, mode='r') as inputfile: + for line in inputfile: + lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None + lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None + tagged = tagged or lineHasTag or lineHasTag2 + # the OpenShift Cluster builder prepends image registry URL + lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None + committed = committed or lineHasCommit + errorandwarnings['errors'] = 0 if committed or tagged else 1 + errorandwarnings['warnings'] = 0 + errorandwarnings['status'] = committed or tagged + files['Target Image Creation'] = errorandwarnings collectInfo[image] = files return collectInfo @@ -328,11 +286,12 @@ class Containerize(): raise ValueError(f'Insufficient Parameter: IP/node {ip}, path {path}') return (ip, path) - def BuildImage(self, HTML): + def BuildImage(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) cmd = cls_cmd.getConnection(lIpAddr) + log_files = [] # Checking the hostname to get adapted on cli and dockerfileprefixes cmd.run('hostnamectl') @@ -424,7 +383,18 @@ class Containerize(): # On when the base image docker file is being modified. if forceBaseImageBuild: cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") - cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600) + logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log' + cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> {logfile}", timeout=1600) + t = ("ran-base", archiveArtifact(cmd, ctx, logfile)) + log_files.append(t) + + # Recover build logs, for the moment only possible when build is successful + cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") + cmd.run("mkdir -p cmake_targets/log/ran-base") + logfile = f'{lSourcePath}/cmake_targets/log/ran-base.log' + cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/all.txt {logfile}") + cmd.run(f"{self.cli} rm -f test") + archiveArtifact(cmd, ctx, logfile) # First verify if the base image was properly created. ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}") allImagesSize = {} @@ -448,17 +418,9 @@ class Containerize(): else: logging.debug('ran-base size is unknown') - # Recover build logs, for the moment only possible when build is successful - cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") - cmd.run("mkdir -p cmake_targets/log/ran-base") - cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base") - cmd.run(f"{self.cli} rm -f test") - # Build the target image(s) status = True - attemptedImages = ['ran-base'] for image,pattern,name,option in imageNames: - attemptedImages += [name] # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}') @@ -472,15 +434,18 @@ class Containerize(): cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') if image == 'oai-gnb-aerial': cmd.run('cp -f /opt/nvidia-ipc/nvipc.src.2025.05.20.tar.gz .') - ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200) + logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log' + ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > {logfile} 2>&1', timeout=1200) + t = (name, archiveArtifact(cmd, ctx, logfile)) + log_files.append(t) if image == 'oai-gnb-aerial': cmd.run('rm -f nvipc.src.2025.05.20.tar.gz') if image == 'ran-build' and ret.returncode == 0: cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true") - cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/") + logfile = f'{lSourcePath}/{image}.log' + cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/all.txt {logfile}") cmd.run(f"docker rm -f test-log") - else: - cmd.run(f"mkdir -p cmake_targets/log/{name}") + archiveArtifact(cmd, ctx, logfile) # check the status of the build ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}") if ret.returncode != 0: @@ -514,13 +479,10 @@ class Containerize(): logging.debug(cmd.run("df -h").stdout) logging.debug(cmd.run("docker system df").stdout) - # create a zip with all logs - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) cmd.close() # Analyze the logs - collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status) + collectInfo = AnalyzeBuildLogs(log_files, status) if status: logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index f201bdbcc6..dcca9b521f 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -138,11 +138,11 @@ def ExecuteActionWithParam(action, ctx): if action == 'Build_eNB': success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) elif action == 'Build_Image': - success = CONTAINERS.BuildImage(HTML) + success = CONTAINERS.BuildImage(ctx, HTML) elif action == 'Build_Proxy': success = CONTAINERS.BuildProxy(ctx, HTML) elif action == 'Build_Cluster_Image': - success = CLUSTER.BuildClusterImage(HTML) + success = CLUSTER.BuildClusterImage(ctx, HTML) elif action == 'Build_Run_Tests': success = CONTAINERS.BuildRunTests(ctx, HTML) From 810dad97ae0a3016ea0f1d65347bd326eaffb3ca Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 17:53:28 +0200 Subject: [PATCH 26/38] Collect the ninja logs for various ran-build images --- ci-scripts/cls_containerize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 3af34dca14..e632349960 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -440,9 +440,9 @@ class Containerize(): log_files.append(t) if image == 'oai-gnb-aerial': cmd.run('rm -f nvipc.src.2025.05.20.tar.gz') - if image == 'ran-build' and ret.returncode == 0: + if image == 'ran-build' or image == 'ran-build-asan' or image == 'ran-build-fhi72': cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true") - logfile = f'{lSourcePath}/{image}.log' + logfile = f'{lSourcePath}/{image}.ninja.log' cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/all.txt {logfile}") cmd.run(f"docker rm -f test-log") archiveArtifact(cmd, ctx, logfile) From 65ec00c5fdf3a1fe87ffdf3fabd8f708ffc76748 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 14:30:05 +0200 Subject: [PATCH 27/38] No need for all the useless docker image rm --- ci-scripts/cls_containerize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index e632349960..0c1bcb530e 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -377,7 +377,7 @@ class Containerize(): # Let's remove any previous run artifacts if still there cmd.run(f"{self.cli} image prune --force") for image,pattern,name,option in imageNames: - cmd.run(f"{self.cli} image rm {name}:{imageTag}") + cmd.run(f"{self.cli} image rm {name}:{imageTag}", reportNonZero=False) # Build the base image only on Push Events (not on Merge Requests) # On when the base image docker file is being modified. From 025a6dbd1dd57ace93715abebb975066d5dbe28e Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 17:43:05 +0200 Subject: [PATCH 28/38] Remove unused testCase_Id --- ci-scripts/cls_containerize.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 0c1bcb530e..306f9a9a47 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -255,8 +255,6 @@ class Containerize(): self.services = ['', '', ''] self.deploymentTag = '' - self.testCase_id = '' - self.cli = '' self.cliBuildOptions = '' self.dockerfileprefix = '' @@ -347,7 +345,6 @@ class Containerize(): imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', '')) imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', '')) - self.testCase_id = HTML.testCase_id cmd.cd(lSourcePath) # if asterix, copy the entitlement and subscription manager configurations if self.host == 'Red Hat': @@ -502,7 +499,6 @@ class Containerize(): logging.debug('Building on server: ' + lIpAddr) ssh = cls_cmd.getConnection(lIpAddr) - self.testCase_id = HTML.testCase_id oldRanCommidID = self.ranCommitID oldRanRepository = self.ranRepository oldRanAllowMerge = self.ranAllowMerge From 27e37a4181feda612ea7c879baf192b90808f910 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 16:14:53 +0200 Subject: [PATCH 29/38] Use archiveArtifact() for LicenceAndFormattingCheck() --- ci-scripts/cls_static_code_analysis.py | 12 ++++++------ ci-scripts/main.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index f2951b0528..33337a4700 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -43,6 +43,7 @@ from pathlib import Path import helpreadme as HELP import constants as CONST import cls_cmd +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Class Declaration @@ -213,7 +214,7 @@ class StaticCodeAnalysis(): return True - def LicenceAndFormattingCheck(self, HTML): + def LicenceAndFormattingCheck(self, ctx, HTML): # Workspace is no longer recreated from scratch. # It implies that this method shall be called last within a build pipeline # where workspace is already created @@ -238,7 +239,7 @@ class StaticCodeAnalysis(): else: check_options += f' --build-arg TARGET_BRANCH={self.ranTargetBranch}' - logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}' + logDir = f'{lSourcePath}/cmake_targets/log/' cmd.run(f'mkdir -p {logDir}') cmd.run('docker image rm oai-formatting-check:latest') cmd.run(f'docker build --target oai-formatting-check --tag oai-formatting-check:latest {check_options} --file {lSourcePath}/ci-scripts/docker/Dockerfile.formatting.bionic {lSourcePath} > {logDir}/oai-formatting-check.txt 2>&1') @@ -247,12 +248,11 @@ class StaticCodeAnalysis(): cmd.run('docker image prune --force') cmd.run('docker volume prune --force') - # Analyzing the logs - cmd.copyin(f'{logDir}/oai-formatting-check.txt', 'oai-formatting-check.txt') + file = archiveArtifact(cmd, ctx, f'{logDir}/oai-formatting-check.txt') cmd.close() finalStatus = 0 - if (os.path.isfile('./oai-formatting-check.txt')): + if (os.path.isfile(file)): analyzed = False nbFilesNotFormatted = 0 listFiles = False @@ -263,7 +263,7 @@ class StaticCodeAnalysis(): gnuGplLicenceFiles = [] suspectLicence = False suspectLicenceFiles = [] - with open('./oai-formatting-check.txt', 'r') as logfile: + with open(file, 'r') as logfile: for line in logfile: ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line)) if ret is not None: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index dcca9b521f..d4ef09668a 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -346,7 +346,7 @@ def ExecuteActionWithParam(action, ctx): success = cls_native.Native.Run_Physim(ctx, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold) elif action == 'LicenceAndFormattingCheck': - success = SCA.LicenceAndFormattingCheck(HTML) + success = SCA.LicenceAndFormattingCheck(ctx, HTML) elif action == 'Cppcheck_Analysis': success = SCA.CppCheckAnalysis(HTML) From 03a47721896bb76a57118b0a7bc9a6f8311b2ac7 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 01:23:25 +0200 Subject: [PATCH 30/38] Remove cppcheck reference file: nobody knows about it, nobody checks --- ci-scripts/cls_static_code_analysis.py | 33 -------------------------- 1 file changed, 33 deletions(-) diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index 33337a4700..bdfb308a09 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -122,19 +122,6 @@ class StaticCodeAnalysis(): CCR_ref = CppCheckResults() vId = 0 for variant in CCR.variants: - refAvailable = False - if self.ranAllowMerge: - refFolder = str(Path.home()) + '/cppcheck-references' - if (os.path.isfile(refFolder + '/cppcheck-'+ variant + '.txt')): - refAvailable = True - with open(refFolder + '/cppcheck-'+ variant + '.txt', 'r') as refFile: - for line in refFile: - ret = re.search(' (?P[0-9\.]+) errors', str(line)) - if ret is not None: - CCR_ref.nbErrors[vId] = int(ret.group('nb_errors')) - ret = re.search(' (?P[0-9\.]+) warnings', str(line)) - if ret is not None: - CCR_ref.nbWarnings[vId] = int(ret.group('nb_warnings')) if (os.path.isfile('./cppcheck-'+ variant + '.txt')): xmlStart = False with open('./cppcheck-'+ variant + '.txt', 'r') as logfile: @@ -186,26 +173,6 @@ class StaticCodeAnalysis(): vMsg += ' Wrong Scanf Nb Args: ' + str(CCR.nbWrongScanfArg[vId]) + '\n' for vLine in vMsg.split('\n'): logging.debug(vLine) - if self.ranAllowMerge and refAvailable: - if CCR_ref.nbErrors[vId] == CCR.nbErrors[vId]: - logging.debug(' No change in number of errors') - elif CCR_ref.nbErrors[vId] > CCR.nbErrors[vId]: - logging.debug(' Good! Decrease in number of errors') - else: - logging.debug(' Bad! increase in number of errors') - if CCR_ref.nbWarnings[vId] == CCR.nbWarnings[vId]: - logging.debug(' No change in number of warnings') - elif CCR_ref.nbWarnings[vId] > CCR.nbWarnings[vId]: - logging.debug(' Good! Decrease in number of warnings') - else: - logging.debug(' Bad! increase in number of warnings') - # Create new reference file - if not self.ranAllowMerge: - refFolder = str(Path.home()) + '/cppcheck-references' - if not os.path.isdir(refFolder): - os.mkdir(refFolder) - with open(refFolder + '/cppcheck-'+ variant + '.txt', 'w') as refFile: - refFile.write(vMsg) vId += 1 HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) From 14389235dfa1e713d3ba1907c3030d8893894f66 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 16:19:36 +0200 Subject: [PATCH 31/38] Use archiveArtifact() for SCA.CppCheckAnalysis() --- ci-scripts/cls_static_code_analysis.py | 17 ++++++++++------- ci-scripts/main.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index bdfb308a09..8b0ec05352 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -81,7 +81,7 @@ class StaticCodeAnalysis(): self.eNBPassword = '' self.eNBSourceCodePath = '' - def CppCheckAnalysis(self, HTML): + def CppCheckAnalysis(self, ctx, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -104,7 +104,7 @@ class StaticCodeAnalysis(): full_ran_repo_name = self.ranRepository + '.git' cmd.cd(lSourcePath) - logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}' + logDir = f'{lSourcePath}/cmake_targets/log' cmd.run(f'mkdir -p {logDir}') cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal') cmd.run(f'sed -e "s@xenial@bionic@" {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.xenial > {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.bionic') @@ -113,18 +113,21 @@ class StaticCodeAnalysis(): cmd.run(f'docker build --tag oai-cppcheck:focal --file {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.focal . > {logDir}/cppcheck-focal.txt 2>&1') cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal') - # Analyzing the logs - cmd.copyin(f'{logDir}/cppcheck-bionic.txt', 'cppcheck-bionic.txt') - cmd.copyin(f'{logDir}/cppcheck-focal.txt', 'cppcheck-focal.txt') + bionic = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-bionic.txt') + focal = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-focal.txt') cmd.close() CCR = CppCheckResults() CCR_ref = CppCheckResults() vId = 0 for variant in CCR.variants: - if (os.path.isfile('./cppcheck-'+ variant + '.txt')): + filename = ctx.baseFilename() + '-cppcheck-'+ variant + '.txt' + logging.info(f"will check file '{filename}'") + if not os.path.isfile(filename): + raise FileNotFoundError(f"{filename} is not a file") + else: xmlStart = False - with open('./cppcheck-'+ variant + '.txt', 'r') as logfile: + with open(filename, 'r') as logfile: for line in logfile: ret = re.search('cppcheck version="(?P[0-9\.]+)"', str(line)) if ret is not None: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index d4ef09668a..836f6aee3b 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -349,7 +349,7 @@ def ExecuteActionWithParam(action, ctx): success = SCA.LicenceAndFormattingCheck(ctx, HTML) elif action == 'Cppcheck_Analysis': - success = SCA.CppCheckAnalysis(HTML) + success = SCA.CppCheckAnalysis(ctx, HTML) elif action == 'Push_Local_Registry': svr_id = test.findtext('svr_id') From 8c2b356c1d3170c6423edb92da335fe6a9ec9a02 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 01:29:30 +0200 Subject: [PATCH 32/38] Remove unused function --- ci-scripts/ran.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 46711b1b73..5c5513a317 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -248,41 +248,6 @@ class RANManagement(): logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m') return enbDidSync - def CheckeNBProcess(self, status_queue): - # At least the instance 0 SHALL be on! - if self.eNBstatuses[0] == 0: - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - elif self.eNBstatuses[0] == 1: - lIpAddr = self.eNB1IPAddress - lUserName = self.eNB1UserName - lPassWord = self.eNB1Password - elif self.eNBstatuses[0] == 2: - lIpAddr = self.eNB2IPAddress - lUserName = self.eNB2UserName - lPassWord = self.eNB2Password - else: - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - if self.air_interface[self.eNB_instance] == '': - pattern = 'softmodem' - else: - pattern = self.air_interface[self.eNB_instance] - mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5) - result = re.search(pattern, mySSH.getBefore()) - success = result is not None - if not success: - logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m') - status_queue.put(CONST.ENB_PROCESS_FAILED) - else: - status_queue.put(CONST.ENB_PROCESS_OK) - mySSH.close() - return success - def TerminateeNB(self, HTML): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress From 5090ce6f0abd4a863928cece4ff02cf55a147be2 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 01:45:02 +0200 Subject: [PATCH 33/38] Ignore main.py TerminateeNB mode No pipeline uses it. With the changes in the next commit, pipelines would start to fail. Remove it for simplicity. --- ci-scripts/main.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 836f6aee3b..8a52b7cf18 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -469,15 +469,7 @@ if py_param_file_present == True: cwd = os.getcwd() if re.match('^TerminateeNB$', mode, re.IGNORECASE): - if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - if RAN.eNBIPAddress == 'none': - sys.exit(0) - RAN.eNB_instance=0 - RAN.eNB_serverId[0]='0' - RAN.eNBSourceCodePath='/tmp/' - RAN.TerminateeNB(HTML) + logging.warning("Option TerminateeNB ignored") elif re.match('^TerminateHSS$', mode, re.IGNORECASE): logging.warning("Option TerminateHSS ignored") elif re.match('^TerminateMME$', mode, re.IGNORECASE): From 244ea8fc22048051fd7a3f5a28bb6a4a7a5c50e6 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 01:37:47 +0200 Subject: [PATCH 34/38] Use archiveArtifact() for TerminateeNB() --- ci-scripts/main.py | 2 +- ci-scripts/ran.py | 32 ++++++++++++++++++-------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 8a52b7cf18..1b06539a2c 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -206,7 +206,7 @@ def ExecuteActionWithParam(action, ctx): RAN.air_interface[RAN.eNB_instance] = 'lte-softmodem' else: RAN.air_interface[RAN.eNB_instance] = air_interface.lower() +'-softmodem' - success = RAN.TerminateeNB(HTML) + success = RAN.TerminateeNB(ctx, HTML) elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE': CiTestObj.ue_ids = test.findtext('id').split(' ') diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 5c5513a317..48039aa2e7 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -46,6 +46,7 @@ import cls_cmd import sshconnection as SSH import helpreadme as HELP import constants as CONST +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Class Declaration @@ -248,7 +249,7 @@ class RANManagement(): logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m') return enbDidSync - def TerminateeNB(self, HTML): + def TerminateeNB(self, ctx, HTML): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName @@ -320,13 +321,11 @@ class RANManagement(): fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)] self.eNBLogFiles[int(self.eNB_instance)] = '' if analyzeFile: - #*stats.log files + pickle + png - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.') - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.') - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.') - # - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.') - if (copyin_res == -1): + with cls_cmd.getConnection(lIpAddr) as cmd: + file = archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/' + fileToAnalyze) + archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrL1_stats.log') + archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrMAC_stats.log') + if file is None: logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m') HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) @@ -339,8 +338,8 @@ class RANManagement(): mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/') mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/') mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/') - logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze) - logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML, self.ran_checkers) + logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + file) + logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers) if (logStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', logStatus) #display rt stats for gNB only @@ -698,11 +697,16 @@ class RANManagement(): datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field - if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'): + # nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file + # currently the link is only implicit as below based on pattern matching + # I will rework this to give the file explicitly + l1_stats_fn = re.sub(r'enb_[0-9]+', 'nrL1_stats', eNBlogFile) + mac_stats_fn = re.sub(r'enb_[0-9]+', 'nrMAC_stats', eNBlogFile) + if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn): # don't use CI-nrL1_stats.log, as this will increase the processing time for # no reason, we just need the last occurence - nrL1_stats = open('./nrL1_stats.log', 'r') - nrMAC_stats = open('./nrMAC_stats.log', 'r') + nrL1_stats = open(l1_stats_fn, 'r') + nrMAC_stats = open(mac_stats_fn, 'r') for line in nrL1_stats.readlines(): for k in rt_keys: result = re.search(k, line) @@ -722,7 +726,7 @@ class RANManagement(): nrL1_stats.close() nrMAC_stats.close() else: - logging.debug("NR Stats files for RT analysis not found") + logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}") #stdout log file and stat log files analysis completed logging.debug(' File analysis (stdout, stats) completed') From cc0d0ae0ab044f5827a2bde09822e8ef1785dc7d Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 12:07:01 +0200 Subject: [PATCH 35/38] gNB caracal USRP build: don't stop if we can't reboot Sometimes the connection fails because of the reboot. Without may_fail, it would stop the build, which will then run the wrong code, possibly. Instead, continue; if there is a problem with USRP, it will be visible when starting nr-softmodem. --- ci-scripts/xml_files/gnb_usrp_build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-scripts/xml_files/gnb_usrp_build.xml b/ci-scripts/xml_files/gnb_usrp_build.xml index f67098116b..cd5e91b02f 100644 --- a/ci-scripts/xml_files/gnb_usrp_build.xml +++ b/ci-scripts/xml_files/gnb_usrp_build.xml @@ -47,6 +47,7 @@ Custom_Command Reboot USRP + true caracal ssh root@192.168.20.2 reboot From 40efa2a545dcb5766748d8a2624cd611daf2e558 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Aug 2025 00:51:35 +0200 Subject: [PATCH 36/38] Remove LogCollectBuild step: not used by any pipeline --- ci-scripts/Jenkinsfile-tmp-full-ran | 18 ------------------ ci-scripts/Jenkinsfile-tmp-multi-enb | 18 ------------------ ci-scripts/Jenkinsfile-tmp-multi-enb-nsa | 18 ------------------ ci-scripts/Jenkinsfile-tmp-ran | 18 ------------------ ci-scripts/cls_oaicitest.py | 18 ------------------ ci-scripts/main.py | 7 +------ 6 files changed, 1 insertion(+), 96 deletions(-) diff --git a/ci-scripts/Jenkinsfile-tmp-full-ran b/ci-scripts/Jenkinsfile-tmp-full-ran index f1baa5b04d..3bc1d6ad8e 100644 --- a/ci-scripts/Jenkinsfile-tmp-full-ran +++ b/ci-scripts/Jenkinsfile-tmp-full-ran @@ -216,24 +216,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb b/ci-scripts/Jenkinsfile-tmp-multi-enb index 2680fbe863..7c7397f30b 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb @@ -222,24 +222,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 5e60b199b4..89daaec8c1 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -210,24 +210,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/Jenkinsfile-tmp-ran b/ci-scripts/Jenkinsfile-tmp-ran index 03c0f9a0cb..82db6641e3 100644 --- a/ci-scripts/Jenkinsfile-tmp-ran +++ b/ci-scripts/Jenkinsfile-tmp-ran @@ -227,24 +227,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index ba322ca608..efdb589cff 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -845,21 +845,3 @@ class OaiCiTest(): message = "Log files:\n" + "\n".join([os.path.basename(l) for l in logs]) HTML.CreateHtmlTestRowQueue(core_name, 'OK', [message]) return True - - def LogCollectBuild(self,RAN): - # Some pipelines are using "none" IP / Credentials - # In that case, just forget about it - if RAN.eNBIPAddress == 'none': - sys.exit(0) - - if (RAN.eNBIPAddress != '' and RAN.eNBUserName != '' and RAN.eNBPassword != ''): - IPAddress = RAN.eNBIPAddress - UserName = RAN.eNBUserName - Password = RAN.eNBPassword - SourceCodePath = RAN.eNBSourceCodePath - else: - sys.exit('Insufficient Parameter') - with cls_cmd.getConnection(IPAddress) as cmd: - d = f'{SourceCodePath}/cmake_targets' - cmd.run(f'rm -f {d}/build.log.zip') - cmd.run(f'cd {d} && zip -r build.log.zip build_log_*/*') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 1b06539a2c..0fdf538052 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -477,12 +477,7 @@ elif re.match('^TerminateMME$', mode, re.IGNORECASE): elif re.match('^TerminateSPGW$', mode, re.IGNORECASE): logging.warning("Option TerminateSPGW ignored") elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): - if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '': - logging.warning("nothing to collect (eNBIPAddress/eNBUserName/eNBPassword/eNBSourceCodePath is '')") - sys.exit(0) - if RAN.eNBIPAddress == 'none': - sys.exit(0) - CiTestObj.LogCollectBuild(RAN) + logging.warning("Option LogCollectBuild ignored") elif re.match('^LogCollecteNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '': HELP.GenericHelp(CONST.Version) From 43e2fd57d4054dcc384028e1ea6dc068ef1129da Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 7 Aug 2025 09:01:55 +0200 Subject: [PATCH 37/38] Use Ninja for FHI72 builds --- docker/Dockerfile.build.fhi72.native_arm.ubuntu22 | 2 +- docker/Dockerfile.build.fhi72.rhel9 | 2 +- docker/Dockerfile.build.fhi72.ubuntu22 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 b/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 index 4ea1bcd7bb..b796122bf3 100644 --- a/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 +++ b/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 @@ -78,7 +78,7 @@ RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ ./build_oai \ - --gNB \ + --ninja --gNB \ --build-lib "telnetsrv enbscope uescope nrscope" \ -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \ --build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \ diff --git a/docker/Dockerfile.build.fhi72.rhel9 b/docker/Dockerfile.build.fhi72.rhel9 index b8ad1560ba..472378935d 100644 --- a/docker/Dockerfile.build.fhi72.rhel9 +++ b/docker/Dockerfile.build.fhi72.rhel9 @@ -58,7 +58,7 @@ RUN /bin/sh oaienv && \ mkdir -p log && \ scl enable gcc-toolset-13 "\ ./build_oai \ - --gNB \ + --ninja --gNB \ --build-lib 'telnetsrv enbscope uescope nrscope' \ -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \ --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION" &&\ diff --git a/docker/Dockerfile.build.fhi72.ubuntu22 b/docker/Dockerfile.build.fhi72.ubuntu22 index 5da6d0b608..4674648285 100644 --- a/docker/Dockerfile.build.fhi72.ubuntu22 +++ b/docker/Dockerfile.build.fhi72.ubuntu22 @@ -68,7 +68,7 @@ RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ ./build_oai \ - --gNB \ + --ninja --gNB \ --build-lib "telnetsrv enbscope uescope nrscope" \ -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \ --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \ From 667bcf920c0b384019f6bf57650dc751bd9bcf47 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 7 Aug 2025 15:15:34 +0200 Subject: [PATCH 38/38] Trace analysis results --- ci-scripts/cls_containerize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 306f9a9a47..34522d9eef 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -100,6 +100,7 @@ def AnalyzeBuildLogs(logfiles, globalStatus): errorandwarnings['errors'] = 0 if committed or tagged else 1 errorandwarnings['warnings'] = 0 errorandwarnings['status'] = committed or tagged + logging.info(f"Analyzing {image}, file {lf}: {errorandwarnings}") files['Target Image Creation'] = errorandwarnings collectInfo[image] = files return collectInfo