diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 3d63de0834..591d6d413d 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -46,6 +46,7 @@ import cls_cmd import helpreadme as HELP import constants as CONST import cls_oaicitest +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Helper functions used here and in other classes @@ -202,12 +203,10 @@ def GetServices(ssh, requested, file): else: return requested -def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename): - remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}" +def CopyinServiceLog(ssh, lSourcePath, svcName, wd_yaml, ctx): + remote_filename = f"{lSourcePath}/cmake_targets/log/{svcName}.logs" ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}') - local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}" - local_filename = f"{local_dir}/{filename}" - return ssh.copyin(remote_filename, local_filename) + return archiveArtifact(ssh, ctx, remote_filename) def GetRunningServices(ssh, file): ret = ssh.run(f'docker compose -f {file} config --services') @@ -229,14 +228,13 @@ def GetRunningServices(ssh, file): logging.info(f'stopping services: {running_services}') return running_services -def CheckLogs(self, yaml, service_name, HTML, RAN): - logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}' - filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log' +def CheckLogs(self, filename, HTML, RAN): success = True - if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): + name = os.path.basename(filename) + if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m') logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN) - opt = f"UE log analysis for service {service_name}" + opt = f"UE log analysis ({name})" # usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files # abuse HTML to store their reports, and we here want to put custom options, # which is not possible with CreateHtmlTestRow @@ -247,13 +245,13 @@ def CheckLogs(self, yaml, service_name, HTML, RAN): else: HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg]) HTML.htmlUEFailureMsg = "" - elif service_name == 'nv-cubb': + elif 'nv-cubb' in name: msg = 'Undeploy PNF/Nvidia CUBB' HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK) - elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])): + elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb'])): logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m') logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) - opt = f"xNB log analysis for service {service_name}" + opt = f"xNB log analysis ({name})" if (logStatus < 0): HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg]) success = False @@ -261,9 +259,9 @@ def CheckLogs(self, yaml, service_name, HTML, RAN): HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg]) HTML.htmleNBFailureMsg = "" else: - logging.info(f'Skipping to analyze log for service name {service_name}') - HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"]) - logging.debug(f"log check: service {service_name} passed analysis {success}") + logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE") + HTML.CreateHtmlTestRowQueue(f"file {name}", 'OK', ["no analysis function"]) + logging.debug(f"log check: file {filename} passed analysis {success}") return success #----------------------------------------------------------- @@ -823,18 +821,14 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"]) return success - def DeployObject(self, HTML): + def DeployObject(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] num_attempts = self.num_attempts lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug(f'Deploying OAI Object on server: {lIpAddr}') yaml = self.yamlPath[self.eNB_instance].strip('/') - # creating the log folder by default - local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}" - os.system(f'mkdir -p {local_dir}') wd = f'{lSourcePath}/{yaml}' wd_yaml = f'{wd}/docker-compose.y*ml' - yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml) if services == [] or services == ' ' or services == None: @@ -861,7 +855,7 @@ class Containerize(): logging.warning(warning_msg) HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [warning_msg]) for svc in services.split(): - CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') + CopyinServiceLog(ssh, lSourcePath, svc, wd_yaml, ctx) ssh.run(f'docker compose -f {wd_yaml} down -- {services}') imagesInfo = info.stdout.splitlines()[1:] logging.debug(f'{info.stdout.splitlines()[1:]}') @@ -871,13 +865,12 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)]) return deployed - def UndeployObject(self, HTML, RAN): + def UndeployObject(self, ctx, HTML, RAN): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m') yaml = self.yamlPath[self.eNB_instance].strip('/') wd = f'{lSourcePath}/{yaml}' - yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: ExistEnvFilePrint(ssh, wd) services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml") @@ -885,7 +878,7 @@ class Containerize(): if services is not None: all_serv = " ".join([s for s, _ in services]) ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}') - copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services) + copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, f"{wd}/docker-compose.y*ml", ctx) for s, _ in services] else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v') @@ -894,7 +887,7 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)']) return False else: - log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services] + log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res] success = all(log_results) if success: logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index a6da7b399d..7f5efefc34 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -330,9 +330,9 @@ def ExecuteActionWithParam(action, ctx): CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) if action == 'Deploy_Object': - success = CONTAINERS.DeployObject(HTML) + success = CONTAINERS.DeployObject(ctx, HTML) elif action == 'Undeploy_Object': - success = CONTAINERS.UndeployObject(HTML, RAN) + success = CONTAINERS.UndeployObject(ctx, HTML, RAN) elif action == 'Create_Workspace': if force_local: # Do not create a working directory when running locally. Current repo directory will be used diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index a7337f6e52..d2fb0cda57 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -6,6 +6,7 @@ logging.basicConfig( format="[%(asctime)s] %(levelname)8s: %(message)s" ) import os +import tempfile os.system(f'rm -rf cmake_targets') os.system(f'mkdir -p cmake_targets/log') import unittest @@ -14,6 +15,7 @@ sys.path.append('./') # to find OAI imports below import cls_oai_html import cls_oaicitest import cls_containerize +from cls_ci_helper import TestCaseCtx import ran import cls_cmd @@ -48,12 +50,16 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.eNBPassword = None self.cont.eNBSourceCodePath = os.getcwd() self.cont.num_attempts = 3 + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') def test_deploy(self): self.cont.yamlPath[0] = 'tests/simple-dep/' self.cont.deploymentTag = "noble" - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -61,8 +67,8 @@ class TestDeploymentMethods(unittest.TestCase): # fails reliably old = self.cont.yamlPath self.cont.yamlPath[0] = 'tests/simple-fail/' - deploy = self.cont.DeployObject(self.html) - self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertFalse(deploy) self.cont.yamlPath = old @@ -70,8 +76,8 @@ class TestDeploymentMethods(unittest.TestCase): # fails reliably old = self.cont.yamlPath self.cont.yamlPath[0] = 'tests/simple-fail-2svc/' - deploy = self.cont.DeployObject(self.html) - self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertFalse(deploy) self.cont.yamlPath = old @@ -79,8 +85,8 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -88,8 +94,8 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb oai-nr-ue" self.cont.deploymentTag = 'develop-12345678' - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -97,10 +103,10 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' - deploy1 = self.cont.DeployObject(self.html) + deploy1 = self.cont.DeployObject(self.ctx, self.html) self.cont.services[0] = "oai-nr-ue" - deploy2 = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy2 = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy1) self.assertTrue(deploy2) self.assertTrue(undeploy)