From fca1561bf7d6dcb81de378401a3ce4e3c24cbd14 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Tue, 9 Jan 2024 19:29:46 +0100 Subject: [PATCH] Modifs of cls_containerize - UndeployObject() - use new ssh class - do proper logging from multiple services - use yaml dir - Use splitlines(), minor modifs of syntax - check that services are running - Check all copyin() operations, not only last - switch to docker compose: with the use of the new ssh class, we cannot (easily) use the old docker-compose anymore (which was an alias to docker compose). Switch to "docker compose". Co-authored-by: Robert Schmidt --- ci-scripts/cls_containerize.py | 57 +++++++++++++++++----------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 708c2a6f3c..cf959e8503 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -951,12 +951,12 @@ class Containerize(): if svcName == '': logging.warning('no service name given: starting all services in ci-docker-compose.yml!') - mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) + mySSH.command(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) # Checking Status grep = '' if svcName != '': grep = f' | grep -A3 {svcName}' - mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5) + mySSH.command(f'docker compose --file ci-docker-compose.yml config {grep}', '\$', 5) result = re.search('container_name: (?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) unhealthyNb = 0 healthyNb = 0 @@ -1058,55 +1058,55 @@ class Containerize(): if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') - logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m') - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - - mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) - + logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m') + mySSH = cls_cmd.getConnection(lIpAddr) + yamlDir = f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}' + mySSH.run(f'cd {yamlDir}') svcName = self.services[self.eNB_instance] forceDown = False if svcName != '': logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!') svcName = '' - mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5) + ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml config --services') + if ret.returncode != 0: + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services") + self.exitStatus = 1 + return # first line has command, last line has next command prompt - allServices = mySSH.getBefore().split('\r\n')[1:-1] + allServices = ret.stdout.splitlines() services = [] for s in allServices: - mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False) - running = mySSH.getBefore().split('\r\n')[2:-1] + # outputs the hash if the container is running + ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml ps --all --quiet -- {s}') + running = ret.stdout.splitlines() logging.debug(f'running services: {running}') - if len(running) > 0: # something is running for that service + if ret.stdout != "" and ret.returncode == 0: # something is running for that service services.append(s) logging.info(f'stopping services {services}') - mySSH.command(f'docker-compose -f ci-docker-compose.yml stop -t3', '\$', 30) - time.sleep(5) # give some time to running containers to stop + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml stop -t3') + copyin_res = True for svcName in services: # head -n -1 suppresses the final "X exited with status code Y" filename = f'{svcName}-{HTML.testCase_id}.log' - mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 120) + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}') + copyin_res = mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', f'{filename}') and copyin_res + # when nv-cubb container is available, copy L1 pcap, OAI Aerial pipeline + if 'nv-cubb' in allServices: + mySSH.run(f'cp {lSourcePath}/cmake_targets/share/gnb_nvipc.pcap {lSourcePath}/cmake_targets/gnb_nvipc.pcap') - mySSH.command('docker-compose -f ci-docker-compose.yml down -v', '\$', 5) - mySSH.close() + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml down -v') # Analyzing log file! - files = ','.join([f'{s}-{HTML.testCase_id}' for s in services]) - if len(services) > 1: - files = '{' + files + '}' - copyin_res = 0 - if len(services) > 0: - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.') - if copyin_res == -1: - HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!' + if not copyin_res: + HTML.htmleNBFailureMsg='Could not copy logfile(s) to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) self.exitStatus = 1 # use function for UE log analysis, when oai-nr-ue container is used elif 'oai-nr-ue' in services or 'lte_ue0' in services: self.exitStatus == 0 - logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m') + logging.debug(f'Analyzing UE logfile {filename}') logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN) if (logStatus < 0): fullStatus = False @@ -1125,11 +1125,12 @@ class Containerize(): HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) # all the xNB run logs shall be on the server 0 for logCollecting if self.eNB_serverId[self.eNB_instance] != '0': - mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/') + mySSH.copyout(f'./*.log', f'{lSourcePath}/cmake_targets/', recursive=True) if self.exitStatus == 0: logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') else: logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m') + mySSH.close() def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0