diff --git a/ci-scripts/cls_cmd.py b/ci-scripts/cls_cmd.py index 30f00cdb02..4eb64e2abe 100644 --- a/ci-scripts/cls_cmd.py +++ b/ci-scripts/cls_cmd.py @@ -86,6 +86,12 @@ class Cmd(metaclass=abc.ABCMeta): return class LocalCmd(Cmd): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def __init__(self, d = None): self.cwd = d if self.cwd is not None: @@ -122,12 +128,40 @@ class LocalCmd(Cmd): if src[0] != '/' or tgt[0] != '/': raise Exception('support only absolute file paths!') opt = '-r' if recursive else '' - self.run(f'cp {opt} {src} {tgt}') + return self.run(f'cp {opt} {src} {tgt}').returncode == 0 def copyout(self, src, tgt, recursive=False): - self.copyin(src, tgt, recursive) + return self.copyin(src, tgt, recursive) + +def PutFile(client, src, tgt): + success = True + sftp = client.open_sftp() + try: + sftp.put(src, tgt) + except FileNotFoundError as error: + logging.error(f"error while putting {src}: {error}") + success = False + sftp.close() + return success + +def GetFile(client, src, tgt): + success = True + sftp = client.open_sftp() + try: + sftp.get(src, tgt) + except FileNotFoundError as error: + logging.error(f"error while getting {src}: {error}") + success = False + sftp.close() + return success class RemoteCmd(Cmd): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def __init__(self, hostname, d=None): cIdx = 0 logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko @@ -193,36 +227,38 @@ class RemoteCmd(Cmd): def getBefore(self): return self.cp.stdout + # if recursive is True, tgt must be a directory (and src is file or directory) + # if recursive is False, tgt and src must be a file name def copyout(self, src, tgt, recursive=False): logging.debug(f"copyout: local:{src} -> remote:{tgt}") if recursive: tmpfile = f"{uuid.uuid4()}.tar" abstmpfile = f"/tmp/{tmpfile}" - cmd = LocalCmd() - cmd.run(f"tar -cf {abstmpfile} {src}") - sftp = self.client.open_sftp() - sftp.put(abstmpfile, abstmpfile) - sftp.close() - cmd.run(f"rm {abstmpfile}") - self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + with LocalCmd() as cmd: + if cmd.run(f"tar -cf {abstmpfile} {src}").returncode != 0: + return False + if not PutFile(self.client, abstmpfile, abstmpfile): + return False + cmd.run(f"rm {abstmpfile}") + ret = self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + return ret.returncode == 0 else: - sftp = self.client.open_sftp() - sftp.put(src, tgt) - sftp.close() + return PutFile(self.client, src, tgt) + # if recursive is True, tgt must be a directory (and src is file or directory) + # if recursive is False, tgt and src must be a file name def copyin(self, src, tgt, recursive=False): logging.debug(f"copyin: remote:{src} -> local:{tgt}") if recursive: tmpfile = f"{uuid.uuid4()}.tar" abstmpfile = f"/tmp/{tmpfile}" - self.run(f"tar -cf {abstmpfile} {src}") - sftp = self.client.open_sftp() - sftp.get(abstmpfile, abstmpfile) - sftp.close() + if self.run(f"tar -cf {abstmpfile} {src}").returncode != 0: + return False + if not GetFile(self.client, abstmpfile, abstmpfile): + return False self.run(f"rm {abstmpfile}") - cmd = LocalCmd() - cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + with LocalCmd() as cmd: + ret = cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + return ret.returncode == 0 else: - sftp = self.client.open_sftp() - sftp.get(src, tgt) - sftp.close() + return GetFile(self.client, src, tgt) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 20ce23ce35..60d0ccf568 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -955,12 +955,12 @@ class Containerize(): if svcName == '': logging.warning('no service name given: starting all services in ci-docker-compose.yml!') - mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) + mySSH.command(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) # Checking Status grep = '' if svcName != '': grep = f' | grep -A3 {svcName}' - mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5) + mySSH.command(f'docker compose --file ci-docker-compose.yml config {grep}', '\$', 5) result = re.search('container_name: (?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) unhealthyNb = 0 healthyNb = 0 @@ -1062,55 +1062,55 @@ class Containerize(): if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') - logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m') - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - - mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) - + logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m') + mySSH = cls_cmd.getConnection(lIpAddr) + yamlDir = f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}' + mySSH.run(f'cd {yamlDir}') svcName = self.services[self.eNB_instance] forceDown = False if svcName != '': logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!') svcName = '' - mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5) + ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml config --services') + if ret.returncode != 0: + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services") + self.exitStatus = 1 + return # first line has command, last line has next command prompt - allServices = mySSH.getBefore().split('\r\n')[1:-1] + allServices = ret.stdout.splitlines() services = [] for s in allServices: - mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False) - running = mySSH.getBefore().split('\r\n')[2:-1] + # outputs the hash if the container is running + ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml ps --all --quiet -- {s}') + running = ret.stdout.splitlines() logging.debug(f'running services: {running}') - if len(running) > 0: # something is running for that service + if ret.stdout != "" and ret.returncode == 0: # something is running for that service services.append(s) logging.info(f'stopping services {services}') - mySSH.command(f'docker-compose -f ci-docker-compose.yml stop -t3', '\$', 30) - time.sleep(5) # give some time to running containers to stop + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml stop -t3') + copyin_res = True for svcName in services: # head -n -1 suppresses the final "X exited with status code Y" filename = f'{svcName}-{HTML.testCase_id}.log' - mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 120) + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}') + copyin_res = mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', f'{filename}') and copyin_res + # when nv-cubb container is available, copy L1 pcap, OAI Aerial pipeline + if 'nv-cubb' in allServices: + mySSH.run(f'cp {lSourcePath}/cmake_targets/share/gnb_nvipc.pcap {lSourcePath}/cmake_targets/gnb_nvipc.pcap') - mySSH.command('docker-compose -f ci-docker-compose.yml down -v', '\$', 5) - mySSH.close() + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml down -v') # Analyzing log file! - files = ','.join([f'{s}-{HTML.testCase_id}' for s in services]) - if len(services) > 1: - files = '{' + files + '}' - copyin_res = 0 - if len(services) > 0: - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.') - if copyin_res == -1: - HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!' + if not copyin_res: + HTML.htmleNBFailureMsg='Could not copy logfile(s) to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) self.exitStatus = 1 # use function for UE log analysis, when oai-nr-ue container is used elif 'oai-nr-ue' in services or 'lte_ue0' in services: self.exitStatus == 0 - logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m') + logging.debug(f'Analyzing UE logfile {filename}') logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN) if (logStatus < 0): fullStatus = False @@ -1129,11 +1129,12 @@ class Containerize(): HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) # all the xNB run logs shall be on the server 0 for logCollecting if self.eNB_serverId[self.eNB_instance] != '0': - mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/') + mySSH.copyout(f'./*.log', f'{lSourcePath}/cmake_targets/', recursive=True) if self.exitStatus == 0: logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') else: logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m') + mySSH.close() def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0