From d5eb5ee0f27ee48f79d9e9a87efe70be8ed2f689 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Thu, 28 Dec 2023 12:00:14 +0100 Subject: [PATCH 1/3] Support for context manager in cls_cmd SSH class --- ci-scripts/cls_cmd.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci-scripts/cls_cmd.py b/ci-scripts/cls_cmd.py index 30f00cdb02..e2a735fa35 100644 --- a/ci-scripts/cls_cmd.py +++ b/ci-scripts/cls_cmd.py @@ -86,6 +86,12 @@ class Cmd(metaclass=abc.ABCMeta): return class LocalCmd(Cmd): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def __init__(self, d = None): self.cwd = d if self.cwd is not None: @@ -128,6 +134,12 @@ class LocalCmd(Cmd): self.copyin(src, tgt, recursive) class RemoteCmd(Cmd): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def __init__(self, hostname, d=None): cIdx = 0 logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko From 3c1ed92b8060a27a0c2b786b3531417672209f7f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 27 Feb 2024 18:02:42 +0100 Subject: [PATCH 2/3] Return success/failure of copyin()/copyout - Return code says if copying succeeded - Improve handling of the recursive code paths, which copy files or directories recursively into another directory - both for RemoteCmd/LocalCmd --- ci-scripts/cls_cmd.py | 66 +++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/ci-scripts/cls_cmd.py b/ci-scripts/cls_cmd.py index e2a735fa35..4eb64e2abe 100644 --- a/ci-scripts/cls_cmd.py +++ b/ci-scripts/cls_cmd.py @@ -128,10 +128,32 @@ class LocalCmd(Cmd): if src[0] != '/' or tgt[0] != '/': raise Exception('support only absolute file paths!') opt = '-r' if recursive else '' - self.run(f'cp {opt} {src} {tgt}') + return self.run(f'cp {opt} {src} {tgt}').returncode == 0 def copyout(self, src, tgt, recursive=False): - self.copyin(src, tgt, recursive) + return self.copyin(src, tgt, recursive) + +def PutFile(client, src, tgt): + success = True + sftp = client.open_sftp() + try: + sftp.put(src, tgt) + except FileNotFoundError as error: + logging.error(f"error while putting {src}: {error}") + success = False + sftp.close() + return success + +def GetFile(client, src, tgt): + success = True + sftp = client.open_sftp() + try: + sftp.get(src, tgt) + except FileNotFoundError as error: + logging.error(f"error while getting {src}: {error}") + success = False + sftp.close() + return success class RemoteCmd(Cmd): def __enter__(self): @@ -205,36 +227,38 @@ class RemoteCmd(Cmd): def getBefore(self): return self.cp.stdout + # if recursive is True, tgt must be a directory (and src is file or directory) + # if recursive is False, tgt and src must be a file name def copyout(self, src, tgt, recursive=False): logging.debug(f"copyout: local:{src} -> remote:{tgt}") if recursive: tmpfile = f"{uuid.uuid4()}.tar" abstmpfile = f"/tmp/{tmpfile}" - cmd = LocalCmd() - cmd.run(f"tar -cf {abstmpfile} {src}") - sftp = self.client.open_sftp() - sftp.put(abstmpfile, abstmpfile) - sftp.close() - cmd.run(f"rm {abstmpfile}") - self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + with LocalCmd() as cmd: + if cmd.run(f"tar -cf {abstmpfile} {src}").returncode != 0: + return False + if not PutFile(self.client, abstmpfile, abstmpfile): + return False + cmd.run(f"rm {abstmpfile}") + ret = self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + return ret.returncode == 0 else: - sftp = self.client.open_sftp() - sftp.put(src, tgt) - sftp.close() + return PutFile(self.client, src, tgt) + # if recursive is True, tgt must be a directory (and src is file or directory) + # if recursive is False, tgt and src must be a file name def copyin(self, src, tgt, recursive=False): logging.debug(f"copyin: remote:{src} -> local:{tgt}") if recursive: tmpfile = f"{uuid.uuid4()}.tar" abstmpfile = f"/tmp/{tmpfile}" - self.run(f"tar -cf {abstmpfile} {src}") - sftp = self.client.open_sftp() - sftp.get(abstmpfile, abstmpfile) - sftp.close() + if self.run(f"tar -cf {abstmpfile} {src}").returncode != 0: + return False + if not GetFile(self.client, abstmpfile, abstmpfile): + return False self.run(f"rm {abstmpfile}") - cmd = LocalCmd() - cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + with LocalCmd() as cmd: + ret = cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + return ret.returncode == 0 else: - sftp = self.client.open_sftp() - sftp.get(src, tgt) - sftp.close() + return GetFile(self.client, src, tgt) From fca1561bf7d6dcb81de378401a3ce4e3c24cbd14 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Tue, 9 Jan 2024 19:29:46 +0100 Subject: [PATCH 3/3] Modifs of cls_containerize - UndeployObject() - use new ssh class - do proper logging from multiple services - use yaml dir - Use splitlines(), minor modifs of syntax - check that services are running - Check all copyin() operations, not only last - switch to docker compose: with the use of the new ssh class, we cannot (easily) use the old docker-compose anymore (which was an alias to docker compose). Switch to "docker compose". Co-authored-by: Robert Schmidt --- ci-scripts/cls_containerize.py | 57 +++++++++++++++++----------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 708c2a6f3c..cf959e8503 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -951,12 +951,12 @@ class Containerize(): if svcName == '': logging.warning('no service name given: starting all services in ci-docker-compose.yml!') - mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) + mySSH.command(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) # Checking Status grep = '' if svcName != '': grep = f' | grep -A3 {svcName}' - mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5) + mySSH.command(f'docker compose --file ci-docker-compose.yml config {grep}', '\$', 5) result = re.search('container_name: (?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) unhealthyNb = 0 healthyNb = 0 @@ -1058,55 +1058,55 @@ class Containerize(): if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') - logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m') - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - - mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) - + logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m') + mySSH = cls_cmd.getConnection(lIpAddr) + yamlDir = f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}' + mySSH.run(f'cd {yamlDir}') svcName = self.services[self.eNB_instance] forceDown = False if svcName != '': logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!') svcName = '' - mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5) + ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml config --services') + if ret.returncode != 0: + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services") + self.exitStatus = 1 + return # first line has command, last line has next command prompt - allServices = mySSH.getBefore().split('\r\n')[1:-1] + allServices = ret.stdout.splitlines() services = [] for s in allServices: - mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False) - running = mySSH.getBefore().split('\r\n')[2:-1] + # outputs the hash if the container is running + ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml ps --all --quiet -- {s}') + running = ret.stdout.splitlines() logging.debug(f'running services: {running}') - if len(running) > 0: # something is running for that service + if ret.stdout != "" and ret.returncode == 0: # something is running for that service services.append(s) logging.info(f'stopping services {services}') - mySSH.command(f'docker-compose -f ci-docker-compose.yml stop -t3', '\$', 30) - time.sleep(5) # give some time to running containers to stop + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml stop -t3') + copyin_res = True for svcName in services: # head -n -1 suppresses the final "X exited with status code Y" filename = f'{svcName}-{HTML.testCase_id}.log' - mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 120) + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}') + copyin_res = mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', f'{filename}') and copyin_res + # when nv-cubb container is available, copy L1 pcap, OAI Aerial pipeline + if 'nv-cubb' in allServices: + mySSH.run(f'cp {lSourcePath}/cmake_targets/share/gnb_nvipc.pcap {lSourcePath}/cmake_targets/gnb_nvipc.pcap') - mySSH.command('docker-compose -f ci-docker-compose.yml down -v', '\$', 5) - mySSH.close() + mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml down -v') # Analyzing log file! - files = ','.join([f'{s}-{HTML.testCase_id}' for s in services]) - if len(services) > 1: - files = '{' + files + '}' - copyin_res = 0 - if len(services) > 0: - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.') - if copyin_res == -1: - HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!' + if not copyin_res: + HTML.htmleNBFailureMsg='Could not copy logfile(s) to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) self.exitStatus = 1 # use function for UE log analysis, when oai-nr-ue container is used elif 'oai-nr-ue' in services or 'lte_ue0' in services: self.exitStatus == 0 - logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m') + logging.debug(f'Analyzing UE logfile {filename}') logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN) if (logStatus < 0): fullStatus = False @@ -1125,11 +1125,12 @@ class Containerize(): HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) # all the xNB run logs shall be on the server 0 for logCollecting if self.eNB_serverId[self.eNB_instance] != '0': - mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/') + mySSH.copyout(f'./*.log', f'{lSourcePath}/cmake_targets/', recursive=True) if self.exitStatus == 0: logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') else: logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m') + mySSH.close() def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0