From ce03b84898eb7d228253e73d948067479ec9ce52 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 15 Dec 2022 07:54:09 +0100 Subject: [PATCH 1/7] Add new SSH/local cmd classes --- ci-scripts/cls_cmd.py | 217 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 ci-scripts/cls_cmd.py diff --git a/ci-scripts/cls_cmd.py b/ci-scripts/cls_cmd.py new file mode 100644 index 0000000000..12eb75e7da --- /dev/null +++ b/ci-scripts/cls_cmd.py @@ -0,0 +1,217 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# +# Required Python Version +# Python 3.x +# +#--------------------------------------------------------------------- + +import abc +import logging +import subprocess as sp +import os +import paramiko +import uuid + +# provides a partial interface for the legacy SSHconnection class (getBefore(), command()) +class Cmd(metaclass=abc.ABCMeta): + def cd(self, d, silent=False): + if d == None or d == '' or d == []: + self.cwd = None + elif d[0] == '/': + self.cwd = d + else: + if not self.cwd: + # no cwd set: get current working directory + self.cwd = self.run('pwd').stdout.strip() + self.cwd += f"/{d}" + if not silent: + logging.debug(f'cd {self.cwd}') + + @abc.abstractmethod + def run(self, line, timeout=300, silent=False): + return + + @abc.abstractmethod + def command(self, commandline, expectedline, timeout, silent=False, resync=False): + return + + @abc.abstractmethod + def close(self): + return + + @abc.abstractmethod + def getBefore(self): + return + + @abc.abstractmethod + def copyin(self, scpIp, scpUser, scpPw, src, tgt): + return + + @abc.abstractmethod + def copyout(self, scpIp, scpUser, scpPw, src, tgt): + return + +class LocalCmd(Cmd): + def __init__(self, d = None): + self.cwd = d + self.cp = sp.CompletedProcess(args='', returncode=0, stdout='') + + def run(self, line, timeout=300, silent=False, reportNonZero=True): + if type(line) is str: + line = [s for s in line.split(' ') if len(s) > 0] + if not silent: + logging.debug(' '.join(line)) + try: + ret = sp.run(line, cwd=self.cwd, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=timeout) + except Exception as e: + ret = sp.CompletedProcess(args=line, returncode=255, stdout=f'Exception: {str(e)}'.encode('utf-8')) + if ret.stdout is None: + ret.stdout = b'' + ret.stdout = ret.stdout.decode('utf-8').strip() + if reportNonZero and ret.returncode != 0: + cmd = ' '.join(ret.args) + logging.warning(f'command "{cmd}" returned non-zero returncode {ret.returncode}: output:\n{ret.stdout}') + self.cp = ret + return ret + + def command(self, commandline, expectedline=None, timeout=300, silent=False, resync=False): + line = [s for s in commandline.split(' ') if len(s) > 0] + if line[0] == 'cd': + self.cd(line[1], silent) + else: + self.run(line, timeout, silent) + return 0 + + def close(self): + pass + + def getBefore(self): + return self.cp.stdout + + def copyin(self, scpIp, scpUser, scpPw, src, tgt): + logging.warning("LocalCmd emulating sshconnection.copyin() function") + self.run(f'cp -r {src} {tgt}') + + def copyout(self, scpIp, scpUser, scpPw, src, tgt): + logging.warning("LocalCmd emulating sshconnection.copyout() function") + self.run(f'cp -r {src} {tgt}') + +class RemoteCmd(Cmd): + def __init__(self, hostname, d=None): + logging.getLogger('paramiko').setLevel(logging.INFO) # prevent spamming through Paramiko + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + cfg = RemoteCmd._lookup_ssh_config(hostname) + self.client.connect(**cfg) + self.cwd = d + self.cp = sp.CompletedProcess(args='', returncode=0, stdout='') + + def _lookup_ssh_config(hostname): + ssh_config = paramiko.SSHConfig() + user_config_file = os.path.expanduser("~/.ssh/config") + if os.path.exists(user_config_file): + with open(user_config_file) as f: + ssh_config.parse(f) + else: + raise FileNotFoundError('class needs SSH config at ~/.ssh/config') + ucfg = ssh_config.lookup(hostname) + if 'identityfile' not in ucfg or 'user' not in ucfg: + raise KeyError(f'no identityfile or user in SSH config for host {hostname}') + cfg = {'hostname':hostname, 'username':ucfg['user'], 'key_filename':ucfg['identityfile']} + if 'hostname' in ucfg: + cfg['hostname'] = ucfg['hostname'] # override user-given hostname with what is in config + if 'port' in ucfg: + cfg['port'] = int(ucfg['port']) + if 'proxycommand' in ucfg: + cfg['sock'] = paramiko.ProxyCommand(ucfg['proxycommand']) + return cfg + + def run(self, line, timeout=300, silent=False, reportNonZero=True): + if type(line) is list: + line = ' '.join(line) + if not silent: + logging.debug(line) + if self.cwd: + line = f"cd {self.cwd} && {line}" + args = line.split(' ') + try: + stdin, stdout, stderr = self.client.exec_command(line, timeout=timeout) + ret = sp.CompletedProcess(args=args, returncode=stdout.channel.recv_exit_status(), stdout=stdout.read(size=None) + stderr.read(size=None)) + except Exception as e: + ret = sp.CompletedProcess(args=args, returncode=255, stdout=f'Exception: {str(e)}'.encode('utf-8')) + ret.stdout = ret.stdout.decode('utf-8').strip() + if reportNonZero and ret.returncode != 0: + cmd = ' '.join(ret.args) + logging.warning(f'command "{cmd}" returned non-zero returncode {ret.returncode}: output:\n{ret.stdout}') + self.cp = ret + return ret + + def command(self, commandline, expectedline=None, timeout=300, silent=False, resync=False): + line = [s for s in commandline.split(' ') if len(s) > 0] + if line[0] == 'cd': + self.cd(line[1], silent) + else: + self.run(line, timeout, silent) + return 0 + + def close(self): + self.client.close() + + def getBefore(self): + return self.cp.stdout + + def copyout(self, src, tgt, recursive=False): + logging.warning("RemoteCmd emulating sshconnection.copyout() function, ignoring scpIp") + logging.debug(f"copyout: local:{src} -> remote:{tgt}") + if recursive: + tmpfile = f"{uuid.uuid4()}.tar" + abstmpfile = f"/tmp/{tmpfile}" + cmd = LocalCmd() + cmd.run(f"tar -cf {abstmpfile} {src}") + sftp = self.client.open_sftp() + sftp.put(abstmpfile, abstmpfile) + sftp.close() + cmd.run(f"rm {abstmpfile}") + self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + else: + sftp = self.client.open_sftp() + sftp.put(src, tgt) + sftp.close() + + def copyin(self, src, tgt, recursive=False): + logging.warning("RemoteCmd emulating sshconnection.copyout() function") + logging.debug(f"copyin: remote:{src} -> local:{tgt}") + if recursive: + tmpfile = f"{uuid.uuid4()}.tar" + abstmpfile = f"/tmp/{tmpfile}" + self.run(f"tar -cf {abstmpfile} {src}") + sftp = self.client.open_sftp() + sftp.get(abstmpfile, abstmpfile) + sftp.close() + self.run(f"rm {abstmpfile}") + cmd = LocalCmd() + cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}") + else: + sftp = self.client.open_sftp() + sftp.get(src, tgt) + sftp.close() From de4a9e1d5a5ea09253dec2ce5f1b79f12c111f89 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 15 Dec 2022 07:54:17 +0100 Subject: [PATCH 2/7] Use new SSH class to execute cluster build --- ci-scripts/cls_cluster.py | 287 ++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 150 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 5eea6a1953..80f7326338 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -32,16 +32,13 @@ # Import #----------------------------------------------------------- import logging -import sshconnection as SSH import cls_oai_html -#import os import re import time -#import subprocess -import sys import constants as CONST import helpreadme as HELP import cls_containerize +import cls_cmd class Cluster: def __init__(self): @@ -58,86 +55,82 @@ class Cluster: self.ranCommitID = "" self.ranAllowMerge = False self.ranTargetBranch = "" + self.cmd = None - def _recreate_entitlements(self, sshSession): + + def _recreate_entitlements(self): # recreating entitlements, don't care if deletion fails - sshSession.command('oc delete secret etc-pki-entitlement', '\$', 5) - if re.search(r"not found", sshSession.getBefore()): - logging.warning("no secrets etc-pki-entitlement found, recreating") - sshSession.command('ls /etc/pki/entitlement/???????????????????.pem | tail -1', '\$', 5, silent=True) - regres1 = re.search(r"/etc/pki/entitlement/[0-9]+.pem", sshSession.getBefore()) - sshSession.command('ls /etc/pki/entitlement/???????????????????-key.pem | tail -1', '\$', 5, silent=True) - regres2 = re.search(r"/etc/pki/entitlement/[0-9]+-key.pem", sshSession.getBefore()) + self.cmd.run('oc delete secret etc-pki-entitlement') + ret = self.cmd.run('ls /etc/pki/entitlement/???????????????????.pem | tail -1', silent=True) + regres1 = re.search(r"/etc/pki/entitlement/[0-9]+.pem", ret.stdout) + ret = self.cmd.run('ls /etc/pki/entitlement/???????????????????-key.pem | tail -1', silent=True) + regres2 = re.search(r"/etc/pki/entitlement/[0-9]+-key.pem", ret.stdout) if regres1 is None or regres2 is None: logging.error("could not find entitlements") return False file1 = regres1.group(0) file2 = regres2.group(0) - sshSession.command(f'oc create secret generic etc-pki-entitlement --from-file {file1} --from-file {file2}', '\$', 5) - regres = re.search(r"secret/etc-pki-entitlement created", sshSession.getBefore()) - if regres is None: + ret = self.cmd.run(f'oc create secret generic etc-pki-entitlement --from-file {file1} --from-file {file2}') + regres = re.search(r"secret/etc-pki-entitlement created", ret.stdout) + if ret.returncode != 0 or regres is None: logging.error("could not create secret/etc-pki-entitlement") return False return True - def _recreate_bc(self, sshSession, name, newTag, filename): - self._retag_image_statement(sshSession, name, name, newTag, filename) - sshSession.command(f'oc delete -f {filename}', '\$', 5) - sshSession.command(f'oc create -f {filename}', '\$', 5) - before = sshSession.getBefore() - if re.search('buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', before) is not None: + def _recreate_bc(self, name, newTag, filename): + self._retag_image_statement(name, name, newTag, filename) + self.cmd.run(f'oc delete -f {filename}') + ret = self.cmd.run(f'oc create -f {filename}') + if re.search('buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', ret.stdout) is not None: return True - logging.error('error while creating buildconfig: ' + sshSession.getBefore()) + logging.error('error while creating buildconfig: ' + ret.stdout) return False - def _recreate_is_tag(self, sshSession, name, newTag, filename): - sshSession.command(f'oc describe is {name}', '\$', 5) - if sshSession.getBefore().count('NotFound') > 0: - sshSession.command(f'oc create -f {filename}', '\$', 5) - before = sshSession.getBefore() - if re.search(f'imagestream.image.openshift.io/{name} created', before) is None: - logging.error('error while creating imagestream: ' + sshSession.getBefore()) + def _recreate_is_tag(self, name, newTag, filename): + ret = self.cmd.run(f'oc describe is {name}') + if ret.returncode != 0: + ret = self.cmd.run(f'oc create -f {filename}') + if ret.returncode != 0: + logging.error(f'error while creating imagestream: {ret.stdout}') return False else: logging.debug(f'-> imagestream {name} found') image = f'{name}:{newTag}' - sshSession.command(f'oc delete istag {image}', '\$', 5) # we don't care if this fails, e.g., if it is missing - sshSession.command(f'oc create istag {image}', '\$', 5) - before = sshSession.getBefore() - if re.search(f'imagestreamtag.image.openshift.io/{image} created', before) is not None: + self.cmd.run(f'oc delete istag {image}', reportNonZero=False) # we don't care if this fails, e.g., if it is missing + ret = self.cmd.run(f'oc create istag {image}') + if ret.returncode == 0: return True - logging.error('error while creating imagestreamtag: ' + sshSession.getBefore()) + logging.error(f'error while creating imagestreamtag: {ret.stdout}') return False - def _start_build(self, sshSession, name): + def _start_build(self, name): # will return "immediately" but build runs in background # if multiple builds are started at the same time, this can take some time, however - ret = sshSession.command(f'oc start-build {name} --from-file={self.eNBSourceCodePath}', '\$', 300) - before = sshSession.getBefore() - regres = re.search(r'build.build.openshift.io/(?P[a-zA-Z0-9\-]+) started', str(before)) - if ret != 0 or before.count('Uploading finished') != 1 or regres is None: - logging.error("error during oc start-build: " + sshSession.getBefore()) - self._delete_pod(sshSession, name) + ret = self.cmd.run(f'oc start-build {name} --from-file={self.eNBSourceCodePath}') + regres = re.search(r'build.build.openshift.io/(?P[a-zA-Z0-9\-]+) started', ret.stdout) + if ret.returncode != 0 or ret.stdout.count('Uploading finished') != 1 or regres is None: + logging.error(f"error during oc start-build: {ret.stdout}") + self._delete_pod(name) return None return regres.group('jobname') + '-build' - def _delete_pod(self, sshSession, shortName): - sshSession.command(f"oc get pods | grep {shortName}", '\$', 5) - regres = re.search(rf'{shortName}-[0-9]+-build', sshSession.getBefore()) + def _delete_pod(self, shortName): + ret = self.cmd.run(f"oc get pods | grep {shortName}") + regres = re.search(rf'{shortName}-[0-9]+-build', ret.stdout) if regres is not None: - sshSession.command(f"oc delete pod {regres.group(0)}", '\$', 5) + self.cmd.run(f"oc delete pod {regres.group(0)}") else: logging.warning(f"no pod found with name {shortName}") - def _wait_build_end(self, sshSession, jobs, timeout_sec, check_interval_sec = 5): + def _wait_build_end(self, jobs, timeout_sec, check_interval_sec = 5): logging.debug(f"waiting for jobs {jobs} to finish building") while timeout_sec > 0: # check status for j in jobs: - sshSession.command(f'oc get pods | grep {j}', '\$', 30, silent = True) - if sshSession.getBefore().count('Completed') > 0: jobs.remove(j) - if sshSession.getBefore().count('Error') > 0: - logging.error(f'error for job {j}: ' + sshSession.getBefore()) + ret = self.cmd.run(f'oc get pods | grep {j}', silent = True) + if ret.stdout.count('Completed') > 0: jobs.remove(j) + if ret.stdout.count('Error') > 0: + logging.error(f'error for job {j}: {ret.stdout}') return False if jobs == []: logging.debug('all jobs completed') @@ -147,44 +140,43 @@ class Cluster: logging.error(f"timeout while waiting for end of build of {jobs}") return False - def _retag_image_statement(self, sshSession, oldImage, newImage, newTag, filename): - sshSession.command(f'sed -i -e "s#{oldImage}:latest#{newImage}:{newTag}#" {filename}', '\$', 5) + def _retag_image_statement(self, oldImage, newImage, newTag, filename): + self.cmd.run(f'sed -i -e "s#{oldImage}:latest#{newImage}:{newTag}#" {filename}') - def _get_image_size(self, sshSession, image, tag): + def _get_image_size(self, image, tag): # get the SHA of the image we built using the image name and its tag - sshSession.command(f'oc describe is {image} | grep -A4 {tag}', '\$', 5) - result = re.search(f'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/(?P{image}@sha256:[a-f0-9]+)', sshSession.getBefore()) + ret = self.cmd.run(f'oc describe is {image} | grep -A4 {tag}') + result = re.search(f'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/(?P{image}@sha256:[a-f0-9]+)', ret.stdout) if result is None: return -1 imageSha = result.group("imageSha") # retrieve the size - sshSession.command(f'oc get -o json isimage {imageSha} | jq -Mc "{{dockerImageSize: .image.dockerImageMetadata.Size}}"', '\$', 5) - result = re.search('{"dockerImageSize":(?P[0-9]+)}', str(sshSession.getBefore())) + ret = self.cmd.run(f'oc get -o json isimage {imageSha} | jq -Mc "{{dockerImageSize: .image.dockerImageMetadata.Size}}"') + result = re.search('{"dockerImageSize":(?P[0-9]+)}', ret.stdout) if result is None: return -1 return int(result.group("size")) - def _deploy_pod(self, sshSession, filename, timeout = 30): - sshSession.command(f'oc create -f {filename}', '\$', 10) - result = re.search(f'pod/(?P[a-zA-Z0-9_\-]+) created', sshSession.getBefore()) + def _deploy_pod(self, filename, timeout = 30): + ret = self.cmd.run(f'oc create -f {filename}') + result = re.search(f'pod/(?P[a-zA-Z0-9_\-]+) created', ret.stdout) if result is None: - logging.error(f'could not deploy pod: {sshSession.getBefore()}') + logging.error(f'could not deploy pod: {ret.stdout}') return None pod = result.group("pod") logging.debug(f'checking if pod {pod} is in Running state') while timeout > 0: - sshSession.command(f'oc get pod {pod} -o json | jq -Mc .status.phase', '\$', 5, silent=True) - if re.search('"Running"', sshSession.getBefore()) is not None: return pod + ret = self.cmd.run(f'oc get pod {pod} -o json | jq -Mc .status.phase', silent=True) + if re.search('"Running"', ret.stdout) is not None: return pod timeout -= 1 time.sleep(1) logging.error(f'pod {pod} did not reach Running state') - self._undeploy_pod(sshSession, filename) + self._undeploy_pod(filename) return None - def _undeploy_pod(self, sshSession, filename): - # to save time we start this in the background and trust that oc stops correctly - sshSession.command(f'oc delete -f {filename} &', '\$', 5) + def _undeploy_pod(self, filename): + self.cmd.run(f'oc delete -f {filename}') def BuildClusterImage(self, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': @@ -194,8 +186,7 @@ class Cluster: lIpAddr = self.eNBIPAddress lSourcePath = self.eNBSourceCodePath if lIpAddr == '' or lSourcePath == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter: no SSH Credentials') + sys.exit('Insufficient Parameter: eNBSourceCodePath missing') ocUserName = self.OCUserName ocPassword = self.OCPassword ocProjectName = self.OCProjectName @@ -206,20 +197,18 @@ class Cluster: sys.exit(f'ocRegistry {self.OCRegistry} should not start with http:// or https:// and end on a slash /') logging.debug(f'Building on cluster triggered from server: {lIpAddr}') - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, "oaicicd", "SHOULD NOT BE NECESSARY") + self.cmd = cls_cmd.RemoteCmd(lIpAddr) self.testCase_id = HTML.testCase_id # Workaround for some servers, we need to erase completely the workspace if self.forcedWorkspaceCleanup: - mySSH.command(f'rm -Rf {lSourcePath}', '\$', 15) - cls_containerize.CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) + self.cmd.run(f'rm -Rf {lSourcePath}') + cls_containerize.CreateWorkspace(self.cmd, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) - # we don't necessarily need a forced workspace cleanup, but in - # order to reduce the amount of data send to OpenShift, we + # to reduce the amount of data send to OpenShift, we # manually delete all generated files in the workspace - mySSH.command(f'rm -rf {lSourcePath}/cmake_targets/ran_build', '\$', 30); + self.cmd.run(f'rm -rf {lSourcePath}/cmake_targets/ran_build'); baseTag = 'develop' forceBaseImageBuild = False @@ -227,8 +216,8 @@ class Cluster: if self.ranAllowMerge: # merging MR branch into develop -> temporary image imageTag = 'ci-temp' if self.ranTargetBranch == 'develop': - mySSH.command(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel8.2 | grep --colour=never -i INDEX', '\$', 5) - result = re.search('index', mySSH.getBefore()) + ret = self.cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel8.2 | grep --colour=never -i INDEX') + result = re.search('index', ret.stdout) if result is not None: forceBaseImageBuild = True baseTag = 'ci-temp' @@ -236,122 +225,120 @@ class Cluster: forceBaseImageBuild = True # logging to OC Cluster and then switch to corresponding project - mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server {self.OCUrl}', '\$', 31) - if mySSH.getBefore().count('Login successful.') == 0: + ret = self.cmd.run(f'oc login -u {ocUserName} -p {ocPassword}') + if ret.returncode != 0: logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m') - mySSH.close() HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL) return False - mySSH.command(f'oc project {ocProjectName}', '\$', 30) - if mySSH.getBefore().count(f'Already on project "{ocProjectName}"') == 0 and mySSH.getBefore().count(f'Now using project "{self.OCProjectName}"') == 0: + ret = self.cmd.run(f'oc project {ocProjectName}') + if ret.returncode != 0: logging.error(f'\u001B[1mUnable to access OC project {ocProjectName}\u001B[0m') - mySSH.command('oc logout', '\$', 30) - mySSH.close() + self.cmd.run('oc logout') HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PROJECT_FAIL) return False - self._recreate_entitlements(mySSH) + self._recreate_entitlements() status = True # flag to abandon compiling if any image fails attemptedImages = [] if forceBaseImageBuild: - self._recreate_is_tag(mySSH, 'ran-base', baseTag, 'openshift/ran-base-is.yaml') - self._recreate_bc(mySSH, 'ran-base', baseTag, 'openshift/ran-base-bc.yaml') - ranbase_job = self._start_build(mySSH, 'ran-base') + self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml') + self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml') + ranbase_job = self._start_build('ran-base') attemptedImages += ['ran-base'] - status = ranbase_job is not None and self._wait_build_end(mySSH, [ranbase_job], 600) + status = ranbase_job is not None and self._wait_build_end([ranbase_job], 600) if not status: logging.error('failure during build of ran-base') - mySSH.command(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log', '\$', 10) + self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect # recover logs by mounting image - self._retag_image_statement(mySSH, 'ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml') - pod = self._deploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml') + self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml') + pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml') if pod is not None: - mySSH.command(f'mkdir -p cmake_targets/log/ran-base', '\$', 5) - mySSH.command(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base', '\$', 5) - self._undeploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml') + self.cmd.run(f'mkdir -p cmake_targets/log/ran-base') + self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base') + self._undeploy_pod('openshift/ran-base-log-retrieval.yaml') else: status = False if status: - self._recreate_is_tag(mySSH, 'oai-physim', imageTag, 'openshift/oai-physim-is.yaml') - self._recreate_bc(mySSH, 'oai-physim', imageTag, 'openshift/oai-physim-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel8.2') - physim_job = self._start_build(mySSH, 'oai-physim') + self._recreate_is_tag('oai-physim', imageTag, 'openshift/oai-physim-is.yaml') + self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel8.2') + physim_job = self._start_build('oai-physim') attemptedImages += ['oai-physim'] - self._recreate_is_tag(mySSH, 'ran-build', imageTag, 'openshift/ran-build-is.yaml') - self._recreate_bc(mySSH, 'ran-build', imageTag, 'openshift/ran-build-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel8.2') - ranbuild_job = self._start_build(mySSH, 'ran-build') + self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml') + self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel8.2') + ranbuild_job = self._start_build('ran-build') attemptedImages += ['ran-build'] - wait = ranbuild_job is not None and physim_job is not None and self._wait_build_end(mySSH, [ranbuild_job, physim_job], 1200) + wait = ranbuild_job is not None and physim_job is not None and self._wait_build_end([ranbuild_job, physim_job], 1200) if not wait: logging.error('error during build of ranbuild_job or physim_job') status = status and wait - mySSH.command(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log', '\$', 10) - mySSH.command(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log', '\$', 10) - mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10) + self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log') + self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log') + self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10) if status: - self._recreate_is_tag(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-is.yaml') - self._recreate_bc(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel8.2') - self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel8.2') - enb_job = self._start_build(mySSH, 'oai-enb') + self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml') + self._recreate_bc('oai-enb', imageTag, 'openshift/oai-enb-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel8.2') + self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel8.2') + enb_job = self._start_build('oai-enb') attemptedImages += ['oai-enb'] - self._recreate_is_tag(mySSH, 'oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml') - self._recreate_bc(mySSH, 'oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel8.2') - self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel8.2') - gnb_job = self._start_build(mySSH, 'oai-gnb') + self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml') + self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel8.2') + self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel8.2') + gnb_job = self._start_build('oai-gnb') attemptedImages += ['oai-gnb'] - self._recreate_is_tag(mySSH, 'oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml') - self._recreate_bc(mySSH, 'oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2') - self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2') - gnb_aw2s_job = self._start_build(mySSH, 'oai-gnb-aw2s') + self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml') + self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2') + self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2') + gnb_aw2s_job = self._start_build('oai-gnb-aw2s') attemptedImages += ['oai-gnb-aw2s'] - wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end(mySSH, [enb_job, gnb_job, gnb_aw2s_job], 600) + wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 600) if not wait: logging.error('error during build of eNB/gNB') status = status and wait # recover logs - mySSH.command(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log', '\$', 10) - mySSH.command(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log', '\$', 10) - mySSH.command(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log', '\$', 10) + self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log') + self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log') + self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log') - self._recreate_is_tag(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml') - self._recreate_bc(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel8.2') - self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel8.2') - lteue_job = self._start_build(mySSH, 'oai-lte-ue') + self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml') + self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel8.2') + self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel8.2') + lteue_job = self._start_build('oai-lte-ue') attemptedImages += ['oai-lte-ue'] - self._recreate_is_tag(mySSH, 'oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml') - self._recreate_bc(mySSH, 'oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml') - self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel8.2') - self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel8.2') - nrue_job = self._start_build(mySSH, 'oai-nr-ue') + self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml') + self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml') + self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel8.2') + self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel8.2') + nrue_job = self._start_build('oai-nr-ue') attemptedImages += ['oai-nr-ue'] - wait = lteue_job is not None and nrue_job is not None and self._wait_build_end(mySSH, [lteue_job, nrue_job], 600) + wait = lteue_job is not None and nrue_job is not None and self._wait_build_end([lteue_job, nrue_job], 600) if not wait: logging.error('error during build of lteUE/nrUE') status = status and wait # recover logs - mySSH.command(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log', '\$', 10) - mySSH.command(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log', '\$', 10) - mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10) + self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log') + self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log') + self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10) # split and analyze logs imageSize = {} for image in attemptedImages: - mySSH.command(f'mkdir -p cmake_targets/log/{image}', '\$', 5) - mySSH.command(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log', '\$', 5) + self.cmd.run(f'mkdir -p cmake_targets/log/{image}') + self.cmd.run(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log') tag = imageTag if image != 'ran-base' else baseTag - size = self._get_image_size(mySSH, image, tag) + size = self._get_image_size(image, tag) if size <= 0: imageSize[image] = 'unknown -- BUILD FAILED' status = False @@ -361,17 +348,17 @@ class Cluster: logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m') grep_exp = "\|".join(attemptedImages) - mySSH.command(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log', '\$', 10); - mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc get pod $pod -o json >> cmake_targets/log/build_pod_summary.log; done', '\$', 60) + self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log'); + self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done', '\$', 60) build_log_name = f'build_log_{self.testCase_id}' - cls_containerize.CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD) + cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD) - mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done', '\$', 60) + self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') # logout will return eventually, but we don't care when -> start in background - mySSH.command('oc logout &', '\$', 5) - mySSH.close() + self.cmd.run(f'oc logout &') + self.cmd.close() # Analyze the logs collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status) From 7361c1689ecafd7eb86832efe442a7afdbb97e8e Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 15 Dec 2022 09:27:43 +0100 Subject: [PATCH 3/7] Use new SSH class to execute image build --- ci-scripts/cls_cluster.py | 2 +- ci-scripts/cls_containerize.py | 100 ++++++++++++++++----------------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 80f7326338..892d9e847e 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -352,7 +352,7 @@ class Cluster: self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done', '\$', 60) build_log_name = f'build_log_{self.testCase_id}' - cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD) + cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name) self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 4540f2a8a0..d766f02b8b 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -44,6 +44,7 @@ from zipfile import ZipFile #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- +import cls_cmd import sshconnection as SSH import helpreadme as HELP import constants as CONST @@ -83,20 +84,20 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge logging.debug(f'Merging with the target branch: {ranTargetBranch}') sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30) -def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw): - sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5) - sshSession.command(f'rm -f {log_name}.zip', '\$', 5) - sshSession.command(f'mkdir -p {log_name}', '\$', 5) - sshSession.command(f'mv log/* {log_name}', '\$', 5) - sshSession.command(f'zip -r -qq {log_name}.zip {log_name}', '\$', 5) +def CopyLogsToExecutor(cmd, sourcePath, log_name): + cmd.cd(f'{sourcePath}/cmake_targets') + cmd.run(f'rm -f {log_name}.zip') + cmd.run(f'mkdir -p {log_name}') + cmd.run(f'mv log/* {log_name}') + cmd.run(f'zip -r -qq {log_name}.zip {log_name}') # copy zip to executor for analysis if (os.path.isfile(f'./{log_name}.zip')): os.remove(f'./{log_name}.zip') if (os.path.isdir(f'./{log_name}')): shutil.rmtree(f'./{log_name}') - sshSession.copyin(scpIp, scpUser, scpPw, f'{sourcePath}/cmake_targets/{log_name}.zip', '.') - sshSession.command(f'rm -f {log_name}.zip','\$', 5) + cmd.copyin(f'{sourcePath}/cmake_targets/{log_name}.zip', f'./{log_name}.zip') + cmd.run(f'rm -f {log_name}.zip') ZipFile(f'{log_name}.zip').extractall('.') def AnalyzeBuildLogs(buildRoot, images, globalStatus): @@ -329,12 +330,11 @@ class Containerize(): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') logging.debug('Building on server: ' + lIpAddr) - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) + cmd = cls_cmd.RemoteCmd(lIpAddr) # Checking the hostname to get adapted on cli and dockerfileprefixes - mySSH.command('hostnamectl', '\$', 5) - result = re.search('Ubuntu|Red Hat', mySSH.getBefore()) + cmd.run('hostnamectl') + result = re.search('Ubuntu|Red Hat', cmd.getBefore()) self.host = result.group(0) if self.host == 'Ubuntu': self.cli = 'docker' @@ -369,18 +369,18 @@ class Containerize(): # Workaround for some servers, we need to erase completely the workspace if self.forcedWorkspaceCleanup: - mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) + cmd.run(f'sudo -S rm -Rf {lSourcePath}') self.testCase_id = HTML.testCase_id - CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) + CreateWorkspace(cmd, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) # if asterix, copy the entitlement and subscription manager configurations if self.host == 'Red Hat': - mySSH.command('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', '\$', 5) - mySSH.command('sudo cp /etc/rhsm/rhsm.conf ./rhsm-conf/', '\$', 5) - mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/', '\$', 5) - mySSH.command('sudo cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/', '\$', 5) + cmd.run('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca') + cmd.run('cp /etc/rhsm/rhsm.conf ./rhsm-conf/') + cmd.run('cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/') + cmd.run('cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/') baseImage = 'ran-base' baseTag = 'develop' @@ -389,8 +389,8 @@ class Containerize(): if (self.ranAllowMerge): imageTag = 'ci-temp' if self.ranTargetBranch == 'develop': - mySSH.command('git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) - result = re.search('index', mySSH.getBefore()) + cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX') + result = re.search('index', cmd.getBefore()) if result is not None: forceBaseImageBuild = True baseTag = 'ci-temp' @@ -398,31 +398,31 @@ class Containerize(): forceBaseImageBuild = True # Let's remove any previous run artifacts if still there - mySSH.command(self.cli + ' image prune --force', '\$', 30) + cmd.run(f"{self.cli} image prune --force") if forceBaseImageBuild: - mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30) + cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") for image,pattern in imageNames: - mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30) + cmd.run(f"{self.cli} image rm {image}:{imageTag}") # Build the base image only on Push Events (not on Merge Requests) # On when the base image docker file is being modified. if forceBaseImageBuild: - mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600) + cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600) # First verify if the base image was properly created. - mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5) + ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}") allImagesSize = {} - if mySSH.getBefore().count('o such image') != 0: + if ret.returncode != 0: logging.error('\u001B[1m Could not build properly ran-base\u001B[0m') # Recover the name of the failed container? - mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) - mySSH.command(self.cli + ' image prune --force', '\$', 30) - mySSH.close() + cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs {self.cli} rm -f") + cmd.run(f"{self.cli} image prune --force") + cmd.close() logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTabFooter(False) sys.exit(1) else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) + result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 imageSizeStr = f'{size:.1f}' @@ -432,10 +432,10 @@ class Containerize(): logging.debug('ran-base size is unknown') # Recover build logs, for the moment only possible when build is successful - mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5) - mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5) - mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5) - mySSH.command(self.cli + ' rm -f test', '\$', 5) + cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") + cmd.run("mkdir -p cmake_targets/log/ran-base") + cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base") + cmd.run(f"{self.cli} rm -f test") # Build the target image(s) status = True @@ -444,24 +444,24 @@ class Containerize(): attemptedImages += [image] # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag - mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5) + cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') if image != 'ran-build': - mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5) - mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200) + cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') + cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', timeout=1200) # split the log - mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) - mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5) - # checking the status of the build - mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) - if mySSH.getBefore().count('o such image') != 0: + cmd.run(f"mkdir -p cmake_targets/log/{image}") + cmd.run(f"python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log") + # check the status of the build + ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {image}:{imageTag}") + if ret.returncode != 0: logging.error('\u001B[1m Could not build properly ' + image + '\u001B[0m') status = False # Here we should check if the last container corresponds to a failed command and destroy it - mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) + cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs {self.cli} rm -f") allImagesSize[image] = 'N/A -- Build Failed' break else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) + result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 imageSizeStr = f'{size:.1f}' @@ -471,18 +471,18 @@ class Containerize(): logging.debug(f'{image} size is unknown') allImagesSize[image] = 'unknown' # Now pruning dangling images in between target builds - mySSH.command(self.cli + ' image prune --force', '\$', 30) + cmd.run(f"{self.cli} image prune --force") # Remove all intermediate build images and clean up if self.ranAllowMerge and forceBaseImageBuild: - mySSH.command(f'{self.cli} image rm {baseImage}:{baseTag} || true', '\$', 30) - mySSH.command(f'{self.cli} image rm ran-build:{imageTag}','\$', 30) - mySSH.command(f'{self.cli} volume prune --force','\$', 15) + cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") + cmd.run(f"{self.cli} image rm ran-build:{imageTag}") + cmd.run(f"{self.cli} volume prune --force") # create a zip with all logs build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, lUserName, lPassWord) - mySSH.close() + CopyLogsToExecutor(cmd, lSourcePath, build_log_name) + cmd.close() # Analyze the logs collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status) From 941acb2e8d886e03bfc31fa2db698f776a796d6b Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 15 Dec 2022 13:43:16 +0100 Subject: [PATCH 4/7] Tag cluster images with branch and commit ID --- ci-scripts/cls_cluster.py | 2 +- ci-scripts/cls_physim1.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 892d9e847e..c42d112ceb 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -214,7 +214,7 @@ class Cluster: forceBaseImageBuild = False imageTag = 'develop' if self.ranAllowMerge: # merging MR branch into develop -> temporary image - imageTag = 'ci-temp' + imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}' if self.ranTargetBranch == 'develop': ret = self.cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel8.2 | grep --colour=never -i INDEX') result = re.search('index', ret.stdout) diff --git a/ci-scripts/cls_physim1.py b/ci-scripts/cls_physim1.py index 5bb64d250c..775eeb5405 100644 --- a/ci-scripts/cls_physim1.py +++ b/ci-scripts/cls_physim1.py @@ -105,7 +105,7 @@ class PhySim: if self.ranCommitID != '': mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30) if self.ranAllowMerge: - imageTag = "ci-temp" + imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}' if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) From 074cf91bec10d76da4f340f99e086ed2a67d5bdb Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 11 Jan 2023 15:09:15 +0100 Subject: [PATCH 5/7] cls_cluster: various fixes after review - Mention python version and separate Python/own modules - Use constants for registry and namespace name - Provide OC url in oc login - Do not perform oc logout in background --- ci-scripts/cls_cluster.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index c42d112ceb..2096404204 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -19,27 +19,28 @@ # * contact@openairinterface.org # */ #--------------------------------------------------------------------- -# Python for CI of OAI-eNB + COTS-UE # # Required Python Version # Python 3.x # -# Required Python Package -# pexpect #--------------------------------------------------------------------- #----------------------------------------------------------- # Import #----------------------------------------------------------- import logging -import cls_oai_html import re import time + +import cls_oai_html import constants as CONST import helpreadme as HELP import cls_containerize import cls_cmd +IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc" +NAMESPACE = "oaicicd-ran" + class Cluster: def __init__(self): self.eNBIPAddress = "" @@ -110,18 +111,9 @@ class Cluster: regres = re.search(r'build.build.openshift.io/(?P[a-zA-Z0-9\-]+) started', ret.stdout) if ret.returncode != 0 or ret.stdout.count('Uploading finished') != 1 or regres is None: logging.error(f"error during oc start-build: {ret.stdout}") - self._delete_pod(name) return None return regres.group('jobname') + '-build' - def _delete_pod(self, shortName): - ret = self.cmd.run(f"oc get pods | grep {shortName}") - regres = re.search(rf'{shortName}-[0-9]+-build', ret.stdout) - if regres is not None: - self.cmd.run(f"oc delete pod {regres.group(0)}") - else: - logging.warning(f"no pod found with name {shortName}") - def _wait_build_end(self, jobs, timeout_sec, check_interval_sec = 5): logging.debug(f"waiting for jobs {jobs} to finish building") while timeout_sec > 0: @@ -146,7 +138,7 @@ class Cluster: def _get_image_size(self, image, tag): # get the SHA of the image we built using the image name and its tag ret = self.cmd.run(f'oc describe is {image} | grep -A4 {tag}') - result = re.search(f'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/(?P{image}@sha256:[a-f0-9]+)', ret.stdout) + result = re.search(f'{IMAGE_REGISTRY_SERVICE_NAME}:5000/{NAMESPACE}/(?P{image}@sha256:[a-f0-9]+)', ret.stdout) if result is None: return -1 imageSha = result.group("imageSha") @@ -225,7 +217,7 @@ class Cluster: forceBaseImageBuild = True # logging to OC Cluster and then switch to corresponding project - ret = self.cmd.run(f'oc login -u {ocUserName} -p {ocPassword}') + ret = self.cmd.run(f'oc login -u {ocUserName} -p {ocPassword} --server {self.OCUrl}') if ret.returncode != 0: logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m') HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL) @@ -357,7 +349,7 @@ class Cluster: self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') # logout will return eventually, but we don't care when -> start in background - self.cmd.run(f'oc logout &') + self.cmd.run(f'oc logout') self.cmd.close() # Analyze the logs From 57dbc796686388e28b06c12b8214bdb7377a3f52 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 11 Jan 2023 15:09:38 +0100 Subject: [PATCH 6/7] cls_physim1: Mention python version and separate Python/own modules --- ci-scripts/cls_physim1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_physim1.py b/ci-scripts/cls_physim1.py index 775eeb5405..0a45f945a2 100644 --- a/ci-scripts/cls_physim1.py +++ b/ci-scripts/cls_physim1.py @@ -32,13 +32,14 @@ # Import #----------------------------------------------------------- import logging -import sshconnection as SSH -import cls_oai_html import os import re import time import subprocess import sys + +import sshconnection as SSH +import cls_oai_html import constants as CONST import helpreadme as HELP From 3b9107228a774cffac41e97a5923eab7adfad284 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 16 Dec 2022 18:20:50 +0100 Subject: [PATCH 7/7] cls_containerize: Add comment to help understand the magnitude of size --- ci-scripts/cls_containerize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index d766f02b8b..c9114227bb 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -463,7 +463,7 @@ class Containerize(): else: result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: - size = float(result.group("size")) / 1000000 + size = float(result.group("size")) / 1000000 # convert to MB imageSizeStr = f'{size:.1f}' logging.debug(f'\u001B[1m {image} size is {imageSizeStr} Mbytes\u001B[0m') allImagesSize[image] = f'{imageSizeStr} Mbytes'