Merge remote-tracking branch 'origin/ci-new-ssh' into integration_2023_w02
This commit is contained in:
Robert Schmidt
+138
-159
@@ -19,29 +19,27 @@
|
||||
# * contact@openairinterface.org
|
||||
# */
|
||||
#---------------------------------------------------------------------
|
||||
# Python for CI of OAI-eNB + COTS-UE
|
||||
#
|
||||
# Required Python Version
|
||||
# Python 3.x
|
||||
#
|
||||
# Required Python Package
|
||||
# pexpect
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Import
|
||||
#-----------------------------------------------------------
|
||||
import logging
|
||||
import sshconnection as SSH
|
||||
import cls_oai_html
|
||||
#import os
|
||||
import re
|
||||
import time
|
||||
#import subprocess
|
||||
import sys
|
||||
|
||||
import cls_oai_html
|
||||
import constants as CONST
|
||||
import helpreadme as HELP
|
||||
import cls_containerize
|
||||
import cls_cmd
|
||||
|
||||
IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc"
|
||||
NAMESPACE = "oaicicd-ran"
|
||||
|
||||
class Cluster:
|
||||
def __init__(self):
|
||||
@@ -58,86 +56,73 @@ class Cluster:
|
||||
self.ranCommitID = ""
|
||||
self.ranAllowMerge = False
|
||||
self.ranTargetBranch = ""
|
||||
self.cmd = None
|
||||
|
||||
def _recreate_entitlements(self, sshSession):
|
||||
|
||||
def _recreate_entitlements(self):
|
||||
# recreating entitlements, don't care if deletion fails
|
||||
sshSession.command('oc delete secret etc-pki-entitlement', '\$', 5)
|
||||
if re.search(r"not found", sshSession.getBefore()):
|
||||
logging.warning("no secrets etc-pki-entitlement found, recreating")
|
||||
sshSession.command('ls /etc/pki/entitlement/???????????????????.pem | tail -1', '\$', 5, silent=True)
|
||||
regres1 = re.search(r"/etc/pki/entitlement/[0-9]+.pem", sshSession.getBefore())
|
||||
sshSession.command('ls /etc/pki/entitlement/???????????????????-key.pem | tail -1', '\$', 5, silent=True)
|
||||
regres2 = re.search(r"/etc/pki/entitlement/[0-9]+-key.pem", sshSession.getBefore())
|
||||
self.cmd.run('oc delete secret etc-pki-entitlement')
|
||||
ret = self.cmd.run('ls /etc/pki/entitlement/???????????????????.pem | tail -1', silent=True)
|
||||
regres1 = re.search(r"/etc/pki/entitlement/[0-9]+.pem", ret.stdout)
|
||||
ret = self.cmd.run('ls /etc/pki/entitlement/???????????????????-key.pem | tail -1', silent=True)
|
||||
regres2 = re.search(r"/etc/pki/entitlement/[0-9]+-key.pem", ret.stdout)
|
||||
if regres1 is None or regres2 is None:
|
||||
logging.error("could not find entitlements")
|
||||
return False
|
||||
file1 = regres1.group(0)
|
||||
file2 = regres2.group(0)
|
||||
sshSession.command(f'oc create secret generic etc-pki-entitlement --from-file {file1} --from-file {file2}', '\$', 5)
|
||||
regres = re.search(r"secret/etc-pki-entitlement created", sshSession.getBefore())
|
||||
if regres is None:
|
||||
ret = self.cmd.run(f'oc create secret generic etc-pki-entitlement --from-file {file1} --from-file {file2}')
|
||||
regres = re.search(r"secret/etc-pki-entitlement created", ret.stdout)
|
||||
if ret.returncode != 0 or regres is None:
|
||||
logging.error("could not create secret/etc-pki-entitlement")
|
||||
return False
|
||||
return True
|
||||
|
||||
def _recreate_bc(self, sshSession, name, newTag, filename):
|
||||
self._retag_image_statement(sshSession, name, name, newTag, filename)
|
||||
sshSession.command(f'oc delete -f {filename}', '\$', 5)
|
||||
sshSession.command(f'oc create -f {filename}', '\$', 5)
|
||||
before = sshSession.getBefore()
|
||||
if re.search('buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', before) is not None:
|
||||
def _recreate_bc(self, name, newTag, filename):
|
||||
self._retag_image_statement(name, name, newTag, filename)
|
||||
self.cmd.run(f'oc delete -f {filename}')
|
||||
ret = self.cmd.run(f'oc create -f {filename}')
|
||||
if re.search('buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', ret.stdout) is not None:
|
||||
return True
|
||||
logging.error('error while creating buildconfig: ' + sshSession.getBefore())
|
||||
logging.error('error while creating buildconfig: ' + ret.stdout)
|
||||
return False
|
||||
|
||||
def _recreate_is_tag(self, sshSession, name, newTag, filename):
|
||||
sshSession.command(f'oc describe is {name}', '\$', 5)
|
||||
if sshSession.getBefore().count('NotFound') > 0:
|
||||
sshSession.command(f'oc create -f {filename}', '\$', 5)
|
||||
before = sshSession.getBefore()
|
||||
if re.search(f'imagestream.image.openshift.io/{name} created', before) is None:
|
||||
logging.error('error while creating imagestream: ' + sshSession.getBefore())
|
||||
def _recreate_is_tag(self, name, newTag, filename):
|
||||
ret = self.cmd.run(f'oc describe is {name}')
|
||||
if ret.returncode != 0:
|
||||
ret = self.cmd.run(f'oc create -f {filename}')
|
||||
if ret.returncode != 0:
|
||||
logging.error(f'error while creating imagestream: {ret.stdout}')
|
||||
return False
|
||||
else:
|
||||
logging.debug(f'-> imagestream {name} found')
|
||||
image = f'{name}:{newTag}'
|
||||
sshSession.command(f'oc delete istag {image}', '\$', 5) # we don't care if this fails, e.g., if it is missing
|
||||
sshSession.command(f'oc create istag {image}', '\$', 5)
|
||||
before = sshSession.getBefore()
|
||||
if re.search(f'imagestreamtag.image.openshift.io/{image} created', before) is not None:
|
||||
self.cmd.run(f'oc delete istag {image}', reportNonZero=False) # we don't care if this fails, e.g., if it is missing
|
||||
ret = self.cmd.run(f'oc create istag {image}')
|
||||
if ret.returncode == 0:
|
||||
return True
|
||||
logging.error('error while creating imagestreamtag: ' + sshSession.getBefore())
|
||||
logging.error(f'error while creating imagestreamtag: {ret.stdout}')
|
||||
return False
|
||||
|
||||
def _start_build(self, sshSession, name):
|
||||
def _start_build(self, name):
|
||||
# will return "immediately" but build runs in background
|
||||
# if multiple builds are started at the same time, this can take some time, however
|
||||
ret = sshSession.command(f'oc start-build {name} --from-file={self.eNBSourceCodePath}', '\$', 300)
|
||||
before = sshSession.getBefore()
|
||||
regres = re.search(r'build.build.openshift.io/(?P<jobname>[a-zA-Z0-9\-]+) started', str(before))
|
||||
if ret != 0 or before.count('Uploading finished') != 1 or regres is None:
|
||||
logging.error("error during oc start-build: " + sshSession.getBefore())
|
||||
self._delete_pod(sshSession, name)
|
||||
ret = self.cmd.run(f'oc start-build {name} --from-file={self.eNBSourceCodePath}')
|
||||
regres = re.search(r'build.build.openshift.io/(?P<jobname>[a-zA-Z0-9\-]+) started', ret.stdout)
|
||||
if ret.returncode != 0 or ret.stdout.count('Uploading finished') != 1 or regres is None:
|
||||
logging.error(f"error during oc start-build: {ret.stdout}")
|
||||
return None
|
||||
return regres.group('jobname') + '-build'
|
||||
|
||||
def _delete_pod(self, sshSession, shortName):
|
||||
sshSession.command(f"oc get pods | grep {shortName}", '\$', 5)
|
||||
regres = re.search(rf'{shortName}-[0-9]+-build', sshSession.getBefore())
|
||||
if regres is not None:
|
||||
sshSession.command(f"oc delete pod {regres.group(0)}", '\$', 5)
|
||||
else:
|
||||
logging.warning(f"no pod found with name {shortName}")
|
||||
|
||||
def _wait_build_end(self, sshSession, jobs, timeout_sec, check_interval_sec = 5):
|
||||
def _wait_build_end(self, jobs, timeout_sec, check_interval_sec = 5):
|
||||
logging.debug(f"waiting for jobs {jobs} to finish building")
|
||||
while timeout_sec > 0:
|
||||
# check status
|
||||
for j in jobs:
|
||||
sshSession.command(f'oc get pods | grep {j}', '\$', 30, silent = True)
|
||||
if sshSession.getBefore().count('Completed') > 0: jobs.remove(j)
|
||||
if sshSession.getBefore().count('Error') > 0:
|
||||
logging.error(f'error for job {j}: ' + sshSession.getBefore())
|
||||
ret = self.cmd.run(f'oc get pods | grep {j}', silent = True)
|
||||
if ret.stdout.count('Completed') > 0: jobs.remove(j)
|
||||
if ret.stdout.count('Error') > 0:
|
||||
logging.error(f'error for job {j}: {ret.stdout}')
|
||||
return False
|
||||
if jobs == []:
|
||||
logging.debug('all jobs completed')
|
||||
@@ -147,44 +132,43 @@ class Cluster:
|
||||
logging.error(f"timeout while waiting for end of build of {jobs}")
|
||||
return False
|
||||
|
||||
def _retag_image_statement(self, sshSession, oldImage, newImage, newTag, filename):
|
||||
sshSession.command(f'sed -i -e "s#{oldImage}:latest#{newImage}:{newTag}#" {filename}', '\$', 5)
|
||||
def _retag_image_statement(self, oldImage, newImage, newTag, filename):
|
||||
self.cmd.run(f'sed -i -e "s#{oldImage}:latest#{newImage}:{newTag}#" {filename}')
|
||||
|
||||
def _get_image_size(self, sshSession, image, tag):
|
||||
def _get_image_size(self, image, tag):
|
||||
# get the SHA of the image we built using the image name and its tag
|
||||
sshSession.command(f'oc describe is {image} | grep -A4 {tag}', '\$', 5)
|
||||
result = re.search(f'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/(?P<imageSha>{image}@sha256:[a-f0-9]+)', sshSession.getBefore())
|
||||
ret = self.cmd.run(f'oc describe is {image} | grep -A4 {tag}')
|
||||
result = re.search(f'{IMAGE_REGISTRY_SERVICE_NAME}:5000/{NAMESPACE}/(?P<imageSha>{image}@sha256:[a-f0-9]+)', ret.stdout)
|
||||
if result is None:
|
||||
return -1
|
||||
imageSha = result.group("imageSha")
|
||||
|
||||
# retrieve the size
|
||||
sshSession.command(f'oc get -o json isimage {imageSha} | jq -Mc "{{dockerImageSize: .image.dockerImageMetadata.Size}}"', '\$', 5)
|
||||
result = re.search('{"dockerImageSize":(?P<size>[0-9]+)}', str(sshSession.getBefore()))
|
||||
ret = self.cmd.run(f'oc get -o json isimage {imageSha} | jq -Mc "{{dockerImageSize: .image.dockerImageMetadata.Size}}"')
|
||||
result = re.search('{"dockerImageSize":(?P<size>[0-9]+)}', ret.stdout)
|
||||
if result is None:
|
||||
return -1
|
||||
return int(result.group("size"))
|
||||
|
||||
def _deploy_pod(self, sshSession, filename, timeout = 30):
|
||||
sshSession.command(f'oc create -f {filename}', '\$', 10)
|
||||
result = re.search(f'pod/(?P<pod>[a-zA-Z0-9_\-]+) created', sshSession.getBefore())
|
||||
def _deploy_pod(self, filename, timeout = 30):
|
||||
ret = self.cmd.run(f'oc create -f {filename}')
|
||||
result = re.search(f'pod/(?P<pod>[a-zA-Z0-9_\-]+) created', ret.stdout)
|
||||
if result is None:
|
||||
logging.error(f'could not deploy pod: {sshSession.getBefore()}')
|
||||
logging.error(f'could not deploy pod: {ret.stdout}')
|
||||
return None
|
||||
pod = result.group("pod")
|
||||
logging.debug(f'checking if pod {pod} is in Running state')
|
||||
while timeout > 0:
|
||||
sshSession.command(f'oc get pod {pod} -o json | jq -Mc .status.phase', '\$', 5, silent=True)
|
||||
if re.search('"Running"', sshSession.getBefore()) is not None: return pod
|
||||
ret = self.cmd.run(f'oc get pod {pod} -o json | jq -Mc .status.phase', silent=True)
|
||||
if re.search('"Running"', ret.stdout) is not None: return pod
|
||||
timeout -= 1
|
||||
time.sleep(1)
|
||||
logging.error(f'pod {pod} did not reach Running state')
|
||||
self._undeploy_pod(sshSession, filename)
|
||||
self._undeploy_pod(filename)
|
||||
return None
|
||||
|
||||
def _undeploy_pod(self, sshSession, filename):
|
||||
# to save time we start this in the background and trust that oc stops correctly
|
||||
sshSession.command(f'oc delete -f {filename} &', '\$', 5)
|
||||
def _undeploy_pod(self, filename):
|
||||
self.cmd.run(f'oc delete -f {filename}')
|
||||
|
||||
def BuildClusterImage(self, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
@@ -194,8 +178,7 @@ class Cluster:
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lSourcePath = self.eNBSourceCodePath
|
||||
if lIpAddr == '' or lSourcePath == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter: no SSH Credentials')
|
||||
sys.exit('Insufficient Parameter: eNBSourceCodePath missing')
|
||||
ocUserName = self.OCUserName
|
||||
ocPassword = self.OCPassword
|
||||
ocProjectName = self.OCProjectName
|
||||
@@ -206,29 +189,27 @@ class Cluster:
|
||||
sys.exit(f'ocRegistry {self.OCRegistry} should not start with http:// or https:// and end on a slash /')
|
||||
|
||||
logging.debug(f'Building on cluster triggered from server: {lIpAddr}')
|
||||
mySSH = SSH.SSHConnection()
|
||||
mySSH.open(lIpAddr, "oaicicd", "SHOULD NOT BE NECESSARY")
|
||||
self.cmd = cls_cmd.RemoteCmd(lIpAddr)
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
|
||||
# Workaround for some servers, we need to erase completely the workspace
|
||||
if self.forcedWorkspaceCleanup:
|
||||
mySSH.command(f'rm -Rf {lSourcePath}', '\$', 15)
|
||||
cls_containerize.CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
self.cmd.run(f'rm -Rf {lSourcePath}')
|
||||
cls_containerize.CreateWorkspace(self.cmd, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
|
||||
# we don't necessarily need a forced workspace cleanup, but in
|
||||
# order to reduce the amount of data send to OpenShift, we
|
||||
# to reduce the amount of data send to OpenShift, we
|
||||
# manually delete all generated files in the workspace
|
||||
mySSH.command(f'rm -rf {lSourcePath}/cmake_targets/ran_build', '\$', 30);
|
||||
self.cmd.run(f'rm -rf {lSourcePath}/cmake_targets/ran_build');
|
||||
|
||||
baseTag = 'develop'
|
||||
forceBaseImageBuild = False
|
||||
imageTag = 'develop'
|
||||
if self.ranAllowMerge: # merging MR branch into develop -> temporary image
|
||||
imageTag = 'ci-temp'
|
||||
imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}'
|
||||
if self.ranTargetBranch == 'develop':
|
||||
mySSH.command(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel8.2 | grep --colour=never -i INDEX', '\$', 5)
|
||||
result = re.search('index', mySSH.getBefore())
|
||||
ret = self.cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel8.2 | grep --colour=never -i INDEX')
|
||||
result = re.search('index', ret.stdout)
|
||||
if result is not None:
|
||||
forceBaseImageBuild = True
|
||||
baseTag = 'ci-temp'
|
||||
@@ -236,122 +217,120 @@ class Cluster:
|
||||
forceBaseImageBuild = True
|
||||
|
||||
# logging to OC Cluster and then switch to corresponding project
|
||||
mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server {self.OCUrl}', '\$', 31)
|
||||
if mySSH.getBefore().count('Login successful.') == 0:
|
||||
ret = self.cmd.run(f'oc login -u {ocUserName} -p {ocPassword} --server {self.OCUrl}')
|
||||
if ret.returncode != 0:
|
||||
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL)
|
||||
return False
|
||||
|
||||
mySSH.command(f'oc project {ocProjectName}', '\$', 30)
|
||||
if mySSH.getBefore().count(f'Already on project "{ocProjectName}"') == 0 and mySSH.getBefore().count(f'Now using project "{self.OCProjectName}"') == 0:
|
||||
ret = self.cmd.run(f'oc project {ocProjectName}')
|
||||
if ret.returncode != 0:
|
||||
logging.error(f'\u001B[1mUnable to access OC project {ocProjectName}\u001B[0m')
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
mySSH.close()
|
||||
self.cmd.run('oc logout')
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PROJECT_FAIL)
|
||||
return False
|
||||
|
||||
self._recreate_entitlements(mySSH)
|
||||
self._recreate_entitlements()
|
||||
|
||||
status = True # flag to abandon compiling if any image fails
|
||||
attemptedImages = []
|
||||
if forceBaseImageBuild:
|
||||
self._recreate_is_tag(mySSH, 'ran-base', baseTag, 'openshift/ran-base-is.yaml')
|
||||
self._recreate_bc(mySSH, 'ran-base', baseTag, 'openshift/ran-base-bc.yaml')
|
||||
ranbase_job = self._start_build(mySSH, 'ran-base')
|
||||
self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml')
|
||||
self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml')
|
||||
ranbase_job = self._start_build('ran-base')
|
||||
attemptedImages += ['ran-base']
|
||||
status = ranbase_job is not None and self._wait_build_end(mySSH, [ranbase_job], 600)
|
||||
status = ranbase_job is not None and self._wait_build_end([ranbase_job], 600)
|
||||
if not status: logging.error('failure during build of ran-base')
|
||||
mySSH.command(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log', '\$', 10)
|
||||
self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect
|
||||
# recover logs by mounting image
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml')
|
||||
pod = self._deploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml')
|
||||
self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml')
|
||||
pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml')
|
||||
if pod is not None:
|
||||
mySSH.command(f'mkdir -p cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base', '\$', 5)
|
||||
self._undeploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml')
|
||||
self.cmd.run(f'mkdir -p cmake_targets/log/ran-base')
|
||||
self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base')
|
||||
self._undeploy_pod('openshift/ran-base-log-retrieval.yaml')
|
||||
else:
|
||||
status = False
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag(mySSH, 'oai-physim', imageTag, 'openshift/oai-physim-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-physim', imageTag, 'openshift/oai-physim-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel8.2')
|
||||
physim_job = self._start_build(mySSH, 'oai-physim')
|
||||
self._recreate_is_tag('oai-physim', imageTag, 'openshift/oai-physim-is.yaml')
|
||||
self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel8.2')
|
||||
physim_job = self._start_build('oai-physim')
|
||||
attemptedImages += ['oai-physim']
|
||||
|
||||
self._recreate_is_tag(mySSH, 'ran-build', imageTag, 'openshift/ran-build-is.yaml')
|
||||
self._recreate_bc(mySSH, 'ran-build', imageTag, 'openshift/ran-build-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel8.2')
|
||||
ranbuild_job = self._start_build(mySSH, 'ran-build')
|
||||
self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml')
|
||||
self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel8.2')
|
||||
ranbuild_job = self._start_build('ran-build')
|
||||
attemptedImages += ['ran-build']
|
||||
|
||||
wait = ranbuild_job is not None and physim_job is not None and self._wait_build_end(mySSH, [ranbuild_job, physim_job], 1200)
|
||||
wait = ranbuild_job is not None and physim_job is not None and self._wait_build_end([ranbuild_job, physim_job], 1200)
|
||||
if not wait: logging.error('error during build of ranbuild_job or physim_job')
|
||||
status = status and wait
|
||||
mySSH.command(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log', '\$', 10)
|
||||
mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10)
|
||||
self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log')
|
||||
self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10)
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel8.2')
|
||||
enb_job = self._start_build(mySSH, 'oai-enb')
|
||||
self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
|
||||
self._recreate_bc('oai-enb', imageTag, 'openshift/oai-enb-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel8.2')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel8.2')
|
||||
enb_job = self._start_build('oai-enb')
|
||||
attemptedImages += ['oai-enb']
|
||||
|
||||
self._recreate_is_tag(mySSH, 'oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel8.2')
|
||||
gnb_job = self._start_build(mySSH, 'oai-gnb')
|
||||
self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml')
|
||||
self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel8.2')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel8.2')
|
||||
gnb_job = self._start_build('oai-gnb')
|
||||
attemptedImages += ['oai-gnb']
|
||||
|
||||
self._recreate_is_tag(mySSH, 'oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2')
|
||||
gnb_aw2s_job = self._start_build(mySSH, 'oai-gnb-aw2s')
|
||||
self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml')
|
||||
self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel8.2')
|
||||
gnb_aw2s_job = self._start_build('oai-gnb-aw2s')
|
||||
attemptedImages += ['oai-gnb-aw2s']
|
||||
|
||||
wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end(mySSH, [enb_job, gnb_job, gnb_aw2s_job], 600)
|
||||
wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 600)
|
||||
if not wait: logging.error('error during build of eNB/gNB')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
mySSH.command(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log', '\$', 10)
|
||||
self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log')
|
||||
self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log')
|
||||
self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log')
|
||||
|
||||
self._recreate_is_tag(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel8.2')
|
||||
lteue_job = self._start_build(mySSH, 'oai-lte-ue')
|
||||
self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml')
|
||||
self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel8.2')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel8.2')
|
||||
lteue_job = self._start_build('oai-lte-ue')
|
||||
attemptedImages += ['oai-lte-ue']
|
||||
|
||||
self._recreate_is_tag(mySSH, 'oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel8.2')
|
||||
nrue_job = self._start_build(mySSH, 'oai-nr-ue')
|
||||
self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml')
|
||||
self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel8.2')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel8.2')
|
||||
nrue_job = self._start_build('oai-nr-ue')
|
||||
attemptedImages += ['oai-nr-ue']
|
||||
|
||||
wait = lteue_job is not None and nrue_job is not None and self._wait_build_end(mySSH, [lteue_job, nrue_job], 600)
|
||||
wait = lteue_job is not None and nrue_job is not None and self._wait_build_end([lteue_job, nrue_job], 600)
|
||||
if not wait: logging.error('error during build of lteUE/nrUE')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
mySSH.command(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log', '\$', 10)
|
||||
mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10)
|
||||
self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log')
|
||||
self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10)
|
||||
|
||||
# split and analyze logs
|
||||
imageSize = {}
|
||||
for image in attemptedImages:
|
||||
mySSH.command(f'mkdir -p cmake_targets/log/{image}', '\$', 5)
|
||||
mySSH.command(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log', '\$', 5)
|
||||
self.cmd.run(f'mkdir -p cmake_targets/log/{image}')
|
||||
self.cmd.run(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log')
|
||||
tag = imageTag if image != 'ran-base' else baseTag
|
||||
size = self._get_image_size(mySSH, image, tag)
|
||||
size = self._get_image_size(image, tag)
|
||||
if size <= 0:
|
||||
imageSize[image] = 'unknown -- BUILD FAILED'
|
||||
status = False
|
||||
@@ -361,17 +340,17 @@ class Cluster:
|
||||
logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
|
||||
|
||||
grep_exp = "\|".join(attemptedImages)
|
||||
mySSH.command(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log', '\$', 10);
|
||||
mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc get pod $pod -o json >> cmake_targets/log/build_pod_summary.log; done', '\$', 60)
|
||||
self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log');
|
||||
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done', '\$', 60)
|
||||
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
cls_containerize.CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD)
|
||||
cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name)
|
||||
|
||||
mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done', '\$', 60)
|
||||
self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done')
|
||||
|
||||
# logout will return eventually, but we don't care when -> start in background
|
||||
mySSH.command('oc logout &', '\$', 5)
|
||||
mySSH.close()
|
||||
self.cmd.run(f'oc logout')
|
||||
self.cmd.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#/*
|
||||
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
# * contributor license agreements. See the NOTICE file distributed with
|
||||
# * this work for additional information regarding copyright ownership.
|
||||
# * The OpenAirInterface Software Alliance licenses this file to You under
|
||||
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
# * except in compliance with the License.
|
||||
# * You may obtain a copy of the License at
|
||||
# *
|
||||
# * http://www.openairinterface.org/?page_id=698
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# * See the License for the specific language governing permissions and
|
||||
# * limitations under the License.
|
||||
# *-------------------------------------------------------------------------------
|
||||
# * For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
# * contact@openairinterface.org
|
||||
# */
|
||||
#---------------------------------------------------------------------
|
||||
#
|
||||
# Required Python Version
|
||||
# Python 3.x
|
||||
#
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import subprocess as sp
|
||||
import os
|
||||
import paramiko
|
||||
import uuid
|
||||
|
||||
# provides a partial interface for the legacy SSHconnection class (getBefore(), command())
|
||||
class Cmd(metaclass=abc.ABCMeta):
|
||||
def cd(self, d, silent=False):
|
||||
if d == None or d == '' or d == []:
|
||||
self.cwd = None
|
||||
elif d[0] == '/':
|
||||
self.cwd = d
|
||||
else:
|
||||
if not self.cwd:
|
||||
# no cwd set: get current working directory
|
||||
self.cwd = self.run('pwd').stdout.strip()
|
||||
self.cwd += f"/{d}"
|
||||
if not silent:
|
||||
logging.debug(f'cd {self.cwd}')
|
||||
|
||||
@abc.abstractmethod
|
||||
def run(self, line, timeout=300, silent=False):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def command(self, commandline, expectedline, timeout, silent=False, resync=False):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def close(self):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def getBefore(self):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def copyin(self, scpIp, scpUser, scpPw, src, tgt):
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def copyout(self, scpIp, scpUser, scpPw, src, tgt):
|
||||
return
|
||||
|
||||
class LocalCmd(Cmd):
|
||||
def __init__(self, d = None):
|
||||
self.cwd = d
|
||||
self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
|
||||
|
||||
def run(self, line, timeout=300, silent=False, reportNonZero=True):
|
||||
if type(line) is str:
|
||||
line = [s for s in line.split(' ') if len(s) > 0]
|
||||
if not silent:
|
||||
logging.debug(' '.join(line))
|
||||
try:
|
||||
ret = sp.run(line, cwd=self.cwd, stdout=sp.PIPE, stderr=sp.STDOUT, timeout=timeout)
|
||||
except Exception as e:
|
||||
ret = sp.CompletedProcess(args=line, returncode=255, stdout=f'Exception: {str(e)}'.encode('utf-8'))
|
||||
if ret.stdout is None:
|
||||
ret.stdout = b''
|
||||
ret.stdout = ret.stdout.decode('utf-8').strip()
|
||||
if reportNonZero and ret.returncode != 0:
|
||||
cmd = ' '.join(ret.args)
|
||||
logging.warning(f'command "{cmd}" returned non-zero returncode {ret.returncode}: output:\n{ret.stdout}')
|
||||
self.cp = ret
|
||||
return ret
|
||||
|
||||
def command(self, commandline, expectedline=None, timeout=300, silent=False, resync=False):
|
||||
line = [s for s in commandline.split(' ') if len(s) > 0]
|
||||
if line[0] == 'cd':
|
||||
self.cd(line[1], silent)
|
||||
else:
|
||||
self.run(line, timeout, silent)
|
||||
return 0
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def getBefore(self):
|
||||
return self.cp.stdout
|
||||
|
||||
def copyin(self, scpIp, scpUser, scpPw, src, tgt):
|
||||
logging.warning("LocalCmd emulating sshconnection.copyin() function")
|
||||
self.run(f'cp -r {src} {tgt}')
|
||||
|
||||
def copyout(self, scpIp, scpUser, scpPw, src, tgt):
|
||||
logging.warning("LocalCmd emulating sshconnection.copyout() function")
|
||||
self.run(f'cp -r {src} {tgt}')
|
||||
|
||||
class RemoteCmd(Cmd):
|
||||
def __init__(self, hostname, d=None):
|
||||
logging.getLogger('paramiko').setLevel(logging.INFO) # prevent spamming through Paramiko
|
||||
self.client = paramiko.SSHClient()
|
||||
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
cfg = RemoteCmd._lookup_ssh_config(hostname)
|
||||
self.client.connect(**cfg)
|
||||
self.cwd = d
|
||||
self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
|
||||
|
||||
def _lookup_ssh_config(hostname):
|
||||
ssh_config = paramiko.SSHConfig()
|
||||
user_config_file = os.path.expanduser("~/.ssh/config")
|
||||
if os.path.exists(user_config_file):
|
||||
with open(user_config_file) as f:
|
||||
ssh_config.parse(f)
|
||||
else:
|
||||
raise FileNotFoundError('class needs SSH config at ~/.ssh/config')
|
||||
ucfg = ssh_config.lookup(hostname)
|
||||
if 'identityfile' not in ucfg or 'user' not in ucfg:
|
||||
raise KeyError(f'no identityfile or user in SSH config for host {hostname}')
|
||||
cfg = {'hostname':hostname, 'username':ucfg['user'], 'key_filename':ucfg['identityfile']}
|
||||
if 'hostname' in ucfg:
|
||||
cfg['hostname'] = ucfg['hostname'] # override user-given hostname with what is in config
|
||||
if 'port' in ucfg:
|
||||
cfg['port'] = int(ucfg['port'])
|
||||
if 'proxycommand' in ucfg:
|
||||
cfg['sock'] = paramiko.ProxyCommand(ucfg['proxycommand'])
|
||||
return cfg
|
||||
|
||||
def run(self, line, timeout=300, silent=False, reportNonZero=True):
|
||||
if type(line) is list:
|
||||
line = ' '.join(line)
|
||||
if not silent:
|
||||
logging.debug(line)
|
||||
if self.cwd:
|
||||
line = f"cd {self.cwd} && {line}"
|
||||
args = line.split(' ')
|
||||
try:
|
||||
stdin, stdout, stderr = self.client.exec_command(line, timeout=timeout)
|
||||
ret = sp.CompletedProcess(args=args, returncode=stdout.channel.recv_exit_status(), stdout=stdout.read(size=None) + stderr.read(size=None))
|
||||
except Exception as e:
|
||||
ret = sp.CompletedProcess(args=args, returncode=255, stdout=f'Exception: {str(e)}'.encode('utf-8'))
|
||||
ret.stdout = ret.stdout.decode('utf-8').strip()
|
||||
if reportNonZero and ret.returncode != 0:
|
||||
cmd = ' '.join(ret.args)
|
||||
logging.warning(f'command "{cmd}" returned non-zero returncode {ret.returncode}: output:\n{ret.stdout}')
|
||||
self.cp = ret
|
||||
return ret
|
||||
|
||||
def command(self, commandline, expectedline=None, timeout=300, silent=False, resync=False):
|
||||
line = [s for s in commandline.split(' ') if len(s) > 0]
|
||||
if line[0] == 'cd':
|
||||
self.cd(line[1], silent)
|
||||
else:
|
||||
self.run(line, timeout, silent)
|
||||
return 0
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
|
||||
def getBefore(self):
|
||||
return self.cp.stdout
|
||||
|
||||
def copyout(self, src, tgt, recursive=False):
|
||||
logging.warning("RemoteCmd emulating sshconnection.copyout() function, ignoring scpIp")
|
||||
logging.debug(f"copyout: local:{src} -> remote:{tgt}")
|
||||
if recursive:
|
||||
tmpfile = f"{uuid.uuid4()}.tar"
|
||||
abstmpfile = f"/tmp/{tmpfile}"
|
||||
cmd = LocalCmd()
|
||||
cmd.run(f"tar -cf {abstmpfile} {src}")
|
||||
sftp = self.client.open_sftp()
|
||||
sftp.put(abstmpfile, abstmpfile)
|
||||
sftp.close()
|
||||
cmd.run(f"rm {abstmpfile}")
|
||||
self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
|
||||
else:
|
||||
sftp = self.client.open_sftp()
|
||||
sftp.put(src, tgt)
|
||||
sftp.close()
|
||||
|
||||
def copyin(self, src, tgt, recursive=False):
|
||||
logging.warning("RemoteCmd emulating sshconnection.copyout() function")
|
||||
logging.debug(f"copyin: remote:{src} -> local:{tgt}")
|
||||
if recursive:
|
||||
tmpfile = f"{uuid.uuid4()}.tar"
|
||||
abstmpfile = f"/tmp/{tmpfile}"
|
||||
self.run(f"tar -cf {abstmpfile} {src}")
|
||||
sftp = self.client.open_sftp()
|
||||
sftp.get(abstmpfile, abstmpfile)
|
||||
sftp.close()
|
||||
self.run(f"rm {abstmpfile}")
|
||||
cmd = LocalCmd()
|
||||
cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
|
||||
else:
|
||||
sftp = self.client.open_sftp()
|
||||
sftp.get(src, tgt)
|
||||
sftp.close()
|
||||
@@ -44,6 +44,7 @@ from zipfile import ZipFile
|
||||
#-----------------------------------------------------------
|
||||
# OAI Testing modules
|
||||
#-----------------------------------------------------------
|
||||
import cls_cmd
|
||||
import sshconnection as SSH
|
||||
import helpreadme as HELP
|
||||
import constants as CONST
|
||||
@@ -83,20 +84,20 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge
|
||||
logging.debug(f'Merging with the target branch: {ranTargetBranch}')
|
||||
sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30)
|
||||
|
||||
def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw):
|
||||
sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5)
|
||||
sshSession.command(f'rm -f {log_name}.zip', '\$', 5)
|
||||
sshSession.command(f'mkdir -p {log_name}', '\$', 5)
|
||||
sshSession.command(f'mv log/* {log_name}', '\$', 5)
|
||||
sshSession.command(f'zip -r -qq {log_name}.zip {log_name}', '\$', 5)
|
||||
def CopyLogsToExecutor(cmd, sourcePath, log_name):
|
||||
cmd.cd(f'{sourcePath}/cmake_targets')
|
||||
cmd.run(f'rm -f {log_name}.zip')
|
||||
cmd.run(f'mkdir -p {log_name}')
|
||||
cmd.run(f'mv log/* {log_name}')
|
||||
cmd.run(f'zip -r -qq {log_name}.zip {log_name}')
|
||||
|
||||
# copy zip to executor for analysis
|
||||
if (os.path.isfile(f'./{log_name}.zip')):
|
||||
os.remove(f'./{log_name}.zip')
|
||||
if (os.path.isdir(f'./{log_name}')):
|
||||
shutil.rmtree(f'./{log_name}')
|
||||
sshSession.copyin(scpIp, scpUser, scpPw, f'{sourcePath}/cmake_targets/{log_name}.zip', '.')
|
||||
sshSession.command(f'rm -f {log_name}.zip','\$', 5)
|
||||
cmd.copyin(f'{sourcePath}/cmake_targets/{log_name}.zip', f'./{log_name}.zip')
|
||||
cmd.run(f'rm -f {log_name}.zip')
|
||||
ZipFile(f'{log_name}.zip').extractall('.')
|
||||
|
||||
def AnalyzeBuildLogs(buildRoot, images, globalStatus):
|
||||
@@ -329,12 +330,11 @@ class Containerize():
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter')
|
||||
logging.debug('Building on server: ' + lIpAddr)
|
||||
mySSH = SSH.SSHConnection()
|
||||
mySSH.open(lIpAddr, lUserName, lPassWord)
|
||||
cmd = cls_cmd.RemoteCmd(lIpAddr)
|
||||
|
||||
# Checking the hostname to get adapted on cli and dockerfileprefixes
|
||||
mySSH.command('hostnamectl', '\$', 5)
|
||||
result = re.search('Ubuntu|Red Hat', mySSH.getBefore())
|
||||
cmd.run('hostnamectl')
|
||||
result = re.search('Ubuntu|Red Hat', cmd.getBefore())
|
||||
self.host = result.group(0)
|
||||
if self.host == 'Ubuntu':
|
||||
self.cli = 'docker'
|
||||
@@ -369,18 +369,18 @@ class Containerize():
|
||||
|
||||
# Workaround for some servers, we need to erase completely the workspace
|
||||
if self.forcedWorkspaceCleanup:
|
||||
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
|
||||
cmd.run(f'sudo -S rm -Rf {lSourcePath}')
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
|
||||
CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
CreateWorkspace(cmd, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
|
||||
# if asterix, copy the entitlement and subscription manager configurations
|
||||
if self.host == 'Red Hat':
|
||||
mySSH.command('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/rhsm/rhsm.conf ./rhsm-conf/', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/', '\$', 5)
|
||||
cmd.run('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca')
|
||||
cmd.run('cp /etc/rhsm/rhsm.conf ./rhsm-conf/')
|
||||
cmd.run('cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/')
|
||||
cmd.run('cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/')
|
||||
|
||||
baseImage = 'ran-base'
|
||||
baseTag = 'develop'
|
||||
@@ -389,8 +389,8 @@ class Containerize():
|
||||
if (self.ranAllowMerge):
|
||||
imageTag = 'ci-temp'
|
||||
if self.ranTargetBranch == 'develop':
|
||||
mySSH.command('git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5)
|
||||
result = re.search('index', mySSH.getBefore())
|
||||
cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX')
|
||||
result = re.search('index', cmd.getBefore())
|
||||
if result is not None:
|
||||
forceBaseImageBuild = True
|
||||
baseTag = 'ci-temp'
|
||||
@@ -398,31 +398,31 @@ class Containerize():
|
||||
forceBaseImageBuild = True
|
||||
|
||||
# Let's remove any previous run artifacts if still there
|
||||
mySSH.command(self.cli + ' image prune --force', '\$', 30)
|
||||
cmd.run(f"{self.cli} image prune --force")
|
||||
if forceBaseImageBuild:
|
||||
mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30)
|
||||
cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
|
||||
for image,pattern in imageNames:
|
||||
mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30)
|
||||
cmd.run(f"{self.cli} image rm {image}:{imageTag}")
|
||||
|
||||
# Build the base image only on Push Events (not on Merge Requests)
|
||||
# On when the base image docker file is being modified.
|
||||
if forceBaseImageBuild:
|
||||
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600)
|
||||
cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600)
|
||||
# First verify if the base image was properly created.
|
||||
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5)
|
||||
ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
|
||||
allImagesSize = {}
|
||||
if mySSH.getBefore().count('o such image') != 0:
|
||||
if ret.returncode != 0:
|
||||
logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
|
||||
# Recover the name of the failed container?
|
||||
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
|
||||
mySSH.command(self.cli + ' image prune --force', '\$', 30)
|
||||
mySSH.close()
|
||||
cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs {self.cli} rm -f")
|
||||
cmd.run(f"{self.cli} image prune --force")
|
||||
cmd.close()
|
||||
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
|
||||
HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
sys.exit(1)
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
||||
if result is not None:
|
||||
size = float(result.group("size")) / 1000000
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
@@ -432,10 +432,10 @@ class Containerize():
|
||||
logging.debug('ran-base size is unknown')
|
||||
|
||||
# Recover build logs, for the moment only possible when build is successful
|
||||
mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5)
|
||||
mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(self.cli + ' rm -f test', '\$', 5)
|
||||
cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
|
||||
cmd.run("mkdir -p cmake_targets/log/ran-base")
|
||||
cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base")
|
||||
cmd.run(f"{self.cli} rm -f test")
|
||||
|
||||
# Build the target image(s)
|
||||
status = True
|
||||
@@ -444,26 +444,26 @@ class Containerize():
|
||||
attemptedImages += [image]
|
||||
# the archived Dockerfiles have "ran-base:latest" as base image
|
||||
# we need to update them with proper tag
|
||||
mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
|
||||
cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
||||
if image != 'ran-build':
|
||||
mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
|
||||
mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200)
|
||||
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
||||
cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', timeout=1200)
|
||||
# split the log
|
||||
mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
|
||||
mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5)
|
||||
# checking the status of the build
|
||||
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5)
|
||||
if mySSH.getBefore().count('o such image') != 0:
|
||||
cmd.run(f"mkdir -p cmake_targets/log/{image}")
|
||||
cmd.run(f"python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log")
|
||||
# check the status of the build
|
||||
ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {image}:{imageTag}")
|
||||
if ret.returncode != 0:
|
||||
logging.error('\u001B[1m Could not build properly ' + image + '\u001B[0m')
|
||||
status = False
|
||||
# Here we should check if the last container corresponds to a failed command and destroy it
|
||||
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
|
||||
cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs {self.cli} rm -f")
|
||||
allImagesSize[image] = 'N/A -- Build Failed'
|
||||
break
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
||||
if result is not None:
|
||||
size = float(result.group("size")) / 1000000
|
||||
size = float(result.group("size")) / 1000000 # convert to MB
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
logging.debug(f'\u001B[1m {image} size is {imageSizeStr} Mbytes\u001B[0m')
|
||||
allImagesSize[image] = f'{imageSizeStr} Mbytes'
|
||||
@@ -471,18 +471,18 @@ class Containerize():
|
||||
logging.debug(f'{image} size is unknown')
|
||||
allImagesSize[image] = 'unknown'
|
||||
# Now pruning dangling images in between target builds
|
||||
mySSH.command(self.cli + ' image prune --force', '\$', 30)
|
||||
cmd.run(f"{self.cli} image prune --force")
|
||||
|
||||
# Remove all intermediate build images and clean up
|
||||
if self.ranAllowMerge and forceBaseImageBuild:
|
||||
mySSH.command(f'{self.cli} image rm {baseImage}:{baseTag} || true', '\$', 30)
|
||||
mySSH.command(f'{self.cli} image rm ran-build:{imageTag}','\$', 30)
|
||||
mySSH.command(f'{self.cli} volume prune --force','\$', 15)
|
||||
cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
|
||||
cmd.run(f"{self.cli} image rm ran-build:{imageTag}")
|
||||
cmd.run(f"{self.cli} volume prune --force")
|
||||
|
||||
# create a zip with all logs
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, lUserName, lPassWord)
|
||||
mySSH.close()
|
||||
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
||||
cmd.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
|
||||
@@ -32,13 +32,14 @@
|
||||
# Import
|
||||
#-----------------------------------------------------------
|
||||
import logging
|
||||
import sshconnection as SSH
|
||||
import cls_oai_html
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import sshconnection as SSH
|
||||
import cls_oai_html
|
||||
import constants as CONST
|
||||
import helpreadme as HELP
|
||||
|
||||
@@ -105,7 +106,7 @@ class PhySim:
|
||||
if self.ranCommitID != '':
|
||||
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
|
||||
if self.ranAllowMerge:
|
||||
imageTag = "ci-temp"
|
||||
imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}'
|
||||
if self.ranTargetBranch == '':
|
||||
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
|
||||
mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30)
|
||||
|
||||
Reference in New Issue
Block a user