Use archiveArtifact() for Build()/BuildCluster()
I have to make both together, since they both depend on AnalyzeBuildLogs(). - remove mkdir for legacy sub-logfiles in build_oai, that does not exist anymore; also, remove corresponding log analysis, I think we don't use it and it is overkill (either build works or not) - manually try to copy "all.txt" - explicitly list log files to copy, then analyze - AnalyzeBuildLogs(): pass log files to inspect explicitly
This commit is contained in:
Robert Schmidt
+50
-41
@@ -39,6 +39,7 @@ import constants as CONST
|
||||
import helpreadme as HELP
|
||||
import cls_containerize
|
||||
import cls_cmd
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc"
|
||||
NAMESPACE = "oaicicd-ran"
|
||||
@@ -207,7 +208,12 @@ class Cluster:
|
||||
HTML.CreateHtmlTestRowQueue(param, 'KO', [msg])
|
||||
return success
|
||||
|
||||
def BuildClusterImage(self, HTML):
|
||||
def _retrieveOCLog(self, ctx, job, lSourcePath, image):
|
||||
fn = f'{lSourcePath}/cmake_targets/log/{image}.log'
|
||||
self.cmd.run(f'oc logs {job} &> {fn}')
|
||||
return (image, archiveArtifact(self.cmd, ctx, fn))
|
||||
|
||||
def BuildClusterImage(self, ctx, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}')
|
||||
@@ -280,22 +286,27 @@ class Cluster:
|
||||
self._recreate_entitlements()
|
||||
|
||||
status = True # flag to abandon compiling if any image fails
|
||||
attemptedImages = []
|
||||
log_files = []
|
||||
build_metrics = f"{lSourcePath}/cmake_targets/log/build-metrics.log"
|
||||
if forceBaseImageBuild:
|
||||
self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml')
|
||||
self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml')
|
||||
ranbase_job = self._start_build('ran-base')
|
||||
attemptedImages += ['ran-base']
|
||||
status = ranbase_job is not None and self._wait_build_end([ranbase_job], 1000)
|
||||
if not status: logging.error('failure during build of ran-base')
|
||||
self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect
|
||||
log_files.append(self._retrieveOCLog(ctx, ranbase_job, lSourcePath, 'ran-base'))
|
||||
|
||||
# recover logs by mounting image
|
||||
self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml')
|
||||
pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml')
|
||||
if pod is not None:
|
||||
self.cmd.run(f'mkdir -p cmake_targets/log/ran-base')
|
||||
self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base')
|
||||
logdir = f'{lSourcePath}/cmake_targets/log/ran-base'
|
||||
self.cmd.run(f'mkdir -p {logdir}')
|
||||
self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ {logdir}')
|
||||
self._undeploy_pod('openshift/ran-base-log-retrieval.yaml')
|
||||
ret = self.cmd.run(f'ls {logdir}')
|
||||
for f in ret.stdout.splitlines():
|
||||
archiveArtifact(self.cmd, ctx, f"{logdir}/{f}")
|
||||
else:
|
||||
status = False
|
||||
|
||||
@@ -304,27 +315,24 @@ class Cluster:
|
||||
self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel9')
|
||||
physim_job = self._start_build('oai-physim')
|
||||
attemptedImages += ['oai-physim']
|
||||
|
||||
self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml')
|
||||
self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel9')
|
||||
ranbuild_job = self._start_build('ran-build')
|
||||
attemptedImages += ['ran-build']
|
||||
|
||||
self._recreate_is_tag('oai-clang', imageTag, 'openshift/oai-clang-is.yaml')
|
||||
self._recreate_bc('oai-clang', imageTag, 'openshift/oai-clang-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.clang.rhel9')
|
||||
clang_job = self._start_build('oai-clang')
|
||||
attemptedImages += ['oai-clang']
|
||||
|
||||
wait = ranbuild_job is not None and physim_job is not None and clang_job is not None and self._wait_build_end([ranbuild_job, physim_job, clang_job], 1200)
|
||||
if not wait: logging.error('error during build of ranbuild_job or physim_job or clang_job')
|
||||
status = status and wait
|
||||
self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log')
|
||||
self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log')
|
||||
self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, ranbuild_job, lSourcePath, 'ran-build'))
|
||||
log_files.append(self._retrieveOCLog(ctx, physim_job, lSourcePath, 'oai-physim'))
|
||||
log_files.append(self._retrieveOCLog(ctx, clang_job, lSourcePath, 'oai-clang'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
|
||||
@@ -332,72 +340,66 @@ class Cluster:
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel9')
|
||||
enb_job = self._start_build('oai-enb')
|
||||
attemptedImages += ['oai-enb']
|
||||
|
||||
self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml')
|
||||
self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel9')
|
||||
gnb_job = self._start_build('oai-gnb')
|
||||
attemptedImages += ['oai-gnb']
|
||||
|
||||
self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml')
|
||||
self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel9')
|
||||
gnb_aw2s_job = self._start_build('oai-gnb-aw2s')
|
||||
attemptedImages += ['oai-gnb-aw2s']
|
||||
|
||||
wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 800)
|
||||
if not wait: logging.error('error during build of eNB/gNB')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log')
|
||||
self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log')
|
||||
self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, enb_job, lSourcePath, 'oai-enb'))
|
||||
log_files.append(self._retrieveOCLog(ctx, gnb_job, lSourcePath, 'oai-gnb'))
|
||||
log_files.append(self._retrieveOCLog(ctx, gnb_aw2s_job, lSourcePath, 'oai-gnb-aw2s'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
self._recreate_is_tag('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-is.yaml')
|
||||
self._recreate_bc('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nr-cuup.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nr-cuup.rhel9')
|
||||
nr_cuup_job = self._start_build('oai-nr-cuup')
|
||||
attemptedImages += ['oai-nr-cuup']
|
||||
|
||||
self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml')
|
||||
self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel9')
|
||||
lteue_job = self._start_build('oai-lte-ue')
|
||||
attemptedImages += ['oai-lte-ue']
|
||||
|
||||
self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml')
|
||||
self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel9')
|
||||
nrue_job = self._start_build('oai-nr-ue')
|
||||
attemptedImages += ['oai-nr-ue']
|
||||
|
||||
wait = nr_cuup_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end([nr_cuup_job, lteue_job, nrue_job], 800)
|
||||
if not wait: logging.error('error during build of nr-cuup/lteUE/nrUE')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log')
|
||||
self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log')
|
||||
self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, nr_cuup_job, lSourcePath, 'oai-nr-cuup'))
|
||||
log_files.append(self._retrieveOCLog(ctx, lteue_job, lSourcePath, 'oai-lte-ue'))
|
||||
log_files.append(self._retrieveOCLog(ctx, nrue_job, lSourcePath, 'oai-nr-ue'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml')
|
||||
self._recreate_bc('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.fhi72.rhel9')
|
||||
ranbuildfhi72_job = self._start_build('ran-build-fhi72')
|
||||
attemptedImages += ['ran-build-fhi72']
|
||||
|
||||
wait = ranbuildfhi72_job is not None and self._wait_build_end([ranbuildfhi72_job], 1200)
|
||||
if not wait: logging.error('error during build of ranbuildfhi72_job')
|
||||
status = status and wait
|
||||
self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, ranbuildfhi72_job, lSourcePath, 'ran-build-fhi72'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml')
|
||||
@@ -405,19 +407,17 @@ class Cluster:
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.fhi72.rhel9')
|
||||
self._retag_image_statement('ran-build-fhi72', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build-fhi72', imageTag, 'docker/Dockerfile.gNB.fhi72.rhel9')
|
||||
gnb_fhi72_job = self._start_build('oai-gnb-fhi72')
|
||||
attemptedImages += ['oai-gnb-fhi72']
|
||||
|
||||
wait = gnb_fhi72_job is not None and self._wait_build_end([gnb_fhi72_job], 600)
|
||||
if not wait: logging.error('error during build of gNB-fhi72')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, gnb_fhi72_job, lSourcePath, 'oai-gnb-fhi72'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
# split and analyze logs
|
||||
imageSize = {}
|
||||
for image in attemptedImages:
|
||||
self.cmd.run(f'mkdir -p cmake_targets/log/{image}')
|
||||
for image, _ in log_files:
|
||||
tag = imageTag if image != 'ran-base' else baseTag
|
||||
size = self._get_image_size(image, tag)
|
||||
if size <= 0:
|
||||
@@ -428,12 +428,14 @@ class Cluster:
|
||||
imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)'
|
||||
logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
|
||||
|
||||
grep_exp = r"\|".join(attemptedImages)
|
||||
self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log');
|
||||
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done')
|
||||
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name)
|
||||
archiveArtifact(self.cmd, ctx, build_metrics)
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/image_registry.log'
|
||||
grep_exp = r"\|".join([i for i,f in log_files])
|
||||
self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> {logfile}');
|
||||
archiveArtifact(self.cmd, ctx, logfile)
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/build_pod_summary.log'
|
||||
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> {logfile}; done')
|
||||
archiveArtifact(self.cmd, ctx, logfile)
|
||||
|
||||
self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done')
|
||||
|
||||
@@ -442,7 +444,7 @@ class Cluster:
|
||||
self.cmd.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
collectInfo = cls_containerize.AnalyzeBuildLogs(log_files, status)
|
||||
for img in collectInfo:
|
||||
for f in collectInfo[img]:
|
||||
status = status and collectInfo[img][f]['status']
|
||||
@@ -458,6 +460,13 @@ class Cluster:
|
||||
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
|
||||
|
||||
# TODO fix groovy script, remove the following.
|
||||
# the groovy scripts expects all logs in
|
||||
# <jenkins-workspace>/<pipeline>/ci-scripts, so copy it there
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f'mkdir -p {os.getcwd()}/test_log_{ctx.test_id}/')
|
||||
c.run(f'cp -r {ctx.logPath} {os.getcwd()}/test_log_{ctx.test_id}/')
|
||||
|
||||
return status
|
||||
|
||||
def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id):
|
||||
|
||||
Reference in New Issue
Block a user