From 5ffa4315141d8dc689b7ceddcdc09d81909f09c2 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 5 Aug 2025 17:41:34 +0200 Subject: [PATCH] Use archiveArtifact() for Build()/BuildCluster() I have to make both together, since they both depend on AnalyzeBuildLogs(). - remove mkdir for legacy sub-logfiles in build_oai, that does not exist anymore; also, remove corresponding log analysis, I think we don't use it and it is overkill (either build works or not) - manually try to copy "all.txt" - explicitly list log files to copy, then analyze - AnalyzeBuildLogs(): pass log files to inspect explicitly --- ci-scripts/cls_cluster.py | 91 ++++++++++++++------------ ci-scripts/cls_containerize.py | 116 +++++++++++---------------------- ci-scripts/main.py | 4 +- 3 files changed, 91 insertions(+), 120 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 7d41e9d9cc..185e90a13e 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -39,6 +39,7 @@ import constants as CONST import helpreadme as HELP import cls_containerize import cls_cmd +from cls_ci_helper import archiveArtifact IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc" NAMESPACE = "oaicicd-ran" @@ -207,7 +208,12 @@ class Cluster: HTML.CreateHtmlTestRowQueue(param, 'KO', [msg]) return success - def BuildClusterImage(self, HTML): + def _retrieveOCLog(self, ctx, job, lSourcePath, image): + fn = f'{lSourcePath}/cmake_targets/log/{image}.log' + self.cmd.run(f'oc logs {job} &> {fn}') + return (image, archiveArtifact(self.cmd, ctx, fn)) + + def BuildClusterImage(self, ctx, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}') @@ -280,22 +286,27 @@ class Cluster: self._recreate_entitlements() status = True # flag to abandon compiling if any image fails - attemptedImages = [] + log_files = [] + build_metrics = f"{lSourcePath}/cmake_targets/log/build-metrics.log" if forceBaseImageBuild: self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml') self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml') ranbase_job = self._start_build('ran-base') - attemptedImages += ['ran-base'] status = ranbase_job is not None and self._wait_build_end([ranbase_job], 1000) if not status: logging.error('failure during build of ran-base') - self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect + log_files.append(self._retrieveOCLog(ctx, ranbase_job, lSourcePath, 'ran-base')) + # recover logs by mounting image self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml') pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml') if pod is not None: - self.cmd.run(f'mkdir -p cmake_targets/log/ran-base') - self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base') + logdir = f'{lSourcePath}/cmake_targets/log/ran-base' + self.cmd.run(f'mkdir -p {logdir}') + self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ {logdir}') self._undeploy_pod('openshift/ran-base-log-retrieval.yaml') + ret = self.cmd.run(f'ls {logdir}') + for f in ret.stdout.splitlines(): + archiveArtifact(self.cmd, ctx, f"{logdir}/{f}") else: status = False @@ -304,27 +315,24 @@ class Cluster: self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel9') physim_job = self._start_build('oai-physim') - attemptedImages += ['oai-physim'] self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml') self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel9') ranbuild_job = self._start_build('ran-build') - attemptedImages += ['ran-build'] self._recreate_is_tag('oai-clang', imageTag, 'openshift/oai-clang-is.yaml') self._recreate_bc('oai-clang', imageTag, 'openshift/oai-clang-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.clang.rhel9') clang_job = self._start_build('oai-clang') - attemptedImages += ['oai-clang'] wait = ranbuild_job is not None and physim_job is not None and clang_job is not None and self._wait_build_end([ranbuild_job, physim_job, clang_job], 1200) if not wait: logging.error('error during build of ranbuild_job or physim_job or clang_job') status = status and wait - self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log') - self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log') - self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, ranbuild_job, lSourcePath, 'ran-build')) + log_files.append(self._retrieveOCLog(ctx, physim_job, lSourcePath, 'oai-physim')) + log_files.append(self._retrieveOCLog(ctx, clang_job, lSourcePath, 'oai-clang')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml') @@ -332,72 +340,66 @@ class Cluster: self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel9') enb_job = self._start_build('oai-enb') - attemptedImages += ['oai-enb'] self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml') self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel9') gnb_job = self._start_build('oai-gnb') - attemptedImages += ['oai-gnb'] self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml') self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel9') gnb_aw2s_job = self._start_build('oai-gnb-aw2s') - attemptedImages += ['oai-gnb-aw2s'] wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 800) if not wait: logging.error('error during build of eNB/gNB') status = status and wait # recover logs - self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log') - self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log') - self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log') + log_files.append(self._retrieveOCLog(ctx, enb_job, lSourcePath, 'oai-enb')) + log_files.append(self._retrieveOCLog(ctx, gnb_job, lSourcePath, 'oai-gnb')) + log_files.append(self._retrieveOCLog(ctx, gnb_aw2s_job, lSourcePath, 'oai-gnb-aw2s')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') self._recreate_is_tag('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-is.yaml') self._recreate_bc('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nr-cuup.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nr-cuup.rhel9') nr_cuup_job = self._start_build('oai-nr-cuup') - attemptedImages += ['oai-nr-cuup'] self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml') self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel9') lteue_job = self._start_build('oai-lte-ue') - attemptedImages += ['oai-lte-ue'] self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml') self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel9') nrue_job = self._start_build('oai-nr-ue') - attemptedImages += ['oai-nr-ue'] wait = nr_cuup_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end([nr_cuup_job, lteue_job, nrue_job], 800) if not wait: logging.error('error during build of nr-cuup/lteUE/nrUE') status = status and wait # recover logs - self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log') - self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log') - self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, nr_cuup_job, lSourcePath, 'oai-nr-cuup')) + log_files.append(self._retrieveOCLog(ctx, lteue_job, lSourcePath, 'oai-lte-ue')) + log_files.append(self._retrieveOCLog(ctx, nrue_job, lSourcePath, 'oai-nr-ue')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml') self._recreate_bc('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.fhi72.rhel9') ranbuildfhi72_job = self._start_build('ran-build-fhi72') - attemptedImages += ['ran-build-fhi72'] wait = ranbuildfhi72_job is not None and self._wait_build_end([ranbuildfhi72_job], 1200) if not wait: logging.error('error during build of ranbuildfhi72_job') status = status and wait - self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, ranbuildfhi72_job, lSourcePath, 'ran-build-fhi72')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml') @@ -405,19 +407,17 @@ class Cluster: self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.fhi72.rhel9') self._retag_image_statement('ran-build-fhi72', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build-fhi72', imageTag, 'docker/Dockerfile.gNB.fhi72.rhel9') gnb_fhi72_job = self._start_build('oai-gnb-fhi72') - attemptedImages += ['oai-gnb-fhi72'] wait = gnb_fhi72_job is not None and self._wait_build_end([gnb_fhi72_job], 600) if not wait: logging.error('error during build of gNB-fhi72') status = status and wait # recover logs - self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, gnb_fhi72_job, lSourcePath, 'oai-gnb-fhi72')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') # split and analyze logs imageSize = {} - for image in attemptedImages: - self.cmd.run(f'mkdir -p cmake_targets/log/{image}') + for image, _ in log_files: tag = imageTag if image != 'ran-base' else baseTag size = self._get_image_size(image, tag) if size <= 0: @@ -428,12 +428,14 @@ class Cluster: imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)' logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m') - grep_exp = r"\|".join(attemptedImages) - self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log'); - self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done') - - build_log_name = f'build_log_{self.testCase_id}' - cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name) + archiveArtifact(self.cmd, ctx, build_metrics) + logfile = f'{lSourcePath}/cmake_targets/log/image_registry.log' + grep_exp = r"\|".join([i for i,f in log_files]) + self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> {logfile}'); + archiveArtifact(self.cmd, ctx, logfile) + logfile = f'{lSourcePath}/cmake_targets/log/build_pod_summary.log' + self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> {logfile}; done') + archiveArtifact(self.cmd, ctx, logfile) self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') @@ -442,7 +444,7 @@ class Cluster: self.cmd.close() # Analyze the logs - collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status) + collectInfo = cls_containerize.AnalyzeBuildLogs(log_files, status) for img in collectInfo: for f in collectInfo[img]: status = status and collectInfo[img][f]['status'] @@ -458,6 +460,13 @@ class Cluster: HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize) + # TODO fix groovy script, remove the following. + # the groovy scripts expects all logs in + # //ci-scripts, so copy it there + with cls_cmd.LocalCmd() as c: + c.run(f'mkdir -p {os.getcwd()}/test_log_{ctx.test_id}/') + c.run(f'cp -r {ctx.logPath} {os.getcwd()}/test_log_{ctx.test_id}/') + return status def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id): diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 381e955127..3af34dca14 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -82,67 +82,25 @@ def CreateTag(ranCommitID, ranBranch, ranAllowMerge): tagToUse = f'develop-{shortCommit}' return tagToUse -def CopyLogsToExecutor(cmd, sourcePath, log_name): - cmd.cd(f'{sourcePath}/cmake_targets') - cmd.run(f'rm -f {log_name}.zip') - cmd.run(f'mkdir -p {log_name}') - cmd.run(f'mv log/* {log_name}') - cmd.run(f'zip -r -qq {log_name}.zip {log_name}') - - # copy zip to executor for analysis - if (os.path.isfile(f'./{log_name}.zip')): - os.remove(f'./{log_name}.zip') - if (os.path.isdir(f'./{log_name}')): - shutil.rmtree(f'./{log_name}') - cmd.copyin(src=f'{sourcePath}/cmake_targets/{log_name}.zip', tgt=f'{os.getcwd()}/{log_name}.zip') - cmd.run(f'rm -f {log_name}.zip') - ZipFile(f'{log_name}.zip').extractall('.') - -def AnalyzeBuildLogs(buildRoot, images, globalStatus): +def AnalyzeBuildLogs(logfiles, globalStatus): collectInfo = {} - for image in images: + for image, lf in logfiles: files = {} - file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')] - # Analyze the "sub-logs" of every target image - for fil in file_list: - errorandwarnings = {} - warningsNo = 0 - errorsNo = 0 - with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile: - for line in inputfile: - result = re.search(' ERROR ', str(line)) - if result is not None: - errorsNo += 1 - result = re.search(' error:', str(line)) - if result is not None: - errorsNo += 1 - result = re.search(' WARNING ', str(line)) - if result is not None: - warningsNo += 1 - result = re.search(' warning:', str(line)) - if result is not None: - warningsNo += 1 - errorandwarnings['errors'] = errorsNo - errorandwarnings['warnings'] = warningsNo - errorandwarnings['status'] = globalStatus - files[fil] = errorandwarnings - # Analyze the target image - if os.path.isfile(f'{buildRoot}/{image}.log'): - errorandwarnings = {} - committed = False - tagged = False - with open(f'{buildRoot}/{image}.log', mode='r') as inputfile: - for line in inputfile: - lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None - lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None - tagged = tagged or lineHasTag or lineHasTag2 - # the OpenShift Cluster builder prepends image registry URL - lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None - committed = committed or lineHasCommit - errorandwarnings['errors'] = 0 if committed or tagged else 1 - errorandwarnings['warnings'] = 0 - errorandwarnings['status'] = committed or tagged - files['Target Image Creation'] = errorandwarnings + errorandwarnings = {} + committed = False + tagged = False + with open(lf, mode='r') as inputfile: + for line in inputfile: + lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None + lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None + tagged = tagged or lineHasTag or lineHasTag2 + # the OpenShift Cluster builder prepends image registry URL + lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None + committed = committed or lineHasCommit + errorandwarnings['errors'] = 0 if committed or tagged else 1 + errorandwarnings['warnings'] = 0 + errorandwarnings['status'] = committed or tagged + files['Target Image Creation'] = errorandwarnings collectInfo[image] = files return collectInfo @@ -328,11 +286,12 @@ class Containerize(): raise ValueError(f'Insufficient Parameter: IP/node {ip}, path {path}') return (ip, path) - def BuildImage(self, HTML): + def BuildImage(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) cmd = cls_cmd.getConnection(lIpAddr) + log_files = [] # Checking the hostname to get adapted on cli and dockerfileprefixes cmd.run('hostnamectl') @@ -424,7 +383,18 @@ class Containerize(): # On when the base image docker file is being modified. if forceBaseImageBuild: cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") - cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600) + logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log' + cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> {logfile}", timeout=1600) + t = ("ran-base", archiveArtifact(cmd, ctx, logfile)) + log_files.append(t) + + # Recover build logs, for the moment only possible when build is successful + cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") + cmd.run("mkdir -p cmake_targets/log/ran-base") + logfile = f'{lSourcePath}/cmake_targets/log/ran-base.log' + cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/all.txt {logfile}") + cmd.run(f"{self.cli} rm -f test") + archiveArtifact(cmd, ctx, logfile) # First verify if the base image was properly created. ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}") allImagesSize = {} @@ -448,17 +418,9 @@ class Containerize(): else: logging.debug('ran-base size is unknown') - # Recover build logs, for the moment only possible when build is successful - cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") - cmd.run("mkdir -p cmake_targets/log/ran-base") - cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base") - cmd.run(f"{self.cli} rm -f test") - # Build the target image(s) status = True - attemptedImages = ['ran-base'] for image,pattern,name,option in imageNames: - attemptedImages += [name] # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}') @@ -472,15 +434,18 @@ class Containerize(): cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') if image == 'oai-gnb-aerial': cmd.run('cp -f /opt/nvidia-ipc/nvipc.src.2025.05.20.tar.gz .') - ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200) + logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log' + ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > {logfile} 2>&1', timeout=1200) + t = (name, archiveArtifact(cmd, ctx, logfile)) + log_files.append(t) if image == 'oai-gnb-aerial': cmd.run('rm -f nvipc.src.2025.05.20.tar.gz') if image == 'ran-build' and ret.returncode == 0: cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true") - cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/") + logfile = f'{lSourcePath}/{image}.log' + cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/all.txt {logfile}") cmd.run(f"docker rm -f test-log") - else: - cmd.run(f"mkdir -p cmake_targets/log/{name}") + archiveArtifact(cmd, ctx, logfile) # check the status of the build ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}") if ret.returncode != 0: @@ -514,13 +479,10 @@ class Containerize(): logging.debug(cmd.run("df -h").stdout) logging.debug(cmd.run("docker system df").stdout) - # create a zip with all logs - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) cmd.close() # Analyze the logs - collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status) + collectInfo = AnalyzeBuildLogs(log_files, status) if status: logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index f201bdbcc6..dcca9b521f 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -138,11 +138,11 @@ def ExecuteActionWithParam(action, ctx): if action == 'Build_eNB': success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) elif action == 'Build_Image': - success = CONTAINERS.BuildImage(HTML) + success = CONTAINERS.BuildImage(ctx, HTML) elif action == 'Build_Proxy': success = CONTAINERS.BuildProxy(ctx, HTML) elif action == 'Build_Cluster_Image': - success = CLUSTER.BuildClusterImage(HTML) + success = CLUSTER.BuildClusterImage(ctx, HTML) elif action == 'Build_Run_Tests': success = CONTAINERS.BuildRunTests(ctx, HTML)