From 668d0865b8b08daa884bdde221ee4ef0ff4b7ca7 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 5 Sep 2022 15:27:36 +0200 Subject: [PATCH] Simplify cls_containerize.BuildImage() and drop allImagesSize class member --- ci-scripts/cls_containerize.py | 92 +++++++++++++--------------------- 1 file changed, 36 insertions(+), 56 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 6d9b20bc68..d083b69be8 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -163,7 +163,6 @@ class Containerize(): self.cliBuildOptions = '' self.dockerfileprefix = '' self.host = '' - self.allImagesSize = {} self.deployedContainers = [] self.tsharkStarted = False @@ -318,32 +317,10 @@ class Containerize(): if forceBaseImageBuild: mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600) # First verify if the base image was properly created. - status = True mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5) + allImagesSize = {} if mySSH.getBefore().count('o such image') != 0: logging.error('\u001B[1m Could not build properly ran-base\u001B[0m') - status = False - else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) - if result is not None: - imageSize = float(result.group('size')) - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') - self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' kbytes' - else: - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') - self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Mbytes' - else: - imageSize = imageSize / 1000 - logging.debug('\u001B[1m ran-base size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') - self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Gbytes' - else: - logging.debug('ran-base size is unknown') - # If the base image failed, no need to continue - if not status: # Recover the name of the failed container? mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) mySSH.command(self.cli + ' image prune --force', '\$', 30) @@ -353,20 +330,32 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) sys.exit(1) else: - # Recover build logs, for the moment only possible when build is successful - mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5) - mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5) - mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5) - mySSH.command(self.cli + ' rm -f test', '\$', 5) + result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) + if result is not None: + size = float(result.group("size")) / 1000000 + imageSizeStr = f'{size:.1f}' + logging.debug(f'\u001B[1m ran-base size is {imageSizeStr} Mbytes\u001B[0m') + allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes' + else: + logging.debug('ran-base size is unknown') + + # Recover build logs, for the moment only possible when build is successful + mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5) + mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5) + mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5) + mySSH.command(self.cli + ' rm -f test', '\$', 5) # Build the target image(s) + status = True + attemptedImages = ['ran-base'] for image,pattern in imageNames: + attemptedImages += [image] # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag - mySSH.command('sed -i -e "s#' + baseImage + ':latest#' + baseImage + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) + mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5) if image != 'ran-build': - mySSH.command('sed -i -e "s#' + "ran-build" + ':latest#' + "ran-build" + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) - mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) + mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5) + mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200) # Flatten Image if image != 'ran-build': mySSH.command('python3 ./ci-scripts/flatten_image.py --tag ' + image + ':' + imageTag, '\$', 300) @@ -380,27 +369,18 @@ class Containerize(): status = False # Here we should check if the last container corresponds to a failed command and destroy it mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) - self.allImagesSize[image] = 'N/A -- Build Failed' + allImagesSize[image] = 'N/A -- Build Failed' + break else: result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) if result is not None: - imageSize = float(result.group('size')) - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') - self.allImagesSize[image] = str(round(imageSize,1)) + ' kbytes' - else: - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') - self.allImagesSize[image] = str(round(imageSize,1)) + ' Mbytes' - else: - imageSize = imageSize / 1000 - logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') - self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes' + size = float(result.group("size")) / 1000000 + imageSizeStr = f'{size:.1f}' + logging.debug(f'\u001B[1m {image} size is {imageSizeStr} Mbytes\u001B[0m') + allImagesSize[image] = f'{imageSizeStr} Mbytes' else: - logging.debug('ran-base size is unknown') - self.allImagesSize[image] = 'unknown' + logging.debug(f'{image} size is unknown') + allImagesSize[image] = 'unknown' # Now pruning dangling images in between target builds mySSH.command(self.cli + ' image prune --force', '\$', 30) @@ -416,17 +396,16 @@ class Containerize(): mySSH.close() # Analyze the logs - images = ['ran-base'] + [tpl[0] for tpl in imageNames] - collectInfo = AnalyzeBuildLogs(build_log_name, images, status) + collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status) if status: logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) - HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, self.allImagesSize) + HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) else: logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) - HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, self.allImagesSize) + HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) HTML.CreateHtmlTabFooter(False) sys.exit(1) @@ -552,13 +531,14 @@ class Containerize(): collectInfo['proxy'] = files mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5) result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) + allImagesSize = {} if result is not None: imageSize = float(result.group('size')) / 1000000 logging.debug('\u001B[1m proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') - self.allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes' + allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes' else: logging.debug('proxy size is unknown') - self.allImagesSize['proxy'] = 'unknown' + allImagesSize['proxy'] = 'unknown' # Cleaning any created tmp volume mySSH.command(self.cli + ' volume prune --force || true','\$', 15) @@ -566,7 +546,7 @@ class Containerize(): logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m') HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) - HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, self.allImagesSize) + HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) def Copy_Image_to_Test_Server(self, HTML): imageTag = 'develop'