From 42c2fe5b756fb7d0d854d0777ead4c09f419eb82 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 8 Dec 2021 16:53:15 +0100 Subject: [PATCH] CI: build intermediary compilation image Change the python script such that we first build the ran-base image (renamed from ran-build, as it does not build anything), followed by the ran-build image compiling all dependencies, and the generate the independent target container images. --- ci-scripts/cls_containerize.py | 71 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 8ee6113628..c09e42b49e 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -177,7 +177,8 @@ class Containerize(): self.dockerfileprefix = '.rhel8.2' self.cliBuildOptions = '--no-cache --disable-compression' - imageNames = [] + # we always build the ran-build image with all targets + imageNames = [('ran-build', 'build')] result = re.search('eNB', self.imageKind) # Creating a tupple with the imageName and the DockerFile prefix pattern on obelix if result is not None: @@ -197,8 +198,6 @@ class Containerize(): imageNames.append(('oai-physim', 'phySim')) if self.host == 'Ubuntu': imageNames.append(('oai-lte-ru', 'lteRU')) - if len(imageNames) == 0: - imageNames.append(('oai-enb', 'eNB')) # Workaround for some servers, we need to erase completely the workspace if self.forcedWorkspaceCleanup: @@ -215,37 +214,37 @@ class Containerize(): mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca/', '\$', 5) mySSH.command('sudo cp /etc/pki/entitlement/*.pem tmp/entitlement/', '\$', 5) - sharedimage = 'ran-build' - sharedTag = 'develop' - forceSharedImageBuild = False + baseImage = 'ran-base' + baseTag = 'develop' + forceBaseImageBuild = False imageTag = 'develop' if (self.ranAllowMerge): imageTag = 'ci-temp' if self.ranTargetBranch == 'develop': - mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.ran' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) + mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) result = re.search('index', mySSH.getBefore()) if result is not None: - forceSharedImageBuild = True - sharedTag = 'ci-temp' + forceBaseImageBuild = True + baseTag = 'ci-temp' else: - forceSharedImageBuild = True + forceBaseImageBuild = True # Let's remove any previous run artifacts if still there mySSH.command(self.cli + ' image prune --force', '\$', 30) - if forceSharedImageBuild: - mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + sharedTag + ' || true', '\$', 30) + if forceBaseImageBuild: + mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30) for image,pattern in imageNames: mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30) - # Build the shared image only on Push Events (not on Merge Requests) - # On when the shared image docker file is being modified. - if forceSharedImageBuild: - mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + sharedimage + ' --tag ' + sharedimage + ':' + sharedTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600) - # First verify if the shared image was properly created. + # Build the base image only on Push Events (not on Merge Requests) + # On when the base image docker file is being modified. + if forceBaseImageBuild: + mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600) + # First verify if the base image was properly created. status = True - mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + sharedimage + ':' + sharedTag, '\$', 5) + mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5) if mySSH.getBefore().count('o such image') != 0: - logging.error('\u001B[1m Could not build properly ran-build\u001B[0m') + logging.error('\u001B[1m Could not build properly ran-base\u001B[0m') status = False else: result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) @@ -253,20 +252,20 @@ class Containerize(): imageSize = float(result.group('size')) imageSize = imageSize / 1000 if imageSize < 1000: - logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') - self.allImagesSize['ran-build'] = str(round(imageSize,1)) + ' kbytes' + logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') + self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' kbytes' else: imageSize = imageSize / 1000 if imageSize < 1000: - logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') - self.allImagesSize['ran-build'] = str(round(imageSize,1)) + ' Mbytes' + logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') + self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Mbytes' else: imageSize = imageSize / 1000 - logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') - self.allImagesSize['ran-build'] = str(round(imageSize,1)) + ' Gbytes' + logging.debug('\u001B[1m ran-base size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') + self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Gbytes' else: - logging.debug('ran-build size is unknown') - # If the shared image failed, no need to continue + logging.debug('ran-base size is unknown') + # If the base image failed, no need to continue if not status: # Recover the name of the failed container? mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) @@ -278,16 +277,18 @@ class Containerize(): sys.exit(1) else: # Recover build logs, for the moment only possible when build is successful - mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + sharedTag, '\$', 5) - mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) - mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) + mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5) + mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5) + mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5) mySSH.command(self.cli + ' rm -f test', '\$', 5) # Build the target image(s) for image,pattern in imageNames: - # the archived Dockerfiles have "ran-build:latest" as base image + # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag - mySSH.command('sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + sharedTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) + mySSH.command('sed -i -e "s#' + baseImage + ':latest#' + baseImage + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) + if image != 'ran-build': + mySSH.command('sed -i -e "s#' + "ran-build" + ':latest#' + "ran-build" + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) # split the log mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) @@ -318,7 +319,7 @@ class Containerize(): logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes' else: - logging.debug('ran-build size is unknown') + logging.debug('ran-base size is unknown') self.allImagesSize[image] = 'unknown' # Now pruning dangling images in between target builds mySSH.command(self.cli + ' image prune --force', '\$', 30) @@ -342,8 +343,8 @@ class Containerize(): #Trying to identify the errors and warnings for each built images imageNames1 = imageNames - shared = ('ran-build','ran') - imageNames1.insert(0, shared) + base = ('ran-base','ran') + imageNames1.insert(0, base) for image,pattern in imageNames1: files = {} file_list = [f for f in os.listdir('build_log_' + self.testCase_id + '/' + image) if os.path.isfile(os.path.join('build_log_' + self.testCase_id + '/' + image, f)) and f.endswith('.txt')]