From 13a204f02775982ade80245d62dd2a35b6c50253 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 19 Mar 2021 17:32:53 +0100 Subject: [PATCH] [CI][DOCKER] handle build failure, and Merge Requests do not build shared image to speed up process Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 4f61faada9..0debe42468 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -174,6 +174,7 @@ class Containerize(): # if the branch is not develop, then it is a merge request and we need to do # the potential merge. Note that merge conflicts should already been checked earlier imageTag = 'develop' + sharedTag = 'develop' if (self.ranAllowMerge): imageTag = 'ci-temp' if self.ranTargetBranch == '': @@ -193,17 +194,19 @@ class Containerize(): sharedimage = 'ran-build' # Let's remove any previous run artifacts if still there mySSH.command(self.cli + ' image prune --force', '\$', 30) - mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + imageTag, '\$', 30) + if (not self.ranAllowMerge): + mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + sharedTag, '\$', 30) for image,pattern in imageNames: mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag, '\$', 30) - # Build the shared image - mySSH.command(self.cli + ' build --target ' + sharedimage + ' --tag ' + sharedimage + ':' + imageTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600) + # Build the shared image only on Push Events (not on Merge Requests) + if (not self.ranAllowMerge): + mySSH.command(self.cli + ' build --target ' + sharedimage + ' --tag ' + sharedimage + ':' + sharedTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600) # First verify if the shared image was properly created. status = True - mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + sharedimage + ':' + imageTag, '\$', 5) - if (mySSH.getBefore().count('No such object') != 0) or (mySSH.getBefore().count('no such image') != 0): - logging.error('Could not build properly ran-build') + mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + sharedimage + ':' + sharedTag, '\$', 5) + if mySSH.getBefore().count('o such image') != 0: + logging.error('\u001B[1m Could not build properly ran-build\u001B[0m') status = False else: result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) @@ -227,6 +230,7 @@ class Containerize(): # If the shared image failed, no need to continue if not status: # Recover the name of the failed container? + mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) mySSH.command(self.cli + ' image prune --force', '\$', 30) mySSH.close() logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') @@ -235,7 +239,7 @@ class Containerize(): sys.exit(1) else: # Recover build logs, for the moment only possible when build is successful - mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + imageTag, '\$', 5) + mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + sharedTag, '\$', 5) mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) mySSH.command(self.cli + ' rm -f test', '\$', 5) @@ -244,17 +248,19 @@ class Containerize(): for image,pattern in imageNames: # the archived Dockerfiles have "ran-build:latest" as base image # we need to update them with proper tag - mySSH.command('sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) + mySSH.command('sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + sharedTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command(self.cli + ' build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) # split the log mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5) # checking the status of the build mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) - if (mySSH.getBefore().count('No such object') != 0) or (mySSH.getBefore().count('no such image') != 0): - logging.error('Could not build properly ' + image) + if mySSH.getBefore().count('o such image') != 0: + logging.error('\u001B[1m Could not build properly ' + image + '\u001B[0m') status = False # Here we should check if the last container corresponds to a failed command and destroy it + mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) + self.allImagesSize[image] = 'N/A -- Build Failed' else: result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) if result is not None: @@ -274,6 +280,7 @@ class Containerize(): self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes' else: logging.debug('ran-build size is unknown') + self.allImagesSize[image] = 'unknown' # Now pruning dangling images in between target builds mySSH.command(self.cli + ' image prune --force', '\$', 30)