From 76638ec283b2e5f8f68177c08971b39d5b3a4cec Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 19 Mar 2021 13:55:01 +0100 Subject: [PATCH] [CI][DOCKER] handling the case of build not successful Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 68 ++++++++++++------------- ci-scripts/docker_log_split.py | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 36 deletions(-) create mode 100644 ci-scripts/docker_log_split.py diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index d58e49b20b..1e9d93ef4a 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -192,10 +192,11 @@ class Containerize(): sharedimage = 'ran-build' # Let's remove any previous run artifacts if still there - mySSH.command(self.cli + ' image prune --force', '\$', 5) - mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + imageTag, '\$', 5) + mySSH.command(self.cli + ' image prune --force', '\$', 30) + mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + imageTag, '\$', 30) for image,pattern in imageNames: - mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag, '\$', 5) + mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag, '\$', 30) + # Build the shared image mySSH.command(self.cli + ' build --target ' + sharedimage + ' --tag ' + sharedimage + ':' + imageTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600) # First verify if the shared image was properly created. @@ -225,31 +226,35 @@ class Containerize(): logging.debug('ran-build size is unknown') # If the shared image failed, no need to continue if not status: + # Recover the name of the failed container? + mySSH.command(self.cli + ' image prune --force', '\$', 30) mySSH.close() logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTabFooter(False) sys.exit(1) + else: + # Recover build logs, for the moment only possible when build is successful + mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + imageTag, '\$', 5) + mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) + mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) + mySSH.command(self.cli + ' rm -f test', '\$', 5) # Build the target image(s) - previousImage = sharedimage + ':' + imageTag - danglingShaOnes=[] for image,pattern in imageNames: # the archived Dockerfiles have "ran-build:latest" as base image # we need to update them with proper tag mySSH.command('sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command(self.cli + ' build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) - # Retrieving the dangling image(s) for the log collection - mySSH.command(self.cli + ' images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5) - result = re.search('sha=(?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) - if result is not None: - danglingShaOnes.append((image, result.group('imageShaOne'))) - previousImage = image + ':' + imageTag # checking the status of the build mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) + # split the log + mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) + mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5) if (mySSH.getBefore().count('No such object') != 0) or (mySSH.getBefore().count('no such image') != 0): logging.error('Could not build properly ' + image) status = False + # Here we should check if the last container corresponds to a failed command and destroy it else: result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) if result is not None: @@ -269,29 +274,13 @@ class Containerize(): self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes' else: logging.debug('ran-build size is unknown') - if not status: - mySSH.close() - logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') - HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) - #HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize) - HTML.CreateHtmlTabFooter(False) - sys.exit(1) + # Now pruning dangling images in between target builds + mySSH.command(self.cli + ' image prune --force', '\$', 30) - # Recover build logs, for the moment only possible when build is successful - mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + imageTag, '\$', 5) - mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) - mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) - mySSH.command(self.cli + ' rm -f test', '\$', 5) - for image,shaone in danglingShaOnes: - mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) - mySSH.command(self.cli + ' create --name test ' + shaone, '\$', 5) - mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/' + image, '\$', 5) - mySSH.command(self.cli + ' rm -f test', '\$', 5) - mySSH.command(self.cli + ' image prune --force', '\$', 5) - mySSH.command('cd cmake_targets', '\$', 5) - mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) + # Analyzing the logs + mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5) mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) - + mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5) mySSH.command('rm -f build_log_' + self.testCase_id + '.zip || true', '\$', 5) if (os.path.isfile('./build_log_' + self.testCase_id + '.zip')): @@ -303,7 +292,7 @@ class Containerize(): mySSH.command('rm -f build_log_' + self.testCase_id + '.zip','\$', 5) mySSH.close() ZipFile('build_log_' + self.testCase_id + '.zip').extractall('.') - + #Trying to identify the errors and warnings for each built images imageNames1 = imageNames shared = ('ran-build','ran') @@ -335,9 +324,16 @@ class Containerize(): files[fil] = errorandwarnings self.collectInfo[image] = files - logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') - HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) - HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize) + if status: + logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') + HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize) + else: + logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') + HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize) + HTML.CreateHtmlTabFooter(False) + sys.exit(1) def DeployObject(self, HTML, EPC): if self.eNB_serverId[self.eNB_instance] == '0': diff --git a/ci-scripts/docker_log_split.py b/ci-scripts/docker_log_split.py new file mode 100644 index 0000000000..8c3c9e41dc --- /dev/null +++ b/ci-scripts/docker_log_split.py @@ -0,0 +1,92 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Python for CI of OAI-eNB + COTS-UE +# +# Required Python Version +# Python 3.x +# +# Required Python Package +# pexpect +#--------------------------------------------------------------------- + + +#----------------------------------------------------------- +# Import Libs +#----------------------------------------------------------- +import sys # arg +import re # reg +import os +import subprocess + + +class SplitReport(): + def __init__(self): + self.logfilename = '' + self.destinationFolder = '' + + def split(self): + self.destinationFolder = self.logfilename.replace(".log","") + if os.path.isfile(self.logfilename): + newImageLog = open(self.logfilename + '.new', 'w') + copyFlag = True + with open(self.logfilename, 'r') as imageLog: + for line in imageLog: + header = False + ret = re.search('====== Start of log for ([0-9\.A-Za-z\-\_]+) ======', line) + if ret is not None: + copyFlag = False + header = True + detailedLogFile = open(self.destinationFolder + '/' + ret.group(1), 'w') + if copyFlag: + newImageLog.write(line) + ret = re.search('====== End of log for ([0-9\.A-Za-z\-\_]+) ======', line) + if ret is not None: + copyFlag = True + detailedLogFile.close() + elif not copyFlag and not header: + detailedLogFile.write(line) + imageLog.close() + newImageLog.close() + os.rename(self.logfilename + '.new', self.logfilename) + else: + print('Cannot split unfound file') + +#-------------------------------------------------------------------------------------------------------- +# +# Start of main +# +#-------------------------------------------------------------------------------------------------------- + +argvs = sys.argv +argc = len(argvs) + +SP = SplitReport() + +while len(argvs) > 1: + myArgv = argvs.pop(1) + if re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE) + SP.logfilename = matchReg.group(1) + +SP.split() + +sys.exit(0)