maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify cls_containerize.BuildImage() and drop allImagesSize class member

Browse Source
This commit is contained in:
Robert Schmidt
2022-09-05 15:29:33 +02:00
parent d78ccd64ca
commit 668d0865b8
1 changed files with 36 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+36 -56
View File
@@ -163,7 +163,6 @@ class Containerize():
self.cliBuildOptions = '' self.cliBuildOptions = ''
self.dockerfileprefix = '' self.dockerfileprefix = ''
self.host = '' self.host = ''
self.allImagesSize = {}
self.deployedContainers = [] self.deployedContainers = []
self.tsharkStarted = False self.tsharkStarted = False
@@ -318,32 +317,10 @@ class Containerize():
if forceBaseImageBuild: if forceBaseImageBuild:
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600) mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600)
# First verify if the base image was properly created. # First verify if the base image was properly created.
status = True
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5) mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5)
allImagesSize = {}
if mySSH.getBefore().count('o such image') != 0: if mySSH.getBefore().count('o such image') != 0:
logging.error('\u001B[1m Could not build properly ran-base\u001B[0m') logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
status = False
else:
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
if result is not None:
imageSize = float(result.group('size'))
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' kbytes'
else:
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Mbytes'
else:
imageSize = imageSize / 1000
logging.debug('\u001B[1m ran-base size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Gbytes'
else:
logging.debug('ran-base size is unknown')
# If the base image failed, no need to continue
if not status:
# Recover the name of the failed container? # Recover the name of the failed container?
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
mySSH.command(self.cli + ' image prune --force', '\$', 30) mySSH.command(self.cli + ' image prune --force', '\$', 30)
@@ -353,20 +330,32 @@ class Containerize():
HTML.CreateHtmlTabFooter(False) HTML.CreateHtmlTabFooter(False)
sys.exit(1) sys.exit(1)
else: else:
# Recover build logs, for the moment only possible when build is successful result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5) if result is not None:
mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5) size = float(result.group("size")) / 1000000
mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5) imageSizeStr = f'{size:.1f}'
mySSH.command(self.cli + ' rm -f test', '\$', 5) logging.debug(f'\u001B[1m ran-base size is {imageSizeStr} Mbytes\u001B[0m')
allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
else:
logging.debug('ran-base size is unknown')
# Recover build logs, for the moment only possible when build is successful
mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5)
mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5)
mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5)
mySSH.command(self.cli + ' rm -f test', '\$', 5)
# Build the target image(s) # Build the target image(s)
status = True
attemptedImages = ['ran-base']
for image,pattern in imageNames: for image,pattern in imageNames:
attemptedImages += [image]
# the archived Dockerfiles have "ran-base:latest" as base image # the archived Dockerfiles have "ran-base:latest" as base image
# we need to update them with proper tag # we need to update them with proper tag
mySSH.command('sed -i -e "s#' + baseImage + ':latest#' + baseImage + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
if image != 'ran-build': if image != 'ran-build':
mySSH.command('sed -i -e "s#' + "ran-build" + ':latest#' + "ran-build" + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200)
# Flatten Image # Flatten Image
if image != 'ran-build': if image != 'ran-build':
mySSH.command('python3 ./ci-scripts/flatten_image.py --tag ' + image + ':' + imageTag, '\$', 300) mySSH.command('python3 ./ci-scripts/flatten_image.py --tag ' + image + ':' + imageTag, '\$', 300)
@@ -380,27 +369,18 @@ class Containerize():
status = False status = False
# Here we should check if the last container corresponds to a failed command and destroy it # Here we should check if the last container corresponds to a failed command and destroy it
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
self.allImagesSize[image] = 'N/A -- Build Failed' allImagesSize[image] = 'N/A -- Build Failed'
break
else: else:
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
if result is not None: if result is not None:
imageSize = float(result.group('size')) size = float(result.group("size")) / 1000000
imageSize = imageSize / 1000 imageSizeStr = f'{size:.1f}'
if imageSize < 1000: logging.debug(f'\u001B[1m {image} size is {imageSizeStr} Mbytes\u001B[0m')
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') allImagesSize[image] = f'{imageSizeStr} Mbytes'
self.allImagesSize[image] = str(round(imageSize,1)) + ' kbytes'
else:
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
self.allImagesSize[image] = str(round(imageSize,1)) + ' Mbytes'
else:
imageSize = imageSize / 1000
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes'
else: else:
logging.debug('ran-base size is unknown') logging.debug(f'{image} size is unknown')
self.allImagesSize[image] = 'unknown' allImagesSize[image] = 'unknown'
# Now pruning dangling images in between target builds # Now pruning dangling images in between target builds
mySSH.command(self.cli + ' image prune --force', '\$', 30) mySSH.command(self.cli + ' image prune --force', '\$', 30)
@@ -416,17 +396,16 @@ class Containerize():
mySSH.close() mySSH.close()
# Analyze the logs # Analyze the logs
images = ['ran-base'] + [tpl[0] for tpl in imageNames] collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
collectInfo = AnalyzeBuildLogs(build_log_name, images, status)
if status: if status:
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, self.allImagesSize) HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
else: else:
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, self.allImagesSize) HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
HTML.CreateHtmlTabFooter(False) HTML.CreateHtmlTabFooter(False)
sys.exit(1) sys.exit(1)
@@ -552,13 +531,14 @@ class Containerize():
collectInfo['proxy'] = files collectInfo['proxy'] = files
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5) mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
allImagesSize = {}
if result is not None: if result is not None:
imageSize = float(result.group('size')) / 1000000 imageSize = float(result.group('size')) / 1000000
logging.debug('\u001B[1m proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') logging.debug('\u001B[1m proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
self.allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes' allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes'
else: else:
logging.debug('proxy size is unknown') logging.debug('proxy size is unknown')
self.allImagesSize['proxy'] = 'unknown' allImagesSize['proxy'] = 'unknown'
# Cleaning any created tmp volume # Cleaning any created tmp volume
mySSH.command(self.cli + ' volume prune --force || true','\$', 15) mySSH.command(self.cli + ' volume prune --force || true','\$', 15)
@@ -566,7 +546,7 @@ class Containerize():
logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m') logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m')
HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, self.allImagesSize) HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
def Copy_Image_to_Test_Server(self, HTML): def Copy_Image_to_Test_Server(self, HTML):
imageTag = 'develop' imageTag = 'develop'