maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: build intermediary compilation image

Browse Source 
Change the python script such that we first build the ran-base image
(renamed from ran-build, as it does not build anything), followed by the
ran-build image compiling all dependencies, and the generate the
independent target container images.
This commit is contained in:
Robert Schmidt
2021-12-09 12:12:35 +01:00
parent cffa73bf45
commit 42c2fe5b75
1 changed files with 36 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+36 -35
View File
@@ -177,7 +177,8 @@ class Containerize():
self.dockerfileprefix = '.rhel8.2' self.dockerfileprefix = '.rhel8.2'
self.cliBuildOptions = '--no-cache --disable-compression' self.cliBuildOptions = '--no-cache --disable-compression'
imageNames = [] # we always build the ran-build image with all targets
imageNames = [('ran-build', 'build')]
result = re.search('eNB', self.imageKind) result = re.search('eNB', self.imageKind)
# Creating a tupple with the imageName and the DockerFile prefix pattern on obelix # Creating a tupple with the imageName and the DockerFile prefix pattern on obelix
if result is not None: if result is not None:
@@ -197,8 +198,6 @@ class Containerize():
imageNames.append(('oai-physim', 'phySim')) imageNames.append(('oai-physim', 'phySim'))
if self.host == 'Ubuntu': if self.host == 'Ubuntu':
imageNames.append(('oai-lte-ru', 'lteRU')) imageNames.append(('oai-lte-ru', 'lteRU'))
if len(imageNames) == 0:
imageNames.append(('oai-enb', 'eNB'))
# Workaround for some servers, we need to erase completely the workspace # Workaround for some servers, we need to erase completely the workspace
if self.forcedWorkspaceCleanup: if self.forcedWorkspaceCleanup:
@@ -215,37 +214,37 @@ class Containerize():
mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca/', '\$', 5) mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca/', '\$', 5)
mySSH.command('sudo cp /etc/pki/entitlement/*.pem tmp/entitlement/', '\$', 5) mySSH.command('sudo cp /etc/pki/entitlement/*.pem tmp/entitlement/', '\$', 5)
sharedimage = 'ran-build' baseImage = 'ran-base'
sharedTag = 'develop' baseTag = 'develop'
forceSharedImageBuild = False forceBaseImageBuild = False
imageTag = 'develop' imageTag = 'develop'
if (self.ranAllowMerge): if (self.ranAllowMerge):
imageTag = 'ci-temp' imageTag = 'ci-temp'
if self.ranTargetBranch == 'develop': if self.ranTargetBranch == 'develop':
mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.ran' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5)
result = re.search('index', mySSH.getBefore()) result = re.search('index', mySSH.getBefore())
if result is not None: if result is not None:
forceSharedImageBuild = True forceBaseImageBuild = True
sharedTag = 'ci-temp' baseTag = 'ci-temp'
else: else:
forceSharedImageBuild = True forceBaseImageBuild = True
# Let's remove any previous run artifacts if still there # Let's remove any previous run artifacts if still there
mySSH.command(self.cli + ' image prune --force', '\$', 30) mySSH.command(self.cli + ' image prune --force', '\$', 30)
if forceSharedImageBuild: if forceBaseImageBuild:
mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + sharedTag + ' || true', '\$', 30) mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30)
for image,pattern in imageNames: for image,pattern in imageNames:
mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30) mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30)
# Build the shared image only on Push Events (not on Merge Requests) # Build the base image only on Push Events (not on Merge Requests)
# On when the shared image docker file is being modified. # On when the base image docker file is being modified.
if forceSharedImageBuild: if forceBaseImageBuild:
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + sharedimage + ' --tag ' + sharedimage + ':' + sharedTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600) mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600)
# First verify if the shared image was properly created. # First verify if the base image was properly created.
status = True status = True
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + sharedimage + ':' + sharedTag, '\$', 5) mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5)
if mySSH.getBefore().count('o such image') != 0: if mySSH.getBefore().count('o such image') != 0:
logging.error('\u001B[1m Could not build properly ran-build\u001B[0m') logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
status = False status = False
else: else:
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
@@ -253,20 +252,20 @@ class Containerize():
imageSize = float(result.group('size')) imageSize = float(result.group('size'))
imageSize = imageSize / 1000 imageSize = imageSize / 1000
if imageSize < 1000: if imageSize < 1000:
logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
self.allImagesSize['ran-build'] = str(round(imageSize,1)) + ' kbytes' self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' kbytes'
else: else:
imageSize = imageSize / 1000 imageSize = imageSize / 1000
if imageSize < 1000: if imageSize < 1000:
logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
self.allImagesSize['ran-build'] = str(round(imageSize,1)) + ' Mbytes' self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Mbytes'
else: else:
imageSize = imageSize / 1000 imageSize = imageSize / 1000
logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') logging.debug('\u001B[1m ran-base size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
self.allImagesSize['ran-build'] = str(round(imageSize,1)) + ' Gbytes' self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Gbytes'
else: else:
logging.debug('ran-build size is unknown') logging.debug('ran-base size is unknown')
# If the shared image failed, no need to continue # If the base image failed, no need to continue
if not status: if not status:
# Recover the name of the failed container? # Recover the name of the failed container?
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
@@ -278,16 +277,18 @@ class Containerize():
sys.exit(1) sys.exit(1)
else: else:
# Recover build logs, for the moment only possible when build is successful # Recover build logs, for the moment only possible when build is successful
mySSH.command(self.cli + ' create --name test ' + sharedimage + ':' + sharedTag, '\$', 5) mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5)
mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5)
mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5)
mySSH.command(self.cli + ' rm -f test', '\$', 5) mySSH.command(self.cli + ' rm -f test', '\$', 5)
# Build the target image(s) # Build the target image(s)
for image,pattern in imageNames: for image,pattern in imageNames:
# the archived Dockerfiles have "ran-build:latest" as base image # the archived Dockerfiles have "ran-base:latest" as base image
# we need to update them with proper tag # we need to update them with proper tag
mySSH.command('sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + sharedTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command('sed -i -e "s#' + baseImage + ':latest#' + baseImage + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5)
if image != 'ran-build':
mySSH.command('sed -i -e "s#' + "ran-build" + ':latest#' + "ran-build" + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5)
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200)
# split the log # split the log
mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
@@ -318,7 +319,7 @@ class Containerize():
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes' self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes'
else: else:
logging.debug('ran-build size is unknown') logging.debug('ran-base size is unknown')
self.allImagesSize[image] = 'unknown' self.allImagesSize[image] = 'unknown'
# Now pruning dangling images in between target builds # Now pruning dangling images in between target builds
mySSH.command(self.cli + ' image prune --force', '\$', 30) mySSH.command(self.cli + ' image prune --force', '\$', 30)
@@ -342,8 +343,8 @@ class Containerize():
#Trying to identify the errors and warnings for each built images #Trying to identify the errors and warnings for each built images
imageNames1 = imageNames imageNames1 = imageNames
shared = ('ran-build','ran') base = ('ran-base','ran')
imageNames1.insert(0, shared) imageNames1.insert(0, base)
for image,pattern in imageNames1: for image,pattern in imageNames1:
files = {} files = {}
file_list = [f for f in os.listdir('build_log_' + self.testCase_id + '/' + image) if os.path.isfile(os.path.join('build_log_' + self.testCase_id + '/' + image, f)) and f.endswith('.txt')] file_list = [f for f in os.listdir('build_log_' + self.testCase_id + '/' + image) if os.path.isfile(os.path.join('build_log_' + self.testCase_id + '/' + image, f)) and f.endswith('.txt')]