diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 6594452ca4..08d8938080 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -98,8 +98,9 @@ def AssignParams(params_dict): def GetParametersFromXML(action): - if action == 'Build_eNB': + if action == 'Build_eNB' or action == 'Build_Image': RAN.Build_eNB_args=test.findtext('Build_eNB_args') + RAN.imageKind=test.findtext('kind') forced_workspace_cleanup = test.findtext('forced_workspace_cleanup') if (forced_workspace_cleanup is None): RAN.Build_eNB_forced_workspace_cleanup=False @@ -127,8 +128,9 @@ def GetParametersFromXML(action): else: RAN.backgroundBuild=False - elif action == 'WaitEndBuild_eNB': + elif action == 'WaitEndBuild_eNB' or action == 'WaitEndBuild_Image': RAN.Build_eNB_args=test.findtext('Build_eNB_args') + RAN.imageKind=test.findtext('kind') eNB_instance=test.findtext('eNB_instance') if (eNB_instance is None): RAN.eNB_instance=0 @@ -640,8 +642,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re break if action == 'Build_eNB': RAN.BuildeNB() + elif action == 'Build_Image': + RAN.BuildImage() elif action == 'WaitEndBuild_eNB': RAN.WaitBuildeNBisFinished() + elif action == 'WaitEndBuild_Image': + RAN.WaitBuildImageisFinished() elif action == 'Initialize_eNB': check_eNB = False check_OAI_UE = False diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 96ec8acf0e..e534b31b9c 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -77,6 +77,7 @@ class RANManagement(): self.backgroundBuildTestId = ['', '', ''] self.Build_eNB_forced_workspace_cleanup = False self.Initialize_eNB_args = '' + self.imageKind = '' self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main self.eNB_instance = 0 self.eNB_serverId = ['', '', ''] @@ -318,6 +319,171 @@ class RANManagement(): self.htmlObj.CreateHtmlTabFooter(False) sys.exit(1) + def BuildImage(self): + if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + if self.eNB_serverId[self.eNB_instance] == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Building on server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + + imageNames = [] + result = re.search('eNB', self.imageKind) + if result is not None: + imageNames.append(('oai-enb', 'eNB')) + else: + result = re.search('gNB', self.imageKind) + if result is not None: + imageNames.append(('oai-gnb', 'gNB')) + else: + result = re.search('all', self.imageKind) + if result is not None: + imageNames.append(('oai-enb', 'eNB')) + imageNames.append(('oai-gnb', 'gNB')) + if len(imageNames) == 0: + imageNames.append(('oai-enb', 'eNB')) + # Workaround for some servers, we need to erase completely the workspace + if self.Build_eNB_forced_workspace_cleanup: + mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) + if self.htmlObj is not None: + self.testCase_id = self.htmlObj.testCase_id + else: + self.testCase_id = '000000' + # on RedHat/CentOS .git extension is mandatory + result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) + if result is not None: + full_ran_repo_name = self.ranRepository + else: + full_ran_repo_name = self.ranRepository + '.git' + mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) + mySSH.command('cd ' + lSourcePath, '\$', 5) + mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) + # Raphael: here add a check if git clone or git fetch went smoothly + mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) + + mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + mySSH.command('mkdir -p cmake_targets/log', '\$', 5) + # if the commit ID is provided use it to point to it + if self.ranCommitID != '': + mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should already been checked earlier + imageTag = 'develop' + if (self.ranAllowMerge): + imageTag = 'ci-temp' + if self.ranTargetBranch == '': + if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + else: + logging.debug('Merging with the target branch: ' + self.ranTargetBranch) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + # Let's remove any previous run artifacts if still there + mySSH.command('docker image prune --force', '\$', 5) + mySSH.command('docker image rm ran-build:' + imageTag, '\$', 5) + for image,pattern in imageNames: + mySSH.command('docker image rm ' + image + ':' + imageTag, '\$', 5) + # Build the shared image + mySSH.command('docker build --target ran-build --tag ran-build:' + imageTag + ' --file docker/Dockerfile.ran.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 800) + # Build the target image(s) + previousImage='ran-build:' + imageTag + danglingShaOnes=[] + for image,pattern in imageNames: + mySSH.command('docker build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + '.ubuntu18 . > cmake_targets/log/' + image + '.log 2>&1', '\$', 600) + # Retrieving the dangling image(s) for the log collection + mySSH.command('docker images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5) + result = re.search('sha=(?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) + if result is not None: + danglingShaOnes.append((image, result.group('imageShaOne'))) + previousImage=image + ':' + imageTag + + imageTag = 'ci-temp' + # First verify if images were properly created. + status = True + mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ran-build:' + imageTag, '\$', 5) + if mySSH.getBefore().count('No such object') != 0: + logging.error('Could not build properly ran-build') + status = False + else: + result = re.search('Size = (?P[0-9\-]+) bytes', mySSH.getBefore()) + if result is not None: + imageSize = float(result.group('size')) + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') + else: + logging.debug('ran-build size is unknown') + for image,pattern in imageNames: + mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) + if mySSH.getBefore().count('No such object') != 0: + logging.error('Could not build properly ' + image) + status = False + else: + result = re.search('Size = (?P[0-9\-]+) bytes', mySSH.getBefore()) + if result is not None: + imageSize = float(result.group('size')) + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') + else: + logging.debug('ran-build size is unknown') + if not status: + mySSH.close() + logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') + if self.htmlObj is not None: + self.htmlObj.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) + self.htmlObj.CreateHtmlTabFooter(False) + sys.exit(1) + + # Recover build logs, for the moment only possible when build is successful + mySSH.command('docker create --name test ran-build:' + imageTag, '\$', 5) + mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) + mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/ran-build', '\$', 5) + mySSH.command('docker rm -f test', '\$', 5) + for image,shaone in danglingShaOnes: + mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) + mySSH.command('docker create --name test ' + shaone, '\$', 5) + mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/' + image, '\$', 5) + mySSH.command('docker rm -f test', '\$', 5) + mySSH.command('docker image prune --force', '\$', 5) + mySSH.close() + + logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') + if self.htmlObj is not None: + self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) + def InitializeeNB(self): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index 0ac30075c2..db4d946d18 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -1,7 +1,9 @@ - Build_PhySim - Run_PhySim - Build_eNB + - Build_Image - WaitEndBuild_eNB + - WaitEndBuild_Image - Initialize_eNB - Terminate_eNB - Initialize_UE diff --git a/ci-scripts/xml_files/fr1_image_build.xml b/ci-scripts/xml_files/fr1_image_build.xml new file mode 100644 index 0000000000..0c8be40f17 --- /dev/null +++ b/ci-scripts/xml_files/fr1_image_build.xml @@ -0,0 +1,50 @@ + + + build-tab + Build + wrench + + 000001 + + + + + Build_Image + Build eNB Image + all + 0 + 0 + True + + + + WaitEndBuild_Image + Wait for end of Build eNB + eNB + 0 + 0 + + +