diff --git a/ci-scripts/args_parse.py b/ci-scripts/args_parse.py index 84b3a84514..248e873cdd 100644 --- a/ci-scripts/args_parse.py +++ b/ci-scripts/args_parse.py @@ -41,7 +41,7 @@ import constants as CONST #----------------------------------------------------------- -def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,HELP): +def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP): py_param_file_present = False @@ -79,6 +79,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,HELP): RAN.ranRepository=matchReg.group(1) HTML.ranRepository=matchReg.group(1) ldpc.ranRepository=matchReg.group(1) + CONTAINERS.ranRepository=matchReg.group(1) elif re.match('^\-\-eNB_AllowMerge=(.+)$|^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE) @@ -90,6 +91,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,HELP): CiTestObj.ranAllowMerge = True RAN.ranAllowMerge=True HTML.ranAllowMerge=True + CONTAINERS.ranAllowMerge=True elif re.match('^\-\-eNBBranch=(.+)$|^\-\-ranBranch=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE) @@ -99,6 +101,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,HELP): RAN.ranBranch=matchReg.group(1) HTML.ranBranch=matchReg.group(1) ldpc.ranBranch=matchReg.group(1) + CONTAINERS.ranBranch=matchReg.group(1) elif re.match('^\-\-eNBCommitID=(.*)$|^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE) @@ -108,6 +111,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,HELP): RAN.ranCommitID=matchReg.group(1) HTML.ranCommitID=matchReg.group(1) ldpc.ranCommitID=matchReg.group(1) + CONTAINERS.ranCommitID=matchReg.group(1) elif re.match('^\-\-eNBTargetBranch=(.*)$|^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE) @@ -117,50 +121,63 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,HELP): RAN.ranTargetBranch=matchReg.group(1) HTML.ranTargetBranch=matchReg.group(1) ldpc.ranTargetBranch=matchReg.group(1) + CONTAINERS.ranTargetBranch=matchReg.group(1) elif re.match('^\-\-eNBIPAddress=(.+)$|^\-\-eNB[1-2]IPAddress=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.eNBIPAddress=matchReg.group(1) ldpc.eNBIpAddr=matchReg.group(1) + CONTAINERS.eNBIPAddress=matchReg.group(1) elif re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.eNB1IPAddress=matchReg.group(1) + CONTAINERS.eNB1IPAddress=matchReg.group(1) elif re.match('^\-\-eNB2IPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB2IPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.eNB2IPAddress=matchReg.group(1) + CONTAINERS.eNB2IPAddress=matchReg.group(1) elif re.match('^\-\-eNBUserName=(.+)$|^\-\-eNB[1-2]UserName=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE) RAN.eNBUserName=matchReg.group(1) ldpc.eNBUserName=matchReg.group(1) + CONTAINERS.eNBUserName=matchReg.group(1) elif re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE) RAN.eNB1UserName=matchReg.group(1) + CONTAINERS.eNB1UserName=matchReg.group(1) elif re.match('^\-\-eNB2UserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB2UserName=(.+)$', myArgv, re.IGNORECASE) RAN.eNB2UserName=matchReg.group(1) + CONTAINERS.eNB2UserName=matchReg.group(1) elif re.match('^\-\-eNBPassword=(.+)$|^\-\-eNB[1-2]Password=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE) RAN.eNBPassword=matchReg.group(1) ldpc.eNBPassWord=matchReg.group(1) + CONTAINERS.eNBPassword=matchReg.group(1) elif re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE) RAN.eNB1Password=matchReg.group(1) + CONTAINERS.eNB1Password=matchReg.group(1) elif re.match('^\-\-eNB2Password=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB2Password=(.+)$', myArgv, re.IGNORECASE) RAN.eNB2Password=matchReg.group(1) + CONTAINERS.eNB2Password=matchReg.group(1) elif re.match('^\-\-eNBSourceCodePath=(.+)$|^\-\-eNB[1-2]SourceCodePath=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.eNBSourceCodePath=matchReg.group(1) ldpc.eNBSourceCodePath=matchReg.group(1) + CONTAINERS.eNBSourceCodePath=matchReg.group(1) elif re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.eNB1SourceCodePath=matchReg.group(1) + CONTAINERS.eNB1SourceCodePath=matchReg.group(1) elif re.match('^\-\-eNB2SourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB2SourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.eNB2SourceCodePath=matchReg.group(1) + CONTAINERS.eNB2SourceCodePath=matchReg.group(1) elif re.match('^\-\-EPCIPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-EPCIPAddress=(.+)$', myArgv, re.IGNORECASE) EPC.IPAddress=matchReg.group(1) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py new file mode 100644 index 0000000000..a4de458af0 --- /dev/null +++ b/ci-scripts/cls_containerize.py @@ -0,0 +1,260 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Python for CI of OAI-eNB + COTS-UE +# +# Required Python Version +# Python 3.x +# +# Required Python Package +# pexpect +#--------------------------------------------------------------------- + +#----------------------------------------------------------- +# Import +#----------------------------------------------------------- +import sys # arg +import re # reg +import logging +import os +import time +from multiprocessing import Process, Lock, SimpleQueue + +#----------------------------------------------------------- +# OAI Testing modules +#----------------------------------------------------------- +import sshconnection as SSH +import epc +import helpreadme as HELP +import constants as CONST +import html + +#----------------------------------------------------------- +# Class Declaration +#----------------------------------------------------------- +class Containerize(): + + def __init__(self): + + self.ranRepository = '' + self.ranBranch = '' + self.ranAllowMerge = False + self.ranCommitID = '' + self.ranTargetBranch = '' + self.eNBIPAddress = '' + self.eNBUserName = '' + self.eNBPassword = '' + self.eNBSourceCodePath = '' + self.eNB1IPAddress = '' + self.eNB1UserName = '' + self.eNB1Password = '' + self.eNB1SourceCodePath = '' + self.eNB2IPAddress = '' + self.eNB2UserName = '' + self.eNB2Password = '' + self.eNB2SourceCodePath = '' + self.forcedWorkspaceCleanup = False + self.imageKind = '' + self.eNB_instance = 0 + self.eNB_serverId = ['', '', ''] + self.testCase_id = '' + self.htmlObj = None + self.epcObj = None + +#----------------------------------------------------------- +# Container management functions +#----------------------------------------------------------- + + def BuildImage(self): + if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + if self.eNB_serverId[self.eNB_instance] == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + logging.debug('lIpAddr = ' + lIpAddr + ' lUserName = ' + lUserName + ' lPassWord = ' + lPassWord + ' lSourcePath = ' + lSourcePath) + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Building on server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + + imageNames = [] + result = re.search('eNB', self.imageKind) + # Creating a tupple with the imageName and the DockerFile prefix pattern + if result is not None: + imageNames.append(('oai-enb', 'eNB')) + else: + result = re.search('gNB', self.imageKind) + if result is not None: + imageNames.append(('oai-gnb', 'gNB')) + else: + result = re.search('all', self.imageKind) + if result is not None: + imageNames.append(('oai-enb', 'eNB')) + imageNames.append(('oai-gnb', 'gNB')) + imageNames.append(('oai-lte-ue', 'lteUE')) + imageNames.append(('oai-nr-ue', 'nrUE')) + if len(imageNames) == 0: + imageNames.append(('oai-enb', 'eNB')) + # Workaround for some servers, we need to erase completely the workspace + if self.forcedWorkspaceCleanup: + mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) + if self.htmlObj is not None: + self.testCase_id = self.htmlObj.testCase_id + else: + self.testCase_id = '000000' + # on RedHat/CentOS .git extension is mandatory + result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) + if result is not None: + full_ran_repo_name = self.ranRepository + else: + full_ran_repo_name = self.ranRepository + '.git' + mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) + mySSH.command('cd ' + lSourcePath, '\$', 5) + mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) + # Raphael: here add a check if git clone or git fetch went smoothly + mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) + + mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + mySSH.command('mkdir -p cmake_targets/log', '\$', 5) + # if the commit ID is provided use it to point to it + if self.ranCommitID != '': + mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should already been checked earlier + imageTag = 'develop' + if (self.ranAllowMerge): + imageTag = 'ci-temp' + if self.ranTargetBranch == '': + if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + else: + logging.debug('Merging with the target branch: ' + self.ranTargetBranch) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + # Let's remove any previous run artifacts if still there + mySSH.command('docker image prune --force', '\$', 5) + mySSH.command('docker image rm ran-build:' + imageTag, '\$', 5) + for image,pattern in imageNames: + mySSH.command('docker image rm ' + image + ':' + imageTag, '\$', 5) + # Build the shared image + mySSH.command('docker build --target ran-build --tag ran-build:' + imageTag + ' --file docker/Dockerfile.ran.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 800) + # Build the target image(s) + previousImage='ran-build:' + imageTag + danglingShaOnes=[] + for image,pattern in imageNames: + # the archived Dockerfiles have "ran-build:latest" as base image + # we need to update them with proper tag + mySSH.command('sed -i -e "s#ran-build:latest#ran-build:' + imageTag + '#" docker/Dockerfile.' + pattern + '.ubuntu18', '\$', 5) + mySSH.command('docker build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + '.ubuntu18 . > cmake_targets/log/' + image + '.log 2>&1', '\$', 600) + # Retrieving the dangling image(s) for the log collection + mySSH.command('docker images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5) + result = re.search('sha=(?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) + if result is not None: + danglingShaOnes.append((image, result.group('imageShaOne'))) + previousImage=image + ':' + imageTag + + imageTag = 'ci-temp' + # First verify if images were properly created. + status = True + mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ran-build:' + imageTag, '\$', 5) + if mySSH.getBefore().count('No such object') != 0: + logging.error('Could not build properly ran-build') + status = False + else: + result = re.search('Size = (?P[0-9\-]+) bytes', mySSH.getBefore()) + if result is not None: + imageSize = float(result.group('size')) + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') + else: + logging.debug('ran-build size is unknown') + for image,pattern in imageNames: + mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) + if mySSH.getBefore().count('No such object') != 0: + logging.error('Could not build properly ' + image) + status = False + else: + result = re.search('Size = (?P[0-9\-]+) bytes', mySSH.getBefore()) + if result is not None: + imageSize = float(result.group('size')) + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + if imageSize < 1000: + logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') + else: + imageSize = imageSize / 1000 + logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') + else: + logging.debug('ran-build size is unknown') + if not status: + mySSH.close() + logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') + if self.htmlObj is not None: + self.htmlObj.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) + self.htmlObj.CreateHtmlTabFooter(False) + sys.exit(1) + + # Recover build logs, for the moment only possible when build is successful + mySSH.command('docker create --name test ran-build:' + imageTag, '\$', 5) + mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) + mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) + mySSH.command('docker rm -f test', '\$', 5) + for image,shaone in danglingShaOnes: + mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) + mySSH.command('docker create --name test ' + shaone, '\$', 5) + mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/' + image, '\$', 5) + mySSH.command('docker rm -f test', '\$', 5) + mySSH.command('docker image prune --force', '\$', 5) + mySSH.command('cd cmake_targets', '\$', 5) + mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) + mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) + mySSH.close() + + logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') + if self.htmlObj is not None: + self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) + diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 1b350ad195..b7583ff753 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -41,6 +41,7 @@ import constants as CONST import cls_oaicitest #main class for OAI CI test framework import cls_physim #class PhySim for physical simulators build and test import cls_cots_ue #class CotsUe for Airplane mode control +import cls_containerize #class Containerize for all container-based operations on RAN/UE objects import sshconnection @@ -100,25 +101,32 @@ def AssignParams(params_dict): def GetParametersFromXML(action): if action == 'Build_eNB' or action == 'Build_Image': RAN.Build_eNB_args=test.findtext('Build_eNB_args') - RAN.imageKind=test.findtext('kind') + CONTAINERS.imageKind=test.findtext('kind') forced_workspace_cleanup = test.findtext('forced_workspace_cleanup') if (forced_workspace_cleanup is None): RAN.Build_eNB_forced_workspace_cleanup=False + CONTAINERS.forcedWorkspaceCleanup=False else: if re.match('true', forced_workspace_cleanup, re.IGNORECASE): RAN.Build_eNB_forced_workspace_cleanup=True + CONTAINERS.forcedWorkspaceCleanup=True else: - RAN.Build_eNB_forced_workspace_cleanup=False + RAN.Build_eNB_forced_workspace_cleanup=True + CONTAINERS.forcedWorkspaceCleanup=False eNB_instance=test.findtext('eNB_instance') if (eNB_instance is None): RAN.eNB_instance=0 + CONTAINERS.eNB_instance=0 else: RAN.eNB_instance=int(eNB_instance) + CONTAINERS.eNB_instance=int(eNB_instance) eNB_serverId=test.findtext('eNB_serverId') if (eNB_serverId is None): RAN.eNB_serverId[RAN.eNB_instance]='0' + CONTAINERS.eNB_serverId[RAN.eNB_instance]='0' else: RAN.eNB_serverId[RAN.eNB_instance]=eNB_serverId + CONTAINERS.eNB_serverId[CONTAINERS.eNB_instance]=eNB_serverId xmlBgBuildField = test.findtext('backgroundBuild') if (xmlBgBuildField is None): RAN.backgroundBuild=False @@ -130,7 +138,6 @@ def GetParametersFromXML(action): elif action == 'WaitEndBuild_eNB': RAN.Build_eNB_args=test.findtext('Build_eNB_args') - RAN.imageKind=test.findtext('kind') eNB_instance=test.findtext('eNB_instance') if (eNB_instance is None): RAN.eNB_instance=0 @@ -354,10 +361,13 @@ SSH = sshconnection.SSHConnection() EPC = epc.EPCManagement() RAN = ran.RANManagement() HTML = html.HTMLManagement() +CONTAINERS = cls_containerize.Containerize() EPC.htmlObj=HTML RAN.htmlObj=HTML RAN.epcObj=EPC +CONTAINERS.htmlObj=HTML +CONTAINERS.epcObj=EPC ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU build @@ -369,7 +379,7 @@ ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU b #----------------------------------------------------------- import args_parse -py_param_file_present, py_params, mode = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,EPC,ldpc,HELP) +py_param_file_present, py_params, mode = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP) @@ -643,7 +653,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re if action == 'Build_eNB': RAN.BuildeNB() elif action == 'Build_Image': - RAN.BuildImage() + CONTAINERS.BuildImage() elif action == 'WaitEndBuild_eNB': RAN.WaitBuildeNBisFinished() elif action == 'Initialize_eNB': diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 1df082a127..f78417e22c 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -77,7 +77,6 @@ class RANManagement(): self.backgroundBuildTestId = ['', '', ''] self.Build_eNB_forced_workspace_cleanup = False self.Initialize_eNB_args = '' - self.imageKind = '' self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main self.eNB_instance = 0 self.eNB_serverId = ['', '', ''] @@ -321,174 +320,6 @@ class RANManagement(): self.htmlObj.CreateHtmlTabFooter(False) sys.exit(1) - def BuildImage(self): - if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - if self.eNB_serverId[self.eNB_instance] == '0': - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - lSourcePath = self.eNBSourceCodePath - elif self.eNB_serverId[self.eNB_instance] == '1': - lIpAddr = self.eNB1IPAddress - lUserName = self.eNB1UserName - lPassWord = self.eNB1Password - lSourcePath = self.eNB1SourceCodePath - elif self.eNB_serverId[self.eNB_instance] == '2': - lIpAddr = self.eNB2IPAddress - lUserName = self.eNB2UserName - lPassWord = self.eNB2Password - lSourcePath = self.eNB2SourceCodePath - if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - logging.debug('Building on server: ' + lIpAddr) - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - - imageNames = [] - result = re.search('eNB', self.imageKind) - if result is not None: - imageNames.append(('oai-enb', 'eNB')) - else: - result = re.search('gNB', self.imageKind) - if result is not None: - imageNames.append(('oai-gnb', 'gNB')) - else: - result = re.search('all', self.imageKind) - if result is not None: - imageNames.append(('oai-enb', 'eNB')) - imageNames.append(('oai-gnb', 'gNB')) - if len(imageNames) == 0: - imageNames.append(('oai-enb', 'eNB')) - # Workaround for some servers, we need to erase completely the workspace - if self.Build_eNB_forced_workspace_cleanup: - mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) - if self.htmlObj is not None: - self.testCase_id = self.htmlObj.testCase_id - else: - self.testCase_id = '000000' - # on RedHat/CentOS .git extension is mandatory - result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) - if result is not None: - full_ran_repo_name = self.ranRepository - else: - full_ran_repo_name = self.ranRepository + '.git' - mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) - mySSH.command('cd ' + lSourcePath, '\$', 5) - mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) - # Raphael: here add a check if git clone or git fetch went smoothly - mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) - mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) - - mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) - mySSH.command('mkdir -p cmake_targets/log', '\$', 5) - # if the commit ID is provided use it to point to it - if self.ranCommitID != '': - mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) - # if the branch is not develop, then it is a merge request and we need to do - # the potential merge. Note that merge conflicts should already been checked earlier - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' - if self.ranTargetBranch == '': - if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) - else: - logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) - # Let's remove any previous run artifacts if still there - mySSH.command('docker image prune --force', '\$', 5) - mySSH.command('docker image rm ran-build:' + imageTag, '\$', 5) - for image,pattern in imageNames: - mySSH.command('docker image rm ' + image + ':' + imageTag, '\$', 5) - # Build the shared image - mySSH.command('docker build --target ran-build --tag ran-build:' + imageTag + ' --file docker/Dockerfile.ran.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 800) - # Build the target image(s) - previousImage='ran-build:' + imageTag - danglingShaOnes=[] - for image,pattern in imageNames: - mySSH.command('docker build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + '.ubuntu18 . > cmake_targets/log/' + image + '.log 2>&1', '\$', 600) - # Retrieving the dangling image(s) for the log collection - mySSH.command('docker images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5) - result = re.search('sha=(?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) - if result is not None: - danglingShaOnes.append((image, result.group('imageShaOne'))) - previousImage=image + ':' + imageTag - - imageTag = 'ci-temp' - # First verify if images were properly created. - status = True - mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ran-build:' + imageTag, '\$', 5) - if mySSH.getBefore().count('No such object') != 0: - logging.error('Could not build properly ran-build') - status = False - else: - result = re.search('Size = (?P[0-9\-]+) bytes', mySSH.getBefore()) - if result is not None: - imageSize = float(result.group('size')) - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') - else: - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') - else: - imageSize = imageSize / 1000 - logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') - else: - logging.debug('ran-build size is unknown') - for image,pattern in imageNames: - mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) - if mySSH.getBefore().count('No such object') != 0: - logging.error('Could not build properly ' + image) - status = False - else: - result = re.search('Size = (?P[0-9\-]+) bytes', mySSH.getBefore()) - if result is not None: - imageSize = float(result.group('size')) - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') - else: - imageSize = imageSize / 1000 - if imageSize < 1000: - logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') - else: - imageSize = imageSize / 1000 - logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') - else: - logging.debug('ran-build size is unknown') - if not status: - mySSH.close() - logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') - if self.htmlObj is not None: - self.htmlObj.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) - self.htmlObj.CreateHtmlTabFooter(False) - sys.exit(1) - - # Recover build logs, for the moment only possible when build is successful - mySSH.command('docker create --name test ran-build:' + imageTag, '\$', 5) - mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) - mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) - mySSH.command('docker rm -f test', '\$', 5) - for image,shaone in danglingShaOnes: - mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) - mySSH.command('docker create --name test ' + shaone, '\$', 5) - mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/' + image, '\$', 5) - mySSH.command('docker rm -f test', '\$', 5) - mySSH.command('docker image prune --force', '\$', 5) - mySSH.command('cd cmake_targets', '\$', 5) - mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) - mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) - mySSH.close() - - logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') - if self.htmlObj is not None: - self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) - def InitializeeNB(self): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress diff --git a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml b/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml new file mode 100644 index 0000000000..21c88bc1ab --- /dev/null +++ b/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml @@ -0,0 +1,43 @@ +version: '3.8' + +services: + enb_mono_fdd: + image: oai-enb:latest + privileged: true + container_name: prod-enb-mono-fdd + environment: + USE_FDD_MONO: 'yes' + USE_B2XX: 'yes' + ENB_NAME: eNB-in-docker + MCC: '222' + MNC: '01' + MNC_LENGTH: 2 + TAC: 1 + UTRA_BAND_ID: 7 + DL_FREQUENCY_IN_MHZ: 2680 + UL_FREQUENCY_OFFSET_IN_MHZ: 120 + NID_CELL: 10 + NB_PRB: 25 + MME_S1C_IP_ADDRESS: CI_MME_IP_ADDR + ENB_S1C_IF_NAME: eth0 + ENB_S1C_IP_ADDRESS: 192.168.61.30 + ENB_S1U_IF_NAME: eth0 + ENB_S1U_IP_ADDRESS: 192.168.61.30 + ENB_X2_IP_ADDRESS: 192.168.61.30 + FLEXRAN_ENABLED: 'no' + FLEXRAN_INTERFACE_NAME: eth0 + FLEXRAN_IPV4_ADDRESS: CI_FLEXRAN_CTL_IP_ADDR + USE_ADDITIONAL_OPTIONS: '--RUs.[0].max_rxgain 118 --RUs.[0].max_pdschReferenceSignalPower -27 --eNBs.[0].component_carriers.[0].pucch_p0_Nominal -96' + volumes: + - /dev:/dev + networks: + public_net: + ipv4_address: 192.168.61.30 + +networks: + public_net: + name: prod-oai-public-net + ipam: + config: + - subnet: 192.168.61.0/26 + diff --git a/docker/Dockerfile.eNB.ubuntu18 b/docker/Dockerfile.eNB.ubuntu18 index 1349a0c7e1..cdfb37ebfc 100644 --- a/docker/Dockerfile.eNB.ubuntu18 +++ b/docker/Dockerfile.eNB.ubuntu18 @@ -38,7 +38,7 @@ RUN /bin/sh oaienv && \ RUN apt-get install -y python3-pip && \ pip3 install --ignore-installed pyyaml && \ - python3 ./docker/scripts/generateTemplate.py ./docker/scripts/parameters.yaml + python3 ./docker/scripts/generateTemplate.py ./docker/scripts/enb_parameters.yaml # debug #RUN ldconfig -v && ldd /oai-ran/targets/bin/lte-softmodem.Rel15 diff --git a/docker/scripts/enb_entrypoint.sh b/docker/scripts/enb_entrypoint.sh index 110030c139..31d99b2a67 100755 --- a/docker/scripts/enb_entrypoint.sh +++ b/docker/scripts/enb_entrypoint.sh @@ -2,7 +2,7 @@ set -euo pipefail -# Based another env var +# Based another env var, pick one template to use if [[ -v USE_FDD_CU ]]; then ln -s /opt/oai-enb/etc/cu.fdd.conf /opt/oai-enb/etc/enb.conf; fi if [[ -v USE_FDD_DU ]]; then ln -s /opt/oai-enb/etc/du.fdd.conf /opt/oai-enb/etc/enb.conf; fi if [[ -v USE_FDD_MONO ]]; then ln -s /opt/oai-enb/etc/enb.fdd.conf /opt/oai-enb/etc/enb.conf; fi @@ -11,6 +11,7 @@ if [[ -v USE_FDD_RCC ]]; then ln -s /opt/oai-enb/etc/rcc.if4p5.enb.fdd.conf /opt if [[ -v USE_FDD_RRU ]]; then ln -s /opt/oai-enb/etc/rru.fdd.conf /opt/oai-enb/etc/enb.conf; fi if [[ -v USE_TDD_RRU ]]; then ln -s /opt/oai-enb/etc/rru.tdd.conf /opt/oai-enb/etc/enb.conf; fi +# Only this template will be manipulated CONFIG_FILES=`ls /opt/oai-enb/etc/enb.conf` for c in ${CONFIG_FILES}; do @@ -21,7 +22,7 @@ for c in ${CONFIG_FILES}; do # with the value of the environment variable "VAR" EXPRESSIONS="" for v in ${VARS}; do - NEW_VAR=`echo $v | sed -e "s#@##g"` + NEW_VAR=`echo $v | sed -e "s#@##g"` if [[ "${!NEW_VAR}x" == "x" ]]; then echo "Error: Environment variable '${NEW_VAR}' is not set." \ "Config file '$(basename $c)' requires all of $VARS." @@ -35,10 +36,30 @@ for c in ${CONFIG_FILES}; do sed -i "${EXPRESSIONS}" ${c} done -# Later-on, how to add command lines options? - # Load the USRP binaries -# --> later conditional (when doing simulators) -/usr/lib/uhd/utils/uhd_images_downloader.py +if [[ -v USE_B2XX ]]; then + /usr/lib/uhd/utils/uhd_images_downloader.py -t b2xx +elif [[ -v USE_X3XX ]]; then + /usr/lib/uhd/utils/uhd_images_downloader.py -t x3xx +elif [[ -v USE_N3XX ]]; then + /usr/lib/uhd/utils/uhd_images_downloader.py -t n3xx +fi -exec "$@" +echo "==================================" +echo "== Starting eNB soft modem" +if [[ -v USE_ADDITIONAL_OPTIONS ]]; then + echo "Additional option(s): ${USE_ADDITIONAL_OPTIONS}" + new_args=() + while [[ $# -gt 0 ]]; do + new_args+=("$1") + shift + done + for word in ${USE_ADDITIONAL_OPTIONS}; do + new_args+=("$word") + done + echo "${new_args[@]}" + exec "${new_args[@]}" +else + echo "$@" + exec "$@" +fi