diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py new file mode 100644 index 0000000000..4ca36b8cf0 --- /dev/null +++ b/ci-scripts/cls_physim.py @@ -0,0 +1,191 @@ +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Python for CI of OAI-eNB + COTS-UE +# +# Required Python Version +# Python 3.x +# +# Required Python Package +# pexpect +#--------------------------------------------------------------------- + +#to use logging.info() +import logging +#to create a SSH object locally in the methods +import sshconnection +#to update the HTML object +import html +from multiprocessing import SimpleQueue +#for log folder maintenance +import os + +class PhySim: + def __init__(self): + self.buildargs = "" + self.runargs = "" + self.eNBIpAddr = "" + self.eNBUserName = "" + self.eNBPassWord = "" + self.eNBSourceCodePath = "" + self.ranRepository = "" + self.ranBranch = "" + self.ranCommitID= "" + self.ranAllowMerge= "" + self.ranTargetBranch= "" + self.exitStatus=0 + self.forced_workspace_cleanup=False + #private attributes + self.__workSpacePath='' + self.__buildLogFile='compile_phy_sim.log' + self.__runLogFile='' + self.__runResults=[] + self.__runLogPath='phy_sim_logs' + + +#----------------- +#PRIVATE Methods +#----------------- + + def __CheckResults_PhySim(self,HTML,CONST,testcase_id): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + #retrieve run log file and store it locally$ + mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.__workSpacePath+self.__runLogFile, '.') + mySSH.close() + #parse results looking for Encoding and Decoding mean values + self.__runResults=[] + with open(self.__runLogFile) as f: + for line in f: + if 'mean' in line: + self.__runResults.append(line) + #the values are appended for each mean value (2), so we take these 2 values from the list + info=self.__runResults[0]+self.__runResults[1] + + #once parsed move the local logfile to its folder for tidiness + os.system('mv '+self.__runLogFile+' '+ self.__runLogPath+'/.') + + #updating the HTML with results + html_cell = '
' + info  + '
' + html_queue=SimpleQueue() + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue(self.runargs, 'OK', 1, html_queue) + return HTML + + + def __CheckBuild_PhySim(self, HTML, CONST): + self.__workSpacePath=self.eNBSourceCodePath+'/cmake_targets/' + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + #retrieve compile log file and store it locally + mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.__workSpacePath+self.__buildLogFile, '.') + #delete older run log file + mySSH.command('rm ' + self.__workSpacePath+self.__runLogFile, '\$', 5) + mySSH.close() + #check build result from local compile log file + buildStatus=False + with open(self.__buildLogFile) as f: + #nr_prachsim is the last compile step + if 'nr_prachsim compiled' in f.read(): + buildStatus=True + #update HTML based on build status + if buildStatus: + HTML.CreateHtmlTestRow(self.buildargs, 'OK', CONST.ALL_PROCESSES_OK, 'LDPC') + self.exitStatus=0 + else: + logging.error('\u001B[1m Building Physical Simulators Failed\u001B[0m') + HTML.CreateHtmlTestRow(self.buildargs, 'KO', CONST.ALL_PROCESSES_OK, 'LDPC') + HTML.CreateHtmlTabFooter(False) + #exitStatus=1 will do a sys.exit in main + self.exitStatus=1 + return HTML + + +#-----------------$ +#PUBLIC Methods$ +#-----------------$ + + def Build_PhySim(self,htmlObj,constObj): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + + #create working dir + mySSH.command('mkdir -p ' + self.eNBSourceCodePath, '\$', 5) + mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) + + if not self.ranRepository.lower().endswith('.git'): + self.ranRepository+='.git' + + #git clone + mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) + #git config + mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) + + #git clean depending on self.forced_workspace_cleanup captured in xml + if self.forced_workspace_cleanup==True: + logging.info('Cleaning workspace ...') + mySSH.command('echo ' + self.eNBPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + else: + logging.info('Workspace cleaning was disabled') + + # if the commit ID is provided, use it to point to it + if self.ranCommitID != '': + mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should have already been checked earlier + if (self.ranAllowMerge): + if self.ranTargetBranch == '': + if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + else: + logging.info('Merging with the target branch: ' + self.ranTargetBranch) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + + #build + mySSH.command('source oaienv', '\$', 5) + mySSH.command('cd cmake_targets', '\$', 5) + mySSH.command('mkdir -p log', '\$', 5) + mySSH.command('chmod 777 log', '\$', 5) + mySSH.command('stdbuf -o0 ./build_oai ' + self.buildargs + ' 2>&1 | stdbuf -o0 tee ' + self.__buildLogFile, 'Bypassing the Tests|build have failed', 1500) + + mySSH.close() + #check build status and update HTML object + lHTML = html.HTMLManagement() + lHTML=self.__CheckBuild_PhySim(htmlObj,constObj) + return lHTML + + + def Run_PhySim(self,htmlObj,constObj,testcase_id): + #create run logs folder locally + os.system('mkdir -p ./'+self.__runLogPath) + #log file is tc_.log remotely + self.__runLogFile='physim_'+str(testcase_id)+'.log' + #open a session for test run + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + mySSH.command('cd '+self.__workSpacePath,'\$',5) + #run and redirect the results to a log file + mySSH.command(self.__workSpacePath+'phy_simulators/build/ldpctest ' + self.runargs + ' >> '+self.__runLogFile, '\$', 30) + mySSH.close() + #return updated HTML to main + lHTML = html.HTMLManagement() + lHTML=self.__CheckResults_PhySim(htmlObj,constObj,testcase_id) + return lHTML diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 56c6ebead8..6bec183255 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -30,6 +30,7 @@ import constants as CONST + #----------------------------------------------------------- # Import #----------------------------------------------------------- @@ -3066,7 +3067,7 @@ class OaiCiTest(): logging.debug('\u001B[1m----------------------------------------\u001B[0m') def CheckClassValidity(action,id): - if action != 'Build_eNB' and action != 'WaitEndBuild_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'DataDisable_UE' and action != 'DataEnable_UE' and action != 'CheckStatusUE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_FlexranCtrl' and action != 'Terminate_FlexranCtrl' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep' and action != 'Perform_X2_Handover': + if action!='Build_PhySim' and action!='Run_PhySim' and action != 'Build_eNB' and action != 'WaitEndBuild_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'DataDisable_UE' and action != 'DataEnable_UE' and action != 'CheckStatusUE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_FlexranCtrl' and action != 'Terminate_FlexranCtrl' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep' and action != 'Perform_X2_Handover': logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action) return False return True @@ -3213,6 +3214,19 @@ def GetParametersFromXML(action): else: CiTestObj.x2_ho_options = string_field + if action == 'Build_PhySim': + ldpc.buildargs = test.findtext('physim_build_args') + forced_workspace_cleanup = test.findtext('forced_workspace_cleanup') + if (forced_workspace_cleanup is None): + ldpc.forced_workspace_cleanup=False + else: + if re.match('true', forced_workspace_cleanup, re.IGNORECASE): + ldpc.forced_workspace_cleanup=True + else: + ldpc.forced_workspace_cleanup=False + + if action == 'Run_PhySim': + ldpc.runargs = test.findtext('physim_run_args') #check if given test is in list #it is in list if one of the strings in 'list' is at the beginning of 'test' @@ -3248,6 +3262,9 @@ EPC.SetHtmlObj(HTML) RAN.SetHtmlObj(HTML) RAN.SetEpcObj(EPC) +import cls_physim #class PhySim for physical simulators build and test +ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU build + argvs = sys.argv argc = len(argvs) cwd = os.getcwd() @@ -3269,12 +3286,14 @@ while len(argvs) > 1: CiTestObj.ranRepository = matchReg.group(1) RAN.SetranRepository(matchReg.group(1)) HTML.SetranRepository(matchReg.group(1)) + ldpc.ranRepository=matchReg.group(1) elif re.match('^\-\-eNB_AllowMerge=(.+)$|^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE) else: matchReg = re.match('^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE) doMerge = matchReg.group(1) + ldpc.ranAllowMerge=matchReg.group(1) if ((doMerge == 'true') or (doMerge == 'True')): CiTestObj.ranAllowMerge = True RAN.SetranAllowMerge(True) @@ -3287,6 +3306,7 @@ while len(argvs) > 1: CiTestObj.ranBranch = matchReg.group(1) RAN.SetranBranch(matchReg.group(1)) HTML.SetranBranch(matchReg.group(1)) + ldpc.ranBranch=matchReg.group(1) elif re.match('^\-\-eNBCommitID=(.*)$|^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE) @@ -3295,6 +3315,7 @@ while len(argvs) > 1: CiTestObj.ranCommitID = matchReg.group(1) RAN.SetranCommitID(matchReg.group(1)) HTML.SetranCommitID(matchReg.group(1)) + ldpc.ranCommitID=matchReg.group(1) elif re.match('^\-\-eNBTargetBranch=(.*)$|^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE) @@ -3303,10 +3324,12 @@ while len(argvs) > 1: CiTestObj.ranTargetBranch = matchReg.group(1) RAN.SetranTargetBranch(matchReg.group(1)) HTML.SetranTargetBranch(matchReg.group(1)) + ldpc.ranTargetBranch=matchReg.group(1) elif re.match('^\-\-eNBIPAddress=(.+)$|^\-\-eNB[1-2]IPAddress=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBIPAddress(matchReg.group(1)) + ldpc.eNBIpAddr=matchReg.group(1) elif re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1IPAddress(matchReg.group(1)) @@ -3317,6 +3340,7 @@ while len(argvs) > 1: if re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBUserName(matchReg.group(1)) + ldpc.eNBUserName=matchReg.group(1) elif re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1UserName(matchReg.group(1)) @@ -3327,6 +3351,7 @@ while len(argvs) > 1: if re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBPassword(matchReg.group(1)) + ldpc.eNBPassWord=matchReg.group(1) elif re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1Password(matchReg.group(1)) @@ -3337,6 +3362,7 @@ while len(argvs) > 1: if re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBSourceCodePath(matchReg.group(1)) + ldpc.eNBSourceCodePath=matchReg.group(1) elif re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1SourceCodePath(matchReg.group(1)) @@ -3504,7 +3530,7 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): foundCount += 1 count += 1 if foundCount != HTML.GetnbTestXMLfiles(): - HTML.SetnbTestXMLfiles(foundcount) + HTML.SetnbTestXMLfiles(foundCount) if (CiTestObj.ADBIPAddress != 'none'): terminate_ue_flag = False @@ -3707,6 +3733,11 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CiTestObj.IdleSleep() elif action == 'Perform_X2_Handover': CiTestObj.Perform_X2_Handover() + elif action == 'Build_PhySim': + HTML=ldpc.Build_PhySim(HTML,CONST) + if ldpc.exitStatus==1:sys.exit() + elif action == 'Run_PhySim': + HTML=ldpc.Run_PhySim(HTML,CONST,id) else: sys.exit('Invalid action') CiTestObj.FailReportCnt += 1 diff --git a/ci-scripts/xml_files/ldpc_gpu_test.xml b/ci-scripts/xml_files/ldpc_gpu_test.xml new file mode 100644 index 0000000000..136cd72b1e --- /dev/null +++ b/ci-scripts/xml_files/ldpc_gpu_test.xml @@ -0,0 +1,161 @@ + + + test-ldpc-gpu + Test-ldpc-GPU + tasks + 1 + 000001 000002 000003 000004 000005 000006 000007 000008 000009 000010 000011 000012 000013 000014 000015 000016 000017 000018 000019 000020 000021 + + + + Build_PhySim + Build for physical simulator + --phy_simulators --ninja + TRUE + + + + Run_PhySim + Run LDPC Test with CPU + -l 3872 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 3872 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 4224 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 4224 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 4576 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 4576 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 4928 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 4928 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 5280 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 5280 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 5632 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 5632 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 6336 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 6336 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 7040 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 7040 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 7744 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 7744 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 8448 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 8448 -s10 -n100 -G 1 + + + + +