From dcadc24ad6cd8b54261fe970ee8b472503fc939f Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 17 Jun 2020 11:58:54 +0200 Subject: [PATCH 1/5] complete and working version supporting phy-simulators --- ci-scripts/cls_physim.py | 155 ++++++++++++++++++++++ ci-scripts/main.py | 25 +++- ci-scripts/xml_files/rh_ldpc_gpu_test.xml | 50 +++++++ 3 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 ci-scripts/cls_physim.py create mode 100644 ci-scripts/xml_files/rh_ldpc_gpu_test.xml diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py new file mode 100644 index 0000000000..c06c818327 --- /dev/null +++ b/ci-scripts/cls_physim.py @@ -0,0 +1,155 @@ +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Python for CI of OAI-eNB + COTS-UE +# +# Required Python Version +# Python 3.x +# +# Required Python Package +# pexpect +#--------------------------------------------------------------------- + +#to create a SSH object locally in the methods +import sshconnection +#to update the HTML object +import html +from multiprocessing import SimpleQueue + + +class PhySim: + def __init__(self): + self.buildargs = "" + self.runargs = "" + self.eNBIpAddr = "" + self.eNBUserName = "" + self.eNBPassWord = "" + self.eNBSourceCodePath = "" + self.ranRepository = "" + self.ranBranch = "" + self.ranCommitID= "" + self.ranAllowMerge= "" + self.ranTargetBranch= "" + self.exitStatus=0 + self.workSpacePath='' + self.buildLogFile='compile_oai_enb.log' + self.runLogFile='ldpctest_run_results.log' + self.runResults=[] + + def __CheckResults_PhySim(self,HTML,CONST): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + #retrieve run log file locally$ + mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.workSpacePath+self.runLogFile, '.') + mySSH.close() + with open(self.runLogFile) as f: + for line in f: + if 'mean' in line: + self.runResults.append(line) + info=self.runResults[-1]+self.runResults[-2] + + html_cell = '
' + info  + '
' + html_queue=SimpleQueue() + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue(self.runargs, 'OK', 1, html_queue) + return HTML + + + def __CheckBuild_PhySim(self, HTML, CONST): + self.workSpacePath=self.eNBSourceCodePath+'/cmake_targets/' + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + #retrieve compile log file locally + mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.workSpacePath+self.buildLogFile, '.') + #delete older run log file + mySSH.command('rm ' + self.workSpacePath+self.runLogFile, '\$', 5) + mySSH.close() + #check build result from local compile log file + buildStatus=False + with open(self.buildLogFile) as f: + if 'nr_prachsim compiled' in f.read(): + buildStatus=True + #update HTML based on build status + if buildStatus: + HTML.CreateHtmlTestRow(self.buildargs, 'OK', CONST.ALL_PROCESSES_OK, 'LDPC') + self.exitStatus=0 + else: + logging.error('\u001B[1m Building OAI UE Failed\u001B[0m') + HTML.CreateHtmlTestRow(self.buildargs, 'KO', CONST.ALL_PROCESSES_OK, 'LDPC') + HTML.CreateHtmlTabFooter(False) + self.exitStatus=1 + return HTML + + def Build_PhySim(self,htmlObj,constObj): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + + #create working dir + mySSH.command('mkdir -p ' + self.eNBSourceCodePath, '\$', 5) + mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) + + if not self.ranRepository.lower().endswith('.git'): + self.ranRepository+='.git' + + #git clone + mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) + #git config + mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) + + #git clean + mySSH.command('echo ' + self.eNBPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + + # if the commit ID is provided, use it to point to it + if self.ranCommitID != '': + mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should have already been checked earlier + if (self.ranAllowMerge): + if self.ranTargetBranch == '': + if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + else: + print('Merging with the target branch: ' + self.ranTargetBranch) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + + #build + mySSH.command('source oaienv', '\$', 5) + mySSH.command('cd cmake_targets', '\$', 5) + mySSH.command('mkdir -p log', '\$', 5) + mySSH.command('chmod 777 log', '\$', 5) + mySSH.command('stdbuf -o0 ./build_oai ' + self.buildargs + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500) + + mySSH.close() + #check build status and update HTML object + lHTML = html.HTMLManagement() + lHTML=self.__CheckBuild_PhySim(htmlObj,constObj) + return lHTML + + + def Run_PhySim(self,htmlObj,constObj): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) + mySSH.command('cd '+self.workSpacePath,'\$',5) + mySSH.command(self.workSpacePath+'phy_simulators/build/ldpctest ' + self.runargs + ' >> '+self.runLogFile, '\$', 30) + mySSH.close() + lHTML = html.HTMLManagement() + lHTML=self.__CheckResults_PhySim(htmlObj,constObj) + return lHTML diff --git a/ci-scripts/main.py b/ci-scripts/main.py index e9313051fc..164b4fc6af 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -30,6 +30,7 @@ import constants as CONST + #----------------------------------------------------------- # Import #----------------------------------------------------------- @@ -3061,7 +3062,7 @@ class OaiCiTest(): logging.debug('\u001B[1m----------------------------------------\u001B[0m') def CheckClassValidity(action,id): - if action != 'Build_eNB' and action != 'WaitEndBuild_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'DataDisable_UE' and action != 'DataEnable_UE' and action != 'CheckStatusUE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_FlexranCtrl' and action != 'Terminate_FlexranCtrl' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep' and action != 'Perform_X2_Handover': + if action!='Build_PhySim' and action!='Run_PhySim' and action != 'Build_eNB' and action != 'WaitEndBuild_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'DataDisable_UE' and action != 'DataEnable_UE' and action != 'CheckStatusUE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_FlexranCtrl' and action != 'Terminate_FlexranCtrl' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep' and action != 'Perform_X2_Handover': logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action) return False return True @@ -3208,6 +3209,11 @@ def GetParametersFromXML(action): else: CiTestObj.x2_ho_options = string_field + if action == 'Build_PhySim': + ldpc.buildargs = test.findtext('physim_build_args') + + if action == 'Run_PhySim': + ldpc.runargs = test.findtext('physim_run_args') #check if given test is in list #it is in list if one of the strings in 'list' is at the beginning of 'test' @@ -3243,6 +3249,9 @@ EPC.SetHtmlObj(HTML) RAN.SetHtmlObj(HTML) RAN.SetEpcObj(EPC) +import cls_physim #class PhySim for physical simulators build and test +ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU build + argvs = sys.argv argc = len(argvs) cwd = os.getcwd() @@ -3264,12 +3273,14 @@ while len(argvs) > 1: CiTestObj.ranRepository = matchReg.group(1) RAN.SetranRepository(matchReg.group(1)) HTML.SetranRepository(matchReg.group(1)) + ldpc.ranRepository=matchReg.group(1) elif re.match('^\-\-eNB_AllowMerge=(.+)$|^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE) else: matchReg = re.match('^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE) doMerge = matchReg.group(1) + ldpc.ranAllowMerge=matchReg.group(1) if ((doMerge == 'true') or (doMerge == 'True')): CiTestObj.ranAllowMerge = True RAN.SetranAllowMerge(True) @@ -3282,6 +3293,7 @@ while len(argvs) > 1: CiTestObj.ranBranch = matchReg.group(1) RAN.SetranBranch(matchReg.group(1)) HTML.SetranBranch(matchReg.group(1)) + ldpc.ranBranch=matchReg.group(1) elif re.match('^\-\-eNBCommitID=(.*)$|^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE) @@ -3290,6 +3302,7 @@ while len(argvs) > 1: CiTestObj.ranCommitID = matchReg.group(1) RAN.SetranCommitID(matchReg.group(1)) HTML.SetranCommitID(matchReg.group(1)) + ldpc.ranCommitID=matchReg.group(1) elif re.match('^\-\-eNBTargetBranch=(.*)$|^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE) @@ -3298,10 +3311,12 @@ while len(argvs) > 1: CiTestObj.ranTargetBranch = matchReg.group(1) RAN.SetranTargetBranch(matchReg.group(1)) HTML.SetranTargetBranch(matchReg.group(1)) + ldpc.ranTargetBranch=matchReg.group(1) elif re.match('^\-\-eNBIPAddress=(.+)$|^\-\-eNB[1-2]IPAddress=(.+)$', myArgv, re.IGNORECASE): if re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBIPAddress(matchReg.group(1)) + ldpc.eNBIpAddr=matchReg.group(1) elif re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1IPAddress(matchReg.group(1)) @@ -3312,6 +3327,7 @@ while len(argvs) > 1: if re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBUserName(matchReg.group(1)) + ldpc.eNBUserName=matchReg.group(1) elif re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1UserName(matchReg.group(1)) @@ -3322,6 +3338,7 @@ while len(argvs) > 1: if re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBPassword(matchReg.group(1)) + ldpc.eNBPassWord=matchReg.group(1) elif re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1Password(matchReg.group(1)) @@ -3332,6 +3349,7 @@ while len(argvs) > 1: if re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNBSourceCodePath(matchReg.group(1)) + ldpc.eNBSourceCodePath=matchReg.group(1) elif re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE) RAN.SeteNB1SourceCodePath(matchReg.group(1)) @@ -3702,6 +3720,11 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CiTestObj.IdleSleep() elif action == 'Perform_X2_Handover': CiTestObj.Perform_X2_Handover() + elif action == 'Build_PhySim': + HTML=ldpc.Build_PhySim(HTML,CONST) + if ldpc.exitStatus==1:sys.exit() + elif action == 'Run_PhySim': + ldpc.Run_PhySim(HTML,CONST) else: sys.exit('Invalid action') CiTestObj.FailReportCnt += 1 diff --git a/ci-scripts/xml_files/rh_ldpc_gpu_test.xml b/ci-scripts/xml_files/rh_ldpc_gpu_test.xml new file mode 100644 index 0000000000..6df90915d3 --- /dev/null +++ b/ci-scripts/xml_files/rh_ldpc_gpu_test.xml @@ -0,0 +1,50 @@ + + + test-ldpc-gpu + Test-ldpc-GPU + tasks + 1 + 000001 000002 000003 + + + + Build_PhySim + Build for physical simulator + --phy_simulators + + + + Run_PhySim + Run LDPC Test with CPU + -l 8448 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 8448 -s10 -n100 -G 1 + + + From a9dbcf6b93477c1474c4694deb6ebd0fad9ddf1d Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 19 Jun 2020 14:18:13 +0200 Subject: [PATCH 2/5] added in-code comments and changed attr to private --- ci-scripts/cls_physim.py | 51 +++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py index c06c818327..baf8144f88 100644 --- a/ci-scripts/cls_physim.py +++ b/ci-scripts/cls_physim.py @@ -48,23 +48,32 @@ class PhySim: self.ranAllowMerge= "" self.ranTargetBranch= "" self.exitStatus=0 - self.workSpacePath='' - self.buildLogFile='compile_oai_enb.log' - self.runLogFile='ldpctest_run_results.log' - self.runResults=[] + #private attributes + self.__workSpacePath='' + self.__buildLogFile='compile_oai_enb.log' + self.__runLogFile='ldpctest_run_results.log' + self.__runResults=[] + + +#----------------- +#PRIVATE Methods +#----------------- def __CheckResults_PhySim(self,HTML,CONST): mySSH = sshconnection.SSHConnection() mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) - #retrieve run log file locally$ - mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.workSpacePath+self.runLogFile, '.') + #retrieve run log file and store it locally$ + mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.__workSpacePath+self.__runLogFile, '.') mySSH.close() - with open(self.runLogFile) as f: + #parse results looking for Encoding and Decoding mean values + with open(self.__runLogFile) as f: for line in f: if 'mean' in line: - self.runResults.append(line) - info=self.runResults[-1]+self.runResults[-2] + self.__runResults.append(line) + #the value are appended for each test, so we take the last 2 values from the list + info=self.__runResults[-1]+self.__runResults[-2] + #updating the HTML with results html_cell = '
' + info  + '
' html_queue=SimpleQueue() html_queue.put(html_cell) @@ -73,17 +82,18 @@ class PhySim: def __CheckBuild_PhySim(self, HTML, CONST): - self.workSpacePath=self.eNBSourceCodePath+'/cmake_targets/' + self.__workSpacePath=self.eNBSourceCodePath+'/cmake_targets/' mySSH = sshconnection.SSHConnection() mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) - #retrieve compile log file locally - mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.workSpacePath+self.buildLogFile, '.') + #retrieve compile log file and store it locally + mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.__workSpacePath+self.__buildLogFile, '.') #delete older run log file - mySSH.command('rm ' + self.workSpacePath+self.runLogFile, '\$', 5) + mySSH.command('rm ' + self.__workSpacePath+self.__runLogFile, '\$', 5) mySSH.close() #check build result from local compile log file buildStatus=False - with open(self.buildLogFile) as f: + with open(self.__buildLogFile) as f: + #nr_prachsim is the last compile step if 'nr_prachsim compiled' in f.read(): buildStatus=True #update HTML based on build status @@ -94,9 +104,15 @@ class PhySim: logging.error('\u001B[1m Building OAI UE Failed\u001B[0m') HTML.CreateHtmlTestRow(self.buildargs, 'KO', CONST.ALL_PROCESSES_OK, 'LDPC') HTML.CreateHtmlTabFooter(False) + #exitStatus=1 will do a sys.exit in main self.exitStatus=1 return HTML + +#-----------------$ +#PUBLIC Methods$ +#-----------------$ + def Build_PhySim(self,htmlObj,constObj): mySSH = sshconnection.SSHConnection() mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) @@ -145,11 +161,14 @@ class PhySim: def Run_PhySim(self,htmlObj,constObj): + #open a session for test run mySSH = sshconnection.SSHConnection() mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) - mySSH.command('cd '+self.workSpacePath,'\$',5) - mySSH.command(self.workSpacePath+'phy_simulators/build/ldpctest ' + self.runargs + ' >> '+self.runLogFile, '\$', 30) + mySSH.command('cd '+self.__workSpacePath,'\$',5) + #run and redirect the results to a log file + mySSH.command(self.__workSpacePath+'phy_simulators/build/ldpctest ' + self.runargs + ' >> '+self.__runLogFile, '\$', 30) mySSH.close() + #return updated HTML to main lHTML = html.HTMLManagement() lHTML=self.__CheckResults_PhySim(htmlObj,constObj) return lHTML From 497c611080a9454710edda07f712f24f83c677a3 Mon Sep 17 00:00:00 2001 From: Remi Hardy Date: Mon, 22 Jun 2020 16:59:15 +0200 Subject: [PATCH 3/5] update 1 after review --- ci-scripts/cls_physim.py | 8 +++++--- ci-scripts/main.py | 2 +- .../xml_files/{rh_ldpc_gpu_test.xml => ldpc_gpu_test.xml} | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) rename ci-scripts/xml_files/{rh_ldpc_gpu_test.xml => ldpc_gpu_test.xml} (96%) diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py index baf8144f88..99c6342697 100644 --- a/ci-scripts/cls_physim.py +++ b/ci-scripts/cls_physim.py @@ -27,6 +27,8 @@ # pexpect #--------------------------------------------------------------------- +#to use logging.info() +import logging #to create a SSH object locally in the methods import sshconnection #to update the HTML object @@ -50,7 +52,7 @@ class PhySim: self.exitStatus=0 #private attributes self.__workSpacePath='' - self.__buildLogFile='compile_oai_enb.log' + self.__buildLogFile='compile_phy_sim.log' self.__runLogFile='ldpctest_run_results.log' self.__runResults=[] @@ -101,7 +103,7 @@ class PhySim: HTML.CreateHtmlTestRow(self.buildargs, 'OK', CONST.ALL_PROCESSES_OK, 'LDPC') self.exitStatus=0 else: - logging.error('\u001B[1m Building OAI UE Failed\u001B[0m') + logging.error('\u001B[1m Building Physical Simulators Failed\u001B[0m') HTML.CreateHtmlTestRow(self.buildargs, 'KO', CONST.ALL_PROCESSES_OK, 'LDPC') HTML.CreateHtmlTabFooter(False) #exitStatus=1 will do a sys.exit in main @@ -143,7 +145,7 @@ class PhySim: if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) else: - print('Merging with the target branch: ' + self.ranTargetBranch) + logging.info('Merging with the target branch: ' + self.ranTargetBranch) mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) #build diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 164b4fc6af..ed2a56e688 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -3517,7 +3517,7 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): foundCount += 1 count += 1 if foundCount != HTML.GetnbTestXMLfiles(): - HTML.SetnbTestXMLfiles(foundcount) + HTML.SetnbTestXMLfiles(foundCount) if (CiTestObj.ADBIPAddress != 'none'): terminate_ue_flag = False diff --git a/ci-scripts/xml_files/rh_ldpc_gpu_test.xml b/ci-scripts/xml_files/ldpc_gpu_test.xml similarity index 96% rename from ci-scripts/xml_files/rh_ldpc_gpu_test.xml rename to ci-scripts/xml_files/ldpc_gpu_test.xml index 6df90915d3..4d398bc995 100644 --- a/ci-scripts/xml_files/rh_ldpc_gpu_test.xml +++ b/ci-scripts/xml_files/ldpc_gpu_test.xml @@ -32,7 +32,7 @@ Build_PhySim Build for physical simulator - --phy_simulators + --phy_simulators --ninja From 03ff1ef3044f6a6fdcedddaf1daf7430abcf23a5 Mon Sep 17 00:00:00 2001 From: Remi Hardy Date: Tue, 23 Jun 2020 22:29:23 +0200 Subject: [PATCH 4/5] update 2 after review --- ci-scripts/cls_physim.py | 26 ++++-- ci-scripts/main.py | 2 +- ci-scripts/xml_files/ldpc_gpu_test.xml | 114 ++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 11 deletions(-) diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py index 99c6342697..09ed1c8363 100644 --- a/ci-scripts/cls_physim.py +++ b/ci-scripts/cls_physim.py @@ -34,7 +34,8 @@ import sshconnection #to update the HTML object import html from multiprocessing import SimpleQueue - +#for log folder maintenance +import os class PhySim: def __init__(self): @@ -53,27 +54,32 @@ class PhySim: #private attributes self.__workSpacePath='' self.__buildLogFile='compile_phy_sim.log' - self.__runLogFile='ldpctest_run_results.log' + self.__runLogFile='' self.__runResults=[] + self.__runLogPath='phy_sim_logs' #----------------- #PRIVATE Methods #----------------- - def __CheckResults_PhySim(self,HTML,CONST): + def __CheckResults_PhySim(self,HTML,CONST,testcase_id): mySSH = sshconnection.SSHConnection() mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) #retrieve run log file and store it locally$ mySSH.copyin(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord, self.__workSpacePath+self.__runLogFile, '.') mySSH.close() #parse results looking for Encoding and Decoding mean values + self.__runResults=[] with open(self.__runLogFile) as f: for line in f: if 'mean' in line: self.__runResults.append(line) - #the value are appended for each test, so we take the last 2 values from the list - info=self.__runResults[-1]+self.__runResults[-2] + #the value are appended for each mean value (2), so we take these 2 values from the list + info=self.__runResults[0]+self.__runResults[1] + + #once parsed move the local logfile to its folder for tidiness + os.system('mv '+self.__runLogFile+' '+ self.__runLogPath+'/.') #updating the HTML with results html_cell = '
' + info  + '
' @@ -153,7 +159,7 @@ class PhySim: mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('mkdir -p log', '\$', 5) mySSH.command('chmod 777 log', '\$', 5) - mySSH.command('stdbuf -o0 ./build_oai ' + self.buildargs + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500) + mySSH.command('stdbuf -o0 ./build_oai ' + self.buildargs + ' 2>&1 | stdbuf -o0 tee ' + self.__buildLogFile, 'Bypassing the Tests|build have failed', 1500) mySSH.close() #check build status and update HTML object @@ -162,7 +168,11 @@ class PhySim: return lHTML - def Run_PhySim(self,htmlObj,constObj): + def Run_PhySim(self,htmlObj,constObj,testcase_id): + #create run logs folder locally + os.system('mkdir -p ./'+self.__runLogPath) + #log file is tc_.log remotely + self.__runLogFile='physim_'+str(testcase_id)+'.log' #open a session for test run mySSH = sshconnection.SSHConnection() mySSH.open(self.eNBIpAddr, self.eNBUserName, self.eNBPassWord) @@ -172,5 +182,5 @@ class PhySim: mySSH.close() #return updated HTML to main lHTML = html.HTMLManagement() - lHTML=self.__CheckResults_PhySim(htmlObj,constObj) + lHTML=self.__CheckResults_PhySim(htmlObj,constObj,testcase_id) return lHTML diff --git a/ci-scripts/main.py b/ci-scripts/main.py index ed2a56e688..a1c1ac7ec0 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -3724,7 +3724,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML=ldpc.Build_PhySim(HTML,CONST) if ldpc.exitStatus==1:sys.exit() elif action == 'Run_PhySim': - ldpc.Run_PhySim(HTML,CONST) + HTML=ldpc.Run_PhySim(HTML,CONST,id) else: sys.exit('Invalid action') CiTestObj.FailReportCnt += 1 diff --git a/ci-scripts/xml_files/ldpc_gpu_test.xml b/ci-scripts/xml_files/ldpc_gpu_test.xml index 4d398bc995..029c49ecf5 100644 --- a/ci-scripts/xml_files/ldpc_gpu_test.xml +++ b/ci-scripts/xml_files/ldpc_gpu_test.xml @@ -26,7 +26,7 @@ Test-ldpc-GPU tasks 1 - 000001 000002 000003 + 000001 000002 000003 000004 000005 000006 000007 000008 000009 000010 000011 000012 000013 000014 000015 000016 000017 000018 000019 000020 000021 @@ -38,13 +38,123 @@ Run_PhySim Run LDPC Test with CPU - -l 8448 -s10 -n100 + -l 3872 -s10 -n100 + Run_PhySim + Run LDPC Test with GPU + -l 3872 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 4224 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 4224 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 4576 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 4576 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 4928 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 4928 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 5280 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 5280 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 5632 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 5632 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 6336 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 6336 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 7040 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 7040 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 7744 -s10 -n100 + + + + Run_PhySim + Run LDPC Test with GPU + -l 7744 -s10 -n100 -G 1 + + + + Run_PhySim + Run LDPC Test with CPU + -l 8448 -s10 -n100 + + + Run_PhySim Run LDPC Test with GPU -l 8448 -s10 -n100 -G 1 + + From ebbff46cde9e2889ddce1656ae60d4dc5839399f Mon Sep 17 00:00:00 2001 From: Remi Hardy Date: Wed, 24 Jun 2020 11:58:41 +0200 Subject: [PATCH 5/5] update 3 after review --- ci-scripts/cls_physim.py | 11 ++++++++--- ci-scripts/main.py | 8 ++++++++ ci-scripts/xml_files/ldpc_gpu_test.xml | 1 + 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py index 09ed1c8363..4ca36b8cf0 100644 --- a/ci-scripts/cls_physim.py +++ b/ci-scripts/cls_physim.py @@ -51,6 +51,7 @@ class PhySim: self.ranAllowMerge= "" self.ranTargetBranch= "" self.exitStatus=0 + self.forced_workspace_cleanup=False #private attributes self.__workSpacePath='' self.__buildLogFile='compile_phy_sim.log' @@ -75,7 +76,7 @@ class PhySim: for line in f: if 'mean' in line: self.__runResults.append(line) - #the value are appended for each mean value (2), so we take these 2 values from the list + #the values are appended for each mean value (2), so we take these 2 values from the list info=self.__runResults[0]+self.__runResults[1] #once parsed move the local logfile to its folder for tidiness @@ -138,8 +139,12 @@ class PhySim: mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) - #git clean - mySSH.command('echo ' + self.eNBPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + #git clean depending on self.forced_workspace_cleanup captured in xml + if self.forced_workspace_cleanup==True: + logging.info('Cleaning workspace ...') + mySSH.command('echo ' + self.eNBPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) + else: + logging.info('Workspace cleaning was disabled') # if the commit ID is provided, use it to point to it if self.ranCommitID != '': diff --git a/ci-scripts/main.py b/ci-scripts/main.py index a1c1ac7ec0..b105c19f13 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -3211,6 +3211,14 @@ def GetParametersFromXML(action): if action == 'Build_PhySim': ldpc.buildargs = test.findtext('physim_build_args') + forced_workspace_cleanup = test.findtext('forced_workspace_cleanup') + if (forced_workspace_cleanup is None): + ldpc.forced_workspace_cleanup=False + else: + if re.match('true', forced_workspace_cleanup, re.IGNORECASE): + ldpc.forced_workspace_cleanup=True + else: + ldpc.forced_workspace_cleanup=False if action == 'Run_PhySim': ldpc.runargs = test.findtext('physim_run_args') diff --git a/ci-scripts/xml_files/ldpc_gpu_test.xml b/ci-scripts/xml_files/ldpc_gpu_test.xml index 029c49ecf5..136cd72b1e 100644 --- a/ci-scripts/xml_files/ldpc_gpu_test.xml +++ b/ci-scripts/xml_files/ldpc_gpu_test.xml @@ -33,6 +33,7 @@ Build_PhySim Build for physical simulator --phy_simulators --ninja + TRUE