Merge remote-tracking branch 'origin/develop' into develop_inria_ci_deployment_gp
This commit is contained in:
Gabriele Perrone
+373
-165
@@ -1,4 +1,4 @@
|
||||
|
||||
#/*
|
||||
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
# * contributor license agreements. See the NOTICE file distributed with
|
||||
# * this work for additional information regarding copyright ownership.
|
||||
@@ -56,6 +56,7 @@ OAI_UE_PROCESS_COULD_NOT_SYNC = -21
|
||||
OAI_UE_PROCESS_ASSERTION = -22
|
||||
OAI_UE_PROCESS_FAILED = -23
|
||||
OAI_UE_PROCESS_NO_TUNNEL_INTERFACE = -24
|
||||
OAI_UE_PROCESS_SEG_FAULT = -25
|
||||
OAI_UE_PROCESS_OK = +6
|
||||
|
||||
UE_STATUS_DETACHED = 0
|
||||
@@ -129,7 +130,9 @@ class OaiCiTest():
|
||||
self.Build_eNB_args = ''
|
||||
self.backgroundBuild = False
|
||||
self.backgroundBuildTestId = ['', '', '']
|
||||
self.Build_eNB_forced_workspace_cleanup = False
|
||||
self.Initialize_eNB_args = ''
|
||||
self.air_interface = 'lte'
|
||||
self.eNB_instance = ''
|
||||
self.eNB_serverId = ''
|
||||
self.eNBLogFiles = ['', '', '']
|
||||
@@ -184,6 +187,7 @@ class OaiCiTest():
|
||||
self.UELogFile = ''
|
||||
self.Build_OAI_UE_args = ''
|
||||
self.Initialize_OAI_UE_args = ''
|
||||
self.clean_repository = True
|
||||
self.flexranCtrlInstalled = False
|
||||
self.flexranCtrlStarted = False
|
||||
self.expectedNbOfConnectedUEs = 0
|
||||
@@ -212,9 +216,24 @@ class OaiCiTest():
|
||||
Usage()
|
||||
sys.exit('Insufficient Parameter')
|
||||
SSH.open(lIpAddr, lUserName, lPassWord)
|
||||
# Check if we build an 5G-NR gNB or an LTE eNB
|
||||
result = re.search('--gNB', self.Build_eNB_args)
|
||||
if result is not None:
|
||||
self.air_interface = 'nr'
|
||||
else:
|
||||
self.air_interface = 'lte'
|
||||
# Worakround for some servers, we need to erase completely the workspace
|
||||
if self.Build_eNB_forced_workspace_cleanup:
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
|
||||
# on RedHat/CentOS .git extension is mandatory
|
||||
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
|
||||
if result is not None:
|
||||
full_ran_repo_name = self.ranRepository
|
||||
else:
|
||||
full_ran_repo_name = self.ranRepository + '.git'
|
||||
SSH.command('mkdir -p ' + lSourcePath, '\$', 5)
|
||||
SSH.command('cd ' + lSourcePath, '\$', 5)
|
||||
SSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
SSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
# Raphael: here add a check if git clone or git fetch went smoothly
|
||||
SSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
||||
SSH.command('git config user.name "OAI Jenkins"', '\$', 5)
|
||||
@@ -248,7 +267,7 @@ class OaiCiTest():
|
||||
SSH.close()
|
||||
self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
|
||||
return
|
||||
|
||||
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
|
||||
# if the commit ID is provided use it to point to it
|
||||
if self.ranCommitID != '':
|
||||
@@ -312,10 +331,14 @@ class OaiCiTest():
|
||||
|
||||
def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId):
|
||||
SSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
|
||||
SSH.command('ls lte_build_oai/build', '\$', 3)
|
||||
SSH.command('ls lte_build_oai/build', '\$', 3)
|
||||
SSH.command('ls ran_build/build', '\$', 3)
|
||||
SSH.command('ls ran_build/build', '\$', 3)
|
||||
if self.air_interface == 'nr':
|
||||
nodeB_prefix = 'g'
|
||||
else:
|
||||
nodeB_prefix = 'e'
|
||||
buildStatus = True
|
||||
result = re.search('lte-softmodem', SSH.getBefore())
|
||||
result = re.search(self.air_interface + '-softmodem', SSH.getBefore())
|
||||
if result is None:
|
||||
buildStatus = False
|
||||
else:
|
||||
@@ -353,9 +376,10 @@ class OaiCiTest():
|
||||
SSH.close()
|
||||
|
||||
if buildStatus:
|
||||
logging.info('\u001B[1m Building OAI ' + nodeB_prefix + 'NB Pass\u001B[0m')
|
||||
self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
|
||||
else:
|
||||
logging.error('\u001B[1m Building OAI eNB Failed\u001B[0m')
|
||||
logging.error('\u001B[1m Building OAI ' + nodeB_prefix + 'NB Failed\u001B[0m')
|
||||
self.CreateHtmlTestRow(self.Build_eNB_args, 'KO', ALL_PROCESSES_OK)
|
||||
self.CreateHtmlTabFooter(False)
|
||||
sys.exit(1)
|
||||
@@ -365,42 +389,56 @@ class OaiCiTest():
|
||||
Usage()
|
||||
sys.exit('Insufficient Parameter')
|
||||
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
|
||||
result = re.search('--nrUE', self.Build_OAI_UE_args)
|
||||
if result is not None:
|
||||
self.air_interface = 'nr'
|
||||
ue_prefix = 'NR '
|
||||
else:
|
||||
self.air_interface = 'lte'
|
||||
ue_prefix = ''
|
||||
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
|
||||
if result is not None:
|
||||
full_ran_repo_name = self.ranRepository
|
||||
else:
|
||||
full_ran_repo_name = self.ranRepository + '.git'
|
||||
SSH.command('mkdir -p ' + self.UESourceCodePath, '\$', 5)
|
||||
SSH.command('cd ' + self.UESourceCodePath, '\$', 5)
|
||||
SSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.ranRepository + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
SSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
# here add a check if git clone or git fetch went smoothly
|
||||
SSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
||||
SSH.command('git config user.name "OAI Jenkins"', '\$', 5)
|
||||
SSH.command('ls *.txt', '\$', 5)
|
||||
result = re.search('LAST_BUILD_INFO', SSH.getBefore())
|
||||
if result is not None:
|
||||
mismatch = False
|
||||
SSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
|
||||
result = re.search(self.ranCommitID, SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
SSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
|
||||
if (self.ranAllowMerge):
|
||||
result = re.search('YES', SSH.getBefore())
|
||||
if self.clean_repository:
|
||||
SSH.command('ls *.txt', '\$', 5)
|
||||
result = re.search('LAST_BUILD_INFO', SSH.getBefore())
|
||||
if result is not None:
|
||||
mismatch = False
|
||||
SSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
|
||||
result = re.search(self.ranCommitID, SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
SSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
|
||||
if self.ranTargetBranch == '':
|
||||
result = re.search('develop', SSH.getBefore())
|
||||
SSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
|
||||
if (self.ranAllowMerge):
|
||||
result = re.search('YES', SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
SSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
|
||||
if self.ranTargetBranch == '':
|
||||
result = re.search('develop', SSH.getBefore())
|
||||
else:
|
||||
result = re.search(self.ranTargetBranch, SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
else:
|
||||
result = re.search(self.ranTargetBranch, SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
else:
|
||||
result = re.search('NO', SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
if not mismatch:
|
||||
SSH.close()
|
||||
self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
|
||||
return
|
||||
result = re.search('NO', SSH.getBefore())
|
||||
if result is None:
|
||||
mismatch = True
|
||||
if not mismatch:
|
||||
SSH.close()
|
||||
self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
|
||||
return
|
||||
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
|
||||
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
|
||||
# if the commit ID is provided use it to point to it
|
||||
if self.ranCommitID != '':
|
||||
SSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
|
||||
@@ -419,10 +457,10 @@ class OaiCiTest():
|
||||
SSH.command('chmod 777 log', '\$', 5)
|
||||
# no need to remove in log (git clean did the trick)
|
||||
SSH.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee compile_oai_ue.log', 'Bypassing the Tests|build have failed', 600)
|
||||
SSH.command('ls lte_build_oai/build', '\$', 3)
|
||||
SSH.command('ls lte_build_oai/build', '\$', 3)
|
||||
SSH.command('ls ran_build/build', '\$', 3)
|
||||
SSH.command('ls ran_build/build', '\$', 3)
|
||||
buildStatus = True
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
result = re.search(self.air_interface + '-uesoftmodem', SSH.getBefore())
|
||||
if result is None:
|
||||
buildStatus = False
|
||||
SSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
|
||||
@@ -662,7 +700,7 @@ class OaiCiTest():
|
||||
SSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED = "no";/\' ' + ci_full_config_file, '\$', 2);
|
||||
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
||||
SSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2);
|
||||
result = re.search('yes', str(self.ssh.before))
|
||||
result = re.search('yes', SSH.getBefore())
|
||||
if result is not None:
|
||||
self.eNBmbmsEnables[int(self.eNB_instance)] = True
|
||||
logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
|
||||
@@ -674,17 +712,21 @@ class OaiCiTest():
|
||||
# Launch eNB with the modified config file
|
||||
SSH.command('source oaienv', '\$', 5)
|
||||
SSH.command('cd cmake_targets', '\$', 5)
|
||||
SSH.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
SSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
SSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface + '-softmodem -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
SSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + lSourcePath + '/cmake_targets -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
SSH.command('hostnamectl','\$', 5)
|
||||
result = re.search('CentOS Linux 7', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo $USER; nohup sudo ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
|
||||
else:
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + lSourcePath + '/cmake_targets -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
|
||||
if extra_options != '':
|
||||
self.eNBOptions[int(self.eNB_instance)] = extra_options
|
||||
time.sleep(6)
|
||||
doLoop = True
|
||||
loopCounter = 10
|
||||
loopCounter = 20
|
||||
enbDidSync = False
|
||||
while (doLoop):
|
||||
loopCounter = loopCounter - 1
|
||||
@@ -728,15 +770,16 @@ class OaiCiTest():
|
||||
time.sleep(10)
|
||||
|
||||
if enbDidSync and eNBinNoS1:
|
||||
self.command('ifconfig oaitun_enb1', '\$', 4)
|
||||
result = re.search('inet addr', str(self.ssh.before))
|
||||
SSH.command('ifconfig oaitun_enb1', '\$', 4)
|
||||
SSH.command('ifconfig oaitun_enb1', '\$', 4)
|
||||
result = re.search('inet addr:1|inet 1', SSH.getBefore())
|
||||
if result is not None:
|
||||
logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
|
||||
else:
|
||||
logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
|
||||
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
||||
self.command('ifconfig oaitun_enm1', '\$', 4)
|
||||
result = re.search('inet addr', str(self.ssh.before))
|
||||
SSH.command('ifconfig oaitun_enm1', '\$', 4)
|
||||
result = re.search('inet addr', SSH.getBefore())
|
||||
if result is not None:
|
||||
logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
|
||||
else:
|
||||
@@ -815,15 +858,19 @@ class OaiCiTest():
|
||||
if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
|
||||
Usage()
|
||||
sys.exit('Insufficient Parameter')
|
||||
result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
|
||||
if result is None:
|
||||
check_eNB = True
|
||||
check_OAI_UE = False
|
||||
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
|
||||
if (pStatus < 0):
|
||||
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus)
|
||||
self.CreateHtmlTabFooter(False)
|
||||
sys.exit(1)
|
||||
if self.air_interface == 'lte':
|
||||
result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
|
||||
if result is None:
|
||||
check_eNB = True
|
||||
check_OAI_UE = False
|
||||
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
|
||||
if (pStatus < 0):
|
||||
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus)
|
||||
self.CreateHtmlTabFooter(False)
|
||||
sys.exit(1)
|
||||
UE_prefix = ''
|
||||
else:
|
||||
UE_prefix = 'NR '
|
||||
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
|
||||
# b2xx_fx3_utils reset procedure
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 60)
|
||||
@@ -832,26 +879,29 @@ class OaiCiTest():
|
||||
logging.debug('Found a B2xx device --> resetting it')
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
|
||||
# Reloading FGPA bin firmware
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 60)
|
||||
else:
|
||||
logging.debug('Did not find any B2xx device')
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 30)
|
||||
result = re.search('type: n3xx', SSH.getBefore())
|
||||
if result is not None:
|
||||
logging.debug('Found a N3xx device --> resetting it')
|
||||
SSH.command('cd ' + self.UESourceCodePath, '\$', 5)
|
||||
# Initialize_OAI_UE_args usually start with -C and followed by the location in repository
|
||||
SSH.command('source oaienv', '\$', 5)
|
||||
SSH.command('cd cmake_targets/lte_build_oai/build', '\$', 5)
|
||||
result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
|
||||
# We may have to regenerate the .u* files
|
||||
if result is None:
|
||||
SSH.command('ls /tmp/*.sed', '\$', 5)
|
||||
result = re.search('adapt_usim_parameters', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('sed -f /tmp/adapt_usim_parameters.sed ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
|
||||
else:
|
||||
SSH.command('sed -e "s#93#92#" -e "s#8baf473f2f8fd09487cccbd7097c6862#fec86ba6eb707ed08905757b1bb44b8f#" -e "s#e734f8734007d6c5ce7a0508809e7e9c#C42449363BBAD02B66D16BC975D77CC1#" ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf .u*', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S ../../../targets/bin/conf2uedata -c ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf -o .', '\$', 5)
|
||||
# Launch UE with the modified config file
|
||||
SSH.command('echo "ulimit -c unlimited && ./lte-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
|
||||
SSH.command('cd cmake_targets/ran_build/build', '\$', 5)
|
||||
if self.air_interface == 'lte':
|
||||
result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
|
||||
# We may have to regenerate the .u* files
|
||||
if result is None:
|
||||
SSH.command('ls /tmp/*.sed', '\$', 5)
|
||||
result = re.search('adapt_usim_parameters', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('sed -f /tmp/adapt_usim_parameters.sed ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
|
||||
else:
|
||||
SSH.command('sed -e "s#93#92#" -e "s#8baf473f2f8fd09487cccbd7097c6862#fec86ba6eb707ed08905757b1bb44b8f#" -e "s#e734f8734007d6c5ce7a0508809e7e9c#C42449363BBAD02B66D16BC975D77CC1#" ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf .u*', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S ../../../targets/bin/conf2uedata -c ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf -o .', '\$', 5)
|
||||
SSH.command('echo "ulimit -c unlimited && ./'+ self.air_interface +'-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
|
||||
SSH.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5)
|
||||
self.UELogFile = 'ue_' + self.testCase_id + '.log'
|
||||
|
||||
# We are now looping several times to hope we really sync w/ an eNB
|
||||
@@ -860,9 +910,11 @@ class OaiCiTest():
|
||||
gotSyncStatus = True
|
||||
fullSyncStatus = True
|
||||
while (doOutterLoop):
|
||||
SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build', '\$', 5)
|
||||
SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets/ran_build/build', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
|
||||
#use nohup instead of daemon
|
||||
#SSH.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/ran_build/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
|
||||
SSH.command('echo $USER; nohup sudo ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh' + ' > ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ' + ' 2>&1 &', self.UEUserName, 5)
|
||||
time.sleep(6)
|
||||
SSH.command('cd ../..', '\$', 5)
|
||||
doLoop = True
|
||||
@@ -878,51 +930,60 @@ class OaiCiTest():
|
||||
doLoop = False
|
||||
continue
|
||||
SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
|
||||
result = re.search('got sync', SSH.getBefore())
|
||||
if self.air_interface == 'nr':
|
||||
result = re.search('Starting sync detection', SSH.getBefore())
|
||||
else:
|
||||
result = re.search('got sync', SSH.getBefore())
|
||||
if result is None:
|
||||
time.sleep(6)
|
||||
time.sleep(10)
|
||||
else:
|
||||
doLoop = False
|
||||
logging.debug('Found "got sync" message!')
|
||||
if gotSyncStatus == False:
|
||||
# we certainly need to stop the lte-uesoftmodem process if it is still running!
|
||||
SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
result = re.search('-uesoftmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT -r *-uesoftmodem', '\$', 4)
|
||||
time.sleep(3)
|
||||
# We are now checking if sync w/ eNB DOES NOT OCCUR
|
||||
# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
|
||||
doLoop = True
|
||||
loopCounter = 10
|
||||
while (doLoop):
|
||||
loopCounter = loopCounter - 1
|
||||
if (loopCounter == 0):
|
||||
# Here we do have a great chance that the UE did cell-sync w/ eNB
|
||||
doLoop = False
|
||||
doOutterLoop = False
|
||||
fullSyncStatus = True
|
||||
continue
|
||||
SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
|
||||
result = re.search('No cell synchronization found', SSH.getBefore())
|
||||
if result is None:
|
||||
time.sleep(6)
|
||||
else:
|
||||
doLoop = False
|
||||
fullSyncStatus = False
|
||||
logging.debug('Found: "No cell synchronization" message! --> try again')
|
||||
time.sleep(6)
|
||||
SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
|
||||
outterLoopCounter = outterLoopCounter - 1
|
||||
if (outterLoopCounter == 0):
|
||||
continue
|
||||
if self.air_interface == 'nr':
|
||||
fullSyncStatus = True
|
||||
doOutterLoop = False
|
||||
else:
|
||||
# We are now checking if sync w/ eNB DOES NOT OCCUR
|
||||
# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
|
||||
doLoop = True
|
||||
loopCounter = 10
|
||||
while (doLoop):
|
||||
loopCounter = loopCounter - 1
|
||||
if (loopCounter == 0):
|
||||
# Here we do have a great chance that the UE did cell-sync w/ eNB
|
||||
doLoop = False
|
||||
doOutterLoop = False
|
||||
fullSyncStatus = True
|
||||
continue
|
||||
SSH.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
|
||||
result = re.search('No cell synchronization found', SSH.getBefore())
|
||||
if result is None:
|
||||
time.sleep(6)
|
||||
else:
|
||||
doLoop = False
|
||||
fullSyncStatus = False
|
||||
logging.debug('Found: "No cell synchronization" message! --> try again')
|
||||
time.sleep(6)
|
||||
SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
|
||||
outterLoopCounter = outterLoopCounter - 1
|
||||
if (outterLoopCounter == 0):
|
||||
doOutterLoop = False
|
||||
|
||||
if fullSyncStatus and gotSyncStatus:
|
||||
if fullSyncStatus and gotSyncStatus and self.air_interface == 'lte':
|
||||
result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
|
||||
if result is None:
|
||||
SSH.command('ifconfig oaitun_ue1', '\$', 4)
|
||||
SSH.command('ifconfig oaitun_ue1', '\$', 4)
|
||||
# ifconfig output is different between ubuntu 16 and ubuntu 18
|
||||
result = re.search('inet addr:1|inet 1', SSH.getBefore())
|
||||
@@ -935,7 +996,7 @@ class OaiCiTest():
|
||||
tunnelInterfaceStatus = False
|
||||
if self.eNBmbmsEnables[0]:
|
||||
self.command('ifconfig oaitun_uem1', '\$', 4)
|
||||
result = re.search('inet addr', str(self.ssh.before))
|
||||
result = re.search('inet addr', SSH.getBefore())
|
||||
if result is not None:
|
||||
logging.debug('\u001B[1m oaitun_uem1 interface is mounted and configured\u001B[0m')
|
||||
tunnelInterfaceStatus = tunnelInterfaceStatus and True
|
||||
@@ -944,6 +1005,8 @@ class OaiCiTest():
|
||||
tunnelInterfaceStatus = False
|
||||
else:
|
||||
tunnelInterfaceStatus = True
|
||||
else:
|
||||
tunnelInterfaceStatus = True
|
||||
|
||||
SSH.close()
|
||||
if fullSyncStatus and gotSyncStatus and tunnelInterfaceStatus:
|
||||
@@ -955,11 +1018,15 @@ class OaiCiTest():
|
||||
self.UEDevicesStatus = []
|
||||
self.UEDevicesStatus.append(UE_STATUS_DETACHED)
|
||||
else:
|
||||
if self.eNBmbmsEnables[0]:
|
||||
self.htmlUEFailureMsg = 'oaitun_ue1/oaitun_uem1 interfaces are either NOT mounted or NOT configured'
|
||||
if self.air_interface == 'lte':
|
||||
if self.eNBmbmsEnables[0]:
|
||||
self.htmlUEFailureMsg = 'oaitun_ue1/oaitun_uem1 interfaces are either NOT mounted or NOT configured'
|
||||
else:
|
||||
self.htmlUEFailureMsg = 'oaitun_ue1 interface is either NOT mounted or NOT configured'
|
||||
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
|
||||
else:
|
||||
self.htmlUEFailureMsg = 'oaitun_ue1 interface is either NOT mounted or NOT configured'
|
||||
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
|
||||
self.htmlUEFailureMsg = 'nr-uesoftmodem did NOT synced'
|
||||
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE')
|
||||
logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
|
||||
self.AutoTerminateUEandeNB()
|
||||
|
||||
@@ -1858,7 +1925,6 @@ class OaiCiTest():
|
||||
else:
|
||||
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
|
||||
SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets/', '\$', 5)
|
||||
SSH.command('cd cmake_targets', '\$', 5)
|
||||
ping_time = re.findall("-c (\d+)",str(self.ping_args))
|
||||
ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '.log', '\$', int(ping_time[0])*1.5)
|
||||
# TIMEOUT CASE
|
||||
@@ -2745,7 +2811,8 @@ class OaiCiTest():
|
||||
multi_jobs = []
|
||||
status_queue = SimpleQueue()
|
||||
# in noS1 config, no need to check status from EPC
|
||||
result = re.search('noS1', str(self.Initialize_eNB_args))
|
||||
# in gNB also currently no need to check
|
||||
result = re.search('noS1|band78', str(self.Initialize_eNB_args))
|
||||
if result is None:
|
||||
p = Process(target = EPC.CheckHSSProcess, args = (status_queue,))
|
||||
p.daemon = True
|
||||
@@ -2826,8 +2893,8 @@ class OaiCiTest():
|
||||
def CheckOAIUEProcess(self, status_queue):
|
||||
try:
|
||||
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
|
||||
SSH.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
SSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface + '-uesoftmodem | grep -v grep', '\$', 5)
|
||||
result = re.search(self.air_interface + '-uesoftmodem', SSH.getBefore())
|
||||
if result is None:
|
||||
logging.debug('\u001B[1;37;41m OAI UE Process Not Found! \u001B[0m')
|
||||
status_queue.put(OAI_UE_PROCESS_FAILED)
|
||||
@@ -2857,8 +2924,8 @@ class OaiCiTest():
|
||||
lUserName = self.eNBUserName
|
||||
lPassWord = self.eNBPassword
|
||||
SSH.open(lIpAddr, lUserName, lPassWord)
|
||||
SSH.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search('lte-softmodem', str(self.ssh.before))
|
||||
SSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface + '-softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search(self.air_interface + '-softmodem', SSH.getBefore())
|
||||
if result is None:
|
||||
logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
|
||||
status_queue.put(ENB_PROCESS_FAILED)
|
||||
@@ -3011,7 +3078,7 @@ class OaiCiTest():
|
||||
result = re.search('[Cc]ore [dD]ump', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
foundSegFault = True
|
||||
result = re.search('./lte_build_oai/build/lte-softmodem', str(line))
|
||||
result = re.search('./ran_build/build/lte-softmodem', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
foundSegFault = True
|
||||
result = re.search('[Aa]ssertion', str(line))
|
||||
@@ -3082,16 +3149,21 @@ class OaiCiTest():
|
||||
mbmsRequestMsg += 1
|
||||
enb_log_file.close()
|
||||
logging.debug(' File analysis completed')
|
||||
self.htmleNBFailureMsg = ''
|
||||
if self.air_interface == 'lte':
|
||||
nodeB_prefix = 'e'
|
||||
else:
|
||||
nodeB_prefix = 'g'
|
||||
if uciStatMsgCount > 0:
|
||||
statMsg = 'eNB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
|
||||
statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += statMsg + '\n'
|
||||
if pdcpFailure > 0:
|
||||
statMsg = 'eNB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
|
||||
statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += statMsg + '\n'
|
||||
if ulschFailure > 0:
|
||||
statMsg = 'eNB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
|
||||
statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += statMsg + '\n'
|
||||
if dropNotEnoughRBs > 0:
|
||||
@@ -3099,22 +3171,25 @@ class OaiCiTest():
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += statMsg + '\n'
|
||||
if rrcSetupComplete > 0:
|
||||
rrcMsg = 'eNB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
|
||||
rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
|
||||
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
if rrcReleaseRequest > 0:
|
||||
rrcMsg = 'eNB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
|
||||
rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
|
||||
rrcMsg = 'eNB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
|
||||
rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
|
||||
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
|
||||
rrcMsg = 'eNB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
|
||||
rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
|
||||
@@ -3148,7 +3223,7 @@ class OaiCiTest():
|
||||
logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rrcMsg + '\n'
|
||||
if rachCanceledProcedure > 0:
|
||||
rachMsg = 'eNB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
|
||||
rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rachMsg + '\n'
|
||||
if isRRU:
|
||||
@@ -3164,18 +3239,18 @@ class OaiCiTest():
|
||||
self.prematureExit = True
|
||||
return ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
|
||||
if foundSegFault:
|
||||
logging.debug('\u001B[1;37;41m eNB ended with a Segmentation Fault! \u001B[0m')
|
||||
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
|
||||
return ENB_PROCESS_SEG_FAULT
|
||||
if foundAssertion:
|
||||
logging.debug('\u001B[1;37;41m eNB ended with an assertion! \u001B[0m')
|
||||
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
|
||||
self.htmleNBFailureMsg += msgAssertion
|
||||
return ENB_PROCESS_ASSERTION
|
||||
if foundRealTimeIssue:
|
||||
logging.debug('\u001B[1;37;41m eNB faced real time issues! \u001B[0m')
|
||||
self.htmleNBFailureMsg += 'eNB faced real time issues!\n'
|
||||
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
|
||||
self.htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues!\n'
|
||||
#return ENB_PROCESS_REALTIME_ISSUE
|
||||
if rlcDiscardBuffer > 0:
|
||||
rlcMsg = 'eNB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
|
||||
rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
|
||||
logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
|
||||
self.htmleNBFailureMsg += rlcMsg + '\n'
|
||||
return ENB_PROCESS_REALTIME_ISSUE
|
||||
@@ -3194,14 +3269,34 @@ class OaiCiTest():
|
||||
uciStatMsgCount = 0
|
||||
pdcpDataReqFailedCount = 0
|
||||
badDciCount = 0
|
||||
f1aRetransmissionCount = 0
|
||||
fatalErrorCount = 0
|
||||
macBsrTimerExpiredCount = 0
|
||||
rrcConnectionRecfgComplete = 0
|
||||
no_cell_sync_found = False
|
||||
mib_found = False
|
||||
frequency_found = False
|
||||
plmn_found = False
|
||||
nrUEFlag = False
|
||||
nrDecodeMib = 0
|
||||
nrFoundDCI = 0
|
||||
nrCRCOK = 0
|
||||
mbms_messages = 0
|
||||
self.htmlUEFailureMsg = ''
|
||||
for line in ue_log_file.readlines():
|
||||
result = re.search('nr_synchro_time', str(line))
|
||||
if result is not None:
|
||||
nrUEFlag = True
|
||||
if nrUEFlag:
|
||||
result = re.search('decode mib', str(line))
|
||||
if result is not None:
|
||||
nrDecodeMib += 1
|
||||
result = re.search('found 1 DCIs', str(line))
|
||||
if result is not None:
|
||||
nrFoundDCI += 1
|
||||
result = re.search('CRC OK', str(line))
|
||||
if result is not None:
|
||||
nrCRCOK += 1
|
||||
result = re.search('Exiting OAI softmodem', str(line))
|
||||
if result is not None:
|
||||
exitSignalReceived = True
|
||||
@@ -3229,9 +3324,18 @@ class OaiCiTest():
|
||||
result = re.search('PDCP data request failed', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
pdcpDataReqFailedCount += 1
|
||||
result = re.search('bad DCI 1A', str(line))
|
||||
result = re.search('bad DCI 1', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
badDciCount += 1
|
||||
result = re.search('Format1A Retransmission but TBS are different', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
f1aRetransmissionCount += 1
|
||||
result = re.search('FATAL ERROR', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
fatalErrorCount += 1
|
||||
result = re.search('MAC BSR Triggered ReTxBSR Timer expiry', str(line))
|
||||
if result is not None and not exitSignalReceived:
|
||||
macBsrTimerExpiredCount += 1
|
||||
result = re.search('Generating RRCConnectionReconfigurationComplete', str(line))
|
||||
if result is not None:
|
||||
rrcConnectionRecfgComplete += 1
|
||||
@@ -3324,6 +3428,23 @@ class OaiCiTest():
|
||||
statMsg = 'UE connected to eNB (' + str(rrcConnectionRecfgComplete) + ' RRCConnectionReconfigurationComplete message(s) generated)'
|
||||
logging.debug('\033[94m' + statMsg + '\033[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if nrUEFlag:
|
||||
if nrDecodeMib > 0:
|
||||
statMsg = 'UE showed ' + str(nrDecodeMib) + ' MIB decode message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if nrFoundDCI > 0:
|
||||
statMsg = 'UE showed ' + str(nrFoundDCI) + ' DCI found message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if nrCRCOK > 0:
|
||||
statMsg = 'UE showed ' + str(nrCRCOK) + ' PDSCH decoding message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if not frequency_found:
|
||||
statMsg = 'NR-UE could NOT synch!'
|
||||
logging.error('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if uciStatMsgCount > 0:
|
||||
statMsg = 'UE showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
@@ -3333,7 +3454,19 @@ class OaiCiTest():
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if badDciCount > 0:
|
||||
statMsg = 'UE showed ' + str(badDciCount) + ' "bad DCI 1A" message(s)'
|
||||
statMsg = 'UE showed ' + str(badDciCount) + ' "bad DCI 1(A)" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if f1aRetransmissionCount > 0:
|
||||
statMsg = 'UE showed ' + str(f1aRetransmissionCount) + ' "Format1A Retransmission but TBS are different" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if fatalErrorCount > 0:
|
||||
statMsg = 'UE showed ' + str(fatalErrorCount) + ' "FATAL ERROR:" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if macBsrTimerExpiredCount > 0:
|
||||
statMsg = 'UE showed ' + str(fatalErrorCount) + ' "MAC BSR Triggered ReTxBSR Timer expiry" message(s)'
|
||||
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if self.eNBmbmsEnables[0]:
|
||||
@@ -3346,20 +3479,32 @@ class OaiCiTest():
|
||||
self.htmlUEFailureMsg += statMsg + '\n'
|
||||
if foundSegFault:
|
||||
logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m')
|
||||
return ENB_PROCESS_SEG_FAULT
|
||||
if not nrUEFlag:
|
||||
return OAI_UE_PROCESS_SEG_FAULT
|
||||
else:
|
||||
if not frequency_found:
|
||||
return OAI_UE_PROCESS_SEG_FAULT
|
||||
if foundAssertion:
|
||||
logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m')
|
||||
self.htmlUEFailureMsg += 'UE showed an assertion!\n'
|
||||
if not mib_found or not frequency_found:
|
||||
return OAI_UE_PROCESS_ASSERTION
|
||||
if not nrUEFlag:
|
||||
if not mib_found or not frequency_found:
|
||||
return OAI_UE_PROCESS_ASSERTION
|
||||
else:
|
||||
if not frequency_found:
|
||||
return OAI_UE_PROCESS_ASSERTION
|
||||
if foundRealTimeIssue:
|
||||
logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m')
|
||||
self.htmlUEFailureMsg += 'UE faced real time issues!\n'
|
||||
#return ENB_PROCESS_REALTIME_ISSUE
|
||||
if no_cell_sync_found and not mib_found:
|
||||
logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m')
|
||||
self.htmlUEFailureMsg += 'UE could not synchronize!\n'
|
||||
return OAI_UE_PROCESS_COULD_NOT_SYNC
|
||||
if nrUEFlag:
|
||||
if not frequency_found:
|
||||
return OAI_UE_PROCESS_COULD_NOT_SYNC
|
||||
else:
|
||||
if no_cell_sync_found and not mib_found:
|
||||
logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m')
|
||||
self.htmlUEFailureMsg += 'UE could not synchronize!\n'
|
||||
return OAI_UE_PROCESS_COULD_NOT_SYNC
|
||||
return 0
|
||||
|
||||
def TerminateeNB(self):
|
||||
@@ -3383,16 +3528,20 @@ class OaiCiTest():
|
||||
sys.exit('Insufficient Parameter')
|
||||
SSH.open(lIpAddr, lUserName, lPassWord)
|
||||
SSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
||||
if self.air_interface == 'lte':
|
||||
nodeB_prefix = 'e'
|
||||
else:
|
||||
nodeB_prefix = 'g'
|
||||
SSH.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search('lte-softmodem', SSH.getBefore())
|
||||
result = re.search('-softmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S daemon --name=enb' + str(self.eNB_instance) + '_daemon --stop', '\$', 5)
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT lte-softmodem || true', '\$', 5)
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem || true', '\$', 5)
|
||||
time.sleep(10)
|
||||
SSH.command('stdbuf -o0 ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search('lte-softmodem', SSH.getBefore())
|
||||
result = re.search('-softmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL lte-softmodem || true', '\$', 5)
|
||||
SSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem || true', '\$', 5)
|
||||
time.sleep(5)
|
||||
SSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
||||
SSH.close()
|
||||
@@ -3434,14 +3583,14 @@ class OaiCiTest():
|
||||
if analyzeFile:
|
||||
copyin_res = SSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
|
||||
if (copyin_res == -1):
|
||||
logging.debug('\u001B[1;37;41m Could not copy eNB logfile to analyze it! \u001B[0m')
|
||||
self.htmleNBFailureMsg = 'Could not copy eNB logfile to analyze it!'
|
||||
logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
|
||||
self.htmleNBFailureMsg = 'Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
|
||||
self.CreateHtmlTestRow('N/A', 'KO', ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
|
||||
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
||||
return
|
||||
if self.eNB_serverId != '0':
|
||||
SSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
|
||||
logging.debug('\u001B[1m Analyzing eNB logfile \u001B[0m ' + fileToAnalyze)
|
||||
logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
|
||||
logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
|
||||
if (logStatus < 0):
|
||||
self.CreateHtmlTestRow('N/A', 'KO', logStatus)
|
||||
@@ -3593,15 +3742,14 @@ class OaiCiTest():
|
||||
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
|
||||
SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5)
|
||||
SSH.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
result = re.search('-uesoftmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S daemon --name=ue' + str(self.UE_instance) + '_daemon --stop', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT lte-uesoftmodem || true', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT -r .*-uesoftmodem || true', '\$', 5)
|
||||
time.sleep(10)
|
||||
SSH.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
|
||||
result = re.search('lte-uesoftmodem', SSH.getBefore())
|
||||
result = re.search('-uesoftmodem', SSH.getBefore())
|
||||
if result is not None:
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL lte-uesoftmodem || true', '\$', 5)
|
||||
SSH.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL -r .*-uesoftmodem || true', '\$', 5)
|
||||
time.sleep(5)
|
||||
SSH.command('rm -f my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5)
|
||||
SSH.close()
|
||||
@@ -3625,11 +3773,16 @@ class OaiCiTest():
|
||||
logging.debug('\u001B[1m' + ueAction + ' Failed \u001B[0m')
|
||||
self.htmlUEFailureMsg = '<b>' + ueAction + ' Failed</b>\n' + self.htmlUEFailureMsg
|
||||
self.CreateHtmlTestRow('N/A', 'KO', logStatus, 'UE')
|
||||
# In case of sniffing on commercial eNBs we have random results
|
||||
# Not an error then
|
||||
if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
|
||||
self.Initialize_OAI_UE_args = ''
|
||||
self.AutoTerminateUEandeNB()
|
||||
if self.air_interface == 'lte':
|
||||
# In case of sniffing on commercial eNBs we have random results
|
||||
# Not an error then
|
||||
if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
|
||||
self.Initialize_OAI_UE_args = ''
|
||||
self.AutoTerminateUEandeNB()
|
||||
else:
|
||||
if (logStatus == OAI_UE_PROCESS_COULD_NOT_SYNC):
|
||||
self.Initialize_OAI_UE_args = ''
|
||||
self.AutoTerminateUEandeNB()
|
||||
else:
|
||||
logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
|
||||
self.htmlUEFailureMsg = '<b>' + ueAction + ' Completed</b>\n' + self.htmlUEFailureMsg
|
||||
@@ -3908,6 +4061,17 @@ class OaiCiTest():
|
||||
if result is not None:
|
||||
self.OsVersion = result.group('os_type')
|
||||
logging.debug('OS is: ' + self.OsVersion)
|
||||
else:
|
||||
SSH.command('hostnamectl', '\$', 5)
|
||||
result = re.search('Operating System: (?P<os_type>[a-zA-Z0-9\-\_\.\ ]+)', SSH.getBefore())
|
||||
if result is not None:
|
||||
self.OsVersion = result.group('os_type')
|
||||
if self.OsVersion == 'CentOS Linux 7 ':
|
||||
SSH.command('cat /etc/redhat-release', '\$', 5)
|
||||
result = re.search('CentOS Linux release (?P<os_version>[0-9\.]+)', SSH.getBefore())
|
||||
if result is not None:
|
||||
self.OsVersion = self.OsVersion.replace('7 ', result.group('os_version'))
|
||||
logging.debug('OS is: ' + self.OsVersion)
|
||||
SSH.command('uname -r', '\$', 5)
|
||||
result = re.search('uname -r\\\\r\\\\n(?P<kernel_version>[a-zA-Z0-9\-\_\.]+)', SSH.getBefore())
|
||||
if result is not None:
|
||||
@@ -3918,11 +4082,23 @@ class OaiCiTest():
|
||||
if result is not None:
|
||||
self.UhdVersion = result.group('uhd_version')
|
||||
logging.debug('UHD Version is: ' + self.UhdVersion)
|
||||
SSH.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 60)
|
||||
result = re.search('product: (?P<usrp_board>[0-9A-Za-z]+)\\\\r\\\\n', SSH.getBefore())
|
||||
if result is not None:
|
||||
self.UsrpBoard = result.group('usrp_board')
|
||||
logging.debug('USRP Board is: ' + self.UsrpBoard)
|
||||
else:
|
||||
SSH.command('uhd_config_info --version', '\$', 5)
|
||||
result = re.search('UHD (?P<uhd_version>[a-zA-Z0-9\.\-]+)', SSH.getBefore())
|
||||
if result is not None:
|
||||
self.UhdVersion = result.group('uhd_version')
|
||||
logging.debug('UHD Version is: ' + self.UhdVersion)
|
||||
SSH.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 30)
|
||||
usrp_boards = re.findall('product: ([0-9A-Za-z]+)\\\\r\\\\n', SSH.getBefore())
|
||||
count = 0
|
||||
for board in usrp_boards:
|
||||
if count == 0:
|
||||
self.UsrpBoard = board
|
||||
else:
|
||||
self.UsrpBoard += ',' + board
|
||||
count += 1
|
||||
if count > 0:
|
||||
logging.debug('USRP Board(s) : ' + self.UsrpBoard)
|
||||
SSH.command('lscpu', '\$', 5)
|
||||
result = re.search('CPU\(s\): *(?P<nb_cpus>[0-9]+).*Model name: *(?P<model>[a-zA-Z0-9\-\_\.\ \(\)]+).*CPU MHz: *(?P<cpu_mhz>[0-9\.]+)', SSH.getBefore())
|
||||
if result is not None:
|
||||
@@ -4141,6 +4317,7 @@ class OaiCiTest():
|
||||
self.htmlFile.write(' <td></td>\n')
|
||||
self.htmlFile.write(' <td></td>\n')
|
||||
self.htmlFile.write(' </tr>\n')
|
||||
|
||||
self.htmlFile.write(' <tr>\n')
|
||||
self.htmlFile.write(' <th colspan=5 bgcolor = "#33CCFF">Final Status</th>\n')
|
||||
if passStatus:
|
||||
@@ -4178,7 +4355,7 @@ class OaiCiTest():
|
||||
self.htmlFile.write(' <td bgcolor = "lightcoral" >KO - eNB process not found</td>\n')
|
||||
elif (processesStatus == OAI_UE_PROCESS_FAILED):
|
||||
self.htmlFile.write(' <td bgcolor = "lightcoral" >KO - OAI UE process not found</td>\n')
|
||||
elif (processesStatus == ENB_PROCESS_SEG_FAULT):
|
||||
elif (processesStatus == ENB_PROCESS_SEG_FAULT) or (processesStatus == OAI_UE_PROCESS_SEG_FAULT):
|
||||
self.htmlFile.write(' <td bgcolor = "lightcoral" >KO - ' + machine + ' process ended in Segmentation Fault</td>\n')
|
||||
elif (processesStatus == ENB_PROCESS_ASSERTION) or (processesStatus == OAI_UE_PROCESS_ASSERTION):
|
||||
self.htmlFile.write(' <td bgcolor = "lightcoral" >KO - ' + machine + ' process ended in Assertion</td>\n')
|
||||
@@ -4324,6 +4501,14 @@ def CheckClassValidity(action,id):
|
||||
def GetParametersFromXML(action):
|
||||
if action == 'Build_eNB':
|
||||
CiTestObj.Build_eNB_args = test.findtext('Build_eNB_args')
|
||||
forced_workspace_cleanup = test.findtext('forced_workspace_cleanup')
|
||||
if (forced_workspace_cleanup is None):
|
||||
CiTestObj.Build_eNB_forced_workspace_cleanup = False
|
||||
else:
|
||||
if re.match('true', forced_workspace_cleanup, re.IGNORECASE):
|
||||
CiTestObj.Build_eNB_forced_workspace_cleanup = True
|
||||
else:
|
||||
CiTestObj.Build_eNB_forced_workspace_cleanup = False
|
||||
CiTestObj.eNB_instance = test.findtext('eNB_instance')
|
||||
if (CiTestObj.eNB_instance is None):
|
||||
CiTestObj.eNB_instance = '0'
|
||||
@@ -4356,6 +4541,11 @@ def GetParametersFromXML(action):
|
||||
CiTestObj.eNB_serverId = test.findtext('eNB_serverId')
|
||||
if (CiTestObj.eNB_serverId is None):
|
||||
CiTestObj.eNB_serverId = '0'
|
||||
CiTestObj.air_interface = test.findtext('air_interface')
|
||||
if (CiTestObj.air_interface is None):
|
||||
CiTestObj.air_interface = 'lte'
|
||||
else:
|
||||
CiTestObj.air_interface = CiTestObj.air_interface.lower()
|
||||
|
||||
if action == 'Terminate_eNB':
|
||||
CiTestObj.eNB_instance = test.findtext('eNB_instance')
|
||||
@@ -4364,6 +4554,11 @@ def GetParametersFromXML(action):
|
||||
CiTestObj.eNB_serverId = test.findtext('eNB_serverId')
|
||||
if (CiTestObj.eNB_serverId is None):
|
||||
CiTestObj.eNB_serverId = '0'
|
||||
CiTestObj.air_interface = test.findtext('air_interface')
|
||||
if (CiTestObj.air_interface is None):
|
||||
CiTestObj.air_interface = 'lte'
|
||||
else:
|
||||
CiTestObj.air_interface = CiTestObj.air_interface.lower()
|
||||
|
||||
if action == 'Attach_UE':
|
||||
nbMaxUEtoAttach = test.findtext('nbMaxUEtoAttach')
|
||||
@@ -4381,12 +4576,22 @@ def GetParametersFromXML(action):
|
||||
|
||||
if action == 'Build_OAI_UE':
|
||||
CiTestObj.Build_OAI_UE_args = test.findtext('Build_OAI_UE_args')
|
||||
CiTestObj.clean_repository = test.findtext('clean_repository')
|
||||
if (CiTestObj.clean_repository == 'false'):
|
||||
CiTestObj.clean_repository = False
|
||||
else:
|
||||
CiTestObj.clean_repository = True
|
||||
|
||||
if action == 'Initialize_OAI_UE':
|
||||
CiTestObj.Initialize_OAI_UE_args = test.findtext('Initialize_OAI_UE_args')
|
||||
CiTestObj.UE_instance = test.findtext('UE_instance')
|
||||
if (CiTestObj.UE_instance is None):
|
||||
CiTestObj.UE_instance = '0'
|
||||
CiTestObj.air_interface = test.findtext('air_interface')
|
||||
if (CiTestObj.air_interface is None):
|
||||
CiTestObj.air_interface = 'lte'
|
||||
else:
|
||||
CiTestObj.air_interface = SSH.air_interface.lower()
|
||||
|
||||
if action == 'Terminate_OAI_UE':
|
||||
CiTestObj.eNB_instance = test.findtext('UE_instance')
|
||||
@@ -4690,10 +4895,13 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE):
|
||||
count = 0
|
||||
foundCount = 0
|
||||
while (count < CiTestObj.nbTestXMLfiles):
|
||||
xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[count]
|
||||
#xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[count]
|
||||
xml_test_file = sys.path[0] + "/" + CiTestObj.testXMLfiles[count]
|
||||
if (os.path.isfile(xml_test_file)):
|
||||
xmlTree = ET.parse(xml_test_file)
|
||||
try:
|
||||
xmlTree = ET.parse(xml_test_file)
|
||||
except:
|
||||
print("Error while parsing file: " + xml_test_file)
|
||||
xmlRoot = xmlTree.getroot()
|
||||
CiTestObj.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count)))
|
||||
CiTestObj.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-' + str(count)))
|
||||
|
||||
Reference in New Issue
Block a user