|
|
|
@@ -43,7 +43,7 @@ import cls_cmd
|
|
|
|
|
#-----------------------------------------------------------
|
|
|
|
|
# OAI Testing modules
|
|
|
|
|
#-----------------------------------------------------------
|
|
|
|
|
import sshconnection as SSH
|
|
|
|
|
import cls_cmd
|
|
|
|
|
import helpreadme as HELP
|
|
|
|
|
import constants as CONST
|
|
|
|
|
from cls_ci_helper import archiveArtifact
|
|
|
|
@@ -72,19 +72,13 @@ class RANManagement():
|
|
|
|
|
self.eNB2UserName = ''
|
|
|
|
|
self.eNB2Password = ''
|
|
|
|
|
self.eNB2SourceCodePath = ''
|
|
|
|
|
self.Build_eNB_args = ''
|
|
|
|
|
self.Build_eNB_forced_workspace_cleanup = False
|
|
|
|
|
self.Initialize_eNB_args = ''
|
|
|
|
|
self.imageKind = ''
|
|
|
|
|
self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
|
|
|
|
|
self.eNB_instance = 0
|
|
|
|
|
self.eNB_serverId = ['', '', '']
|
|
|
|
|
self.eNBLogFiles = ['', '', '']
|
|
|
|
|
self.air_interface = ''
|
|
|
|
|
self.eNBOptions = ['', '', '']
|
|
|
|
|
self.eNBmbmsEnables = [False, False, False]
|
|
|
|
|
self.eNBstatuses = [-1, -1, -1]
|
|
|
|
|
self.testCase_id = ''
|
|
|
|
|
self.epcPcapFile = ''
|
|
|
|
|
self.runtime_stats= ''
|
|
|
|
|
self.datalog_rt_stats={}
|
|
|
|
|
self.datalog_rt_stats_file='datalog_rt_stats.default.yaml'
|
|
|
|
@@ -100,261 +94,98 @@ class RANManagement():
|
|
|
|
|
# RAN management functions
|
|
|
|
|
#-----------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
def InitializeeNB(self, HTML):
|
|
|
|
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
|
|
|
|
lIpAddr = self.eNBIPAddress
|
|
|
|
|
lUserName = self.eNBUserName
|
|
|
|
|
lPassWord = self.eNBPassword
|
|
|
|
|
lSourcePath = self.eNBSourceCodePath
|
|
|
|
|
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
|
|
|
lIpAddr = self.eNB1IPAddress
|
|
|
|
|
lUserName = self.eNB1UserName
|
|
|
|
|
lPassWord = self.eNB1Password
|
|
|
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
|
|
|
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
|
|
|
lIpAddr = self.eNB2IPAddress
|
|
|
|
|
lUserName = self.eNB2UserName
|
|
|
|
|
lPassWord = self.eNB2Password
|
|
|
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
|
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
|
|
|
HELP.GenericHelp(CONST.Version)
|
|
|
|
|
sys.exit('Insufficient Parameter')
|
|
|
|
|
logging.debug('Starting eNB/gNB on server: ' + lIpAddr)
|
|
|
|
|
def InitializeeNB(self, ctx, node, HTML):
|
|
|
|
|
if not node:
|
|
|
|
|
raise ValueError(f"{node=}")
|
|
|
|
|
logging.debug('Starting eNB/gNB on server: ' + node)
|
|
|
|
|
|
|
|
|
|
self.testCase_id = HTML.testCase_id
|
|
|
|
|
mySSH = SSH.SSHConnection()
|
|
|
|
|
lSourcePath = self.eNBSourceCodePath
|
|
|
|
|
cmd = cls_cmd.getConnection(node)
|
|
|
|
|
|
|
|
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
|
|
|
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
|
|
|
|
# Initialize_eNB_args usually start with -O and followed by the location in repository
|
|
|
|
|
full_config_file = self.Initialize_eNB_args.replace('-O ','')
|
|
|
|
|
extra_options = ''
|
|
|
|
|
extIdx = full_config_file.find('.conf')
|
|
|
|
|
if (extIdx > 0):
|
|
|
|
|
extra_options = full_config_file[extIdx + 5:]
|
|
|
|
|
# if tracer options is on, compiling and running T Tracer
|
|
|
|
|
result = re.search('T_stdout', str(extra_options))
|
|
|
|
|
if result is not None:
|
|
|
|
|
logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
|
|
|
|
|
mySSH.command('cd common/utils/T/tracer', '\$', 5)
|
|
|
|
|
mySSH.command('make', '\$', 10)
|
|
|
|
|
mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
|
|
|
|
|
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
|
|
|
|
full_config_file = full_config_file[:extIdx + 5]
|
|
|
|
|
config_path, config_file = os.path.split(full_config_file)
|
|
|
|
|
else:
|
|
|
|
|
sys.exit('Insufficient Parameter')
|
|
|
|
|
ci_full_config_file = config_path + '/ci-' + config_file
|
|
|
|
|
rruCheck = False
|
|
|
|
|
result = re.search('^rru|^rcc|^du.band', str(config_file))
|
|
|
|
|
if result is not None:
|
|
|
|
|
rruCheck = True
|
|
|
|
|
mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
|
|
|
|
|
mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
|
|
|
mySSH.command('grep --colour=never enable_enb_m2 ' + ci_full_config_file, '\$', 2);
|
|
|
|
|
result = re.search('yes', mySSH.getBefore())
|
|
|
|
|
if result is not None:
|
|
|
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = True
|
|
|
|
|
logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
|
|
|
|
|
result = re.search('noS1', str(self.Initialize_eNB_args))
|
|
|
|
|
eNBinNoS1 = False
|
|
|
|
|
if result is not None:
|
|
|
|
|
eNBinNoS1 = True
|
|
|
|
|
logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
|
|
|
|
|
# Launch eNB with the modified config file
|
|
|
|
|
mySSH.command('source oaienv', '\$', 5)
|
|
|
|
|
mySSH.command('cd cmake_targets', '\$', 5)
|
|
|
|
|
if self.air_interface[self.eNB_instance] == 'nr-softmodem':
|
|
|
|
|
mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5)
|
|
|
|
|
mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5)
|
|
|
|
|
# NOTE: WE SHALL do a check if the executable is present (in case build went wrong)
|
|
|
|
|
if (extIdx <= 0):
|
|
|
|
|
raise ValueError(f"no config file in {self.Initialize_eNB_args}")
|
|
|
|
|
extra_options = full_config_file[extIdx + 5:]
|
|
|
|
|
full_config_file = full_config_file[:extIdx + 5]
|
|
|
|
|
config_path, config_file = os.path.split(full_config_file)
|
|
|
|
|
|
|
|
|
|
#hack UHD_RFNOC_DIR variable for gNB / N310 on RHEL8 server:
|
|
|
|
|
#if the USRP address is in the xml then we are using an eth USRP (N3xx)
|
|
|
|
|
if self.air_interface[self.eNB_instance] == 'lte-softmodem':
|
|
|
|
|
gNB = False
|
|
|
|
|
else:
|
|
|
|
|
gNB = True
|
|
|
|
|
mySSH.command(f'echo "ulimit -c unlimited && {self.cmd_prefix} ./ran_build/build/{self.air_interface[self.eNB_instance]} -O {lSourcePath}/{ci_full_config_file} {extra_options}" > ./my-lte-softmodem-run{self.eNB_instance}.sh', '\$', 5)
|
|
|
|
|
logfile = f'{lSourcePath}/cmake_targets/enb.log'
|
|
|
|
|
cmd.cd(f"{lSourcePath}/cmake_targets/") # important: set wd so nrL1_stats.log etc are logged here
|
|
|
|
|
cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/{self.air_interface} -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &')
|
|
|
|
|
|
|
|
|
|
mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
|
|
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
|
|
|
|
|
mySSH.command('echo $USER; nohup sudo -E stdbuf -o0 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
|
|
|
|
|
|
|
|
|
|
self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
|
|
|
|
|
if extra_options != '':
|
|
|
|
|
self.eNBOptions[int(self.eNB_instance)] = extra_options
|
|
|
|
|
time.sleep(6)
|
|
|
|
|
doLoop = True
|
|
|
|
|
loopCounter = 20
|
|
|
|
|
self.eNBOptions = extra_options
|
|
|
|
|
|
|
|
|
|
enbDidSync = False
|
|
|
|
|
while (doLoop):
|
|
|
|
|
loopCounter = loopCounter - 1
|
|
|
|
|
if (loopCounter == 0):
|
|
|
|
|
# In case of T tracer recording, we may need to kill it
|
|
|
|
|
result = re.search('T_stdout', str(self.Initialize_eNB_args))
|
|
|
|
|
if result is not None:
|
|
|
|
|
mySSH.command('killall --signal SIGKILL record', '\$', 5)
|
|
|
|
|
mySSH.close()
|
|
|
|
|
doLoop = False
|
|
|
|
|
logging.error('\u001B[1;37;41m eNB/gNB logging system did not show got sync! \u001B[0m')
|
|
|
|
|
HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | grep -E --text --color=never -i "wait|sync|Starting|Started"', '\$', 4)
|
|
|
|
|
if rruCheck:
|
|
|
|
|
result = re.search('wait RUs', mySSH.getBefore())
|
|
|
|
|
else:
|
|
|
|
|
result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
|
|
|
|
|
if result is None:
|
|
|
|
|
time.sleep(6)
|
|
|
|
|
else:
|
|
|
|
|
doLoop = False
|
|
|
|
|
enbDidSync = True
|
|
|
|
|
time.sleep(10)
|
|
|
|
|
|
|
|
|
|
rruCheck = False
|
|
|
|
|
result = re.search('^rru|^du.band', str(config_file))
|
|
|
|
|
if result is not None:
|
|
|
|
|
rruCheck = True
|
|
|
|
|
if enbDidSync and eNBinNoS1 and not rruCheck:
|
|
|
|
|
mySSH.command('ifconfig oaitun_enb1', '\$', 4)
|
|
|
|
|
mySSH.command('ifconfig oaitun_enb1', '\$', 4)
|
|
|
|
|
result = re.search('inet addr:1|inet 1', mySSH.getBefore())
|
|
|
|
|
for _ in range(10):
|
|
|
|
|
time.sleep(5)
|
|
|
|
|
ret = cmd.run(f'grep --text -E --color=never -i "wait|sync|Starting|Started" {logfile}', reportNonZero=False)
|
|
|
|
|
result = re.search('got sync|Starting F1AP at CU', ret.stdout)
|
|
|
|
|
if result is not None:
|
|
|
|
|
logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
|
|
|
|
|
else:
|
|
|
|
|
logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
|
|
|
|
|
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
|
|
|
|
mySSH.command('ifconfig oaitun_enm1', '\$', 4)
|
|
|
|
|
result = re.search('inet addr', mySSH.getBefore())
|
|
|
|
|
if result is not None:
|
|
|
|
|
logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
|
|
|
|
|
else:
|
|
|
|
|
logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
|
|
|
|
|
enbDidSync = True
|
|
|
|
|
break
|
|
|
|
|
if not enbDidSync:
|
|
|
|
|
cmd.run(f'sudo killall -9 {self.air_interface}') # in case it did not stop automatically
|
|
|
|
|
archiveArtifact(cmd, ctx, logfile)
|
|
|
|
|
|
|
|
|
|
cmd.close()
|
|
|
|
|
|
|
|
|
|
msg = f'{self.cmd_prefix} {self.air_interface} -O {config_file} {extra_options}'
|
|
|
|
|
if enbDidSync:
|
|
|
|
|
self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId[self.eNB_instance])
|
|
|
|
|
logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m')
|
|
|
|
|
HTML.CreateHtmlTestRowQueue(msg, 'OK', [])
|
|
|
|
|
else:
|
|
|
|
|
logging.error('\u001B[1;37;41m eNB/gNB logging system did not show got sync! \u001B[0m')
|
|
|
|
|
HTML.CreateHtmlTestRowQueue(msg, 'KO', [])
|
|
|
|
|
|
|
|
|
|
mySSH.close()
|
|
|
|
|
|
|
|
|
|
HTML.CreateHtmlTestRow(f'{self.cmd_prefix} {self.air_interface[self.eNB_instance]} -O {config_file} {extra_options}', 'OK', CONST.ALL_PROCESSES_OK)
|
|
|
|
|
logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m')
|
|
|
|
|
return enbDidSync
|
|
|
|
|
|
|
|
|
|
def TerminateeNB(self, ctx, HTML):
|
|
|
|
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
|
|
|
|
lIpAddr = self.eNBIPAddress
|
|
|
|
|
lUserName = self.eNBUserName
|
|
|
|
|
lPassWord = self.eNBPassword
|
|
|
|
|
lSourcePath = self.eNBSourceCodePath
|
|
|
|
|
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
|
|
|
lIpAddr = self.eNB1IPAddress
|
|
|
|
|
lUserName = self.eNB1UserName
|
|
|
|
|
lPassWord = self.eNB1Password
|
|
|
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
|
|
|
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
|
|
|
lIpAddr = self.eNB2IPAddress
|
|
|
|
|
lUserName = self.eNB2UserName
|
|
|
|
|
lPassWord = self.eNB2Password
|
|
|
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
|
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
|
|
|
HELP.GenericHelp(CONST.Version)
|
|
|
|
|
sys.exit('Insufficient Parameter')
|
|
|
|
|
logging.debug('Stopping eNB/gNB on server: ' + lIpAddr)
|
|
|
|
|
mySSH = SSH.SSHConnection()
|
|
|
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
|
|
|
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
|
|
|
|
if self.air_interface[self.eNB_instance] == 'lte-softmodem':
|
|
|
|
|
nodeB_prefix = 'e'
|
|
|
|
|
else:
|
|
|
|
|
nodeB_prefix = 'g'
|
|
|
|
|
mySSH.command('stdbuf -o0 ps -aux | grep --color=never -e softmodem | grep -v grep', '\$', 5)
|
|
|
|
|
result = re.search('-softmodem', mySSH.getBefore())
|
|
|
|
|
def TerminateeNB(self, ctx, node, HTML):
|
|
|
|
|
logging.debug('Stopping eNB/gNB on server: ' + node)
|
|
|
|
|
lSourcePath = self.eNBSourceCodePath
|
|
|
|
|
cmd = cls_cmd.getConnection(node)
|
|
|
|
|
ret = cmd.run('ps -aux | grep --color=never -e softmodem | grep -v grep')
|
|
|
|
|
result = re.search('-softmodem', ret.stdout)
|
|
|
|
|
if result is not None:
|
|
|
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem || true', '\$', 5)
|
|
|
|
|
time.sleep(10)
|
|
|
|
|
mySSH.command('stdbuf -o0 ps -aux | grep --color=never -e softmodem | grep -v grep', '\$', 5)
|
|
|
|
|
result = re.search('-softmodem', mySSH.getBefore())
|
|
|
|
|
if result is not None:
|
|
|
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem || true', '\$', 5)
|
|
|
|
|
time.sleep(5)
|
|
|
|
|
mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
|
|
|
|
#stopping tshark (valid if eNB and enabled in xml, will not harm otherwise)
|
|
|
|
|
logging.debug('\u001B[1m Stopping tshark on xNB \u001B[0m')
|
|
|
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
mySSH.close()
|
|
|
|
|
# if T tracer was run with option 0 (no logs), analyze logs
|
|
|
|
|
# from textlog, otherwise do normal analysis (e.g., option 2)
|
|
|
|
|
result = re.search('T_stdout 0', str(self.Initialize_eNB_args))
|
|
|
|
|
if (result is not None):
|
|
|
|
|
logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
|
|
|
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
|
|
|
mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
|
|
|
|
|
enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
|
|
|
|
|
raw_record_file = enbLogFile.replace('.log', '_record.raw')
|
|
|
|
|
replay_log_file = enbLogFile.replace('.log', '_replay.log')
|
|
|
|
|
extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
|
|
|
|
|
extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
|
|
|
|
|
mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
|
|
|
|
|
mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
|
|
|
|
|
mySSH.command('./textlog -d ' + lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
|
|
|
|
|
mySSH.close()
|
|
|
|
|
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
|
|
|
|
|
logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
|
|
|
|
|
logStatus = self.AnalyzeLogFile_eNB(extracted_log_file, HTML, self.ran_checkers)
|
|
|
|
|
cmd.run('sudo -S killall --signal SIGINT -r .*-softmodem')
|
|
|
|
|
time.sleep(6)
|
|
|
|
|
|
|
|
|
|
ret = cmd.run('ps -aux | grep --color=never -e softmodem | grep -v grep')
|
|
|
|
|
result = re.search('-softmodem', ret.stdout)
|
|
|
|
|
if result is not None:
|
|
|
|
|
cmd.run('sudo -S killall --signal SIGKILL -r .*-softmodem')
|
|
|
|
|
time.sleep(5)
|
|
|
|
|
|
|
|
|
|
# see InitializeeNB()
|
|
|
|
|
logfile = f'{lSourcePath}/cmake_targets/enb.log'
|
|
|
|
|
logdir = os.path.dirname(logfile)
|
|
|
|
|
|
|
|
|
|
file = archiveArtifact(cmd, ctx, logfile)
|
|
|
|
|
archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log")
|
|
|
|
|
archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log")
|
|
|
|
|
cmd.close()
|
|
|
|
|
if file is None:
|
|
|
|
|
logging.debug('\u001B[1;37;41m Could not copy xNB logfile to analyze it! \u001B[0m')
|
|
|
|
|
msg = 'Could not copy xNB logfile to analyze it!'
|
|
|
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + file)
|
|
|
|
|
logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers)
|
|
|
|
|
if logStatus < 0:
|
|
|
|
|
HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
|
|
|
|
|
else:
|
|
|
|
|
HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
|
|
|
self.eNBLogFiles[int(self.eNB_instance)] = ''
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
analyzeFile = False
|
|
|
|
|
if self.eNBLogFiles[int(self.eNB_instance)] != '':
|
|
|
|
|
analyzeFile = True
|
|
|
|
|
fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
|
|
|
|
|
self.eNBLogFiles[int(self.eNB_instance)] = ''
|
|
|
|
|
if analyzeFile:
|
|
|
|
|
with cls_cmd.getConnection(lIpAddr) as cmd:
|
|
|
|
|
file = archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/' + fileToAnalyze)
|
|
|
|
|
archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrL1_stats.log')
|
|
|
|
|
archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrMAC_stats.log')
|
|
|
|
|
if file is None:
|
|
|
|
|
logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
|
|
|
|
|
HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
|
|
|
|
|
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
|
|
|
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
|
|
|
return False
|
|
|
|
|
if self.eNB_serverId[self.eNB_instance] != '0':
|
|
|
|
|
#*stats.log files + pickle + png
|
|
|
|
|
|
|
|
|
|
#debug / tentative
|
|
|
|
|
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
|
|
|
|
|
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
|
|
|
|
|
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
|
|
|
|
|
logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + file)
|
|
|
|
|
logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers)
|
|
|
|
|
if (logStatus < 0):
|
|
|
|
|
HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
|
|
|
|
|
#display rt stats for gNB only
|
|
|
|
|
if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g':
|
|
|
|
|
HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
|
|
|
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
|
|
|
else:
|
|
|
|
|
HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
|
|
|
#display rt stats for gNB only
|
|
|
|
|
if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g':
|
|
|
|
|
if len(self.datalog_rt_stats) != 0:
|
|
|
|
|
HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
|
|
|
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
|
|
|
self.eNBstatuses[int(self.eNB_instance)] = -1
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
return logStatus >= 0
|
|
|
|
|
|
|
|
|
|
def LogCollecteNB(self):
|
|
|
|
|
mySSH = SSH.SSHConnection()
|
|
|
|
@@ -526,16 +357,16 @@ class RANManagement():
|
|
|
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
|
|
|
|
|
X2HO_outNbProcedures += 1
|
|
|
|
|
|
|
|
|
|
if self.eNBOptions[int(self.eNB_instance)] != '':
|
|
|
|
|
res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
|
|
|
|
|
if self.eNBOptions[0] != '':
|
|
|
|
|
res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[0])
|
|
|
|
|
res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)', str(line))
|
|
|
|
|
if res1 is not None and res2 is not None:
|
|
|
|
|
requested_option = int(res1.group('requested_option'))
|
|
|
|
|
applied_option = int(res2.group('applied_option'))
|
|
|
|
|
if requested_option == applied_option:
|
|
|
|
|
htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
|
|
|
|
|
htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[0] + '\n\n'
|
|
|
|
|
else:
|
|
|
|
|
htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
|
|
|
|
|
htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[0] + '\n\n'
|
|
|
|
|
result = re.search('Exiting OAI softmodem|Caught SIGTERM, shutting down', str(line))
|
|
|
|
|
if result is not None:
|
|
|
|
|
exitSignalReceived = True
|
|
|
|
@@ -622,7 +453,7 @@ class RANManagement():
|
|
|
|
|
result = re.search('dropping, not enough RBs', str(line))
|
|
|
|
|
if result is not None:
|
|
|
|
|
dropNotEnoughRBs += 1
|
|
|
|
|
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
|
|
|
|
if self.eNBmbmsEnables[0]:
|
|
|
|
|
result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
|
|
|
|
|
if result is not None:
|
|
|
|
|
mbmsRequestMsg += 1
|
|
|
|
@@ -731,7 +562,7 @@ class RANManagement():
|
|
|
|
|
|
|
|
|
|
#post processing depending on the node type
|
|
|
|
|
if not nodeB_prefix_found:
|
|
|
|
|
if self.air_interface[self.eNB_instance] == 'lte-softmodem':
|
|
|
|
|
if self.air_interface == 'lte-softmodem':
|
|
|
|
|
nodeB_prefix = 'e'
|
|
|
|
|
else:
|
|
|
|
|
nodeB_prefix = 'g'
|
|
|
|
@@ -935,7 +766,7 @@ class RANManagement():
|
|
|
|
|
rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
|
|
|
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
|
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
|
|
|
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
|
|
|
|
if self.eNBmbmsEnables[0]:
|
|
|
|
|
if mbmsRequestMsg > 0:
|
|
|
|
|
rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
|
|
|
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
|
|
@@ -948,8 +779,8 @@ class RANManagement():
|
|
|
|
|
rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
|
|
|
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
|
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
|
|
|
if self.eNBOptions[int(self.eNB_instance)] != '':
|
|
|
|
|
res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
|
|
|
|
|
if self.eNBOptions[0] != '':
|
|
|
|
|
res1 = re.search('drx_Config_present prSetup', self.eNBOptions[0])
|
|
|
|
|
if res1 is not None:
|
|
|
|
|
if cdrxActivationMessageCount > 0:
|
|
|
|
|
rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
|
|
|
|
|