maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/ci_improve_reporting_xnb' into integration_2022_wk09

Browse Source
This commit is contained in:
Robert Schmidt
2022-03-01 18:52:42 +01:00
parent d6dc279439 1fc6d12aef
commit 7857522d95
7 changed files with 101 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+6 -2
View File
@@ -106,6 +106,9 @@ class Containerize():
self.registrySvrId = ''
self.testSvrId = ''
#checkers from xml
self.ran_checkers={}
#-----------------------------------------------------------
# Container management functions
#-----------------------------------------------------------
@@ -624,7 +627,7 @@ class Containerize():
else:
if containerToKill:
logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML)
logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML, self.ran_checkers)
else:
logStatus = 0
if (logStatus < 0):
@@ -788,9 +791,10 @@ class Containerize():
continue
logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML)
logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
if (logStatus < 0):
fullStatus = False
self.exitStatus = 1
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
else:
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
ci-scripts/constants.py
+1
View File
@@ -45,6 +45,7 @@ ENB_PROCESS_REALTIME_ISSUE = -13
ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
ENB_PROCESS_SLAVE_RRU_NOT_SYNCED = -15
ENB_REAL_TIME_PROCESSING_ISSUE = -16
ENB_RETX_ISSUE = -17
HSS_PROCESS_FAILED = -2
HSS_PROCESS_OK = +2
MME_PROCESS_FAILED = -3
ci-scripts/main.py
+6
View File
@@ -401,6 +401,12 @@ def GetParametersFromXML(action):
string_field=test.findtext('nb_healthy')
if (string_field is not None):
CONTAINERS.nb_healthy[0] = int(string_field)
string_field=test.findtext('d_retx_th')
if (string_field is not None):
CONTAINERS.ran_checkers['d_retx_th']= string_field
string_field=test.findtext('u_retx_th')
if (string_field is not None):
CONTAINERS.ran_checkers['u_retx_th']= string_field
elif action == 'PingFromContainer':
string_field = test.findtext('container_name')
ci-scripts/ran.py
+82 -17
View File
@@ -39,6 +39,7 @@ import time
from multiprocessing import Process, Lock, SimpleQueue
import yaml
#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
@@ -758,7 +759,7 @@ class RANManagement():
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
mySSH.close()
def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):
if (not os.path.isfile('./' + eNBlogFile)):
return -1
enb_log_file = open('./' + eNBlogFile, 'r')
@@ -815,10 +816,12 @@ class RANManagement():
removing_ue = 0
#count"X2AP-PDU"
x2ap_pdu = 0
#NSA specific log markers
nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []}
#gnb specific log markers
gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]}
nodeB_prefix_found = False
RealTimeProcessingIssue = False
DLRetxIssue = False
ULRetxIssue = False
line_cnt=0 #log file line counter
for line in enb_log_file.readlines():
@@ -1008,11 +1011,11 @@ class RANManagement():
result = re.search('X2AP-PDU', str(line))
if result is not None:
x2ap_pdu += 1
#nsa markers logging
for k in nsa_markers:
#gnb markers logging
for k in gnb_markers:
result = re.search(k, line)
if result is not None:
nsa_markers[k].append(line_cnt)
gnb_markers[k].append(line_cnt)
enb_log_file.close()
@@ -1096,31 +1099,77 @@ class RANManagement():
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
#nsa markers
#gnb markers
statMsg = 'logfile line count = ' + str(line_cnt)
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
if len(nsa_markers['SgNBReleaseRequestAcknowledge'])!=0:
statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequestAcknowledge'][0])
if len(gnb_markers['SgNBReleaseRequestAcknowledge'])!=0:
statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequestAcknowledge'][0])
else:
statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences'
statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences'
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
statMsg = 'FAILURE = ' + str(len(nsa_markers['FAILURE'])) + ' occurences'
statMsg = 'FAILURE = ' + str(len(gnb_markers['FAILURE'])) + ' occurences'
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
statMsg = 'Detected UL Failure on PUSCH = ' + str(len(gnb_markers['Detected UL Failure on PUSCH'])) + ' occurences'
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
#ulsch and dlsch statistics
#ulsch and dlsch statistics and checkers
#print statistics into html
if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
#for each dictionary key, generate the msg for html as information
statMsg=''
for key in dlsch_ulsch_stats: #for each dictionary key
for key in dlsch_ulsch_stats:
statMsg += dlsch_ulsch_stats[key] + '\n'
logging.debug(dlsch_ulsch_stats[key])
htmleNBFailureMsg += statMsg
#checker
if (len(dlsch_ulsch_stats)!=0) and (len(checkers)!=0):
if 'd_retx_th' in checkers:
checkers['d_retx_th'] = [float(x) for x in checkers['d_retx_th'].split(',')]
dlsch_checker_status = list(0 for i in checkers['d_retx_th'])#status 0 / -1
d_perc_retx = list(0 for i in checkers['d_retx_th'])#results in %
if 'u_retx_th' in checkers:
checkers['u_retx_th'] = [float(x) for x in checkers['u_retx_th'].split(',')]
ulsch_checker_status = list(0 for i in checkers['u_retx_th'])
u_perc_retx = list(0 for i in checkers['u_retx_th'])
#ul and dl retransmissions checkers
#NOTICE: DL and UL regex are different
if ('dlsch_rounds' in dlsch_ulsch_stats) and ('d_retx_th' in checkers):
tmp=re.match(r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)',dlsch_ulsch_stats['dlsch_rounds'])
if tmp is not None :
#captures the different groups from the regex
retx_data=[float(x) for x in tmp.groups()]
for i in range(0,len(d_perc_retx)):
#case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0%
d_perc_retx[i] = 0 if (retx_data[i] == 0) else 100*retx_data[i+1]/retx_data[i]
#treating % > 100 , % > requirement
if (d_perc_retx[i] > 100) or (d_perc_retx[i] > checkers['d_retx_th'][i]): dlsch_checker_status[i] = -1
if -1 in dlsch_checker_status:
DLRetxIssue = True
if ('ulsch_rounds' in dlsch_ulsch_stats) and ('u_retx_th' in checkers):
tmp=re.match(r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+.*,\s+ulsch_errors\s+(\d+)',dlsch_ulsch_stats['ulsch_rounds'])
if tmp is not None :
retx_data=[float(x) for x in tmp.groups()]
for i in range(0,len(d_perc_retx)):
u_perc_retx[i] = 0 if (retx_data[i] == 0) else 100*retx_data[i+1]/retx_data[i]
if (u_perc_retx[i] > 100) or (u_perc_retx[i] > checkers['u_retx_th'][i]): ulsch_checker_status[i] = -1
if -1 in ulsch_checker_status:
ULRetxIssue = True
#real time statistics
datalog_rt_stats['Data']={}
if len(real_time_stats)!=0: #check if dictionary is not empty
@@ -1158,18 +1207,34 @@ class RANManagement():
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
if len(nsa_markers['SgNBReleaseRequest'])!=0:
statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequest'][0])
if len(gnb_markers['SgNBReleaseRequest'])!=0:
statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequest'][0])
else:
statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences'
statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences'
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
statMsg = 'scgFailureInformationNR-r15 = ' + str(len(nsa_markers['scgFailureInformationNR-r15'])) + ' occurences'
statMsg = 'scgFailureInformationNR-r15 = ' + str(len(gnb_markers['scgFailureInformationNR-r15'])) + ' occurences'
htmlMsg = statMsg+'\n'
logging.debug(statMsg)
htmleNBFailureMsg += htmlMsg
if DLRetxIssue:
retx_checker_status_str = ''
for status in dlsch_checker_status : retx_checker_status_str+=str(status)+ ' '
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in DL ! \u001B[0m')
logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in DL, status : ' + retx_checker_status_str + '\n'
global_status = CONST.ENB_RETX_ISSUE
if ULRetxIssue:
retx_checker_status_str = ''
for status in ulsch_checker_status : retx_checker_status_str+=str(status)+ ' '
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in UL ! \u001B[0m')
logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in UL, status : ' + retx_checker_status_str + '\n'
global_status = CONST.ENB_RETX_ISSUE
if RealTimeProcessingIssue:
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m')
htmleNBFailureMsg += 'Fail due to real time processing issue\n'
ci-scripts/xml_files/container_5g_f1_rfsim.xml
+2
View File
@@ -116,6 +116,8 @@
<class>UndeployGenObject</class>
<desc>Undeploy all OAI 5G stack</desc>
<yaml_path>yaml_files/5g_f1_rfsimulator</yaml_path>
<d_retx_th>5,0,0,0</d_retx_th>
<u_retx_th>5,0,0,0</u_retx_th>
</testCase>
</testCaseList>
ci-scripts/xml_files/container_5g_fdd_rfsim.xml
+2
View File
@@ -111,6 +111,8 @@
<class>UndeployGenObject</class>
<desc>Undeploy all OAI 5G stack</desc>
<yaml_path>yaml_files/5g_fdd_rfsimulator</yaml_path>
<d_retx_th>5,0,0,0</d_retx_th>
<u_retx_th>5,0,0,0</u_retx_th>
</testCase>
</testCaseList>
ci-scripts/xml_files/container_5g_rfsim.xml
+2
View File
@@ -109,6 +109,8 @@
<class>UndeployGenObject</class>
<desc>Undeploy all OAI 5G stack</desc>
<yaml_path>yaml_files/5g_rfsimulator</yaml_path>
<d_retx_th>5,0,0,0</d_retx_th>
<u_retx_th>5,0,0,0</u_retx_th>
</testCase>
</testCaseList>