Merge remote-tracking branch 'origin/ci_improve_reporting_xnb' into integration_2022_wk09

ci-scripts/cls_containerize.py

@@ -106,6 +106,9 @@ class Containerize():
 		self.registrySvrId = ''
 		self.testSvrId = ''
 
+		#checkers from xml
+		self.ran_checkers={}
+
 #-----------------------------------------------------------
 # Container management functions
 #-----------------------------------------------------------
@@ -624,7 +627,7 @@ class Containerize():
 		else:
 			if containerToKill:
 				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
-				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML)
+				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML, self.ran_checkers)
 			else:
 				logStatus = 0
 			if (logStatus < 0):
@@ -788,9 +791,10 @@ class Containerize():
 					continue
 
 				logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
-				logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML)
+				logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
 				if (logStatus < 0):
 					fullStatus = False
+					self.exitStatus = 1
 					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
 				else:
 					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)

ci-scripts/constants.py

@@ -45,6 +45,7 @@ ENB_PROCESS_REALTIME_ISSUE = -13
 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
 ENB_PROCESS_SLAVE_RRU_NOT_SYNCED = -15
 ENB_REAL_TIME_PROCESSING_ISSUE = -16
+ENB_RETX_ISSUE = -17
 HSS_PROCESS_FAILED = -2
 HSS_PROCESS_OK = +2
 MME_PROCESS_FAILED = -3

ci-scripts/main.py

@@ -401,6 +401,12 @@ def GetParametersFromXML(action):
 		string_field=test.findtext('nb_healthy')
 		if (string_field is not None):
 			CONTAINERS.nb_healthy[0] = int(string_field)
+		string_field=test.findtext('d_retx_th')
+		if (string_field is not None):
+			CONTAINERS.ran_checkers['d_retx_th']= string_field
+		string_field=test.findtext('u_retx_th')
+		if (string_field is not None):
+			CONTAINERS.ran_checkers['u_retx_th']= string_field
 
 	elif action == 'PingFromContainer':
 		string_field = test.findtext('container_name')

ci-scripts/ran.py

@@ -39,6 +39,7 @@ import time
 from multiprocessing import Process, Lock, SimpleQueue
 import yaml
 
+
 #-----------------------------------------------------------
 # OAI Testing modules
 #-----------------------------------------------------------
@@ -758,7 +759,7 @@ class RANManagement():
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
 		mySSH.close()
 
-	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
+	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):
 		if (not os.path.isfile('./' + eNBlogFile)):
 			return -1
 		enb_log_file = open('./' + eNBlogFile, 'r')
@@ -815,10 +816,12 @@ class RANManagement():
 		removing_ue = 0
 		#count"X2AP-PDU"
 		x2ap_pdu = 0
-		#NSA specific log markers
+		#gnb specific log markers
-		nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []}
+		gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]}
 		nodeB_prefix_found = False
 		RealTimeProcessingIssue = False
+		DLRetxIssue = False
+		ULRetxIssue = False
 	
 		line_cnt=0 #log file line counter
 		for line in enb_log_file.readlines():
@@ -1008,11 +1011,11 @@ class RANManagement():
 			result = re.search('X2AP-PDU', str(line))
 			if result is not None:
 				x2ap_pdu += 1
-			#nsa markers logging
+			#gnb markers logging
-			for k in nsa_markers:
+			for k in gnb_markers:
 				result = re.search(k, line)
 				if result is not None:
-					nsa_markers[k].append(line_cnt)					
+					gnb_markers[k].append(line_cnt)
 
 		enb_log_file.close()
 
@@ -1096,31 +1099,77 @@ class RANManagement():
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
-			#nsa markers
+			#gnb markers
 			statMsg = 'logfile line count = ' + str(line_cnt)			
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
-			if len(nsa_markers['SgNBReleaseRequestAcknowledge'])!=0:
+			if len(gnb_markers['SgNBReleaseRequestAcknowledge'])!=0:
-				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequestAcknowledge'][0])
+				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequestAcknowledge'][0])
 			else:
-				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' 
+				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' 
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
-			statMsg = 'FAILURE = ' + str(len(nsa_markers['FAILURE'])) + ' occurences'
+			statMsg = 'FAILURE = ' + str(len(gnb_markers['FAILURE'])) + ' occurences'
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
+			statMsg = 'Detected UL Failure on PUSCH = ' + str(len(gnb_markers['Detected UL Failure on PUSCH'])) + ' occurences'
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
 
-			#ulsch and dlsch statistics
+			#ulsch and dlsch statistics and checkers
+			#print statistics into html
 			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
+				#for each dictionary key, generate the msg for html as information
 				statMsg=''
-				for key in dlsch_ulsch_stats: #for each dictionary key
+				for key in dlsch_ulsch_stats: 
 					statMsg += dlsch_ulsch_stats[key] + '\n' 
 					logging.debug(dlsch_ulsch_stats[key])
 				htmleNBFailureMsg += statMsg
 
+			#checker
+			if (len(dlsch_ulsch_stats)!=0) and (len(checkers)!=0):
+				if 'd_retx_th' in checkers:
+					checkers['d_retx_th'] = [float(x) for x in checkers['d_retx_th'].split(',')]
+					dlsch_checker_status = list(0 for i in checkers['d_retx_th'])#status 0 / -1
+					d_perc_retx = list(0 for i in checkers['d_retx_th'])#results in %
+
+				if 'u_retx_th' in checkers:
+					checkers['u_retx_th'] = [float(x) for x in checkers['u_retx_th'].split(',')]
+					ulsch_checker_status = list(0 for i in checkers['u_retx_th'])
+					u_perc_retx = list(0 for i in checkers['u_retx_th'])
+
+				#ul and dl retransmissions checkers
+				#NOTICE:  DL and UL regex are different 
+
+				if ('dlsch_rounds' in dlsch_ulsch_stats) and ('d_retx_th' in checkers):
+					tmp=re.match(r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)',dlsch_ulsch_stats['dlsch_rounds'])
+					if tmp is not None :
+						#captures the different groups from the regex
+						retx_data=[float(x) for x in tmp.groups()]
+						for i in range(0,len(d_perc_retx)):
+							#case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0%
+							d_perc_retx[i] = 0 if (retx_data[i] == 0)  else 100*retx_data[i+1]/retx_data[i]
+							#treating % > 100 , % > requirement
+							if (d_perc_retx[i] > 100) or (d_perc_retx[i] > checkers['d_retx_th'][i]): dlsch_checker_status[i] = -1
+					if -1 in dlsch_checker_status:
+						DLRetxIssue = True
+
+				if ('ulsch_rounds' in dlsch_ulsch_stats)  and ('u_retx_th' in checkers):
+					tmp=re.match(r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+.*,\s+ulsch_errors\s+(\d+)',dlsch_ulsch_stats['ulsch_rounds'])
+					if tmp is not None :
+						retx_data=[float(x) for x in tmp.groups()]
+						for i in range(0,len(d_perc_retx)):
+							u_perc_retx[i] = 0 if (retx_data[i] == 0) else 100*retx_data[i+1]/retx_data[i]
+							if (u_perc_retx[i] > 100) or (u_perc_retx[i] > checkers['u_retx_th'][i]): ulsch_checker_status[i] = -1
+					if -1 in ulsch_checker_status:
+						ULRetxIssue = True
+
+
+
 			#real time statistics
 			datalog_rt_stats['Data']={}
 			if len(real_time_stats)!=0: #check if dictionary is not empty
@@ -1158,18 +1207,34 @@ class RANManagement():
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
-			if len(nsa_markers['SgNBReleaseRequest'])!=0:
+			if len(gnb_markers['SgNBReleaseRequest'])!=0:
-				statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequest'][0])
+				statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequest'][0])
 			else:
-				statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences'
+				statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences'
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
-			statMsg = 'scgFailureInformationNR-r15 = ' + str(len(nsa_markers['scgFailureInformationNR-r15'])) + ' occurences'
+			statMsg = 'scgFailureInformationNR-r15 = ' + str(len(gnb_markers['scgFailureInformationNR-r15'])) + ' occurences'
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg			
 
+		if DLRetxIssue:
+			retx_checker_status_str = ''
+			for status in dlsch_checker_status : retx_checker_status_str+=str(status)+ ' '
+			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in DL ! \u001B[0m')
+			logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
+			htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in DL, status : ' + retx_checker_status_str + '\n'
+			global_status = CONST.ENB_RETX_ISSUE
+
+		if ULRetxIssue:
+			retx_checker_status_str = ''
+			for status in ulsch_checker_status : retx_checker_status_str+=str(status)+ ' '
+			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in UL ! \u001B[0m')
+			logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
+			htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in UL, status : ' + retx_checker_status_str + '\n'
+			global_status = CONST.ENB_RETX_ISSUE
+
 		if RealTimeProcessingIssue:
 			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m')
 			htmleNBFailureMsg += 'Fail due to real time processing issue\n'

ci-scripts/xml_files/container_5g_f1_rfsim.xml

@@ -116,6 +116,8 @@
                 <class>UndeployGenObject</class>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <yaml_path>yaml_files/5g_f1_rfsimulator</yaml_path>
+                <d_retx_th>5,0,0,0</d_retx_th>
+                <u_retx_th>5,0,0,0</u_retx_th>
         </testCase>
 
 </testCaseList>

ci-scripts/xml_files/container_5g_fdd_rfsim.xml

@@ -111,6 +111,8 @@
                 <class>UndeployGenObject</class>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <yaml_path>yaml_files/5g_fdd_rfsimulator</yaml_path>
+                <d_retx_th>5,0,0,0</d_retx_th>
+                <u_retx_th>5,0,0,0</u_retx_th>
         </testCase>
 
 </testCaseList>

ci-scripts/xml_files/container_5g_rfsim.xml

@@ -109,6 +109,8 @@
                 <class>UndeployGenObject</class>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <yaml_path>yaml_files/5g_rfsimulator</yaml_path>
+                <d_retx_th>5,0,0,0</d_retx_th>
+                <u_retx_th>5,0,0,0</u_retx_th>
         </testCase>
 
 </testCaseList>