CI: fixes and enhancements

-- fix for IF4p5 TDD scenario (missing build test-case)
  -- added automatic UE/eNB termination in main.py when iperf/ping fails
     It allows to have a eNB log analysis for HTML reporting
  -- added statistics on PDCP out-of-resource messages
  -- added statistics on ULSCH error messages

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/conf_files/rru.tdd.band40.conf

@@ -13,7 +13,7 @@ RUs = (
         nb_rx                            = 1;
         max_pdschReferenceSignalPower    = -27;
         max_rxgain                       = 115;
-        bands                            = [7];
+        bands                            = [40];
     }
 );
 

ci-scripts/main.py

@@ -384,7 +384,7 @@ class SSHConnection():
 			loopCounter = loopCounter - 1
 			if (loopCounter == 0):
 				doLoop = False
-				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! See with attach later \u001B[0m')
+				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
 				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
 				self.CreateHtmlFooter()
 				self.close()
@@ -489,6 +489,7 @@ class SSHConnection():
 		pStatus = self.CheckProcessExist(initialize_eNB_flag)
 		if (pStatus < 0):
 			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
+			self.AutoTerminateUEandeNB()
 			self.CreateHtmlFooter()
 			sys.exit(1)
 		multi_jobs = []
@@ -505,6 +506,7 @@ class SSHConnection():
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
 			self.CreateHtmlFooter()
+			self.AutoTerminateUEandeNB()
 			sys.exit(1)
 		else:
 			attach_status = True
@@ -524,6 +526,7 @@ class SSHConnection():
 				self.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue)
 			else:
 				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
+				self.AutoTerminateUEandeNB()
 				self.CreateHtmlFooter()
 				sys.exit(1)
 
@@ -544,6 +547,7 @@ class SSHConnection():
 		pStatus = self.CheckProcessExist(initialize_eNB_flag)
 		if (pStatus < 0):
 			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
+			self.AutoTerminateUEandeNB()
 			self.CreateHtmlFooter()
 			sys.exit(1)
 		multi_jobs = []
@@ -765,6 +769,7 @@ class SSHConnection():
 
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
+			self.AutoTerminateUEandeNB()
 			self.CreateHtmlFooter()
 			sys.exit(1)
 		else:
@@ -783,6 +788,7 @@ class SSHConnection():
 				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
 			else:
 				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
+				self.AutoTerminateUEandeNB()
 				self.CreateHtmlFooter()
 				sys.exit(1)
 
@@ -1217,11 +1223,13 @@ class SSHConnection():
 		pStatus = self.CheckProcessExist(initialize_eNB_flag)
 		if (pStatus < 0):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
+			self.AutoTerminateUEandeNB()
 			self.CreateHtmlFooter()
 			sys.exit(1)
 		ueIpStatus = self.GetAllUEIPAddresses()
 		if (ueIpStatus < 0):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
+			self.AutoTerminateUEandeNB()
 			self.CreateHtmlFooter()
 			sys.exit(1)
 		multi_jobs = []
@@ -1241,6 +1249,7 @@ class SSHConnection():
 
 		if (status_queue.empty()):
 			self.CreateHtmlTestRow(self.iperf_args, 'KO', ALL_PROCESSES_OK)
+			self.AutoTerminateUEandeNB()
 			self.CreateHtmlFooter()
 			sys.exit(1)
 		else:
@@ -1264,6 +1273,7 @@ class SSHConnection():
 				self.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
 			else:
 				self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
+				self.AutoTerminateUEandeNB()
 				self.CreateHtmlFooter()
 				sys.exit(1)
 
@@ -1391,6 +1401,8 @@ class SSHConnection():
 		rrcReestablishComplete = 0
 		rrcReestablishReject = 0
 		uciStatMsgCount = 0
+		pdcpFailure = 0
+		ulschFailure = 0
 		for line in enb_log_file.readlines():
 			result = re.search('[Ss]egmentation [Ff]ault', str(line))
 			if result is not None:
@@ -1434,12 +1446,25 @@ class SSHConnection():
 			result = re.search('uci->stat', str(line))
 			if result is not None:
 				uciStatMsgCount += 1
+			result = re.search('PDCP.*Out of Resources.*reason', str(line))
+			if result is not None:
+				pdcpFailure += 1
+			result = re.search('ULSCH in error in round', str(line))
+			if result is not None:
+				ulschFailure += 1
 		enb_log_file.close()
 		self.htmleNBFailureMsg = ''
 		if uciStatMsgCount > 0:
-			statMsg = 'eNB showed ' + str(uciStatMsgCount) + ' uci->stat message(s)'
+			statMsg = 'eNB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
 			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
 			self.htmleNBFailureMsg += statMsg + '\n'
+		if pdcpFailure > 0:
+			statMsg = 'eNB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
+			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
+			self.htmleNBFailureMsg += statMsg + '\n'
+		if ulschFailure > 0:
+			statMsg = 'eNB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
+			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
 		if rrcSetupRequest > 0 or rrcSetupComplete > 0:
 			rrcMsg = 'eNB requested ' + str(rrcSetupRequest) + ' RRC Connection Setup(s)'
 			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
@@ -1586,6 +1611,17 @@ class SSHConnection():
 			job.join()
 		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
 
+	def AutoTerminateUEandeNB(self):
+		self.testCase_id = 'AUTO-KILL-UE'
+		self.desc = 'Automatic Termination of UE'
+		self.ShowTestID()
+		self.TerminateUE()
+		self.testCase_id = 'AUTO-KILL-eNB'
+		self.desc = 'Automatic Termination of eNB'
+		self.ShowTestID()
+		self.eNB_instance = '0'
+		self.TerminateeNB()
+
 	def LogCollectBuild(self):
 		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
 		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
@@ -1822,6 +1858,15 @@ class SSHConnection():
 				i += 1
 			self.htmlFile.write('      </tr>\n')
 
+#-----------------------------------------------------------
+# ShowTestID()
+#-----------------------------------------------------------
+	def ShowTestID(self):
+		logging.debug('\u001B[1m----------------------------------------\u001B[0m')
+		logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m')
+		logging.debug('\u001B[1m' + self.desc + '\u001B[0m')
+		logging.debug('\u001B[1m----------------------------------------\u001B[0m')
+
 #-----------------------------------------------------------
 # Usage()
 #-----------------------------------------------------------
@@ -1853,15 +1898,6 @@ def Usage():
 	print('  --XMLTestFile=[XML Test File to be run]')
 	print('------------------------------------------------------------')
 
-#-----------------------------------------------------------
-# ShowTestID()
-#-----------------------------------------------------------
-def ShowTestID():
-	logging.debug('\u001B[1m----------------------------------------\u001B[0m')
-	logging.debug('\u001B[1mTest ID:' + SSH.testCase_id + '\u001B[0m')
-	logging.debug('\u001B[1m' + SSH.desc + '\u001B[0m')
-	logging.debug('\u001B[1m----------------------------------------\u001B[0m')
-
 def CheckClassValidity(action,id):
 	if action != 'Build_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW':
 		logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action)
@@ -2118,7 +2154,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE):
 			action = test.findtext('class')
 			if (CheckClassValidity(action, id) == False):
 				continue
-			ShowTestID()
+			SSH.ShowTestID()
 			GetParametersFromXML(action)
 			if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE':
 				terminate_ue_flag = False

ci-scripts/xml_files/if4p5_usrp210_band40.xml

@@ -22,14 +22,14 @@
 -->
 <testCaseList>
 	<!-- Only 5MHz is tested since neither 10MHz nor 20MHz work in this configuration -->
-	<!-- Also no UL iperf in 5MHz -->
+	<!-- Also no UL iperf in 5MHz (040642) -->
         <TestCaseRequestedList>
+ 010101
  050101 060101 070101 040101
  030104 030105 040301 040501 040602 040401 040201 030201 030202
  050201 060201 070201
 	</TestCaseRequestedList>
 	<!--
- 010101
  030104 030105 040301 040501 040602 040642 040401 040201 030201 030202
  030114 030115 040301 040511 040612 040652 040401 040201 030201 030202
  030124 030125 040301 040521 040622 040662 040401 040201 030201 030202
@@ -39,7 +39,7 @@
 
 	<testCase id="010101">
 		<class>Build_eNB</class>
-		<desc>Build eNB (USRP)</desc>
+		<desc>Build eNB (USRP -- Ethernet Fronthaul)</desc>
 		<Build_eNB_args>-t ETHERNET -w USRP -c --eNB</Build_eNB_args>
 	</testCase>
 

ci-scripts/xml_files/if4p5_usrp210_band7.xml

@@ -37,7 +37,7 @@
 
 	<testCase id="010101">
 		<class>Build_eNB</class>
-		<desc>Build eNB (USRP)</desc>
+		<desc>Build eNB (USRP -- Ethernet Fronthaul)</desc>
 		<Build_eNB_args>-t ETHERNET -w USRP -c --eNB</Build_eNB_args>
 	</testCase>