Merge branch 'ci_improve_iperf_pass_fail' into 'develop'

[CI] improve iperf pass fail

See merge request oai/openairinterface5g!1373

ci-scripts/cls_oaicitest.py

@@ -57,12 +57,15 @@ import sshconnection
 import cls_module_ue
 import cls_ci_ueinfra		#class defining the multi Ue infrastrucure
 
+logging.getLogger("matplotlib").setLevel(logging.WARNING)
+import matplotlib.pyplot as plt
+import numpy as np
 
 #-----------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------
 
-def GetPingTimeAnalysis(ping_log_file):
+def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold):
 	#ping time values read from file
 	t_ping=[]
 	#ping stats (dictionary) to be returned by the function
@@ -86,17 +89,46 @@ def GetPingTimeAnalysis(ping_log_file):
 		max_loc=t_ping.index(max(t_ping))
 		ping_stat['max_loc']=max_loc
 		#remove it
-		t_ping.pop(max_loc)
+		t_ping_post=t_ping.copy()
+		t_ping_post.pop(max_loc)
 		#new stats after removing max value
-		ping_stat['min_1']=min(t_ping)
+		ping_stat['min_1']=min(t_ping_post)
-		ping_stat['mean_1']=stat.mean(t_ping)
+		ping_stat['mean_1']=stat.mean(t_ping_post)
-		ping_stat['median_1']=stat.median(t_ping)
+		ping_stat['median_1']=stat.median(t_ping_post)
-		ping_stat['max_1']=max(t_ping)
+		ping_stat['max_1']=max(t_ping_post)
+
+		#plot ping over time and save png for artifacts
+		ticks = np.arange(0, len(t_ping), 1)
+		figure, axis = plt.subplots(figsize=(10, 10))
+		axis.plot(ticks,t_ping,marker='o')
+		axis.set_xlabel('Ping Events')
+		axis.set_ylabel("Ping RTT (in ms)")
+		axis.set_title(ping_log_file)
+		axis.set_xticks(ticks)
+		axis.set_xticklabels([])
+		YMAX=20 #base scale
+		if max(t_ping) > YMAX:
+			y_max=max(t_ping)+1
+		else:
+			y_max=YMAX+1
+		plt.ylim(0,y_max)
+		if ping_rttavg_threshold != '':
+			th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold
+			plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label)
+			axis.legend()
+		plt.savefig(ping_log_file+'.png')
+
+		#copy the png file already to enb to move it move it later into the artifacts
+		try:
+			mySSH = sshconnection.SSHConnection()
+			mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/')
+		except:
+			logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m')
 
 		return ping_stat
 
 	else:
-		logging.error("Ping log file does not exist")
+		logging.error("GetPingTimeAnalysis : Ping log file does not exist")
 		return -1
 
 	
@@ -129,6 +161,7 @@ class OaiCiTest():
 		self.ping_rttavg_threshold =''
 		self.iperf_args = ''
 		self.iperf_packetloss_threshold = ''
+		self.iperf_bitrate_threshold = ''
 		self.iperf_profile = ''
 		self.iperf_options = ''
 		self.iperf_direction = ''
@@ -1531,7 +1564,7 @@ class OaiCiTest():
 		statusQueue.put(message)
 		lock.release()
 
-	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE):
+	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN):
 		try:
 			SSH = sshconnection.SSHConnection()
 			# Launch ping on the EPC side (true for ltebox and old open-air-cn)
@@ -1559,6 +1592,7 @@ class OaiCiTest():
 					ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)				
 				else:
 					ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
+				ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
 				#copy the ping log file to have it locally for analysis (ping stats)
 				SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.')				
 			else:
@@ -1578,6 +1612,7 @@ class OaiCiTest():
 					SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
 					#cat is executed on EPC
 					SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
+					ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log'
 				else: #launch from Module
 					SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 					#target address is different depending on EPC type
@@ -1595,6 +1630,7 @@ class OaiCiTest():
 
 					#cat is executed locally 
 					SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5)
+					ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log'
 					ping_status=0
 
 			# TIMEOUT CASE
@@ -1642,9 +1678,9 @@ class OaiCiTest():
 			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
 
 			#adding extra ping stats from local file
-			ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
+			#ping_log_file variable is defined above in this function, depending on device/ue
 			logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
-			ping_stat=GetPingTimeAnalysis(ping_log_file)
+			ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
 			ping_stat_msg=''
 			if (ping_stat!=-1) and (len(ping_stat)!=0):
 				ping_stat_msg+='Ping stats before removing largest value : \n'
@@ -1834,7 +1870,7 @@ class OaiCiTest():
 				device_id = self.UEDevices[i]
 			else:
 				device_id = Module_UE.ID + "-" + Module_UE.Kind 
-			p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,))
+			p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,))
 			p.daemon = True
 			p.start()
 			multi_jobs.append(p)
@@ -2085,12 +2121,20 @@ class OaiCiTest():
 				pl = float(100 * pl_sum / ps_sum)
 				packetloss = '%2.1f ' % (pl)
 				packetloss += '%'
+				#checking packet loss compliance
 				if float(pl) > float(self.iperf_packetloss_threshold):
-					pal_too_high_msg = 'Packet Loss too high :  actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%\n'
+					pal_too_high_msg = 'Packet Loss too high :  tested = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%'
 				else:
-					pal_too_high_msg=''			
+					pal_too_high_msg='Packet Loss value is within acceptance range'
+				#checking bitrate perf compliance
+				if float(br_loss) < float(self.iperf_bitrate_threshold):
+					bit_too_low_msg = 'Bitrate too low :  tested = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%'
+				else:
+					bit_too_low_msg='Bitrate perf value is within acceptance range'
 			lock.acquire()
-			if (br_loss < 90) or (float(pl) > float(self.iperf_packetloss_threshold)):
+			if (float(br_loss) < float(self.iperf_bitrate_threshold)) and (float(pl) > float(self.iperf_packetloss_threshold)):
+				statusQueue.put(-1)
+			elif (float(br_loss) < float(self.iperf_bitrate_threshold)) or (float(pl) > float(self.iperf_packetloss_threshold)): 
 				statusQueue.put(1)
 			else:
 				statusQueue.put(0)
@@ -2101,7 +2145,7 @@ class OaiCiTest():
 			brl_msg = 'Bitrate Perf: ' + bitperf
 			jit_msg = 'Jitter      : ' + jitter
 			pal_msg = 'Packet Loss : ' + packetloss
-			statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n')
+			statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n' + bit_too_low_msg + '\n')
 			logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m')
 			logging.debug('\u001B[1;35m    ' + req_msg + '\u001B[0m')
 			logging.debug('\u001B[1;35m    ' + bir_msg + '\u001B[0m')
@@ -2109,6 +2153,7 @@ class OaiCiTest():
 			logging.debug('\u001B[1;35m    ' + jit_msg + '\u001B[0m')
 			logging.debug('\u001B[1;35m    ' + pal_msg + '\u001B[0m')
 			logging.debug('\u001B[1;35m    ' + pal_too_high_msg + '\u001B[0m')
+			logging.debug('\u001B[1;35m    ' + bit_too_low_msg + '\u001B[0m')
 			lock.release()
 		else:
 			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
@@ -3391,6 +3436,41 @@ class OaiCiTest():
 					CONTAINERS.UndeployObject(HTML,RAN)
 		RAN.prematureExit=True
 
+	#this function is called only if eNB/gNB fails to start
+	#RH to be re-factored
+	def AutoTerminateeNB(self,HTML,RAN,EPC,CONTAINERS):
+		if (RAN.Initialize_eNB_args != ''):
+			self.testCase_id = 'AUTO-KILL-RAN'
+			HTML.testCase_id = self.testCase_id
+			self.desc = 'Automatic Termination of all RAN nodes'
+			HTML.desc = self.desc
+			self.ShowTestID()
+			#terminate all RAN nodes eNB/gNB/OCP
+			for instance in range(0, len(RAN.air_interface)):
+				if RAN.air_interface[instance]!='':
+					logging.debug('Auto Termination of Instance ' + str(instance) + ' : ' + RAN.air_interface[instance])
+					RAN.eNB_instance=instance
+					RAN.TerminateeNB(HTML,EPC)
+		if RAN.flexranCtrlInstalled and RAN.flexranCtrlStarted:
+			self.testCase_id = 'AUTO-KILL-flexran-ctl'
+			HTML.testCase_id = self.testCase_id
+			self.desc = 'Automatic Termination of FlexRan CTL'
+			HTML.desc = self.desc
+			self.ShowTestID()
+			self.TerminateFlexranCtrl(HTML,RAN,EPC)
+		if CONTAINERS.yamlPath[0] != '':
+			self.testCase_id = 'AUTO-KILL-CONTAINERS'
+			HTML.testCase_id = self.testCase_id
+			self.desc = 'Automatic Termination of all RAN containers'
+			HTML.desc = self.desc
+			self.ShowTestID()
+			for instance in range(0, len(CONTAINERS.yamlPath)):
+				if CONTAINERS.yamlPath[instance]!='':
+					CONTAINERS.eNB_instance=instance
+					CONTAINERS.UndeployObject(HTML,RAN)
+		RAN.prematureExit=True
+
+
 	def IdleSleep(self,HTML):
 		time.sleep(self.idle_sleep_time)
 		HTML.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', CONST.ALL_PROCESSES_OK)

ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf

@@ -237,7 +237,7 @@ RUs = (
          max_pdschReferenceSignalPower = -27;
          max_rxgain                    = 75;
          eNB_instances  = [0];
-         sdr_addrs      = "mgmt_addr=192.168.18.241,addr=192.168.10.2";
+         sdr_addrs      = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2";
 
     }
 );  

ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf

@@ -237,7 +237,7 @@ RUs = (
          max_pdschReferenceSignalPower = -27;
          max_rxgain                    = 75;
          eNB_instances  = [0];
-         sdr_addrs      = "mgmt_addr=192.168.18.241,addr=192.168.10.2";
+         sdr_addrs      = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2";
 
     }
 );  

ci-scripts/main.py

@@ -310,6 +310,13 @@ def GetParametersFromXML(action):
 			CiTestObj.ue_id = ue_id
 		CiTestObj.iperf_direction = test.findtext('direction')#used for modules only	
 		CiTestObj.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold')
+		iperf_bitrate_threshold = test.findtext('iperf_bitrate_threshold')
+		if (iperf_bitrate_threshold is None):
+			CiTestObj.iperf_bitrate_threshold = "90" #if no threshold is specified, default will be 90%
+		else:
+			CiTestObj.iperf_bitrate_threshold = iperf_bitrate_threshold
+
+
 		CiTestObj.iperf_profile = test.findtext('iperf_profile')
 		if (CiTestObj.iperf_profile is None):
 			CiTestObj.iperf_profile = 'balanced'
@@ -811,6 +818,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					check_OAI_UE = False
 					RAN.pStatus=CiTestObj.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 					RAN.InitializeeNB(HTML, EPC)
+					if RAN.prematureExit:
+						CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS)
 				elif action == 'Terminate_eNB':
 					RAN.TerminateeNB(HTML, EPC)
 				elif action == 'Initialize_UE':

ci-scripts/ran.py

@@ -738,8 +738,8 @@ class RANManagement():
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 60)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 15)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
 		mySSH.close()
 
 	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):

ci-scripts/xml_files/container_nsa_b200_quectel.xml

@@ -124,6 +124,7 @@
 		<direction>DL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>3</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 
@@ -134,6 +135,7 @@
 		<direction>UL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 

ci-scripts/xml_files/fr1_lte_2x2_quectel.xml

@@ -107,6 +107,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 
@@ -117,6 +118,7 @@
 		<direction>UL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 

ci-scripts/xml_files/fr1_nsa_quectel.xml

@@ -119,6 +119,7 @@
 		<direction>DL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>3</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 
@@ -129,6 +130,7 @@
 		<direction>UL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 

ci-scripts/xml_files/fr1_sa_quectel.xml

@@ -105,6 +105,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 
@@ -115,6 +116,7 @@
 		<direction>UL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 

ci-scripts/xml_files/fr1_sa_quectel_stages.xml

@@ -104,6 +104,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 	<testCase id="070001">
@@ -113,6 +114,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 	<testCase id="070002">
@@ -122,6 +124,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 	<testCase id="070003">
@@ -131,6 +134,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 	<testCase id="070004">
@@ -140,6 +144,7 @@
 		<direction>DL</direction>
 		<id>nrmodule2_quectel</id>
 		<iperf_packetloss_threshold>5</iperf_packetloss_threshold>
+		<iperf_bitrate_threshold>95</iperf_bitrate_threshold>
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>