Merge remote-tracking branch 'origin/ci_test_nsa_fix_quectel_nic' into integration_2021_wk20_a

ci-scripts/Jenkinsfile-trig-nsa

@@ -37,7 +37,7 @@ pipeline {
             steps {
                 script {     
                     //retrieve MR that are opened nd with tag READY_TO_BE_MERGED             
-                    MR_LIST= sh returnStdout: true, script: 'curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100&milestone=REVIEW_COMPLETED_AND_APPROVED" | jq ".[].iid" || true '
+                    MR_LIST= sh returnStdout: true, script: 'curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100&milestone=NSA" | jq ".[].iid" || true '
                     echo "List of selected MR:\n${MR_LIST}" 
                     def MR_ARRAY = MR_LIST.split('\n') 
                     //for every selected MR, retrieve the branch name and the latest commit              

ci-scripts/ci_ueinfra.yaml

@@ -8,7 +8,7 @@ idefix:
   WakeupScript : ci_ctl_qtel.py /dev/ttyUSB2 wup
   DetachScript : ci_ctl_qtel.py /dev/ttyUSB2 detach
   PLMN : 22201
-  UENetwork : wwan0
+  UENetwork : enxb22a941dbf00 
   HostIPAddress : 192.168.18.188
   HostUsername : oaicicd
   HostPassword : oaicicd

ci-scripts/cls_module_ue.py

@@ -91,7 +91,7 @@ class Module_UE:
 	def Command(self,cmd):
 		mySSH = sshconnection.SSHConnection()
 		mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
-		mySSH.command('echo ' + self.HostPassword + ' | sudo -S python3 ' + self.cmd_dict[cmd],'\$',5)
+		mySSH.command('echo ' + self.HostPassword + ' | sudo -S python3 ' + self.cmd_dict[cmd],'\$',10)
 		time.sleep(5)
 		logging.debug("Module "+ cmd)
 		mySSH.close()

ci-scripts/cls_oaicitest.py

@@ -1513,7 +1513,8 @@ class OaiCiTest():
 					SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
 				else: #launch from Module
 					SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
-					cmd = 'ping -I ' + UE_IPAddress + ' ' + self.ping_args + ' ' +  EPC.IPAddress  + ' 2>&1 > ping_' + self.testCase_id + '_' + self.ue_id + '.log' 
+					#ping from module NIC rather than IP address to make sure round trip is over the air	
+					cmd = 'ping -I ' + Module_UE.UENetwork  + ' ' + self.ping_args + ' ' +  EPC.IPAddress  + ' 2>&1 > ping_' + self.testCase_id + '_' + self.ue_id + '.log' 
 					SSH.command(cmd,'\$',int(ping_time[0])*1.5)
 					#copy the ping log file to have it locally for analysis (ping stats)
 					SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
@@ -1957,7 +1958,7 @@ class OaiCiTest():
 			if type==0:
 				result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)/ +(?P<sentPack>[0-9]+)', str(line))
 			else:
-				result = re.search('^\[  3\].+ +(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)\/(?P<sentPack>[0-9]+)', str(line))
+				result = re.search('^\[  \d\].+ +(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)\/(?P<sentPack>[0-9]+)', str(line))
 
 			if result is not None:
 				bitrate = result.group('bitrate')
@@ -2203,13 +2204,16 @@ class OaiCiTest():
 			logging.debug("Iperf for Module in DL mode detected")
 			#server side UE
 			SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
-			cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u  2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' 
+			cmd = 'rm iperf_server_' +  self.testCase_id + '_' + self.ue_id + '.log'
+			SSH.command(cmd,'\$',5)
+			cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u  2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' 
 			SSH.command(cmd,'\$',5)
 			#client side EPC
 			SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
+			cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
+			SSH.command(cmd,'\$',5)
 			cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' 
 			SSH.command(cmd,'\$',int(iperf_time)*5.0)
-
 			#copy the 2 resulting files locally
 			SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
 			SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
@@ -2217,22 +2221,41 @@ class OaiCiTest():
 			filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
 			self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1)	
 
-		elif self.iperf_direction=="UL":
+		elif self.iperf_direction=="UL":#does not work at the moment
 			logging.debug("Iperf for Module in UL mode detected")
 			#server side EPC
 			SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
-			cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' 
+			cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
 			SSH.command(cmd,'\$',5)
+			
+			#cmd = 'echo $USER; nohup iperf3 -s -i 1 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
+			#SSH.command(cmd,'\$',5)
+
+			HOST=EPC.IPAddress
+			COMMAND='echo $USER; nohup iperf3 -s -i 1 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
+			logging.debug(COMMAND)
+			subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+
 			#client side UE
 			SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
-			cmd = 'iperf -B ' + UE_IPAddress + ' ' + '-c ' + EPC.IPAddress + ' ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' 
+			cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
-			SSH.command(cmd,'\$',int(iperf_time)*5.0)
+			SSH.command(cmd,'\$',5)
+
+
+#			SSH.command('iperf3 -c ' + EPC.IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0)
+
+			HOST=Module_UE.HostIPAddress
+			COMMAND='iperf3 -c ' + EPC.IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
+			logging.debug(COMMAND)
+			subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+
+
 
 			#copy the 2 resulting files locally
-			SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
+			SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
-			SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
+			SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
 			#send for analysis
-			filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
+			filename='iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 			self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1)				
 		else :
 			logging.debug("Incorrect or missing IPERF direction in XML")

ci-scripts/ran.py

@@ -707,6 +707,8 @@ class RANManagement():
 		real_time_stats = {}
 		#count "problem receiving samples" msg
 		pb_receiving_samples_cnt = 0
+		#NSA specific log markers
+		nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []}
 	
 		#the datalog config file has to be loaded
 		datalog_rt_stats_file='datalog_rt_stats.yaml'
@@ -722,8 +724,9 @@ class RANManagement():
 			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
 		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  
 
-
+		line_cnt=0 #log file line counter
 		for line in enb_log_file.readlines():
+			line_cnt+=1
 			# Runtime statistics
 			result = re.search('Run time:' ,str(line))
 			if result is not None:
@@ -901,6 +904,12 @@ class RANManagement():
 			if result is not None:
 				pb_receiving_samples_cnt += 1
 
+			#nsa markers logging
+			for k in nsa_markers:
+				result = re.search(k, line)
+				if result is not None:
+					nsa_markers[k].append(line_cnt)					
+
 		enb_log_file.close()
 		logging.debug('   File analysis completed')
 		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
@@ -908,7 +917,7 @@ class RANManagement():
 		else:
 			nodeB_prefix = 'g'
 
-		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
+		if nodeB_prefix == 'g':
 			if ulschReceiveOK > 0:
 				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
 				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
@@ -935,6 +944,22 @@ class RANManagement():
 			htmlMsg = statMsg+'\n'
 			logging.debug(statMsg)
 			htmleNBFailureMsg += htmlMsg
+			#nsa markers
+			statMsg = 'logfile line count = ' + str(line_cnt)			
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
+			if len(nsa_markers['SgNBReleaseRequestAcknowledge'])!=0:
+				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequestAcknowledge'][0])
+			else:
+				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(nsa_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' 
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
+			statMsg = 'FAILURE = ' + str(len(nsa_markers['FAILURE'])) + ' occurences'
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
 
 			#ulsch and dlsch statistics
 			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
@@ -966,6 +991,25 @@ class RANManagement():
 				logging.debug('No real time stats found in the log file')
 				htmleNBFailureMsg += statMsg
 
+		else:
+			#nsa markers
+			statMsg = 'logfile line count = ' + str(line_cnt)			
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
+			if len(nsa_markers['SgNBReleaseRequest'])!=0:
+				statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(nsa_markers['SgNBReleaseRequest'][0])
+			else:
+				statMsg = 'SgNBReleaseRequest = ' + str(len(nsa_markers['SgNBReleaseRequest'])) + ' occurences'
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
+			statMsg = 'scgFailureInformationNR-r15 = ' + str(len(nsa_markers['scgFailureInformationNR-r15'])) + ' occurences'
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg			
+
+
 		if uciStatMsgCount > 0:
 			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
 			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')

ci-scripts/xml_files/benetel_nsa_quectel.xml

@@ -30,19 +30,17 @@
  040000
  000002
  010000
- 000001
  050000
  050001
  000001
  070000
  000001
- 070001
+ 070000
  000001
  010002
  000001
  080001
  080000
-  
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
@@ -82,13 +80,13 @@
 	<testCase id="000001">
 		<class>IdleSleep</class>
 		<desc>Sleep</desc>
-		<idle_sleep_time_in_sec>10</idle_sleep_time_in_sec>
+		<idle_sleep_time_in_sec>5</idle_sleep_time_in_sec>
 	</testCase>
 
 	<testCase id="000002">
 		<class>IdleSleep</class>
 		<desc>Sleep</desc>
-		<idle_sleep_time_in_sec>30</idle_sleep_time_in_sec>
+		<idle_sleep_time_in_sec>20</idle_sleep_time_in_sec>
 	</testCase>
 
 
@@ -110,8 +108,8 @@
 
 	<testCase id="070000">
 		<class>Iperf</class>
-		<desc>iperf (DL/20Mbps/UDP)(20 sec)(single-ue profile)</desc>
+		<desc>iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile)</desc>
-		<iperf_args>-u -b 20M -t 20 -i 1</iperf_args>
+		<iperf_args>-u -b 20M -t 60 -i 1</iperf_args>
 		<direction>DL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>
@@ -120,8 +118,8 @@
 
 	<testCase id="070001">
 		<class>Iperf</class>
-		<desc>iperf (UL/3Mbps/UDP)(20 sec)(single-ue profile)</desc>
+		<desc>iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile)</desc>
-		<iperf_args>-u -b 3M -t 20 -i 1</iperf_args>
+		<iperf_args>-u -b 3M -t 60 -i 1</iperf_args>
 		<direction>UL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>50</iperf_packetloss_threshold>

ci-scripts/xml_files/fr1_nsa_quectel.xml

@@ -30,19 +30,17 @@
  040000
  000002
  010000
- 000001
  050000
  050001
  000001
  070000
  000001
- 070001
+ 070000
  000001
  010002
  000001
  080001
  080000
-  
 	</TestCaseRequestedList>
 	<TestCaseExclusionList></TestCaseExclusionList>
 
@@ -82,13 +80,13 @@
 	<testCase id="000001">
 		<class>IdleSleep</class>
 		<desc>Sleep</desc>
-		<idle_sleep_time_in_sec>10</idle_sleep_time_in_sec>
+		<idle_sleep_time_in_sec>5</idle_sleep_time_in_sec>
 	</testCase>
 
 	<testCase id="000002">
 		<class>IdleSleep</class>
 		<desc>Sleep</desc>
-		<idle_sleep_time_in_sec>30</idle_sleep_time_in_sec>
+		<idle_sleep_time_in_sec>20</idle_sleep_time_in_sec>
 	</testCase>