fix(ci): some little cosmetics fixes

* The quectel CM program output is now redirected to a file. Easier to follow when running
* I added `-i 1` option to all server iperf commands to have details
* module-based iperf logs are also copied to the RAN enb log folder so they are artifacted
* iperf3 console output was badly formatted
* `docker kill --signal INT` does not stop the container

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/cls_containerize.py

@@ -606,6 +606,8 @@ class Containerize():
 		if containerToKill:
 			mySSH.command('docker kill --signal INT ' + containerName, '\$', 30)
 			time.sleep(5)
+			mySSH.command('docker kill --signal KILL ' + containerName, '\$', 30)
+			time.sleep(5)
 			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
 			mySSH.command('docker rm -f ' + containerName, '\$', 30)
 		# Forcing the down now to remove the networks and any artifacts

ci-scripts/cls_module_ue.py

@@ -76,7 +76,8 @@ class Module_UE:
 			logging.debug('Starting ' + self.Process['Name'])
 			mySSH = sshconnection.SSHConnection()
 			mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
-			mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S ' + self.Process['Cmd'] + ' ' +  self.Process['Apn'][CNType]  + ' > /dev/null 2>&1 &','\$',5)
+			mySSH.command('sudo rm -f /tmp/quectel-cm.log','\$',5)
+			mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S stdbuf -o0 ' + self.Process['Cmd'] + ' ' +  self.Process['Apn'][CNType]  + ' > /tmp/quectel-cm.log 2>&1 &','\$',5)
 			mySSH.close()
 			#checking the process
 			time.sleep(5)

ci-scripts/cls_oaicitest.py

@@ -122,6 +122,7 @@ def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold):
 		try:
 			mySSH = sshconnection.SSHConnection()
 			mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/')
+			mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file, RAN.eNBSourceCodePath + '/cmake_targets/')
 		except:
 			logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m')
 
@@ -1598,7 +1599,7 @@ class OaiCiTest():
 			else:
 				if launchfromModule == False:
 					#ping log file is on the python executor
-					cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 > ping_' + self.testCase_id + '_' + device_id + '.log' 
+					cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' > ping_' + self.testCase_id + '_' + device_id + '.log 2>&1'
 					message = cmd + '\n'
 					logging.debug(cmd)
 					ret = subprocess.run(cmd, shell=True)
@@ -1623,7 +1624,7 @@ class OaiCiTest():
 					else:
 						Target = EPC.IPAddress
 					#ping from module NIC rather than IP address to make sure round trip is over the air	
-					cmd = 'ping -I ' + Module_UE.UENetwork  + ' ' + self.ping_args + ' ' +  Target  + ' 2>&1 > ping_' + self.testCase_id + '_' + self.ue_id + '.log' 
+					cmd = 'ping -I ' + Module_UE.UENetwork  + ' ' + self.ping_args + ' ' +  Target  + ' > ping_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1'
 					SSH.command(cmd,'\$',int(ping_time[0])*1.5)
 					#copy the ping log file to have it locally for analysis (ping stats)
 					SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
@@ -2084,7 +2085,8 @@ class OaiCiTest():
 			statusQueue.put(UE_IPAddress)
 			statusQueue.put(report_msg)
 			logging.debug('\u001B[1;37;45m TCP Bidir Iperf Result (' + UE_IPAddress + ') \u001B[0m')
-			logging.debug('\u001B[1;35m    ' + report_msg + '\u001B[0m')
+			for rLine in report_msg.split('\n'):
+				logging.debug('\u001B[1;35m    ' + rLine + '\u001B[0m')
 			lock.release()
 		else:
 			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Bidir TCP : Could not analyze from Log file')
@@ -2379,10 +2381,12 @@ class OaiCiTest():
 			SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts')
 
 
-	def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE):
+	def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE, RAN):
+		SSH = sshconnection.SSHConnection()
+		server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
+		client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 		if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)):
 			#retrieve trf-gen container IP address
-			SSH = sshconnection.SSHConnection()
 			SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
 			SSH.command('docker inspect --format="TRF_IP_ADDR = {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" prod-trf-gen', '\$', 5)
 			result = re.search('TRF_IP_ADDR = (?P<trf_ip_addr>[0-9\.]+)', SSH.getBefore())
@@ -2404,20 +2408,18 @@ class OaiCiTest():
 			if self.iperf_direction=="DL":
 				logging.debug("Iperf for Module in DL mode detected")
 				##server side UE
-				server_filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
 				SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 				cmd = 'rm ' + server_filename
 				SSH.command(cmd,'\$',5)
-				cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > ' + server_filename + ' &'
+				cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u -i 1 > ' + server_filename + ' 2>&1 &'
 				SSH.command(cmd,'\$',5)
 				SSH.close()
 				##client side EPC
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
-				client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 				#remove old client file in EPC.SourceCodePath
 				cmd = 'rm ' + EPC.SourceCodePath + '/' + client_filename 
 				SSH.command(cmd,'\$',5)
-				iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename
+				iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1'
 				cmd = 'docker exec -w /iperf-2.0.13 -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' 
 				SSH.command(cmd,'\$',int(iperf_time)*5.0)
 				SSH.command('docker cp prod-trf-gen:/iperf-2.0.13/'+ client_filename + ' ' + EPC.SourceCodePath, '\$', 5)
@@ -2433,19 +2435,17 @@ class OaiCiTest():
 				logging.debug("Iperf for Module in UL mode detected")
 				#server side EPC
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
-				server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
 
-				iperf_cmd = 'echo $USER; nohup bin/iperf -s -u 2>&1 > ' + server_filename
+				iperf_cmd = 'echo $USER; nohup bin/iperf -s -u -i 1 > ' + server_filename + ' 2>&1'
 				cmd = 'docker exec -d -w /iperf-2.0.13 prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' 
 				SSH.command(cmd,'\$',5)
 				SSH.close()
 
 				#client side UE
 				SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
-				client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 				cmd = 'rm '+ client_filename
 				SSH.command(cmd,'\$',5)
-				SSH.command('iperf -B ' + UE_IPAddress + ' -c ' +  trf_gen_IP + ' '  + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0)
+				SSH.command('iperf -B ' + UE_IPAddress + ' -c ' +  trf_gen_IP + ' '  + self.iperf_args + ' > ' + client_filename + ' 2>&1', '\$', int(iperf_time)*5.0)
 				SSH.close()
 
 				#once client is done, retrieve the server file from container to EPC Host
@@ -2461,12 +2461,10 @@ class OaiCiTest():
 
 			elif self.iperf_direction=="BIDIR":
 				logging.debug("Iperf for Module in BIDIR mode detected")
-				server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
-				client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 
 				#server side EPC
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
-				iperf_cmd = 'echo $USER; nohup /usr/local/bin/iperf3 -s 2>&1 > ' + server_filename
+				iperf_cmd = 'echo $USER; nohup /usr/local/bin/iperf3 -s -i 1 > ' + server_filename + ' 2>&1'
 				cmd = 'docker exec -d -w /iperf-2.0.13 prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' 
 				SSH.command(cmd,'\$',5)
 				SSH.close()
@@ -2475,7 +2473,7 @@ class OaiCiTest():
 				SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 				cmd = 'rm '+ client_filename
 				SSH.command(cmd,'\$',5)
-				SSH.command('iperf3 -B ' + UE_IPAddress + ' -c ' +  trf_gen_IP + ' '  + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0)
+				SSH.command('iperf3 -B ' + UE_IPAddress + ' -c ' +  trf_gen_IP + ' '  + self.iperf_args + ' > ' + client_filename + ' 2>&1', '\$', int(iperf_time)*5.0)
 				SSH.close()
 
 				#once client is done, retrieve the server file from container to EPC Host
@@ -2499,8 +2497,6 @@ class OaiCiTest():
 
 		else: 		#default is ltebox
 
-			SSH = sshconnection.SSHConnection()
-
 			#kill iperf processes before (in case there are still some remaining)
 			SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 			cmd = 'killall --signal=SIGKILL iperf'
@@ -2524,14 +2520,14 @@ class OaiCiTest():
 				SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 				cmd = 'rm iperf_server_' +  self.testCase_id + '_' + self.ue_id + '.log'
 				SSH.command(cmd,'\$',5)
-				cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log &' 
+				cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u -i 1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1 &' 
 				SSH.command(cmd,'\$',5)
 				SSH.close()
 				#client side EPC
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
 				cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 				SSH.command(cmd,'\$',5)
-				cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' 
+				cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1' 
 				SSH.command(cmd,'\$',int(iperf_time)*5.0)
 				SSH.close()
 				#copy the 2 resulting files locally
@@ -2547,7 +2543,7 @@ class OaiCiTest():
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
 				cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
 				SSH.command(cmd,'\$',5)
-				cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log  &'
+				cmd = 'echo $USER; nohup iperf -s -u -i 1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1 &'
 				SSH.command(cmd,'\$',5)
 				SSH.close()
 
@@ -2555,7 +2551,7 @@ class OaiCiTest():
 				SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 				cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 				SSH.command(cmd,'\$',5)
-				SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0)
+				SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1', '\$', int(iperf_time)*5.0)
 				SSH.close()
 
 				#copy the 2 resulting files locally
@@ -2566,15 +2562,13 @@ class OaiCiTest():
 				self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1)
 			elif self.iperf_direction=="BIDIR":
 				logging.debug("Iperf for Module in BIDIR mode detected")
-				server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
-				client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
 
 
 				#server side EPC
 				SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
 				cmd = 'rm ' + server_filename
 				SSH.command(cmd,'\$',5)
-				cmd = 'echo $USER; nohup iperf3 -s 2>&1 > '+server_filename+' &'
+				cmd = 'echo $USER; nohup iperf3 -s -i 1 > '+server_filename+' 2>&1 &'
 				SSH.command(cmd,'\$',5)
 				SSH.close()
 
@@ -2582,7 +2576,7 @@ class OaiCiTest():
 				SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 				cmd = 'rm ' + client_filename
 				SSH.command(cmd,'\$',5)
-				SSH.command('iperf3 -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > '+client_filename, '\$', int(iperf_time)*5.0)
+				SSH.command('iperf3 -c 192.172.0.1 ' + self.iperf_args + ' > '+client_filename + ' 2>&1', '\$', int(iperf_time)*5.0)
 				SSH.close()
 
 				#copy the 2 resulting files locally
@@ -2593,6 +2587,7 @@ class OaiCiTest():
 			else :
 				logging.debug("Incorrect or missing IPERF direction in XML")
 
+
 			#kill iperf processes after to be clean
 			SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
 			cmd = 'killall --signal=SIGKILL iperf'
@@ -2606,7 +2601,12 @@ class OaiCiTest():
 			cmd = 'killall --signal=SIGKILL iperf3'
 			SSH.command(cmd,'\$',5)
 			SSH.close()
-			return
+
+		# Copying to xNB server for Jenkins artifacting
+		if (os.path.isfile(server_filename)):
+			SSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, server_filename, RAN.eNBSourceCodePath + '/cmake_targets/')
+		if (os.path.isfile(client_filename)):
+			SSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, client_filename, RAN.eNBSourceCodePath + '/cmake_targets/')
 
 	def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC):
 		try:
@@ -2804,12 +2804,12 @@ class OaiCiTest():
 				else:
 					SSH.copyin(self.ADBIPAddress, self.ADBUserName, self.ADBPassword, EPC.SourceCodePath + '/scripts/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.')
 				# fromdos has to be called on the python executor not on ADB server
-				cmd = 'fromdos -o iperf_server_' + self.testCase_id + '_' + device_id + '.log 2>&1 > /dev/null'
+				cmd = 'fromdos -o iperf_server_' + self.testCase_id + '_' + device_id + '.log > /dev/null 2>&1'
 				try:
 					subprocess.run(cmd, shell=True)
 				except:
 					pass
-				cmd = 'dos2unix -o iperf_server_' + self.testCase_id + '_' + device_id + '.log 2>&1 > /dev/null'
+				cmd = 'dos2unix -o iperf_server_' + self.testCase_id + '_' + device_id + '.log > /dev/null 2>&1'
 				try:
 					subprocess.run(cmd, shell=True)
 				except:
@@ -2995,7 +2995,7 @@ class OaiCiTest():
         	#special quick and dirty treatment for modules, iperf to be restructured
 			if self.ue_id!="": #is module
 				device_id = Module_UE.ID + "-" + Module_UE.Kind
-				p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE,))
+				p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE, RAN, ))
 			else: #legacy code
 				p = Process(target = self.Iperf_common, args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, ))
 			p.daemon = True

ci-scripts/ran.py

@@ -756,11 +756,11 @@ class RANManagement():
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 60)
 		result = re.search('core.\d+', mySSH.getBefore())
 		if result is not None:
 			mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip core* ran_build/build/{lte,nr}-softmodem', '\$', 60) # add core and executable to zip
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 15)
 		mySSH.close()
 
 	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):

ci-scripts/xml_files/container_nsa_b200_quectel.xml

@@ -39,8 +39,9 @@
  050001
  070001
  070000
- 050000
+ 070002
- 050001
+ 050002
+ 050003
  010002
  000001
  030202
@@ -117,10 +118,28 @@
 		<ping_rttavg_threshold>15</ping_rttavg_threshold>
 	</testCase>
 
+	<testCase id="050002">
+		<class>Ping</class>
+		<desc>Ping: 20pings in 20sec</desc>
+		<id>idefix</id>
+		<ping_args>-c 20</ping_args>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
+	</testCase>
+
+	<testCase id="050003">
+		<class>Ping</class>
+		<desc>Ping: 100pings in 20sec</desc>
+		<id>idefix</id>
+		<ping_args>-c 100 -i 0.2</ping_args>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+		<ping_rttavg_threshold>15</ping_rttavg_threshold>
+	</testCase>
+
 	<testCase id="070000">
 		<class>Iperf</class>
 		<desc>iperf (DL/40Mbps/UDP)(60 sec)(single-ue profile)</desc>
-		<iperf_args>-u -b 40M -t 60</iperf_args>
+		<iperf_args>-u -b 40M -t 60 -i 1 -fm</iperf_args>
 		<direction>DL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>20</iperf_packetloss_threshold>
@@ -131,7 +150,7 @@
 	<testCase id="070001">
 		<class>Iperf</class>
 		<desc>iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile)</desc>
-		<iperf_args>-u -b 3M -t 60</iperf_args>
+		<iperf_args>-u -b 3M -t 60 -i 1 -fm</iperf_args>
 		<direction>UL</direction>
 		<id>idefix</id>
 		<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
@@ -139,6 +158,15 @@
 		<iperf_profile>single-ue</iperf_profile>
 	</testCase>
 
+    <testCase id="070002">
+        <class>Iperf</class>
+        <desc>iperf (BIDIR TCP)(10 sec)(single-ue profile)</desc>
+        <iperf_args>-t 10 --bidir</iperf_args>
+        <direction>BIDIR</direction>
+        <id>idefix</id>
+        <iperf_profile>single-ue</iperf_profile>
+    </testCase>
+
     <testCase id="030201">
         <class>Undeploy_Object</class>
         <desc>Undeploy eNB</desc>