maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): some little cosmetics fixes

Browse Source 
* The quectel CM program output is now redirected to a file. Easier to follow when running
* I added `-i 1` option to all server iperf commands to have details
* module-based iperf logs are also copied to the RAN enb log folder so they are artifacted
* iperf3 console output was badly formatted
* `docker kill --signal INT` does not stop the container

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
This commit is contained in:
Raphael Defosseux
2022-03-30 14:09:45 +02:00
parent 2c86b745fd
commit 52ead754c0
5 changed files with 69 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+2
View File
@@ -606,6 +606,8 @@ class Containerize():
if containerToKill:
mySSH.command('docker kill --signal INT ' + containerName, '\$', 30)
time.sleep(5)
mySSH.command('docker kill --signal KILL ' + containerName, '\$', 30)
time.sleep(5)
mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
mySSH.command('docker rm -f ' + containerName, '\$', 30)
# Forcing the down now to remove the networks and any artifacts
ci-scripts/cls_module_ue.py
+2 -1
View File
@@ -76,7 +76,8 @@ class Module_UE:
logging.debug('Starting ' + self.Process['Name'])
mySSH = sshconnection.SSHConnection()
mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S ' + self.Process['Cmd'] + ' ' + self.Process['Apn'][CNType] + ' > /dev/null 2>&1 &','\$',5)
mySSH.command('sudo rm -f /tmp/quectel-cm.log','\$',5)
mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S stdbuf -o0 ' + self.Process['Cmd'] + ' ' + self.Process['Apn'][CNType] + ' > /tmp/quectel-cm.log 2>&1 &','\$',5)
mySSH.close()
#checking the process
time.sleep(5)
ci-scripts/cls_oaicitest.py
+31 -31
View File
@@ -122,6 +122,7 @@ def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold):
try:
mySSH = sshconnection.SSHConnection()
mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/')
mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file, RAN.eNBSourceCodePath + '/cmake_targets/')
except:
logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m')
@@ -1598,7 +1599,7 @@ class OaiCiTest():
else:
if launchfromModule == False:
#ping log file is on the python executor
cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 > ping_' + self.testCase_id + '_' + device_id + '.log'
cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' > ping_' + self.testCase_id + '_' + device_id + '.log 2>&1'
message = cmd + '\n'
logging.debug(cmd)
ret = subprocess.run(cmd, shell=True)
@@ -1623,7 +1624,7 @@ class OaiCiTest():
else:
Target = EPC.IPAddress
#ping from module NIC rather than IP address to make sure round trip is over the air
cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' 2>&1 > ping_' + self.testCase_id + '_' + self.ue_id + '.log'
cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' > ping_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1'
SSH.command(cmd,'\$',int(ping_time[0])*1.5)
#copy the ping log file to have it locally for analysis (ping stats)
SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
@@ -2084,7 +2085,8 @@ class OaiCiTest():
statusQueue.put(UE_IPAddress)
statusQueue.put(report_msg)
logging.debug('\u001B[1;37;45m TCP Bidir Iperf Result (' + UE_IPAddress + ') \u001B[0m')
logging.debug('\u001B[1;35m ' + report_msg + '\u001B[0m')
for rLine in report_msg.split('\n'):
logging.debug('\u001B[1;35m ' + rLine + '\u001B[0m')
lock.release()
else:
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Bidir TCP : Could not analyze from Log file')
@@ -2379,10 +2381,12 @@ class OaiCiTest():
SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts')
def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE):
def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE, RAN):
SSH = sshconnection.SSHConnection()
server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)):
#retrieve trf-gen container IP address
SSH = sshconnection.SSHConnection()
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
SSH.command('docker inspect --format="TRF_IP_ADDR = {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" prod-trf-gen', '\$', 5)
result = re.search('TRF_IP_ADDR = (?P<trf_ip_addr>[0-9\.]+)', SSH.getBefore())
@@ -2404,20 +2408,18 @@ class OaiCiTest():
if self.iperf_direction=="DL":
logging.debug("Iperf for Module in DL mode detected")
##server side UE
server_filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'rm ' + server_filename
SSH.command(cmd,'\$',5)
cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > ' + server_filename + ' &'
cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u -i 1 > ' + server_filename + ' 2>&1 &'
SSH.command(cmd,'\$',5)
SSH.close()
##client side EPC
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
#remove old client file in EPC.SourceCodePath
cmd = 'rm ' + EPC.SourceCodePath + '/' + client_filename
SSH.command(cmd,'\$',5)
iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename
iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1'
cmd = 'docker exec -w /iperf-2.0.13 -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"'
SSH.command(cmd,'\$',int(iperf_time)*5.0)
SSH.command('docker cp prod-trf-gen:/iperf-2.0.13/'+ client_filename + ' ' + EPC.SourceCodePath, '\$', 5)
@@ -2433,19 +2435,17 @@ class OaiCiTest():
logging.debug("Iperf for Module in UL mode detected")
#server side EPC
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
iperf_cmd = 'echo $USER; nohup bin/iperf -s -u 2>&1 > ' + server_filename
iperf_cmd = 'echo $USER; nohup bin/iperf -s -u -i 1 > ' + server_filename + ' 2>&1'
cmd = 'docker exec -d -w /iperf-2.0.13 prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"'
SSH.command(cmd,'\$',5)
SSH.close()
#client side UE
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
cmd = 'rm '+ client_filename
SSH.command(cmd,'\$',5)
SSH.command('iperf -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0)
SSH.command('iperf -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1', '\$', int(iperf_time)*5.0)
SSH.close()
#once client is done, retrieve the server file from container to EPC Host
@@ -2461,12 +2461,10 @@ class OaiCiTest():
elif self.iperf_direction=="BIDIR":
logging.debug("Iperf for Module in BIDIR mode detected")
server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
#server side EPC
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
iperf_cmd = 'echo $USER; nohup /usr/local/bin/iperf3 -s 2>&1 > ' + server_filename
iperf_cmd = 'echo $USER; nohup /usr/local/bin/iperf3 -s -i 1 > ' + server_filename + ' 2>&1'
cmd = 'docker exec -d -w /iperf-2.0.13 prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"'
SSH.command(cmd,'\$',5)
SSH.close()
@@ -2475,7 +2473,7 @@ class OaiCiTest():
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'rm '+ client_filename
SSH.command(cmd,'\$',5)
SSH.command('iperf3 -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0)
SSH.command('iperf3 -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1', '\$', int(iperf_time)*5.0)
SSH.close()
#once client is done, retrieve the server file from container to EPC Host
@@ -2499,8 +2497,6 @@ class OaiCiTest():
else: #default is ltebox
SSH = sshconnection.SSHConnection()
#kill iperf processes before (in case there are still some remaining)
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'killall --signal=SIGKILL iperf'
@@ -2524,14 +2520,14 @@ class OaiCiTest():
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
SSH.command(cmd,'\$',5)
cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log &'
cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u -i 1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1 &'
SSH.command(cmd,'\$',5)
SSH.close()
#client side EPC
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
SSH.command(cmd,'\$',5)
cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1'
SSH.command(cmd,'\$',int(iperf_time)*5.0)
SSH.close()
#copy the 2 resulting files locally
@@ -2547,7 +2543,7 @@ class OaiCiTest():
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
SSH.command(cmd,'\$',5)
cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log &'
cmd = 'echo $USER; nohup iperf -s -u -i 1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1 &'
SSH.command(cmd,'\$',5)
SSH.close()
@@ -2555,7 +2551,7 @@ class OaiCiTest():
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
SSH.command(cmd,'\$',5)
SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0)
SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1', '\$', int(iperf_time)*5.0)
SSH.close()
#copy the 2 resulting files locally
@@ -2566,15 +2562,13 @@ class OaiCiTest():
self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1)
elif self.iperf_direction=="BIDIR":
logging.debug("Iperf for Module in BIDIR mode detected")
server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log'
#server side EPC
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
cmd = 'rm ' + server_filename
SSH.command(cmd,'\$',5)
cmd = 'echo $USER; nohup iperf3 -s 2>&1 > '+server_filename+' &'
cmd = 'echo $USER; nohup iperf3 -s -i 1 > '+server_filename+' 2>&1 &'
SSH.command(cmd,'\$',5)
SSH.close()
@@ -2582,7 +2576,7 @@ class OaiCiTest():
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'rm ' + client_filename
SSH.command(cmd,'\$',5)
SSH.command('iperf3 -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > '+client_filename, '\$', int(iperf_time)*5.0)
SSH.command('iperf3 -c 192.172.0.1 ' + self.iperf_args + ' > '+client_filename + ' 2>&1', '\$', int(iperf_time)*5.0)
SSH.close()
#copy the 2 resulting files locally
@@ -2593,6 +2587,7 @@ class OaiCiTest():
else :
logging.debug("Incorrect or missing IPERF direction in XML")
#kill iperf processes after to be clean
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
cmd = 'killall --signal=SIGKILL iperf'
@@ -2606,7 +2601,12 @@ class OaiCiTest():
cmd = 'killall --signal=SIGKILL iperf3'
SSH.command(cmd,'\$',5)
SSH.close()
return
# Copying to xNB server for Jenkins artifacting
if (os.path.isfile(server_filename)):
SSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, server_filename, RAN.eNBSourceCodePath + '/cmake_targets/')
if (os.path.isfile(client_filename)):
SSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, client_filename, RAN.eNBSourceCodePath + '/cmake_targets/')
def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC):
try:
@@ -2804,12 +2804,12 @@ class OaiCiTest():
else:
SSH.copyin(self.ADBIPAddress, self.ADBUserName, self.ADBPassword, EPC.SourceCodePath + '/scripts/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.')
# fromdos has to be called on the python executor not on ADB server
cmd = 'fromdos -o iperf_server_' + self.testCase_id + '_' + device_id + '.log 2>&1 > /dev/null'
cmd = 'fromdos -o iperf_server_' + self.testCase_id + '_' + device_id + '.log > /dev/null 2>&1'
try:
subprocess.run(cmd, shell=True)
except:
pass
cmd = 'dos2unix -o iperf_server_' + self.testCase_id + '_' + device_id + '.log 2>&1 > /dev/null'
cmd = 'dos2unix -o iperf_server_' + self.testCase_id + '_' + device_id + '.log > /dev/null 2>&1'
try:
subprocess.run(cmd, shell=True)
except:
@@ -2995,7 +2995,7 @@ class OaiCiTest():
#special quick and dirty treatment for modules, iperf to be restructured
if self.ue_id!="": #is module
device_id = Module_UE.ID + "-" + Module_UE.Kind
p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE,))
p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE, RAN, ))
else: #legacy code
p = Process(target = self.Iperf_common, args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, ))
p.daemon = True
ci-scripts/ran.py
+2 -2
View File
@@ -756,11 +756,11 @@ class RANManagement():
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 60)
result = re.search('core.\d+', mySSH.getBefore())
if result is not None:
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip core* ran_build/build/{lte,nr}-softmodem', '\$', 60) # add core and executable to zip
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 15)
mySSH.close()
def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):
ci-scripts/xml_files/container_nsa_b200_quectel.xml
+32 -4
View File
@@ -39,8 +39,9 @@
050001
070001
070000
050000
050001
070002
050002
050003
010002
000001
030202
@@ -117,10 +118,28 @@
<ping_rttavg_threshold>15</ping_rttavg_threshold>
</testCase>
<testCase id="050002">
<class>Ping</class>
<desc>Ping: 20pings in 20sec</desc>
<id>idefix</id>
<ping_args>-c 20</ping_args>
<ping_packetloss_threshold>1</ping_packetloss_threshold>
<ping_rttavg_threshold>15</ping_rttavg_threshold>
</testCase>
<testCase id="050003">
<class>Ping</class>
<desc>Ping: 100pings in 20sec</desc>
<id>idefix</id>
<ping_args>-c 100 -i 0.2</ping_args>
<ping_packetloss_threshold>1</ping_packetloss_threshold>
<ping_rttavg_threshold>15</ping_rttavg_threshold>
</testCase>
<testCase id="070000">
<class>Iperf</class>
<desc>iperf (DL/40Mbps/UDP)(60 sec)(single-ue profile)</desc>
<iperf_args>-u -b 40M -t 60</iperf_args>
<iperf_args>-u -b 40M -t 60 -i 1 -fm</iperf_args>
<direction>DL</direction>
<id>idefix</id>
<iperf_packetloss_threshold>20</iperf_packetloss_threshold>
@@ -131,7 +150,7 @@
<testCase id="070001">
<class>Iperf</class>
<desc>iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile)</desc>
<iperf_args>-u -b 3M -t 60</iperf_args>
<iperf_args>-u -b 3M -t 60 -i 1 -fm</iperf_args>
<direction>UL</direction>
<id>idefix</id>
<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
@@ -139,6 +158,15 @@
<iperf_profile>single-ue</iperf_profile>
</testCase>
<testCase id="070002">
<class>Iperf</class>
<desc>iperf (BIDIR TCP)(10 sec)(single-ue profile)</desc>
<iperf_args>-t 10 --bidir</iperf_args>
<direction>BIDIR</direction>
<id>idefix</id>
<iperf_profile>single-ue</iperf_profile>
</testCase>
<testCase id="030201">
<class>Undeploy_Object</class>
<desc>Undeploy eNB</desc>