From 52ead754c0a33ce2560ea7b6df66e67f6d586a04 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Mon, 21 Mar 2022 16:54:00 +0100 Subject: [PATCH] fix(ci): some little cosmetics fixes * The quectel CM program output is now redirected to a file. Easier to follow when running * I added `-i 1` option to all server iperf commands to have details * module-based iperf logs are also copied to the RAN enb log folder so they are artifacted * iperf3 console output was badly formatted * `docker kill --signal INT` does not stop the container Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 2 + ci-scripts/cls_module_ue.py | 3 +- ci-scripts/cls_oaicitest.py | 62 +++++++++---------- ci-scripts/ran.py | 4 +- .../xml_files/container_nsa_b200_quectel.xml | 36 +++++++++-- 5 files changed, 69 insertions(+), 38 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index fdc2bc1597..d85659568b 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -606,6 +606,8 @@ class Containerize(): if containerToKill: mySSH.command('docker kill --signal INT ' + containerName, '\$', 30) time.sleep(5) + mySSH.command('docker kill --signal KILL ' + containerName, '\$', 30) + time.sleep(5) mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) mySSH.command('docker rm -f ' + containerName, '\$', 30) # Forcing the down now to remove the networks and any artifacts diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index cabd0a045d..4e2efaeda5 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -76,7 +76,8 @@ class Module_UE: logging.debug('Starting ' + self.Process['Name']) mySSH = sshconnection.SSHConnection() mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) - mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S ' + self.Process['Cmd'] + ' ' + self.Process['Apn'][CNType] + ' > /dev/null 2>&1 &','\$',5) + mySSH.command('sudo rm -f /tmp/quectel-cm.log','\$',5) + mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S stdbuf -o0 ' + self.Process['Cmd'] + ' ' + self.Process['Apn'][CNType] + ' > /tmp/quectel-cm.log 2>&1 &','\$',5) mySSH.close() #checking the process time.sleep(5) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 5d89a7b2e0..ec1202cb21 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -122,6 +122,7 @@ def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold): try: mySSH = sshconnection.SSHConnection() mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file, RAN.eNBSourceCodePath + '/cmake_targets/') except: logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m') @@ -1598,7 +1599,7 @@ class OaiCiTest(): else: if launchfromModule == False: #ping log file is on the python executor - cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 > ping_' + self.testCase_id + '_' + device_id + '.log' + cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' > ping_' + self.testCase_id + '_' + device_id + '.log 2>&1' message = cmd + '\n' logging.debug(cmd) ret = subprocess.run(cmd, shell=True) @@ -1623,7 +1624,7 @@ class OaiCiTest(): else: Target = EPC.IPAddress #ping from module NIC rather than IP address to make sure round trip is over the air - cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' 2>&1 > ping_' + self.testCase_id + '_' + self.ue_id + '.log' + cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' > ping_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1' SSH.command(cmd,'\$',int(ping_time[0])*1.5) #copy the ping log file to have it locally for analysis (ping stats) SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.') @@ -2084,7 +2085,8 @@ class OaiCiTest(): statusQueue.put(UE_IPAddress) statusQueue.put(report_msg) logging.debug('\u001B[1;37;45m TCP Bidir Iperf Result (' + UE_IPAddress + ') \u001B[0m') - logging.debug('\u001B[1;35m ' + report_msg + '\u001B[0m') + for rLine in report_msg.split('\n'): + logging.debug('\u001B[1;35m ' + rLine + '\u001B[0m') lock.release() else: self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Bidir TCP : Could not analyze from Log file') @@ -2379,10 +2381,12 @@ class OaiCiTest(): SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts') - def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE): + def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE, RAN): + SSH = sshconnection.SSHConnection() + server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): #retrieve trf-gen container IP address - SSH = sshconnection.SSHConnection() SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) SSH.command('docker inspect --format="TRF_IP_ADDR = {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" prod-trf-gen', '\$', 5) result = re.search('TRF_IP_ADDR = (?P[0-9\.]+)', SSH.getBefore()) @@ -2404,20 +2408,18 @@ class OaiCiTest(): if self.iperf_direction=="DL": logging.debug("Iperf for Module in DL mode detected") ##server side UE - server_filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'rm ' + server_filename SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > ' + server_filename + ' &' + cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u -i 1 > ' + server_filename + ' 2>&1 &' SSH.command(cmd,'\$',5) SSH.close() ##client side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' #remove old client file in EPC.SourceCodePath cmd = 'rm ' + EPC.SourceCodePath + '/' + client_filename SSH.command(cmd,'\$',5) - iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename + iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1' cmd = 'docker exec -w /iperf-2.0.13 -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' SSH.command(cmd,'\$',int(iperf_time)*5.0) SSH.command('docker cp prod-trf-gen:/iperf-2.0.13/'+ client_filename + ' ' + EPC.SourceCodePath, '\$', 5) @@ -2433,19 +2435,17 @@ class OaiCiTest(): logging.debug("Iperf for Module in UL mode detected") #server side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - iperf_cmd = 'echo $USER; nohup bin/iperf -s -u 2>&1 > ' + server_filename + iperf_cmd = 'echo $USER; nohup bin/iperf -s -u -i 1 > ' + server_filename + ' 2>&1' cmd = 'docker exec -d -w /iperf-2.0.13 prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' SSH.command(cmd,'\$',5) SSH.close() #client side UE SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' cmd = 'rm '+ client_filename SSH.command(cmd,'\$',5) - SSH.command('iperf -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0) + SSH.command('iperf -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1', '\$', int(iperf_time)*5.0) SSH.close() #once client is done, retrieve the server file from container to EPC Host @@ -2461,12 +2461,10 @@ class OaiCiTest(): elif self.iperf_direction=="BIDIR": logging.debug("Iperf for Module in BIDIR mode detected") - server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' #server side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - iperf_cmd = 'echo $USER; nohup /usr/local/bin/iperf3 -s 2>&1 > ' + server_filename + iperf_cmd = 'echo $USER; nohup /usr/local/bin/iperf3 -s -i 1 > ' + server_filename + ' 2>&1' cmd = 'docker exec -d -w /iperf-2.0.13 prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' SSH.command(cmd,'\$',5) SSH.close() @@ -2475,7 +2473,7 @@ class OaiCiTest(): SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'rm '+ client_filename SSH.command(cmd,'\$',5) - SSH.command('iperf3 -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0) + SSH.command('iperf3 -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' > ' + client_filename + ' 2>&1', '\$', int(iperf_time)*5.0) SSH.close() #once client is done, retrieve the server file from container to EPC Host @@ -2499,8 +2497,6 @@ class OaiCiTest(): else: #default is ltebox - SSH = sshconnection.SSHConnection() - #kill iperf processes before (in case there are still some remaining) SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'killall --signal=SIGKILL iperf' @@ -2524,14 +2520,14 @@ class OaiCiTest(): SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log &' + cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u -i 1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1 &' SSH.command(cmd,'\$',5) SSH.close() #client side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.command(cmd,'\$',5) - cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1' SSH.command(cmd,'\$',int(iperf_time)*5.0) SSH.close() #copy the 2 resulting files locally @@ -2547,7 +2543,7 @@ class OaiCiTest(): SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log &' + cmd = 'echo $USER; nohup iperf -s -u -i 1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1 &' SSH.command(cmd,'\$',5) SSH.close() @@ -2555,7 +2551,7 @@ class OaiCiTest(): SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.command(cmd,'\$',5) - SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0) + SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1', '\$', int(iperf_time)*5.0) SSH.close() #copy the 2 resulting files locally @@ -2566,15 +2562,13 @@ class OaiCiTest(): self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) elif self.iperf_direction=="BIDIR": logging.debug("Iperf for Module in BIDIR mode detected") - server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' #server side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) cmd = 'rm ' + server_filename SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup iperf3 -s 2>&1 > '+server_filename+' &' + cmd = 'echo $USER; nohup iperf3 -s -i 1 > '+server_filename+' 2>&1 &' SSH.command(cmd,'\$',5) SSH.close() @@ -2582,7 +2576,7 @@ class OaiCiTest(): SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'rm ' + client_filename SSH.command(cmd,'\$',5) - SSH.command('iperf3 -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > '+client_filename, '\$', int(iperf_time)*5.0) + SSH.command('iperf3 -c 192.172.0.1 ' + self.iperf_args + ' > '+client_filename + ' 2>&1', '\$', int(iperf_time)*5.0) SSH.close() #copy the 2 resulting files locally @@ -2593,6 +2587,7 @@ class OaiCiTest(): else : logging.debug("Incorrect or missing IPERF direction in XML") + #kill iperf processes after to be clean SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) cmd = 'killall --signal=SIGKILL iperf' @@ -2606,7 +2601,12 @@ class OaiCiTest(): cmd = 'killall --signal=SIGKILL iperf3' SSH.command(cmd,'\$',5) SSH.close() - return + + # Copying to xNB server for Jenkins artifacting + if (os.path.isfile(server_filename)): + SSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, server_filename, RAN.eNBSourceCodePath + '/cmake_targets/') + if (os.path.isfile(client_filename)): + SSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, client_filename, RAN.eNBSourceCodePath + '/cmake_targets/') def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC): try: @@ -2804,12 +2804,12 @@ class OaiCiTest(): else: SSH.copyin(self.ADBIPAddress, self.ADBUserName, self.ADBPassword, EPC.SourceCodePath + '/scripts/iperf_server_' + self.testCase_id + '_' + device_id + '.log', '.') # fromdos has to be called on the python executor not on ADB server - cmd = 'fromdos -o iperf_server_' + self.testCase_id + '_' + device_id + '.log 2>&1 > /dev/null' + cmd = 'fromdos -o iperf_server_' + self.testCase_id + '_' + device_id + '.log > /dev/null 2>&1' try: subprocess.run(cmd, shell=True) except: pass - cmd = 'dos2unix -o iperf_server_' + self.testCase_id + '_' + device_id + '.log 2>&1 > /dev/null' + cmd = 'dos2unix -o iperf_server_' + self.testCase_id + '_' + device_id + '.log > /dev/null 2>&1' try: subprocess.run(cmd, shell=True) except: @@ -2995,7 +2995,7 @@ class OaiCiTest(): #special quick and dirty treatment for modules, iperf to be restructured if self.ue_id!="": #is module device_id = Module_UE.ID + "-" + Module_UE.Kind - p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE,)) + p = Process(target = self.Iperf_Module ,args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, Module_UE, RAN, )) else: #legacy code p = Process(target = self.Iperf_common, args = (lock, UE_IPAddress, device_id, i, ue_num, status_queue, EPC, )) p.daemon = True diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 3b32a75542..f95ce90c66 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -756,11 +756,11 @@ class RANManagement(): mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 60) result = re.search('core.\d+', mySSH.getBefore()) if result is not None: mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip core* ran_build/build/{lte,nr}-softmodem', '\$', 60) # add core and executable to zip - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 15) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}): diff --git a/ci-scripts/xml_files/container_nsa_b200_quectel.xml b/ci-scripts/xml_files/container_nsa_b200_quectel.xml index 55f8b8638d..610f0b019a 100644 --- a/ci-scripts/xml_files/container_nsa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_nsa_b200_quectel.xml @@ -39,8 +39,9 @@ 050001 070001 070000 - 050000 - 050001 + 070002 + 050002 + 050003 010002 000001 030202 @@ -117,10 +118,28 @@ 15 + + Ping + Ping: 20pings in 20sec + idefix + -c 20 + 1 + 15 + + + + Ping + Ping: 100pings in 20sec + idefix + -c 100 -i 0.2 + 1 + 15 + + Iperf iperf (DL/40Mbps/UDP)(60 sec)(single-ue profile) - -u -b 40M -t 60 + -u -b 40M -t 60 -i 1 -fm DL idefix 20 @@ -131,7 +150,7 @@ Iperf iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) - -u -b 3M -t 60 + -u -b 3M -t 60 -i 1 -fm UL idefix 1 @@ -139,6 +158,15 @@ single-ue + + Iperf + iperf (BIDIR TCP)(10 sec)(single-ue profile) + -t 10 --bidir + BIDIR + idefix + single-ue + + Undeploy_Object Undeploy eNB