From 90f150a4a8591c83c44b587fee0208337cbca658 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 21 Dec 2021 16:12:37 +0100 Subject: [PATCH 01/16] improve iperf pass fail checker --- ci-scripts/cls_oaicitest.py | 16 +++++++++++++--- ci-scripts/main.py | 7 +++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index b8b99a053b..68b71da892 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -129,6 +129,7 @@ class OaiCiTest(): self.ping_rttavg_threshold ='' self.iperf_args = '' self.iperf_packetloss_threshold = '' + self.iperf_bitrate_threshold = '' self.iperf_profile = '' self.iperf_options = '' self.iperf_direction = '' @@ -2085,13 +2086,21 @@ class OaiCiTest(): pl = float(100 * pl_sum / ps_sum) packetloss = '%2.1f ' % (pl) packetloss += '%' + #checking packet loss compliance if float(pl) > float(self.iperf_packetloss_threshold): pal_too_high_msg = 'Packet Loss too high : actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%\n' else: - pal_too_high_msg='' + pal_too_high_msg='' + #checking bitrate perf compliance + if float(br_loss) < float(self.iperf_bitrate_threshold): + bit_too_low_msg = 'Bitrate too low : actual = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%\n' + else: + bit_too_low_msg='' lock.acquire() - if (br_loss < 90) or (float(pl) > float(self.iperf_packetloss_threshold)): + if (br_loss < self.iperf_bitrate_threshold) or (float(pl) > float(self.iperf_packetloss_threshold)): statusQueue.put(1) + elif (br_loss < self.iperf_bitrate_threshold) and (float(pl) > float(self.iperf_packetloss_threshold)): + statusQueue.put(-1) else: statusQueue.put(0) statusQueue.put(device_id) @@ -2101,7 +2110,7 @@ class OaiCiTest(): brl_msg = 'Bitrate Perf: ' + bitperf jit_msg = 'Jitter : ' + jitter pal_msg = 'Packet Loss : ' + packetloss - statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n') + statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n' + bit_too_low_msg + '\n') logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m') logging.debug('\u001B[1;35m ' + req_msg + '\u001B[0m') logging.debug('\u001B[1;35m ' + bir_msg + '\u001B[0m') @@ -2109,6 +2118,7 @@ class OaiCiTest(): logging.debug('\u001B[1;35m ' + jit_msg + '\u001B[0m') logging.debug('\u001B[1;35m ' + pal_msg + '\u001B[0m') logging.debug('\u001B[1;35m ' + pal_too_high_msg + '\u001B[0m') + logging.debug('\u001B[1;35m ' + bit_too_low_msg + '\u001B[0m') lock.release() else: self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index ddf9bb531e..415d3c4d3d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -310,6 +310,13 @@ def GetParametersFromXML(action): CiTestObj.ue_id = ue_id CiTestObj.iperf_direction = test.findtext('direction')#used for modules only CiTestObj.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold') + iperf_bitrate_threshold = test.findtext('iperf_bitrate_threshold') + if (iperf_bitrate_threshold is None): + CiTestObj.iperf_bitrate_threshold = "90" #if no threshold is specified, default will be 90% + else: + CiTestObj.iperf_bitrate_threshold = iperf_bitrate_threshold + + CiTestObj.iperf_profile = test.findtext('iperf_profile') if (CiTestObj.iperf_profile is None): CiTestObj.iperf_profile = 'balanced' From 984338e991adb7d8bdf30917ea35f51bdbce6330 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 21 Dec 2021 16:29:04 +0100 Subject: [PATCH 02/16] update 2x2 tests with new iperf pass fail criteria --- ci-scripts/xml_files/container_nsa_b200_quectel.xml | 2 ++ ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 2 ++ ci-scripts/xml_files/fr1_nsa_quectel.xml | 2 ++ ci-scripts/xml_files/fr1_sa_quectel.xml | 2 ++ ci-scripts/xml_files/fr1_sa_quectel_stages.xml | 5 +++++ 5 files changed, 13 insertions(+) diff --git a/ci-scripts/xml_files/container_nsa_b200_quectel.xml b/ci-scripts/xml_files/container_nsa_b200_quectel.xml index 0af0d9bd70..3cac13dad4 100644 --- a/ci-scripts/xml_files/container_nsa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_nsa_b200_quectel.xml @@ -124,6 +124,7 @@ DL idefix 3 + 95 single-ue @@ -134,6 +135,7 @@ UL idefix 1 + 95 single-ue diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index ae7a4c92be..4cb6b88a0c 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -107,6 +107,7 @@ DL nrmodule2_quectel 5 + 95 single-ue @@ -117,6 +118,7 @@ UL nrmodule2_quectel 5 + 95 single-ue diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index 6414f09b6b..6e3d208040 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -119,6 +119,7 @@ DL idefix 3 + 95 single-ue @@ -129,6 +130,7 @@ UL idefix 1 + 95 single-ue diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml index 4db5bb64a5..4cef4d404b 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel.xml @@ -105,6 +105,7 @@ DL nrmodule2_quectel 5 + 95 single-ue @@ -115,6 +116,7 @@ UL nrmodule2_quectel 5 + 95 single-ue diff --git a/ci-scripts/xml_files/fr1_sa_quectel_stages.xml b/ci-scripts/xml_files/fr1_sa_quectel_stages.xml index a9922bec52..754d201fcf 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel_stages.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel_stages.xml @@ -104,6 +104,7 @@ DL nrmodule2_quectel 5 + 95 single-ue @@ -113,6 +114,7 @@ DL nrmodule2_quectel 5 + 95 single-ue @@ -122,6 +124,7 @@ DL nrmodule2_quectel 5 + 95 single-ue @@ -131,6 +134,7 @@ DL nrmodule2_quectel 5 + 95 single-ue @@ -140,6 +144,7 @@ DL nrmodule2_quectel 5 + 95 single-ue From 478f67e47dcc5613cffebeec53b64879d7ed1c25 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 23 Dec 2021 11:58:21 +0100 Subject: [PATCH 03/16] fix type conversion for comparison operation --- ci-scripts/cls_oaicitest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 68b71da892..e0c261df22 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2097,9 +2097,9 @@ class OaiCiTest(): else: bit_too_low_msg='' lock.acquire() - if (br_loss < self.iperf_bitrate_threshold) or (float(pl) > float(self.iperf_packetloss_threshold)): + if (float(br_loss) < float(self.iperf_bitrate_threshold)) or (float(pl) > float(self.iperf_packetloss_threshold)): statusQueue.put(1) - elif (br_loss < self.iperf_bitrate_threshold) and (float(pl) > float(self.iperf_packetloss_threshold)): + elif (float(br_loss) < float(self.iperf_bitrate_threshold)) and (float(pl) > float(self.iperf_packetloss_threshold)): statusQueue.put(-1) else: statusQueue.put(0) From c26d6fd685ca8b2381bcd5c2b1fe7199fe66cd48 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 3 Jan 2022 11:55:26 +0100 Subject: [PATCH 04/16] fix test status depending on results ; cosmetic fix on assertions text --- ci-scripts/cls_oaicitest.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index e0c261df22..9f930e8897 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2088,19 +2088,19 @@ class OaiCiTest(): packetloss += '%' #checking packet loss compliance if float(pl) > float(self.iperf_packetloss_threshold): - pal_too_high_msg = 'Packet Loss too high : actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%\n' + pal_too_high_msg = 'Packet Loss too high : actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%' else: - pal_too_high_msg='' + pal_too_high_msg='Packet Loss value is within acceptance range' #checking bitrate perf compliance if float(br_loss) < float(self.iperf_bitrate_threshold): - bit_too_low_msg = 'Bitrate too low : actual = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%\n' + bit_too_low_msg = 'Bitrate too low : actual = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%' else: - bit_too_low_msg='' + bit_too_low_msg='Bitrate perf value is within acceptance range' lock.acquire() - if (float(br_loss) < float(self.iperf_bitrate_threshold)) or (float(pl) > float(self.iperf_packetloss_threshold)): - statusQueue.put(1) - elif (float(br_loss) < float(self.iperf_bitrate_threshold)) and (float(pl) > float(self.iperf_packetloss_threshold)): + if (float(br_loss) < float(self.iperf_bitrate_threshold)) and (float(pl) > float(self.iperf_packetloss_threshold)): statusQueue.put(-1) + elif (float(br_loss) < float(self.iperf_bitrate_threshold)) or (float(pl) > float(self.iperf_packetloss_threshold)): + statusQueue.put(1) else: statusQueue.put(0) statusQueue.put(device_id) From 535cc3de21de090ab9d5da392e78553946eeb52f Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 3 Jan 2022 16:15:04 +0100 Subject: [PATCH 05/16] fixing ping analysis (extra stats) for module UE --- ci-scripts/cls_oaicitest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 9f930e8897..f5692579ee 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -1579,6 +1579,7 @@ class OaiCiTest(): SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) #cat is executed on EPC SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) + ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log' else: #launch from Module SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) #target address is different depending on EPC type @@ -1596,6 +1597,7 @@ class OaiCiTest(): #cat is executed locally SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5) + ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log' ping_status=0 # TIMEOUT CASE @@ -1643,7 +1645,7 @@ class OaiCiTest(): logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m') #adding extra ping stats from local file - ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log' + #ping_log_file variable is defined above in this function, depending on device/ue logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) ping_stat=GetPingTimeAnalysis(ping_log_file) ping_stat_msg='' @@ -2088,12 +2090,12 @@ class OaiCiTest(): packetloss += '%' #checking packet loss compliance if float(pl) > float(self.iperf_packetloss_threshold): - pal_too_high_msg = 'Packet Loss too high : actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%' + pal_too_high_msg = 'Packet Loss too high : tested = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%' else: pal_too_high_msg='Packet Loss value is within acceptance range' #checking bitrate perf compliance if float(br_loss) < float(self.iperf_bitrate_threshold): - bit_too_low_msg = 'Bitrate too low : actual = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%' + bit_too_low_msg = 'Bitrate too low : tested = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%' else: bit_too_low_msg='Bitrate perf value is within acceptance range' lock.acquire() From 76f0990352574007ca763027f787cf22ab746426 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 3 Jan 2022 17:46:38 +0100 Subject: [PATCH 06/16] add/fix autoterminate when a node does not start (gnb in nsa for ex) --- ci-scripts/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 415d3c4d3d..f448447e44 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -818,6 +818,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re check_OAI_UE = False RAN.pStatus=CiTestObj.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) RAN.InitializeeNB(HTML, EPC) + if RAN.prematureExit: + CiTestObj.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) elif action == 'Terminate_eNB': RAN.TerminateeNB(HTML, EPC) elif action == 'Initialize_UE': From 4a263215ada6a6a4ccd9a34f567c1af79811f295 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 3 Jan 2022 23:27:04 +0100 Subject: [PATCH 07/16] add ping events graph into artifacts as png --- ci-scripts/cls_oaicitest.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index f5692579ee..7e02a51b34 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -62,7 +62,7 @@ import cls_ci_ueinfra #class defining the multi Ue infrastrucure # Utility functions #----------------------------------------------------------- -def GetPingTimeAnalysis(ping_log_file): +def GetPingTimeAnalysis(ping_log_file,ping_rttavg_threshold): #ping time values read from file t_ping=[] #ping stats (dictionary) to be returned by the function @@ -93,6 +93,24 @@ def GetPingTimeAnalysis(ping_log_file): ping_stat['median_1']=stat.median(t_ping) ping_stat['max_1']=max(t_ping) + #plot ping over time and save png for artifacts + ticks = np.arange(0, len(t_ping), 1) + figure, axis = plt.subplots(figsize=(10, 10)) + axis.plot(ticks,t_ping,marker='o') + axis.set_xlabel('Ping Events') + axis.set_ylabel("Ping RTT (in ms)") + axis.set_title(ping_log_file) + YMAX=20 #base scale + if max(t_ping) > YMAX: + y_max=max(t_ping)+1 + else: + y_max=YMAX+1 + plt.ylim(0,y_max) + th_label="Ping Fail Threshold="+ping_rttavg_threshold + plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label) + axis.legend() + plt.savefig(ping_log_file+'.png') + return ping_stat else: @@ -1647,7 +1665,7 @@ class OaiCiTest(): #adding extra ping stats from local file #ping_log_file variable is defined above in this function, depending on device/ue logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) - ping_stat=GetPingTimeAnalysis(ping_log_file) + ping_stat=GetPingTimeAnalysis(ping_log_file,self.ping_rttavg_threshold) ping_stat_msg='' if (ping_stat!=-1) and (len(ping_stat)!=0): ping_stat_msg+='Ping stats before removing largest value : \n' From a0fd26e8dff5d7be767f58149059a63185390870 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 4 Jan 2022 08:50:49 +0100 Subject: [PATCH 08/16] fix matplotlib numpy lib import --- ci-scripts/cls_oaicitest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 7e02a51b34..1e3b15cd08 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -57,6 +57,8 @@ import sshconnection import cls_module_ue import cls_ci_ueinfra #class defining the multi Ue infrastrucure +import matplotlib.pyplot as plt +import numpy as np #----------------------------------------------------------- # Utility functions From af554f66ede8243076c2b19a58b36d073de66520 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 4 Jan 2022 10:54:34 +0100 Subject: [PATCH 09/16] fix matplotlib logging level --- ci-scripts/cls_oaicitest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 1e3b15cd08..799e5283c4 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -57,6 +57,7 @@ import sshconnection import cls_module_ue import cls_ci_ueinfra #class defining the multi Ue infrastrucure +logging.getLogger("matplotlib").setLevel(logging.WARNING) import matplotlib.pyplot as plt import numpy as np From ada154f852c368d15298bb7d85bb39d8139967a6 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 4 Jan 2022 15:15:28 +0100 Subject: [PATCH 10/16] add ping png to artifacts --- ci-scripts/cls_oaicitest.py | 8 +++++++- ci-scripts/ran.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 799e5283c4..310b71f8c0 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -103,17 +103,23 @@ def GetPingTimeAnalysis(ping_log_file,ping_rttavg_threshold): axis.set_xlabel('Ping Events') axis.set_ylabel("Ping RTT (in ms)") axis.set_title(ping_log_file) + axis.set_xticks(ticks) + axis.set_xticklabels([]) YMAX=20 #base scale if max(t_ping) > YMAX: y_max=max(t_ping)+1 else: y_max=YMAX+1 plt.ylim(0,y_max) - th_label="Ping Fail Threshold="+ping_rttavg_threshold + th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label) axis.legend() plt.savefig(ping_log_file+'.png') + #copy the png file already to enb to move it move it later into the artifacts + mySSH = sshconnection.SSHConnection() + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, ping_log_file+'.png', self.eNBSourceCodePath + '/cmake_targets/') + return ping_stat else: diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 95066d168d..32913f1ad0 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -738,8 +738,8 @@ class RANManagement(): mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 15) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): From f34d5e2f1bd187df5820f54d8fcd8d2dddb76fe4 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 24 Dec 2021 12:23:34 +0100 Subject: [PATCH 11/16] Change obelix N300 IP addresses We suspected a faulty N310 on obelix. It has been replaced with another N300 with the same .10.2 addr, but the mgmt_addr changed to .18.16, corrected in this commit. --- ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf | 2 +- ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf | 2 +- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 2 +- ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml | 2 +- ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml | 2 +- ci-scripts/xml_files/fr1_sa_oaiue_n310.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf index d2e91fb872..bc29f1bd37 100644 --- a/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf +++ b/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf @@ -237,7 +237,7 @@ RUs = ( max_pdschReferenceSignalPower = -27; max_rxgain = 75; eNB_instances = [0]; - sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2"; + sdr_addrs = "mgmt_addr=192.168.18.16,addr=192.168.10.2"; } ); diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf index 6c8d5ba6f0..f9d42acaa1 100644 --- a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf +++ b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf @@ -237,7 +237,7 @@ RUs = ( max_pdschReferenceSignalPower = -27; max_rxgain = 75; eNB_instances = [0]; - sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2"; + sdr_addrs = "mgmt_addr=192.168.18.16,addr=192.168.10.2"; } ); diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index 4cb6b88a0c..aa500f27de 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -65,7 +65,7 @@ lte yes yes - 192.168.18.241 + 192.168.18.16 diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml index 86af12b53e..6018fcd16c 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml @@ -67,7 +67,7 @@ lte yes yes - 192.168.18.241 + 192.168.18.16 diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml index 2d17ff38e2..e56ae2b8cd 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml @@ -87,7 +87,7 @@ lte yes yes - 192.168.18.241 + 192.168.18.16 diff --git a/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml b/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml index 28eee1e224..d6f30c0909 100644 --- a/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml +++ b/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml @@ -42,7 +42,7 @@ Initialize_OAI_UE Initialize OAI UE nr - --sa -O ../../../ci-scripts/conf_files/ue.sa.conf --usrp-args "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2" --numerology 1 -r 106 --band 78 -C 3319680000 --nokrnmod 1 --ue-txgain 0 --ue-rxgain 70 --ue-fo-compensation + --sa -O ../../../ci-scripts/conf_files/ue.sa.conf --usrp-args "mgmt_addr=192.168.18.16,second_addr=192.168.10.2" --numerology 1 -r 106 --band 78 -C 3319680000 --nokrnmod 1 --ue-txgain 0 --ue-rxgain 70 --ue-fo-compensation From 02a1b883a6cb8de492d0bf7917264ba9813d4afe Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 4 Jan 2022 22:23:12 +0100 Subject: [PATCH 12/16] fix copy of ping graph --- ci-scripts/cls_oaicitest.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 310b71f8c0..59993034a3 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -65,7 +65,7 @@ import numpy as np # Utility functions #----------------------------------------------------------- -def GetPingTimeAnalysis(ping_log_file,ping_rttavg_threshold): +def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold): #ping time values read from file t_ping=[] #ping stats (dictionary) to be returned by the function @@ -89,12 +89,13 @@ def GetPingTimeAnalysis(ping_log_file,ping_rttavg_threshold): max_loc=t_ping.index(max(t_ping)) ping_stat['max_loc']=max_loc #remove it - t_ping.pop(max_loc) + t_ping_post=t_ping.copy() + t_ping_post.pop(max_loc) #new stats after removing max value - ping_stat['min_1']=min(t_ping) - ping_stat['mean_1']=stat.mean(t_ping) - ping_stat['median_1']=stat.median(t_ping) - ping_stat['max_1']=max(t_ping) + ping_stat['min_1']=min(t_ping_post) + ping_stat['mean_1']=stat.mean(t_ping_post) + ping_stat['median_1']=stat.median(t_ping_post) + ping_stat['max_1']=max(t_ping_post) #plot ping over time and save png for artifacts ticks = np.arange(0, len(t_ping), 1) @@ -118,7 +119,7 @@ def GetPingTimeAnalysis(ping_log_file,ping_rttavg_threshold): #copy the png file already to enb to move it move it later into the artifacts mySSH = sshconnection.SSHConnection() - mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, ping_log_file+'.png', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/') return ping_stat @@ -1559,7 +1560,7 @@ class OaiCiTest(): statusQueue.put(message) lock.release() - def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE): + def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN): try: SSH = sshconnection.SSHConnection() # Launch ping on the EPC side (true for ltebox and old open-air-cn) @@ -1674,7 +1675,7 @@ class OaiCiTest(): #adding extra ping stats from local file #ping_log_file variable is defined above in this function, depending on device/ue logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) - ping_stat=GetPingTimeAnalysis(ping_log_file,self.ping_rttavg_threshold) + ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold) ping_stat_msg='' if (ping_stat!=-1) and (len(ping_stat)!=0): ping_stat_msg+='Ping stats before removing largest value : \n' @@ -1864,7 +1865,7 @@ class OaiCiTest(): device_id = self.UEDevices[i] else: device_id = Module_UE.ID + "-" + Module_UE.Kind - p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,)) + p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,)) p.daemon = True p.start() multi_jobs.append(p) From 6ccc419dfa784700d9e475676a84cbc10467588f Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 5 Jan 2022 15:30:50 +0100 Subject: [PATCH 13/16] fix autoterminate --- ci-scripts/cls_oaicitest.py | 35 +++++++++++++++++++++++++++++++++++ ci-scripts/main.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 59993034a3..42ed46cca5 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -3434,6 +3434,41 @@ class OaiCiTest(): CONTAINERS.UndeployObject(HTML,RAN) RAN.prematureExit=True + #this function is called only if eNB/gNB fails to start + #RH to be re-factored + def AutoTerminateeNB(self,HTML,RAN,EPC,CONTAINERS): + if (RAN.Initialize_eNB_args != ''): + self.testCase_id = 'AUTO-KILL-RAN' + HTML.testCase_id = self.testCase_id + self.desc = 'Automatic Termination of all RAN nodes' + HTML.desc = self.desc + self.ShowTestID() + #terminate all RAN nodes eNB/gNB/OCP + for instance in range(0, len(RAN.air_interface)): + if RAN.air_interface[instance]!='': + logging.debug('Auto Termination of Instance ' + str(instance) + ' : ' + RAN.air_interface[instance]) + RAN.eNB_instance=instance + RAN.TerminateeNB(HTML,EPC) + if RAN.flexranCtrlInstalled and RAN.flexranCtrlStarted: + self.testCase_id = 'AUTO-KILL-flexran-ctl' + HTML.testCase_id = self.testCase_id + self.desc = 'Automatic Termination of FlexRan CTL' + HTML.desc = self.desc + self.ShowTestID() + self.TerminateFlexranCtrl(HTML,RAN,EPC) + if CONTAINERS.yamlPath[0] != '': + self.testCase_id = 'AUTO-KILL-CONTAINERS' + HTML.testCase_id = self.testCase_id + self.desc = 'Automatic Termination of all RAN containers' + HTML.desc = self.desc + self.ShowTestID() + for instance in range(0, len(CONTAINERS.yamlPath)): + if CONTAINERS.yamlPath[instance]!='': + CONTAINERS.eNB_instance=instance + CONTAINERS.UndeployObject(HTML,RAN) + RAN.prematureExit=True + + def IdleSleep(self,HTML): time.sleep(self.idle_sleep_time) HTML.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', CONST.ALL_PROCESSES_OK) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index f448447e44..3a5a69573d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -819,7 +819,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re RAN.pStatus=CiTestObj.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) RAN.InitializeeNB(HTML, EPC) if RAN.prematureExit: - CiTestObj.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) + CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS) elif action == 'Terminate_eNB': RAN.TerminateeNB(HTML, EPC) elif action == 'Initialize_UE': From 5afec8db5756130a0bc88ce95c00759e1acbfd3a Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 5 Jan 2022 16:05:51 +0100 Subject: [PATCH 14/16] fix ping analysis if no threshold defined + escape if scp fails --- ci-scripts/cls_oaicitest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 42ed46cca5..3ba014b7c9 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -112,14 +112,18 @@ def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold): else: y_max=YMAX+1 plt.ylim(0,y_max) - th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold - plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label) - axis.legend() + if ping_rttavg_threshold != '': + th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold + plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label) + axis.legend() plt.savefig(ping_log_file+'.png') #copy the png file already to enb to move it move it later into the artifacts - mySSH = sshconnection.SSHConnection() - mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/') + try: + mySSH = sshconnection.SSHConnection() + mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/') + except: + logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m') return ping_stat From 33f93804c62d9f620a44eaa8b446c44850b38615 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 6 Jan 2022 08:31:17 +0100 Subject: [PATCH 15/16] bugfix ping analysis --- ci-scripts/cls_oaicitest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 3ba014b7c9..ac1d89893f 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -128,7 +128,7 @@ def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold): return ping_stat else: - logging.error("Ping log file does not exist") + logging.error("GetPingTimeAnalysis : Ping log file does not exist") return -1 @@ -1592,6 +1592,7 @@ class OaiCiTest(): ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) else: ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) + ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log' #copy the ping log file to have it locally for analysis (ping stats) SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') else: @@ -1611,7 +1612,7 @@ class OaiCiTest(): SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) #cat is executed on EPC SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) - ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log' + ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log' else: #launch from Module SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) #target address is different depending on EPC type From 8e9f7d9477a4bf32e0603f8decbfe8c4d0609e32 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 7 Jan 2022 09:49:01 +0100 Subject: [PATCH 16/16] Revert "Change obelix N300 IP addresses" This reverts commit f34d5e2f1bd187df5820f54d8fcd8d2dddb76fe4. --- ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf | 2 +- ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf | 2 +- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 2 +- ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml | 2 +- ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml | 2 +- ci-scripts/xml_files/fr1_sa_oaiue_n310.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf index bc29f1bd37..d2e91fb872 100644 --- a/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf +++ b/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf @@ -237,7 +237,7 @@ RUs = ( max_pdschReferenceSignalPower = -27; max_rxgain = 75; eNB_instances = [0]; - sdr_addrs = "mgmt_addr=192.168.18.16,addr=192.168.10.2"; + sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2"; } ); diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf index f9d42acaa1..6c8d5ba6f0 100644 --- a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf +++ b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf @@ -237,7 +237,7 @@ RUs = ( max_pdschReferenceSignalPower = -27; max_rxgain = 75; eNB_instances = [0]; - sdr_addrs = "mgmt_addr=192.168.18.16,addr=192.168.10.2"; + sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2"; } ); diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index aa500f27de..4cb6b88a0c 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -65,7 +65,7 @@ lte yes yes - 192.168.18.16 + 192.168.18.241 diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml index c453d78e37..2516af9429 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml @@ -67,7 +67,7 @@ lte yes yes - 192.168.18.16 + 192.168.18.241 diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml index b6153d50ad..52cf1ec6e8 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel_attach_detach.xml @@ -87,7 +87,7 @@ lte yes yes - 192.168.18.16 + 192.168.18.241 diff --git a/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml b/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml index 2e30f25020..00eeeb0c46 100644 --- a/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml +++ b/ci-scripts/xml_files/fr1_sa_oaiue_n310.xml @@ -42,7 +42,7 @@ Initialize_OAI_UE Initialize OAI UE (N310) nr - --sa -O ../../../ci-scripts/conf_files/ue.sa.conf --usrp-args "mgmt_addr=192.168.18.16,second_addr=192.168.10.2" --numerology 1 -r 106 --band 78 -C 3319680000 --nokrnmod 1 --ue-txgain 0 --ue-rxgain 70 --ue-fo-compensation + --sa -O ../../../ci-scripts/conf_files/ue.sa.conf --usrp-args "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2" --numerology 1 -r 106 --band 78 -C 3319680000 --nokrnmod 1 --ue-txgain 0 --ue-rxgain 70 --ue-fo-compensation