diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py
index 097cfd476d..9d2c75363d 100644
--- a/ci-scripts/cls_oaicitest.py
+++ b/ci-scripts/cls_oaicitest.py
@@ -57,12 +57,15 @@ import sshconnection
import cls_module_ue
import cls_ci_ueinfra #class defining the multi Ue infrastrucure
+logging.getLogger("matplotlib").setLevel(logging.WARNING)
+import matplotlib.pyplot as plt
+import numpy as np
#-----------------------------------------------------------
# Utility functions
#-----------------------------------------------------------
-def GetPingTimeAnalysis(ping_log_file):
+def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold):
#ping time values read from file
t_ping=[]
#ping stats (dictionary) to be returned by the function
@@ -86,17 +89,46 @@ def GetPingTimeAnalysis(ping_log_file):
max_loc=t_ping.index(max(t_ping))
ping_stat['max_loc']=max_loc
#remove it
- t_ping.pop(max_loc)
+ t_ping_post=t_ping.copy()
+ t_ping_post.pop(max_loc)
#new stats after removing max value
- ping_stat['min_1']=min(t_ping)
- ping_stat['mean_1']=stat.mean(t_ping)
- ping_stat['median_1']=stat.median(t_ping)
- ping_stat['max_1']=max(t_ping)
+ ping_stat['min_1']=min(t_ping_post)
+ ping_stat['mean_1']=stat.mean(t_ping_post)
+ ping_stat['median_1']=stat.median(t_ping_post)
+ ping_stat['max_1']=max(t_ping_post)
+
+ #plot ping over time and save png for artifacts
+ ticks = np.arange(0, len(t_ping), 1)
+ figure, axis = plt.subplots(figsize=(10, 10))
+ axis.plot(ticks,t_ping,marker='o')
+ axis.set_xlabel('Ping Events')
+ axis.set_ylabel("Ping RTT (in ms)")
+ axis.set_title(ping_log_file)
+ axis.set_xticks(ticks)
+ axis.set_xticklabels([])
+ YMAX=20 #base scale
+ if max(t_ping) > YMAX:
+ y_max=max(t_ping)+1
+ else:
+ y_max=YMAX+1
+ plt.ylim(0,y_max)
+ if ping_rttavg_threshold != '':
+ th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold
+ plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label)
+ axis.legend()
+ plt.savefig(ping_log_file+'.png')
+
+ #copy the png file already to enb to move it move it later into the artifacts
+ try:
+ mySSH = sshconnection.SSHConnection()
+ mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/')
+ except:
+ logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m')
return ping_stat
else:
- logging.error("Ping log file does not exist")
+ logging.error("GetPingTimeAnalysis : Ping log file does not exist")
return -1
@@ -129,6 +161,7 @@ class OaiCiTest():
self.ping_rttavg_threshold =''
self.iperf_args = ''
self.iperf_packetloss_threshold = ''
+ self.iperf_bitrate_threshold = ''
self.iperf_profile = ''
self.iperf_options = ''
self.iperf_direction = ''
@@ -1531,7 +1564,7 @@ class OaiCiTest():
statusQueue.put(message)
lock.release()
- def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE):
+ def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN):
try:
SSH = sshconnection.SSHConnection()
# Launch ping on the EPC side (true for ltebox and old open-air-cn)
@@ -1559,6 +1592,7 @@ class OaiCiTest():
ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
else:
ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
+ ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
#copy the ping log file to have it locally for analysis (ping stats)
SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.')
else:
@@ -1578,6 +1612,7 @@ class OaiCiTest():
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
#cat is executed on EPC
SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
+ ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log'
else: #launch from Module
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
#target address is different depending on EPC type
@@ -1595,6 +1630,7 @@ class OaiCiTest():
#cat is executed locally
SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5)
+ ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log'
ping_status=0
# TIMEOUT CASE
@@ -1642,9 +1678,9 @@ class OaiCiTest():
logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m')
#adding extra ping stats from local file
- ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
+ #ping_log_file variable is defined above in this function, depending on device/ue
logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
- ping_stat=GetPingTimeAnalysis(ping_log_file)
+ ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
ping_stat_msg=''
if (ping_stat!=-1) and (len(ping_stat)!=0):
ping_stat_msg+='Ping stats before removing largest value : \n'
@@ -1834,7 +1870,7 @@ class OaiCiTest():
device_id = self.UEDevices[i]
else:
device_id = Module_UE.ID + "-" + Module_UE.Kind
- p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,))
+ p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,))
p.daemon = True
p.start()
multi_jobs.append(p)
@@ -2085,12 +2121,20 @@ class OaiCiTest():
pl = float(100 * pl_sum / ps_sum)
packetloss = '%2.1f ' % (pl)
packetloss += '%'
+ #checking packet loss compliance
if float(pl) > float(self.iperf_packetloss_threshold):
- pal_too_high_msg = 'Packet Loss too high : actual = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%\n'
+ pal_too_high_msg = 'Packet Loss too high : tested = '+packetloss+', target = '+self.iperf_packetloss_threshold+'%'
else:
- pal_too_high_msg=''
+ pal_too_high_msg='Packet Loss value is within acceptance range'
+ #checking bitrate perf compliance
+ if float(br_loss) < float(self.iperf_bitrate_threshold):
+ bit_too_low_msg = 'Bitrate too low : tested = '+bitperf+', target = '+self.iperf_bitrate_threshold+'%'
+ else:
+ bit_too_low_msg='Bitrate perf value is within acceptance range'
lock.acquire()
- if (br_loss < 90) or (float(pl) > float(self.iperf_packetloss_threshold)):
+ if (float(br_loss) < float(self.iperf_bitrate_threshold)) and (float(pl) > float(self.iperf_packetloss_threshold)):
+ statusQueue.put(-1)
+ elif (float(br_loss) < float(self.iperf_bitrate_threshold)) or (float(pl) > float(self.iperf_packetloss_threshold)):
statusQueue.put(1)
else:
statusQueue.put(0)
@@ -2101,7 +2145,7 @@ class OaiCiTest():
brl_msg = 'Bitrate Perf: ' + bitperf
jit_msg = 'Jitter : ' + jitter
pal_msg = 'Packet Loss : ' + packetloss
- statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n')
+ statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n' + pal_too_high_msg + '\n' + bit_too_low_msg + '\n')
logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m')
logging.debug('\u001B[1;35m ' + req_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + bir_msg + '\u001B[0m')
@@ -2109,6 +2153,7 @@ class OaiCiTest():
logging.debug('\u001B[1;35m ' + jit_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + pal_msg + '\u001B[0m')
logging.debug('\u001B[1;35m ' + pal_too_high_msg + '\u001B[0m')
+ logging.debug('\u001B[1;35m ' + bit_too_low_msg + '\u001B[0m')
lock.release()
else:
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
@@ -3391,6 +3436,41 @@ class OaiCiTest():
CONTAINERS.UndeployObject(HTML,RAN)
RAN.prematureExit=True
+ #this function is called only if eNB/gNB fails to start
+ #RH to be re-factored
+ def AutoTerminateeNB(self,HTML,RAN,EPC,CONTAINERS):
+ if (RAN.Initialize_eNB_args != ''):
+ self.testCase_id = 'AUTO-KILL-RAN'
+ HTML.testCase_id = self.testCase_id
+ self.desc = 'Automatic Termination of all RAN nodes'
+ HTML.desc = self.desc
+ self.ShowTestID()
+ #terminate all RAN nodes eNB/gNB/OCP
+ for instance in range(0, len(RAN.air_interface)):
+ if RAN.air_interface[instance]!='':
+ logging.debug('Auto Termination of Instance ' + str(instance) + ' : ' + RAN.air_interface[instance])
+ RAN.eNB_instance=instance
+ RAN.TerminateeNB(HTML,EPC)
+ if RAN.flexranCtrlInstalled and RAN.flexranCtrlStarted:
+ self.testCase_id = 'AUTO-KILL-flexran-ctl'
+ HTML.testCase_id = self.testCase_id
+ self.desc = 'Automatic Termination of FlexRan CTL'
+ HTML.desc = self.desc
+ self.ShowTestID()
+ self.TerminateFlexranCtrl(HTML,RAN,EPC)
+ if CONTAINERS.yamlPath[0] != '':
+ self.testCase_id = 'AUTO-KILL-CONTAINERS'
+ HTML.testCase_id = self.testCase_id
+ self.desc = 'Automatic Termination of all RAN containers'
+ HTML.desc = self.desc
+ self.ShowTestID()
+ for instance in range(0, len(CONTAINERS.yamlPath)):
+ if CONTAINERS.yamlPath[instance]!='':
+ CONTAINERS.eNB_instance=instance
+ CONTAINERS.UndeployObject(HTML,RAN)
+ RAN.prematureExit=True
+
+
def IdleSleep(self,HTML):
time.sleep(self.idle_sleep_time)
HTML.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', CONST.ALL_PROCESSES_OK)
diff --git a/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf
index 758bb44879..d2e91fb872 100644
--- a/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf
+++ b/ci-scripts/conf_files/enb.band38.lte_2x2.100PRB.usrpn310.conf
@@ -237,7 +237,7 @@ RUs = (
max_pdschReferenceSignalPower = -27;
max_rxgain = 75;
eNB_instances = [0];
- sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.10.2";
+ sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2";
}
);
diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf
index d1d32e2b58..6c8d5ba6f0 100644
--- a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf
+++ b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf
@@ -237,7 +237,7 @@ RUs = (
max_pdschReferenceSignalPower = -27;
max_rxgain = 75;
eNB_instances = [0];
- sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.10.2";
+ sdr_addrs = "mgmt_addr=192.168.18.241,addr=192.168.20.2,second_addr=192.168.10.2";
}
);
diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index ddf9bb531e..3a5a69573d 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -310,6 +310,13 @@ def GetParametersFromXML(action):
CiTestObj.ue_id = ue_id
CiTestObj.iperf_direction = test.findtext('direction')#used for modules only
CiTestObj.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold')
+ iperf_bitrate_threshold = test.findtext('iperf_bitrate_threshold')
+ if (iperf_bitrate_threshold is None):
+ CiTestObj.iperf_bitrate_threshold = "90" #if no threshold is specified, default will be 90%
+ else:
+ CiTestObj.iperf_bitrate_threshold = iperf_bitrate_threshold
+
+
CiTestObj.iperf_profile = test.findtext('iperf_profile')
if (CiTestObj.iperf_profile is None):
CiTestObj.iperf_profile = 'balanced'
@@ -811,6 +818,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
check_OAI_UE = False
RAN.pStatus=CiTestObj.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
RAN.InitializeeNB(HTML, EPC)
+ if RAN.prematureExit:
+ CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS)
elif action == 'Terminate_eNB':
RAN.TerminateeNB(HTML, EPC)
elif action == 'Initialize_UE':
diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py
index 94d37a01dd..f01b85015b 100644
--- a/ci-scripts/ran.py
+++ b/ci-scripts/ran.py
@@ -738,8 +738,8 @@ class RANManagement():
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
- mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 60)
- mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 15)
+ mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60)
+ mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15)
mySSH.close()
def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
diff --git a/ci-scripts/xml_files/container_nsa_b200_quectel.xml b/ci-scripts/xml_files/container_nsa_b200_quectel.xml
index 0af0d9bd70..3cac13dad4 100644
--- a/ci-scripts/xml_files/container_nsa_b200_quectel.xml
+++ b/ci-scripts/xml_files/container_nsa_b200_quectel.xml
@@ -124,6 +124,7 @@
DL
idefix
3
+ 95
single-ue
@@ -134,6 +135,7 @@
UL
idefix
1
+ 95
single-ue
diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml
index 45a4f065e9..57a66ece76 100644
--- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml
+++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml
@@ -107,6 +107,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue
@@ -117,6 +118,7 @@
UL
nrmodule2_quectel
5
+ 95
single-ue
diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml
index 9d430417c8..5e5853ab7f 100644
--- a/ci-scripts/xml_files/fr1_nsa_quectel.xml
+++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml
@@ -119,6 +119,7 @@
DL
idefix
3
+ 95
single-ue
@@ -129,6 +130,7 @@
UL
idefix
1
+ 95
single-ue
diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml
index 518e9dfe81..72b4c0a273 100644
--- a/ci-scripts/xml_files/fr1_sa_quectel.xml
+++ b/ci-scripts/xml_files/fr1_sa_quectel.xml
@@ -105,6 +105,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue
@@ -115,6 +116,7 @@
UL
nrmodule2_quectel
5
+ 95
single-ue
diff --git a/ci-scripts/xml_files/fr1_sa_quectel_stages.xml b/ci-scripts/xml_files/fr1_sa_quectel_stages.xml
index 4b169f85e9..dabe53f233 100644
--- a/ci-scripts/xml_files/fr1_sa_quectel_stages.xml
+++ b/ci-scripts/xml_files/fr1_sa_quectel_stages.xml
@@ -104,6 +104,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue
@@ -113,6 +114,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue
@@ -122,6 +124,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue
@@ -131,6 +134,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue
@@ -140,6 +144,7 @@
DL
nrmodule2_quectel
5
+ 95
single-ue