Merge remote-tracking branch 'origin/ci-ping-improvements' into integration_2023_w20
This commit is contained in:
Robert Schmidt
@@ -879,8 +879,8 @@ class Containerize():
|
|||||||
cnt += 1
|
cnt += 1
|
||||||
mySSH.command('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName, '\$', 5)
|
mySSH.command('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName, '\$', 5)
|
||||||
for stdoutLine in mySSH.getBefore().split('\n'):
|
for stdoutLine in mySSH.getBefore().split('\n'):
|
||||||
if stdoutLine.count('ImageUsed: porcepix'):
|
if stdoutLine.count('ImageUsed: oai-ci'):
|
||||||
usedImage = stdoutLine.replace('ImageUsed: porcepix', 'porcepix').strip()
|
usedImage = stdoutLine.replace('ImageUsed: oai-ci', 'oai-ci').strip()
|
||||||
logging.debug('Used image is ' + usedImage)
|
logging.debug('Used image is ' + usedImage)
|
||||||
if usedImage != '':
|
if usedImage != '':
|
||||||
mySSH.command('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage, '\$', 5, silent=True)
|
mySSH.command('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage, '\$', 5, silent=True)
|
||||||
|
|||||||
+112
-350
@@ -58,79 +58,6 @@ logging.getLogger("matplotlib").setLevel(logging.WARNING)
|
|||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
#-----------------------------------------------------------
|
|
||||||
# Utility functions
|
|
||||||
#-----------------------------------------------------------
|
|
||||||
|
|
||||||
def GetPingTimeAnalysis(RAN,ping_log_file,ping_rttavg_threshold):
|
|
||||||
#ping time values read from file
|
|
||||||
t_ping=[]
|
|
||||||
#ping stats (dictionary) to be returned by the function
|
|
||||||
ping_stat={}
|
|
||||||
if (os.path.isfile(ping_log_file)):
|
|
||||||
with open(ping_log_file,"r") as f:
|
|
||||||
for line in f:
|
|
||||||
#looking for time=xxx ms field
|
|
||||||
result=re.match('^.+time=(?P<ping_time>[0-9\.]+)',line)
|
|
||||||
if result != None:
|
|
||||||
t_ping.append(float(result.group('ping_time')))
|
|
||||||
|
|
||||||
#initial stats
|
|
||||||
ping_stat['min_0']=min(t_ping)
|
|
||||||
ping_stat['mean_0']=stat.mean(t_ping)
|
|
||||||
ping_stat['median_0']=stat.median(t_ping)
|
|
||||||
ping_stat['max_0']=max(t_ping)
|
|
||||||
|
|
||||||
#get index of max value
|
|
||||||
|
|
||||||
max_loc=t_ping.index(max(t_ping))
|
|
||||||
ping_stat['max_loc']=max_loc
|
|
||||||
#remove it
|
|
||||||
t_ping_post=t_ping.copy()
|
|
||||||
t_ping_post.pop(max_loc)
|
|
||||||
#new stats after removing max value
|
|
||||||
ping_stat['min_1']=min(t_ping_post)
|
|
||||||
ping_stat['mean_1']=stat.mean(t_ping_post)
|
|
||||||
ping_stat['median_1']=stat.median(t_ping_post)
|
|
||||||
ping_stat['max_1']=max(t_ping_post)
|
|
||||||
|
|
||||||
#plot ping over time and save png for artifacts
|
|
||||||
ticks = np.arange(0, len(t_ping), 1)
|
|
||||||
figure, axis = plt.subplots(figsize=(10, 10))
|
|
||||||
axis.plot(ticks,t_ping,marker='o')
|
|
||||||
axis.set_xlabel('Ping Events')
|
|
||||||
axis.set_ylabel("Ping RTT (in ms)")
|
|
||||||
axis.set_title(ping_log_file)
|
|
||||||
axis.set_xticks(ticks)
|
|
||||||
axis.set_xticklabels([])
|
|
||||||
YMAX=20 #base scale
|
|
||||||
if max(t_ping) > YMAX:
|
|
||||||
y_max=max(t_ping)+1
|
|
||||||
else:
|
|
||||||
y_max=YMAX+1
|
|
||||||
plt.ylim(0,y_max)
|
|
||||||
if ping_rttavg_threshold != '':
|
|
||||||
th_label="AVG Ping Fail Threshold="+ping_rttavg_threshold
|
|
||||||
plt.axhline(y=float(ping_rttavg_threshold), color='r', linestyle='-',label=th_label)
|
|
||||||
axis.legend()
|
|
||||||
plt.savefig(ping_log_file+'.png')
|
|
||||||
|
|
||||||
#copy the png file already to enb to move it move it later into the artifacts
|
|
||||||
try:
|
|
||||||
mySSH = sshconnection.SSHConnection()
|
|
||||||
mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file+'.png', RAN.eNBSourceCodePath + '/cmake_targets/')
|
|
||||||
mySSH.copyout(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, ping_log_file, RAN.eNBSourceCodePath + '/cmake_targets/')
|
|
||||||
except:
|
|
||||||
logging.debug('\u001B[1;37;41m Ping PNG SCP to eNB FAILED\u001B[0m')
|
|
||||||
|
|
||||||
return ping_stat
|
|
||||||
|
|
||||||
else:
|
|
||||||
logging.error("GetPingTimeAnalysis : Ping log file does not exist")
|
|
||||||
return -1
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#-----------------------------------------------------------
|
#-----------------------------------------------------------
|
||||||
# OaiCiTest Class Definition
|
# OaiCiTest Class Definition
|
||||||
#-----------------------------------------------------------
|
#-----------------------------------------------------------
|
||||||
@@ -576,298 +503,133 @@ class OaiCiTest():
|
|||||||
statusQueue.put(message)
|
statusQueue.put(message)
|
||||||
lock.release()
|
lock.release()
|
||||||
|
|
||||||
def Ping_common(self, lock, statusQueue,EPC, ue, RAN):
|
def Ping_common(self, EPC, ue, RAN):
|
||||||
try:
|
# Launch ping on the EPC side (true for ltebox and old open-air-cn)
|
||||||
# Launch ping on the EPC side (true for ltebox and old open-air-cn)
|
# But for OAI-Rel14-CUPS, we launch from python executor
|
||||||
# But for OAI-Rel14-CUPS, we launch from python executor
|
ping_status = 0
|
||||||
ping_status = 0
|
ueIP = ue.getIP()
|
||||||
|
if not ueIP:
|
||||||
|
return (False, f"UE {ue.getName()} has no IP address")
|
||||||
|
launchFromEpc = False
|
||||||
|
ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
|
||||||
|
if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE):
|
||||||
launchFromEpc = False
|
launchFromEpc = False
|
||||||
ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
|
ping_time = re.findall("-c *(\d+)",str(self.ping_args))
|
||||||
if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE):
|
if launchFromEpc:
|
||||||
launchFromEpc = False
|
SSH = sshconnection.SSHConnection()
|
||||||
ping_time = re.findall("-c *(\d+)",str(self.ping_args))
|
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
|
||||||
if launchFromEpc:
|
SSH.command('cd ' + EPC.SourceCodePath, '\$', 5)
|
||||||
SSH = sshconnection.SSHConnection()
|
SSH.command('cd scripts', '\$', 5)
|
||||||
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
|
# In case of a docker-based deployment, we need to ping from the trf-gen container
|
||||||
SSH.command('cd ' + EPC.SourceCodePath, '\$', 5)
|
launchFromTrfContainer = False
|
||||||
SSH.command('cd scripts', '\$', 5)
|
if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)):
|
||||||
# In case of a docker-based deployment, we need to ping from the trf-gen container
|
launchFromTrfContainer = True
|
||||||
launchFromTrfContainer = False
|
if launchFromTrfContainer:
|
||||||
if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)):
|
ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ueIP}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
|
||||||
launchFromTrfContainer = True
|
|
||||||
if launchFromTrfContainer:
|
|
||||||
ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ue.getIP()}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
|
|
||||||
else:
|
|
||||||
ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ue.getIP()} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
|
|
||||||
#copy the ping log file to have it locally for analysis (ping stats)
|
|
||||||
SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.')
|
|
||||||
SSH.close()
|
|
||||||
else:
|
else:
|
||||||
#target address is different depending on EPC type
|
ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ueIP} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
|
||||||
if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
|
#copy the ping log file to have it locally for analysis (ping stats)
|
||||||
Target = EPC.MmeIPAddress
|
SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.')
|
||||||
elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
|
|
||||||
Target = EPC.MmeIPAddress
|
|
||||||
elif re.match('OC-OAI-CN5G', EPC.Type, re.IGNORECASE):
|
|
||||||
Target = "172.21.6.100"
|
|
||||||
else:
|
|
||||||
Target = EPC.IPAddress
|
|
||||||
#ping from module NIC rather than IP address to make sure round trip is over the air
|
|
||||||
interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
|
|
||||||
ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}'
|
|
||||||
cmd = cls_cmd.getConnection(ue.getHost())
|
|
||||||
response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
|
|
||||||
if response.returncode != 0:
|
|
||||||
ping_status = -1
|
|
||||||
else:
|
|
||||||
#copy the ping log file to have it locally for analysis (ping stats)
|
|
||||||
cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file)
|
|
||||||
|
|
||||||
cmd.close()
|
|
||||||
|
|
||||||
# TIMEOUT CASE
|
|
||||||
if ping_status < 0:
|
|
||||||
message = 'Ping with UE (' + str(ue.getIP()) + ') crashed due to TIMEOUT!'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
|
|
||||||
return
|
|
||||||
#search is done on cat result
|
|
||||||
with open(ping_log_file, 'r') as f:
|
|
||||||
ping_output = "".join(f.readlines())
|
|
||||||
result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output)
|
|
||||||
if result is None:
|
|
||||||
message = 'Packet Loss Not Found!'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
|
|
||||||
return
|
|
||||||
packetloss = result.group('packetloss')
|
|
||||||
if float(packetloss) == 100:
|
|
||||||
message = 'Packet Loss is 100%'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
|
|
||||||
return
|
|
||||||
result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', ping_output)
|
|
||||||
if result is None:
|
|
||||||
message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
|
|
||||||
return
|
|
||||||
rtt_min = result.group('rtt_min')
|
|
||||||
rtt_avg = result.group('rtt_avg')
|
|
||||||
rtt_max = result.group('rtt_max')
|
|
||||||
pal_msg = 'Packet Loss : ' + packetloss + '%'
|
|
||||||
min_msg = 'RTT(Min) : ' + rtt_min + ' ms'
|
|
||||||
avg_msg = 'RTT(Avg) : ' + rtt_avg + ' ms'
|
|
||||||
max_msg = 'RTT(Max) : ' + rtt_max + ' ms'
|
|
||||||
|
|
||||||
lock.acquire()
|
|
||||||
logging.debug('\u001B[1;37;44m ping result (' + ue.getIP() + ') \u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m')
|
|
||||||
|
|
||||||
#adding extra ping stats from local file
|
|
||||||
#ping_log_file variable is defined above in this function, depending on device/ue
|
|
||||||
ping_stat_msg=''
|
|
||||||
logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
|
|
||||||
ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
|
|
||||||
|
|
||||||
if (ping_stat!=-1) and (len(ping_stat)!=0):
|
|
||||||
ping_stat_msg+='Ping stats before removing largest value : \n'
|
|
||||||
ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='Max Index : ' + str(ping_stat['max_loc']) + '\n'
|
|
||||||
ping_stat_msg+='Ping stats after removing largest value : \n'
|
|
||||||
ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n'
|
|
||||||
ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n'
|
|
||||||
|
|
||||||
#building html message
|
|
||||||
qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n' + ping_stat_msg
|
|
||||||
|
|
||||||
#checking packet loss compliance
|
|
||||||
packetLossOK = True
|
|
||||||
if (packetloss is not None) :
|
|
||||||
if float(packetloss) > float(self.ping_packetloss_threshold):
|
|
||||||
qMsg += '\nPacket Loss too high'
|
|
||||||
logging.debug('\u001B[1;37;41m Packet Loss too high; Target: '+ self.ping_packetloss_threshold + '%\u001B[0m')
|
|
||||||
packetLossOK = False
|
|
||||||
elif float(packetloss) > 0:
|
|
||||||
qMsg += '\nPacket Loss is not 0%'
|
|
||||||
logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
|
|
||||||
|
|
||||||
#checking RTT avg compliance
|
|
||||||
rttavgOK = True
|
|
||||||
if self.ping_rttavg_threshold != '':
|
|
||||||
if float(rtt_avg)>float(self.ping_rttavg_threshold):
|
|
||||||
ping_rttavg_error_msg = 'RTT(Avg) too high: ' + rtt_avg + ' ms; Target: '+ self.ping_rttavg_threshold+ ' ms'
|
|
||||||
qMsg += '\n'+ping_rttavg_error_msg
|
|
||||||
logging.debug('\u001B[1;37;41m'+ ping_rttavg_error_msg +' \u001B[0m')
|
|
||||||
rttavgOK = False
|
|
||||||
|
|
||||||
if packetLossOK and rttavgOK:
|
|
||||||
statusQueue.put(0)
|
|
||||||
else:
|
|
||||||
statusQueue.put(-1)
|
|
||||||
statusQueue.put(ue.getName())
|
|
||||||
statusQueue.put(ue.getIP())
|
|
||||||
statusQueue.put(qMsg)
|
|
||||||
lock.release()
|
|
||||||
except:
|
|
||||||
logging.debug('exit from Ping_Common except')
|
|
||||||
os.kill(os.getppid(),signal.SIGUSR1)
|
|
||||||
|
|
||||||
def PingNoS1_wrong_exit(self, qMsg,HTML):
|
|
||||||
message = 'OAI UE ping result\n{qMsg}'
|
|
||||||
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', [message])
|
|
||||||
|
|
||||||
def PingNoS1(self,HTML,RAN,EPC,CONTAINERS):
|
|
||||||
SSH=sshconnection.SSHConnection()
|
|
||||||
check_eNB = True
|
|
||||||
check_OAI_UE = True
|
|
||||||
ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
|
|
||||||
if ping_from_eNB is not None:
|
|
||||||
if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '':
|
|
||||||
HELP.GenericHelp(CONST.Version)
|
|
||||||
sys.exit('Insufficient Parameter')
|
|
||||||
else:
|
|
||||||
if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
|
|
||||||
HELP.GenericHelp(CONST.Version)
|
|
||||||
sys.exit('Insufficient Parameter')
|
|
||||||
try:
|
|
||||||
if ping_from_eNB is not None:
|
|
||||||
SSH.open(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword)
|
|
||||||
SSH.command('cd ' + RAN.eNBSourceCodePath + '/cmake_targets/', '\$', 5)
|
|
||||||
else:
|
|
||||||
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
|
|
||||||
SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets/', '\$', 5)
|
|
||||||
ping_time = re.findall("-c (\d+)",str(self.ping_args))
|
|
||||||
ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '.log', '\$', int(ping_time[0])*1.5)
|
|
||||||
# TIMEOUT CASE
|
|
||||||
if ping_status < 0:
|
|
||||||
message = 'Ping with OAI UE crashed due to TIMEOUT!'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.PingNoS1_wrong_exit(message,HTML)
|
|
||||||
return
|
|
||||||
result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', SSH.getBefore())
|
|
||||||
if result is None:
|
|
||||||
message = 'Packet Loss Not Found!'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.PingNoS1_wrong_exit(message,HTML)
|
|
||||||
return
|
|
||||||
packetloss = result.group('packetloss')
|
|
||||||
if float(packetloss) == 100:
|
|
||||||
message = 'Packet Loss is 100%'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.PingNoS1_wrong_exit(message,HTML)
|
|
||||||
return
|
|
||||||
result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', SSH.getBefore())
|
|
||||||
if result is None:
|
|
||||||
message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
|
|
||||||
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
|
||||||
self.PingNoS1_wrong_exit(message,HTML)
|
|
||||||
return
|
|
||||||
rtt_min = result.group('rtt_min')
|
|
||||||
rtt_avg = result.group('rtt_avg')
|
|
||||||
rtt_max = result.group('rtt_max')
|
|
||||||
pal_msg = 'Packet Loss : ' + packetloss + '%'
|
|
||||||
min_msg = 'RTT(Min) : ' + rtt_min + ' ms'
|
|
||||||
avg_msg = 'RTT(Avg) : ' + rtt_avg + ' ms'
|
|
||||||
max_msg = 'RTT(Max) : ' + rtt_max + ' ms'
|
|
||||||
logging.debug('\u001B[1;37;44m OAI UE ping result \u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m')
|
|
||||||
logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m')
|
|
||||||
qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
|
|
||||||
packetLossOK = True
|
|
||||||
if packetloss is not None:
|
|
||||||
if float(packetloss) > float(self.ping_packetloss_threshold):
|
|
||||||
qMsg += '\nPacket Loss too high'
|
|
||||||
logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
|
|
||||||
packetLossOK = False
|
|
||||||
elif float(packetloss) > 0:
|
|
||||||
qMsg += '\nPacket Loss is not 0%'
|
|
||||||
logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
|
|
||||||
SSH.close()
|
SSH.close()
|
||||||
message = f'OAI UE ping result\n{qMsg}'
|
else:
|
||||||
if packetLossOK:
|
#target address is different depending on EPC type
|
||||||
HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', [message])
|
if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
|
||||||
|
Target = EPC.MmeIPAddress
|
||||||
|
elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
|
||||||
|
Target = EPC.MmeIPAddress
|
||||||
|
elif re.match('OC-OAI-CN5G', EPC.Type, re.IGNORECASE):
|
||||||
|
Target = "172.21.6.100"
|
||||||
else:
|
else:
|
||||||
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', [message])
|
Target = EPC.IPAddress
|
||||||
|
#ping from module NIC rather than IP address to make sure round trip is over the air
|
||||||
|
interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
|
||||||
|
ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}'
|
||||||
|
cmd = cls_cmd.getConnection(ue.getHost())
|
||||||
|
response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
|
||||||
|
if response.returncode != 0:
|
||||||
|
ping_status = -1
|
||||||
|
else:
|
||||||
|
#copy the ping log file to have it locally for analysis (ping stats)
|
||||||
|
cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file)
|
||||||
|
|
||||||
# copying on the EPC server for logCollection
|
cmd.close()
|
||||||
if ping_from_eNB is not None:
|
|
||||||
copyin_res = SSH.copyin(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, RAN.eNBSourceCodePath + '/cmake_targets/ping_' + self.testCase_id + '.log', '.')
|
# TIMEOUT CASE
|
||||||
else:
|
ue_header = f'UE {ue.getName()} ({ueIP})'
|
||||||
copyin_res = SSH.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/ping_' + self.testCase_id + '.log', '.')
|
if ping_status < 0:
|
||||||
if (copyin_res == 0):
|
message = ue_header + ': ping crashed: TIMEOUT?'
|
||||||
SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'ping_' + self.testCase_id + '.log', EPC.SourceCodePath + '/scripts')
|
logging.error('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
||||||
except:
|
return (False, message)
|
||||||
os.kill(os.getppid(),signal.SIGUSR1)
|
#search is done on cat result
|
||||||
|
logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
|
||||||
|
with open(ping_log_file, 'r') as f:
|
||||||
|
ping_output = "".join(f.readlines())
|
||||||
|
result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output)
|
||||||
|
if result is None:
|
||||||
|
message = ue_header + ': Packet Loss Not Found!'
|
||||||
|
logging.error('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
||||||
|
return (False, message)
|
||||||
|
packetloss = result.group('packetloss')
|
||||||
|
result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', ping_output)
|
||||||
|
if result is None:
|
||||||
|
message = ue_header + ': Ping RTT_Min RTT_Avg RTT_Max Not Found!'
|
||||||
|
logging.error('\u001B[1;37;41m ' + message + ' \u001B[0m')
|
||||||
|
return (False, message)
|
||||||
|
rtt_min = result.group('rtt_min')
|
||||||
|
rtt_avg = result.group('rtt_avg')
|
||||||
|
rtt_max = result.group('rtt_max')
|
||||||
|
|
||||||
|
pal_msg = f'Packet Loss: {packetloss}%'
|
||||||
|
min_msg = f'RTT(Min) : {rtt_min} ms'
|
||||||
|
avg_msg = f'RTT(Avg) : {rtt_avg} ms'
|
||||||
|
max_msg = f'RTT(Max) : {rtt_max} ms'
|
||||||
|
|
||||||
|
logging.info(f'\u001B[1;37;44m ping result for {ue_header} \u001B[0m')
|
||||||
|
logging.info(f'\u001B[1;34m ' + pal_msg + '\u001B[0m')
|
||||||
|
logging.info(f'\u001B[1;34m ' + min_msg + '\u001B[0m')
|
||||||
|
logging.info(f'\u001B[1;34m ' + avg_msg + '\u001B[0m')
|
||||||
|
logging.info(f'\u001B[1;34m ' + max_msg + '\u001B[0m')
|
||||||
|
|
||||||
|
message = f'{ue_header}\n{pal_msg}\n{min_msg}\n{avg_msg}\n{max_msg}'
|
||||||
|
|
||||||
|
#checking packet loss compliance
|
||||||
|
if float(packetloss) > float(self.ping_packetloss_threshold):
|
||||||
|
message += '\nPacket Loss too high'
|
||||||
|
logging.error(f'\u001B[1;37;41m Packet Loss too high; Target: {self.ping_packetloss_threshold}%\u001B[0m')
|
||||||
|
return (False, message)
|
||||||
|
elif float(packetloss) > 0:
|
||||||
|
message += '\nPacket Loss is not 0%'
|
||||||
|
logging.info('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
|
||||||
|
|
||||||
|
if self.ping_rttavg_threshold != '':
|
||||||
|
if float(rtt_avg) > float(self.ping_rttavg_threshold):
|
||||||
|
ping_rttavg_error_msg = f'RTT(Avg) too high: {rtt_avg} ms; Target: {self.ping_rttavg_threshold} ms'
|
||||||
|
message += '\n' + ping_rttavg_error_msg
|
||||||
|
logging.error('\u001B[1;37;41m'+ ping_rttavg_error_msg +' \u001B[0m')
|
||||||
|
return (False, message)
|
||||||
|
|
||||||
|
return (True, message)
|
||||||
|
|
||||||
def Ping(self,HTML,RAN,EPC,CONTAINERS):
|
def Ping(self,HTML,RAN,EPC,CONTAINERS):
|
||||||
result = re.search('noS1', str(RAN.Initialize_eNB_args))
|
|
||||||
if result is not None:
|
|
||||||
self.PingNoS1(HTML,RAN,EPC,CONTAINERS)
|
|
||||||
return
|
|
||||||
if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
|
if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
|
||||||
HELP.GenericHelp(CONST.Version)
|
HELP.GenericHelp(CONST.Version)
|
||||||
sys.exit('Insufficient Parameter')
|
sys.exit('Insufficient Parameter')
|
||||||
check_eNB = True
|
|
||||||
if (len(self.UEDevices) == 1) and (self.UEDevices[0] == 'OAI-UE'):
|
|
||||||
check_OAI_UE = True
|
|
||||||
else:
|
|
||||||
check_OAI_UE = False
|
|
||||||
|
|
||||||
if self.ue_ids == []:
|
if self.ue_ids == []:
|
||||||
raise Exception("no module names in self.ue_ids provided")
|
raise Exception("no module names in self.ue_ids provided")
|
||||||
ues = []
|
|
||||||
for ue_id in self.ue_ids:
|
ues = [cls_module_ue.Module_UE(n.strip()) for n in self.ue_ids]
|
||||||
ue = cls_module_ue.Module_UE(ue_id)
|
|
||||||
if not ue.getIP():
|
|
||||||
logging.error("no IP addresses returned")
|
|
||||||
HTML.CreateHtmlTestRow(self.ping_args, 'KO', len(self.UEDevices), html_queue)
|
|
||||||
self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
|
|
||||||
ues.append(ue)
|
|
||||||
logging.debug(ues)
|
logging.debug(ues)
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
multi_jobs = []
|
futures = [executor.submit(self.Ping_common, EPC, ue, RAN) for ue in ues]
|
||||||
i = 0
|
results = [f.result() for f in futures]
|
||||||
lock = Lock()
|
# each result in results is a tuple, first member goes to successes, second to messages
|
||||||
status_queue = SimpleQueue()
|
successes, messages = map(list, zip(*results))
|
||||||
for ue in ues:
|
if len(successes) == len(ues) and all(successes):
|
||||||
p = Process(target = self.Ping_common, args = (lock,status_queue,EPC,ue,RAN,))
|
HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', messages)
|
||||||
p.daemon = True
|
|
||||||
p.start()
|
|
||||||
multi_jobs.append(p)
|
|
||||||
i = i + 1
|
|
||||||
for job in multi_jobs:
|
|
||||||
job.join()
|
|
||||||
|
|
||||||
if (status_queue.empty()):
|
|
||||||
HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
|
|
||||||
else:
|
else:
|
||||||
ping_status = True
|
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages)
|
||||||
messages = []
|
self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
|
||||||
while (not status_queue.empty()):
|
|
||||||
count = status_queue.get()
|
|
||||||
if (count < 0):
|
|
||||||
ping_status = False
|
|
||||||
device_id = status_queue.get()
|
|
||||||
ip_addr = status_queue.get()
|
|
||||||
message = status_queue.get()
|
|
||||||
messages.append(f'UE ({device_id})\nIP Address : {ip_addr}\n{message}')
|
|
||||||
if (ping_status):
|
|
||||||
HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', messages)
|
|
||||||
else:
|
|
||||||
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages)
|
|
||||||
self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
|
|
||||||
|
|
||||||
def Iperf_ComputeTime(self):
|
def Iperf_ComputeTime(self):
|
||||||
result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
|
result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
|
||||||
|
|||||||
@@ -228,8 +228,8 @@
|
|||||||
<yaml_path>ci-scripts/yaml_files/nsa_b200_gnb</yaml_path>
|
<yaml_path>ci-scripts/yaml_files/nsa_b200_gnb</yaml_path>
|
||||||
<eNB_instance>1</eNB_instance>
|
<eNB_instance>1</eNB_instance>
|
||||||
<eNB_serverId>1</eNB_serverId>
|
<eNB_serverId>1</eNB_serverId>
|
||||||
<d_retx_th>1,100,100,100</d_retx_th>
|
<d_retx_th>10,100,100,100</d_retx_th>
|
||||||
<u_retx_th>1,100,100,100</u_retx_th>
|
<u_retx_th>10,100,100,100</u_retx_th>
|
||||||
</testCase>
|
</testCase>
|
||||||
|
|
||||||
</testCaseList>
|
</testCaseList>
|
||||||
|
|||||||
@@ -179,8 +179,8 @@
|
|||||||
<yaml_path>ci-scripts/yaml_files/sa_b200_gnb</yaml_path>
|
<yaml_path>ci-scripts/yaml_files/sa_b200_gnb</yaml_path>
|
||||||
<eNB_instance>0</eNB_instance>
|
<eNB_instance>0</eNB_instance>
|
||||||
<eNB_serverId>0</eNB_serverId>
|
<eNB_serverId>0</eNB_serverId>
|
||||||
<d_retx_th>1,100,100,100</d_retx_th>
|
<d_retx_th>10,100,100,100</d_retx_th>
|
||||||
<u_retx_th>1,100,100,100</u_retx_th>
|
<u_retx_th>10,100,100,100</u_retx_th>
|
||||||
</testCase>
|
</testCase>
|
||||||
|
|
||||||
</testCaseList>
|
</testCaseList>
|
||||||
|
|||||||
Reference in New Issue
Block a user