maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Ping to always handle Module_UE

Browse Source
This commit is contained in:
Robert Schmidt
2023-02-23 17:38:10 +01:00
parent 08ea58b243
commit 2825d2bf5d
1 changed files with 66 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_oaicitest.py
+66 -121
View File
@@ -51,6 +51,7 @@ import constants as CONST
import sshconnection import sshconnection
import cls_module_ue import cls_module_ue
import cls_cmd
logging.getLogger("matplotlib").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING)
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@@ -744,30 +745,18 @@ class OaiCiTest():
statusQueue.put(message) statusQueue.put(message)
lock.release() lock.release()
def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN): def Ping_common(self, lock, statusQueue,EPC, ue, RAN):
try: try:
SSH = sshconnection.SSHConnection()
# Launch ping on the EPC side (true for ltebox and old open-air-cn) # Launch ping on the EPC side (true for ltebox and old open-air-cn)
# But for OAI-Rel14-CUPS, we launch from python executor # But for OAI-Rel14-CUPS, we launch from python executor
ping_status = 0 ping_status = 0
launchFromEpc = True launchFromEpc = False
launchFromModule = False ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
launchFromASUE = False
if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE): if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE):
launchFromEpc = False launchFromEpc = False
#if module, ping from module to EPC ping_time = re.findall("-c *(\d+)",str(self.ping_args))
if self.ue_id!='':
if (re.match('amarisoft', self.ue_id, re.IGNORECASE)):
launchFromEpc = False
launchFromASUE = True
else:
launchFromEpc = False
launchFromModule = True
#no ping args for ASUE
if self.ping_args!='':
ping_time = re.findall("-c (\d+)",str(self.ping_args))
if launchFromEpc: if launchFromEpc:
SSH = sshconnection.SSHConnection()
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
SSH.command('cd ' + EPC.SourceCodePath, '\$', 5) SSH.command('cd ' + EPC.SourceCodePath, '\$', 5)
SSH.command('cd scripts', '\$', 5) SSH.command('cd scripts', '\$', 5)
@@ -776,93 +765,59 @@ class OaiCiTest():
if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)):
launchFromTrfContainer = True launchFromTrfContainer = True
if launchFromTrfContainer: if launchFromTrfContainer:
ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ue.getIP()}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
else: else:
ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ue.getIP()} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5)
ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log'
#copy the ping log file to have it locally for analysis (ping stats) #copy the ping log file to have it locally for analysis (ping stats)
SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.')
SSH.close()
else: else:
if (launchFromModule == False) and (launchFromASUE == False): #target address is different depending on EPC type
#ping log file is on the python executor if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' > ping_' + self.testCase_id + '_' + device_id + '.log 2>&1' Target = EPC.MmeIPAddress
message = cmd + '\n' elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
logging.debug(cmd) Target = EPC.MmeIPAddress
ret = subprocess.run(cmd, shell=True)
ping_status = ret.returncode
#copy the ping log file to an other folder for log collection (source and desti elif (launchfromModule == True) and (launchfromASUE == False): #launch from Modulenation are EPC)
SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'ping_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts')
#copy the ping log file to have it locally for analysis (ping stats)
logging.debug(EPC.SourceCodePath + 'ping_' + self.testCase_id + '_' + device_id + '.log')
SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath +'/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.')
SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
#cat is executed on EPC
SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5)
ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log'
elif (launchFromModule == True) and (launchFromASUE == False): #launch from Module
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
#target address is different depending on EPC type
if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
Target = EPC.MmeIPAddress
elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
Target = EPC.MmeIPAddress
else:
Target = EPC.IPAddress
#ping from module NIC rather than IP address to make sure round trip is over the air
cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' > ping_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1'
SSH.command(cmd,'\$',int(ping_time[0])*1.5)
#copy the ping log file to have it locally for analysis (ping stats)
SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.')
#cat is executed locally
SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5)
ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log'
elif (launchFromASUE == True):
#ping was already executed when running scenario
#we only need to retrieve ping log file, whose location is in the ci_ueinfra.yaml
logging.debug("Get logs from AS server : " + Module_UE.Ping + ", " + Module_UE.UELog)
SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, Module_UE.Ping, '.')
SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, Module_UE.UELog, '.')
logging.debug("Ping analysis from Amarisoft scenario")
path,ping_log_file = os.path.split(Module_UE.Ping)
SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword)
SSH.command('cat ' + Module_UE.Ping, '\$', 5)
else: else:
ping_status=-1 Target = EPC.IPAddress
#ping from module NIC rather than IP address to make sure round trip is over the air
interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}'
cmd = cls_cmd.getConnection(ue.getHost())
response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
if response.returncode != 0:
ping_status = -1
else:
#copy the ping log file to have it locally for analysis (ping stats)
cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file)
cmd.close()
# TIMEOUT CASE # TIMEOUT CASE
if ping_status < 0: if ping_status < 0:
message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!' message = 'Ping with UE (' + str(ue.getIP()) + ') crashed due to TIMEOUT!'
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
SSH.close() self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
return return
#search is done on cat result #search is done on cat result
result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', SSH.getBefore()) with open(ping_log_file, 'r') as f:
ping_output = "".join(f.readlines())
result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output)
if result is None: if result is None:
message = 'Packet Loss Not Found!' message = 'Packet Loss Not Found!'
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
SSH.close() self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
return return
packetloss = result.group('packetloss') packetloss = result.group('packetloss')
if float(packetloss) == 100: if float(packetloss) == 100:
message = 'Packet Loss is 100%' message = 'Packet Loss is 100%'
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
SSH.close() self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
return return
result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', SSH.getBefore()) result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', ping_output)
if result is None: if result is None:
message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!' message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
SSH.close() self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message)
self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
return return
rtt_min = result.group('rtt_min') rtt_min = result.group('rtt_min')
rtt_avg = result.group('rtt_avg') rtt_avg = result.group('rtt_avg')
@@ -873,7 +828,7 @@ class OaiCiTest():
max_msg = 'RTT(Max) : ' + rtt_max + ' ms' max_msg = 'RTT(Max) : ' + rtt_max + ' ms'
lock.acquire() lock.acquire()
logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m') logging.debug('\u001B[1;37;44m ping result (' + ue.getIP() + ') \u001B[0m')
logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m') logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m')
logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m') logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m')
logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m') logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m')
@@ -882,22 +837,21 @@ class OaiCiTest():
#adding extra ping stats from local file #adding extra ping stats from local file
#ping_log_file variable is defined above in this function, depending on device/ue #ping_log_file variable is defined above in this function, depending on device/ue
ping_stat_msg='' ping_stat_msg=''
if launchFromASUE == False : #skip in case of AS UE (for the moment) logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file)
logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold)
if (ping_stat!=-1) and (len(ping_stat)!=0): if (ping_stat!=-1) and (len(ping_stat)!=0):
ping_stat_msg+='Ping stats before removing largest value : \n' ping_stat_msg+='Ping stats before removing largest value : \n'
ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n' ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n'
ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n' ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n'
ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n' ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n'
ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n' ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n'
ping_stat_msg+='Max Index : ' + str(ping_stat['max_loc']) + '\n' ping_stat_msg+='Max Index : ' + str(ping_stat['max_loc']) + '\n'
ping_stat_msg+='Ping stats after removing largest value : \n' ping_stat_msg+='Ping stats after removing largest value : \n'
ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n' ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n'
ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n' ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n'
ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n' ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n'
ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n' ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n'
#building html message #building html message
qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n' + ping_stat_msg qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n' + ping_stat_msg
@@ -926,11 +880,10 @@ class OaiCiTest():
statusQueue.put(0) statusQueue.put(0)
else: else:
statusQueue.put(-1) statusQueue.put(-1)
statusQueue.put(device_id) statusQueue.put(ue.getName())
statusQueue.put(UE_IPAddress) statusQueue.put(ue.getIP())
statusQueue.put(qMsg) statusQueue.put(qMsg)
lock.release() lock.release()
SSH.close()
except: except:
logging.debug('exit from Ping_Common except') logging.debug('exit from Ping_Common except')
os.kill(os.getppid(),signal.SIGUSR1) os.kill(os.getppid(),signal.SIGUSR1)
@@ -1049,31 +1002,23 @@ class OaiCiTest():
return return
if self.ue_id == "": if self.ue_id == "":
logging.error("no module ID!!!") raise Exception("no module names in self.ue_id provided")
exit(1) ues = []
for ue_name in self.ue_id.split(' '):
ue = cls_module_ue.Module_UE(ue_name.strip())
if not ue.getIP():
logging.error("no IP addresses returned")
HTML.CreateHtmlTestRow(self.ping_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,CONTAINERS)
ues.append(ue)
logging.debug(ues)
self.UEIPAddresses = []
ue = cls_module_ue.Module_UE(self.ue_id)
ip = ue.getIP()
if ip is None:
logging.error("no IP addresses returned")
html_queue = SimpleQueue()
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,CONTAINERS)
self.UEIPAddresses.append(ip)
logging.debug(self.UEIPAddresses)
multi_jobs = [] multi_jobs = []
i = 0 i = 0
lock = Lock() lock = Lock()
status_queue = SimpleQueue() status_queue = SimpleQueue()
for UE_IPAddress in self.UEIPAddresses: for ue in ues:
if self.ue_id=="": p = Process(target = self.Ping_common, args = (lock,status_queue,EPC,ue,RAN,))
device_id = self.UEDevices[i]
else:
device_id = Module_UE.ID + "-" + Module_UE.Kind
logging.debug(device_id)
p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,))
p.daemon = True p.daemon = True
p.start() p.start()
multi_jobs.append(p) multi_jobs.append(p)