From 2825d2bf5d62b70e6401bb44635bbe4d28a759d7 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 13 Feb 2023 15:05:09 +0100 Subject: [PATCH] Refactor Ping to always handle Module_UE --- ci-scripts/cls_oaicitest.py | 187 +++++++++++++----------------------- 1 file changed, 66 insertions(+), 121 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 97359e9db6..dea6542af1 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -51,6 +51,7 @@ import constants as CONST import sshconnection import cls_module_ue +import cls_cmd logging.getLogger("matplotlib").setLevel(logging.WARNING) import matplotlib.pyplot as plt @@ -744,30 +745,18 @@ class OaiCiTest(): statusQueue.put(message) lock.release() - def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue,EPC, Module_UE,RAN): + def Ping_common(self, lock, statusQueue,EPC, ue, RAN): try: - SSH = sshconnection.SSHConnection() # Launch ping on the EPC side (true for ltebox and old open-air-cn) # But for OAI-Rel14-CUPS, we launch from python executor ping_status = 0 - launchFromEpc = True - launchFromModule = False - launchFromASUE = False + launchFromEpc = False + ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log' if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE): launchFromEpc = False - #if module, ping from module to EPC - if self.ue_id!='': - if (re.match('amarisoft', self.ue_id, re.IGNORECASE)): - launchFromEpc = False - launchFromASUE = True - else: - launchFromEpc = False - launchFromModule = True - #no ping args for ASUE - if self.ping_args!='': - ping_time = re.findall("-c (\d+)",str(self.ping_args)) - + ping_time = re.findall("-c *(\d+)",str(self.ping_args)) if launchFromEpc: + SSH = sshconnection.SSHConnection() SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) SSH.command('cd ' + EPC.SourceCodePath, '\$', 5) SSH.command('cd scripts', '\$', 5) @@ -776,93 +765,59 @@ class OaiCiTest(): if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): launchFromTrfContainer = True if launchFromTrfContainer: - ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) + ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ue.getIP()}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5) else: - ping_status = SSH.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) - ping_log_file='ping_' + self.testCase_id + '_' + device_id + '.log' + ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ue.getIP()} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5) #copy the ping log file to have it locally for analysis (ping stats) - SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.') + SSH.close() else: - if (launchFromModule == False) and (launchFromASUE == False): - #ping log file is on the python executor - cmd = 'ping ' + self.ping_args + ' ' + UE_IPAddress + ' > ping_' + self.testCase_id + '_' + device_id + '.log 2>&1' - message = cmd + '\n' - logging.debug(cmd) - ret = subprocess.run(cmd, shell=True) - ping_status = ret.returncode - #copy the ping log file to an other folder for log collection (source and desti elif (launchfromModule == True) and (launchfromASUE == False): #launch from Modulenation are EPC) - SSH.copyout(EPC.IPAddress, EPC.UserName, EPC.Password, 'ping_' + self.testCase_id + '_' + device_id + '.log', EPC.SourceCodePath + '/scripts') - #copy the ping log file to have it locally for analysis (ping stats) - logging.debug(EPC.SourceCodePath + 'ping_' + self.testCase_id + '_' + device_id + '.log') - SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath +'/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '.') - - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - #cat is executed on EPC - SSH.command('cat ' + EPC.SourceCodePath + '/scripts/ping_' + self.testCase_id + '_' + device_id + '.log', '\$', 5) - ping_log_file='/scripts/ping_' + self.testCase_id + '_' + device_id + '.log' - - elif (launchFromModule == True) and (launchFromASUE == False): #launch from Module - SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - #target address is different depending on EPC type - if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): - Target = EPC.MmeIPAddress - elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): - Target = EPC.MmeIPAddress - else: - Target = EPC.IPAddress - #ping from module NIC rather than IP address to make sure round trip is over the air - cmd = 'ping -I ' + Module_UE.UENetwork + ' ' + self.ping_args + ' ' + Target + ' > ping_' + self.testCase_id + '_' + self.ue_id + '.log 2>&1' - SSH.command(cmd,'\$',int(ping_time[0])*1.5) - - #copy the ping log file to have it locally for analysis (ping stats) - SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'ping_' + self.testCase_id + '_' + self.ue_id + '.log', '.') - - #cat is executed locally - SSH.command('cat ping_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', 5) - ping_log_file='ping_' + self.testCase_id + '_' + self.ue_id + '.log' - - elif (launchFromASUE == True): - #ping was already executed when running scenario - #we only need to retrieve ping log file, whose location is in the ci_ueinfra.yaml - logging.debug("Get logs from AS server : " + Module_UE.Ping + ", " + Module_UE.UELog) - SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, Module_UE.Ping, '.') - SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, Module_UE.UELog, '.') - logging.debug("Ping analysis from Amarisoft scenario") - path,ping_log_file = os.path.split(Module_UE.Ping) - SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - SSH.command('cat ' + Module_UE.Ping, '\$', 5) - + #target address is different depending on EPC type + if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): + Target = EPC.MmeIPAddress + elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): + Target = EPC.MmeIPAddress else: - ping_status=-1 + Target = EPC.IPAddress + #ping from module NIC rather than IP address to make sure round trip is over the air + interface = f'-I {ue.getIFName()}' if ue.getIFName() else '' + ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}' + cmd = cls_cmd.getConnection(ue.getHost()) + response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5) + if response.returncode != 0: + ping_status = -1 + else: + #copy the ping log file to have it locally for analysis (ping stats) + cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file) + + cmd.close() # TIMEOUT CASE if ping_status < 0: - message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!' + message = 'Ping with UE (' + str(ue.getIP()) + ') crashed due to TIMEOUT!' logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - SSH.close() - self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message) + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) return #search is done on cat result - result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', SSH.getBefore()) + with open(ping_log_file, 'r') as f: + ping_output = "".join(f.readlines()) + result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output) if result is None: message = 'Packet Loss Not Found!' logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - SSH.close() - self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message) + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) return packetloss = result.group('packetloss') if float(packetloss) == 100: message = 'Packet Loss is 100%' logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - SSH.close() - self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message) + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) return - result = re.search('rtt min\/avg\/max\/mdev = (?P[0-9\.]+)\/(?P[0-9\.]+)\/(?P[0-9\.]+)\/[0-9\.]+ ms', SSH.getBefore()) + result = re.search('rtt min\/avg\/max\/mdev = (?P[0-9\.]+)\/(?P[0-9\.]+)\/(?P[0-9\.]+)\/[0-9\.]+ ms', ping_output) if result is None: message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!' logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - SSH.close() - self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message) + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) return rtt_min = result.group('rtt_min') rtt_avg = result.group('rtt_avg') @@ -873,7 +828,7 @@ class OaiCiTest(): max_msg = 'RTT(Max) : ' + rtt_max + ' ms' lock.acquire() - logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m') + logging.debug('\u001B[1;37;44m ping result (' + ue.getIP() + ') \u001B[0m') logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m') logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m') logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m') @@ -882,22 +837,21 @@ class OaiCiTest(): #adding extra ping stats from local file #ping_log_file variable is defined above in this function, depending on device/ue ping_stat_msg='' - if launchFromASUE == False : #skip in case of AS UE (for the moment) - logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) - ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold) + logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) + ping_stat=GetPingTimeAnalysis(RAN,ping_log_file,self.ping_rttavg_threshold) - if (ping_stat!=-1) and (len(ping_stat)!=0): - ping_stat_msg+='Ping stats before removing largest value : \n' - ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n' - ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n' - ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n' - ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n' - ping_stat_msg+='Max Index : ' + str(ping_stat['max_loc']) + '\n' - ping_stat_msg+='Ping stats after removing largest value : \n' - ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n' - ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n' - ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n' - ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n' + if (ping_stat!=-1) and (len(ping_stat)!=0): + ping_stat_msg+='Ping stats before removing largest value : \n' + ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_0'])) + 'ms \n' + ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_0'])) + 'ms \n' + ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_0'])) + 'ms \n' + ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_0'])) + 'ms \n' + ping_stat_msg+='Max Index : ' + str(ping_stat['max_loc']) + '\n' + ping_stat_msg+='Ping stats after removing largest value : \n' + ping_stat_msg+='RTT(Min) : ' + str("{:.2f}".format(ping_stat['min_1'])) + 'ms \n' + ping_stat_msg+='RTT(Mean) : ' + str("{:.2f}".format(ping_stat['mean_1'])) + 'ms \n' + ping_stat_msg+='RTT(Median) : ' + str("{:.2f}".format(ping_stat['median_1'])) + 'ms \n' + ping_stat_msg+='RTT(Max) : ' + str("{:.2f}".format(ping_stat['max_1'])) + 'ms \n' #building html message qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + '\n' + ping_stat_msg @@ -926,11 +880,10 @@ class OaiCiTest(): statusQueue.put(0) else: statusQueue.put(-1) - statusQueue.put(device_id) - statusQueue.put(UE_IPAddress) + statusQueue.put(ue.getName()) + statusQueue.put(ue.getIP()) statusQueue.put(qMsg) lock.release() - SSH.close() except: logging.debug('exit from Ping_Common except') os.kill(os.getppid(),signal.SIGUSR1) @@ -1049,31 +1002,23 @@ class OaiCiTest(): return if self.ue_id == "": - logging.error("no module ID!!!") - exit(1) + raise Exception("no module names in self.ue_id provided") + ues = [] + for ue_name in self.ue_id.split(' '): + ue = cls_module_ue.Module_UE(ue_name.strip()) + if not ue.getIP(): + logging.error("no IP addresses returned") + HTML.CreateHtmlTestRow(self.ping_args, 'KO', len(self.UEDevices), html_queue) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,CONTAINERS) + ues.append(ue) + logging.debug(ues) - self.UEIPAddresses = [] - ue = cls_module_ue.Module_UE(self.ue_id) - ip = ue.getIP() - if ip is None: - logging.error("no IP addresses returned") - html_queue = SimpleQueue() - HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,CONTAINERS) - self.UEIPAddresses.append(ip) - - logging.debug(self.UEIPAddresses) multi_jobs = [] i = 0 lock = Lock() status_queue = SimpleQueue() - for UE_IPAddress in self.UEIPAddresses: - if self.ue_id=="": - device_id = self.UEDevices[i] - else: - device_id = Module_UE.ID + "-" + Module_UE.Kind - logging.debug(device_id) - p = Process(target = self.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,EPC,Module_UE,RAN,)) + for ue in ues: + p = Process(target = self.Ping_common, args = (lock,status_queue,EPC,ue,RAN,)) p.daemon = True p.start() multi_jobs.append(p)