diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 93e8cb6382..550d2eeb15 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -504,132 +504,128 @@ class OaiCiTest(): lock.release() def Ping_common(self, lock, statusQueue,EPC, ue, RAN): - try: - # Launch ping on the EPC side (true for ltebox and old open-air-cn) - # But for OAI-Rel14-CUPS, we launch from python executor - ping_status = 0 + # Launch ping on the EPC side (true for ltebox and old open-air-cn) + # But for OAI-Rel14-CUPS, we launch from python executor + ping_status = 0 + launchFromEpc = False + ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log' + if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE): launchFromEpc = False - ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log' - if re.match('OAI-Rel14-CUPS', EPC.Type, re.IGNORECASE): - launchFromEpc = False - ping_time = re.findall("-c *(\d+)",str(self.ping_args)) - if launchFromEpc: - SSH = sshconnection.SSHConnection() - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - SSH.command('cd ' + EPC.SourceCodePath, '\$', 5) - SSH.command('cd scripts', '\$', 5) - # In case of a docker-based deployment, we need to ping from the trf-gen container - launchFromTrfContainer = False - if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): - launchFromTrfContainer = True - if launchFromTrfContainer: - ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ue.getIP()}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5) - else: - ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ue.getIP()} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5) + ping_time = re.findall("-c *(\d+)",str(self.ping_args)) + if launchFromEpc: + SSH = sshconnection.SSHConnection() + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + SSH.command('cd ' + EPC.SourceCodePath, '\$', 5) + SSH.command('cd scripts', '\$', 5) + # In case of a docker-based deployment, we need to ping from the trf-gen container + launchFromTrfContainer = False + if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): + launchFromTrfContainer = True + if launchFromTrfContainer: + ping_status = SSH.command(f'docker exec -it prod-trf-gen /bin/bash -c "ping {self.ping_args} {ue.getIP()}" 2>&1 | tee {ping_log_file}', '\$', int(ping_time[0])*1.5) + else: + ping_status = SSH.command(f'stdbuf -o0 ping {self.ping_args} {ue.getIP()} 2>&1 | stdbuf -o0 tee {ping_log_file}', '\$', int(ping_time[0])*1.5) + #copy the ping log file to have it locally for analysis (ping stats) + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.') + SSH.close() + else: + #target address is different depending on EPC type + if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): + Target = EPC.MmeIPAddress + elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): + Target = EPC.MmeIPAddress + elif re.match('OC-OAI-CN5G', EPC.Type, re.IGNORECASE): + Target = "172.21.6.100" + else: + Target = EPC.IPAddress + #ping from module NIC rather than IP address to make sure round trip is over the air + interface = f'-I {ue.getIFName()}' if ue.getIFName() else '' + ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}' + cmd = cls_cmd.getConnection(ue.getHost()) + response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5) + if response.returncode != 0: + ping_status = -1 + else: #copy the ping log file to have it locally for analysis (ping stats) - SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/scripts/{ping_log_file}', '.') - SSH.close() - else: - #target address is different depending on EPC type - if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): - Target = EPC.MmeIPAddress - elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): - Target = EPC.MmeIPAddress - elif re.match('OC-OAI-CN5G', EPC.Type, re.IGNORECASE): - Target = "172.21.6.100" - else: - Target = EPC.IPAddress - #ping from module NIC rather than IP address to make sure round trip is over the air - interface = f'-I {ue.getIFName()}' if ue.getIFName() else '' - ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {Target} &> /tmp/{ping_log_file}' - cmd = cls_cmd.getConnection(ue.getHost()) - response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5) - if response.returncode != 0: - ping_status = -1 - else: - #copy the ping log file to have it locally for analysis (ping stats) - cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file) + cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=ping_log_file) - cmd.close() + cmd.close() - # TIMEOUT CASE - if ping_status < 0: - message = 'Ping with UE (' + str(ue.getIP()) + ') crashed due to TIMEOUT!' - logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) - return - #search is done on cat result - with open(ping_log_file, 'r') as f: - ping_output = "".join(f.readlines()) - result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output) - if result is None: - message = 'Packet Loss Not Found!' - logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) - return - packetloss = result.group('packetloss') - if float(packetloss) == 100: - message = 'Packet Loss is 100%' - logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) - return - result = re.search('rtt min\/avg\/max\/mdev = (?P[0-9\.]+)\/(?P[0-9\.]+)\/(?P[0-9\.]+)\/[0-9\.]+ ms', ping_output) - if result is None: - message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!' - logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') - self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) - return - rtt_min = result.group('rtt_min') - rtt_avg = result.group('rtt_avg') - rtt_max = result.group('rtt_max') - pal_msg = 'Packet Loss : ' + packetloss + '%' - min_msg = 'RTT(Min) : ' + rtt_min + ' ms' - avg_msg = 'RTT(Avg) : ' + rtt_avg + ' ms' - max_msg = 'RTT(Max) : ' + rtt_max + ' ms' + # TIMEOUT CASE + if ping_status < 0: + message = 'Ping with UE (' + str(ue.getIP()) + ') crashed due to TIMEOUT!' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) + return + #search is done on cat result + with open(ping_log_file, 'r') as f: + ping_output = "".join(f.readlines()) + result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output) + if result is None: + message = 'Packet Loss Not Found!' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) + return + packetloss = result.group('packetloss') + if float(packetloss) == 100: + message = 'Packet Loss is 100%' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) + return + result = re.search('rtt min\/avg\/max\/mdev = (?P[0-9\.]+)\/(?P[0-9\.]+)\/(?P[0-9\.]+)\/[0-9\.]+ ms', ping_output) + if result is None: + message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!' + logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m') + self.ping_iperf_wrong_exit(lock, ue.getIP(), ue.getName(), statusQueue, message) + return + rtt_min = result.group('rtt_min') + rtt_avg = result.group('rtt_avg') + rtt_max = result.group('rtt_max') + pal_msg = 'Packet Loss : ' + packetloss + '%' + min_msg = 'RTT(Min) : ' + rtt_min + ' ms' + avg_msg = 'RTT(Avg) : ' + rtt_avg + ' ms' + max_msg = 'RTT(Max) : ' + rtt_max + ' ms' - lock.acquire() - logging.debug('\u001B[1;37;44m ping result (' + ue.getIP() + ') \u001B[0m') - logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m') - logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m') - logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m') - logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m') + lock.acquire() + logging.debug('\u001B[1;37;44m ping result (' + ue.getIP() + ') \u001B[0m') + logging.debug('\u001B[1;34m ' + pal_msg + '\u001B[0m') + logging.debug('\u001B[1;34m ' + min_msg + '\u001B[0m') + logging.debug('\u001B[1;34m ' + avg_msg + '\u001B[0m') + logging.debug('\u001B[1;34m ' + max_msg + '\u001B[0m') - logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) + logging.debug('Analyzing Ping log file : ' + os.getcwd() + '/' + ping_log_file) - #building html message - qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg + #building html message + qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg - #checking packet loss compliance - packetLossOK = True - if (packetloss is not None) : - if float(packetloss) > float(self.ping_packetloss_threshold): - qMsg += '\nPacket Loss too high' - logging.debug('\u001B[1;37;41m Packet Loss too high; Target: '+ self.ping_packetloss_threshold + '%\u001B[0m') - packetLossOK = False - elif float(packetloss) > 0: - qMsg += '\nPacket Loss is not 0%' - logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m') + #checking packet loss compliance + packetLossOK = True + if (packetloss is not None) : + if float(packetloss) > float(self.ping_packetloss_threshold): + qMsg += '\nPacket Loss too high' + logging.debug('\u001B[1;37;41m Packet Loss too high; Target: '+ self.ping_packetloss_threshold + '%\u001B[0m') + packetLossOK = False + elif float(packetloss) > 0: + qMsg += '\nPacket Loss is not 0%' + logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m') - #checking RTT avg compliance - rttavgOK = True - if self.ping_rttavg_threshold != '': - if float(rtt_avg)>float(self.ping_rttavg_threshold): - ping_rttavg_error_msg = 'RTT(Avg) too high: ' + rtt_avg + ' ms; Target: '+ self.ping_rttavg_threshold+ ' ms' - qMsg += '\n'+ping_rttavg_error_msg - logging.debug('\u001B[1;37;41m'+ ping_rttavg_error_msg +' \u001B[0m') - rttavgOK = False + #checking RTT avg compliance + rttavgOK = True + if self.ping_rttavg_threshold != '': + if float(rtt_avg)>float(self.ping_rttavg_threshold): + ping_rttavg_error_msg = 'RTT(Avg) too high: ' + rtt_avg + ' ms; Target: '+ self.ping_rttavg_threshold+ ' ms' + qMsg += '\n'+ping_rttavg_error_msg + logging.debug('\u001B[1;37;41m'+ ping_rttavg_error_msg +' \u001B[0m') + rttavgOK = False - if packetLossOK and rttavgOK: - statusQueue.put(0) - else: - statusQueue.put(-1) - statusQueue.put(ue.getName()) - statusQueue.put(ue.getIP()) - statusQueue.put(qMsg) - lock.release() - except: - logging.debug('exit from Ping_Common except') - os.kill(os.getppid(),signal.SIGUSR1) + if packetLossOK and rttavgOK: + statusQueue.put(0) + else: + statusQueue.put(-1) + statusQueue.put(ue.getName()) + statusQueue.put(ue.getIP()) + statusQueue.put(qMsg) + lock.release() def PingNoS1_wrong_exit(self, qMsg,HTML): message = 'OAI UE ping result\n{qMsg}'