From cb1f72a8a6f65a536ebf57edfbf35ea5aae5f97c Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Wed, 20 Mar 2024 23:58:14 +0100 Subject: [PATCH] Refactor Iperf_Module --- ci-scripts/cls_oaicitest.py | 158 +++++++++++------------------------- 1 file changed, 49 insertions(+), 109 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 4da7ac3c75..8a1903797c 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -45,6 +45,7 @@ import signal import statistics as stat from multiprocessing import SimpleQueue, Lock import concurrent.futures +import json #import our libs import helpreadme as HELP @@ -949,125 +950,61 @@ class OaiCiTest(): else: return (False, 'Could not analyze from server log') - def Iperf_Module(self, EPC, ue, RAN, idx, ue_num): + def Iperf_Module(self, EPC, ue, svr, RAN, idx, ue_num, CONTAINERS): ueIP = ue.getIP() if not ueIP: return (False, f"UE {ue.getName()} has no IP address") - SSH = sshconnection.SSHConnection() - server_filename = f'iperf_server_{self.testCase_id}_{ue.getName()}.log' - client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log' - if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): - #retrieve trf-gen container IP address - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - SSH.command('docker inspect --format="TRF_IP_ADDR = {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" prod-trf-gen', '\$', 5) - result = re.search('TRF_IP_ADDR = (?P[0-9\.]+)', SSH.getBefore()) - if result is None: - raise Exception("could not corver prod-trf-gen IP address") - cn_target_ip = result.group('trf_ip_addr') - SSH.close() - cn_iperf_prefix = "docker exec prod-trf-gen" # -w /iperf-2.0.13 necessary? - elif (re.match('OC-OAI-CN5G', EPC.Type, re.IGNORECASE)): - cn_target_ip = "172.21.6.102" - else: # lteboix, sabox - cn_target_ip = "192.172.0.1" - cn_iperf_prefix = "" + svrIP = svr.getIP() + if not svrIP: + return (False, f"Iperf server {ue.getName()} has no IP address") + runIperf3Server = svr.getRunIperf3Server() iperf_opt = self.iperf_args + jsonReport = "--json" + serverReport = "" udpIperf = re.search('-u', iperf_opt) is not None - udpSwitch = '-u' if udpIperf else '' + bidirIperf = re.search('--bidir', iperf_opt) is not None + client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log' if udpIperf: - iperf_opt = self.Iperf_ComputeModifiedBW(idx, ue_num) + iperf_opt = Iperf_ComputeModifiedBW(idx, ue_num, self.iperf_profile, self.iperf_args) + # note: for UDP testing we don't want to use json report - reports 0 Mbps received bitrate + jsonReport = "" + # note: enable server report collection on the UE side, no need to store and collect server report separately on the server side + serverReport = "--get-server-output" logging.info(f'iperf options modified from "{self.iperf_args}" to "{iperf_opt}" for {ue.getName()}') - iperf_time = int(self.Iperf_ComputeTime()) - port = f'-p {5001+idx}' + iperf_time = Iperf_ComputeTime(self.iperf_args) # hack: the ADB UEs don't have iperf in $PATH, so we need to hardcode for the moment - iperf_ue = '/data/local/tmp/iperf' if re.search('adb', ue.getName()) else 'iperf' - + iperf_ue = '/data/local/tmp/iperf3' if re.search('adb', ue.getName()) else 'iperf3' ue_header = f'UE {ue.getName()} ({ueIP})' - if self.iperf_direction == "DL": - logging.debug("Iperf in DL requested") - cmd = cls_cmd.getConnection(ue.getHost()) - cmd.run(f'rm {server_filename}') - cmd.run(f'{ue.getCmdPrefix()} {iperf_ue} -s -B {ueIP} {udpSwitch} -i 1 -t {iperf_time * 1.5} {port} &> /tmp/{server_filename} &') - cmd.close() - - cmd = cls_cmd.getConnection(EPC.IPAddress) - cmd.run(f'rm {EPC.SourceCodePath}/{client_filename}') - cmd.run(f'{cn_iperf_prefix} iperf -c {ueIP} {iperf_opt} {port} &> {EPC.SourceCodePath}/{client_filename}', timeout=iperf_time * 1.5) - cmd.copyin(f'{EPC.SourceCodePath}/{client_filename}', client_filename) - cmd.close() - - cmd = cls_cmd.getConnection(ue.getHost()) - cmd.copyin(f'/tmp/{server_filename}', server_filename) - cmd.close() - + if svr.getName() == "rfsim4g_enb_fembms": + with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(EPC.IPAddress) as cmd_svr: + port = 5002 + idx + cmd_ue.run(f'{ue.getCmdPrefix()} iperf -B {ueIP} -s -u -i1 >> {server_filename} &', timeout=iperf_time*1.5) + cmd_svr.run(f'{svr.getCmdPrefix()} iperf -c {ueIP} -B {svrIP} {iperf_opt} -i1 2>&1 | tee {client_filename}', timeout=iperf_time*1.5) + cmd_ue.run(f'cp {client_filename} {logPath}/{client_filename}') + cmd_ue.run(f'cp {server_filename} {logPath}/{server_filename}') + status, msg = Iperf_analyzeV2UDP(server_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, iperf_opt) + else: + with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(EPC.IPAddress) as cmd_svr: + port = 5002 + idx + # note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False` + if runIperf3Server: + cmd_svr.run(f'{svr.getCmdPrefix()} nohup iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} &', timeout=iperf_time*1.5) + cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False) + cmd_ue.run(f'{ue.getCmdPrefix()} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=iperf_time*1.5) + if svr.getHost() == 'localhost': + cmd_ue.run(f'mkdir -p {logPath}') + cmd_ue.run(f'cp /tmp/{client_filename} {logPath}/{client_filename}') + cmd_ue.run(f'cp /tmp/{client_filename} {client_filename}') + else: + cmd_ue.copyin(f'/tmp/{client_filename}', client_filename) if udpIperf: - status, msg = self.Iperf_analyzeV2Server(iperf_opt, server_filename, 1) + status, msg = Iperf_analyzeV3UDP(client_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold) + elif bidirIperf: + status, msg = Iperf_analyzeV3BIDIRJson(client_filename) else: - cmd = cls_cmd.getConnection(EPC.IPAddress) - status, msg = self.Iperf_analyzeV2TCPOutput(cmd, f"{EPC.SourceCodePath}/{client_filename}") - cmd.close() - - elif self.iperf_direction == "UL": - logging.debug("Iperf in UL requested") - cmd = cls_cmd.getConnection(EPC.IPAddress) - cmd.run(f'rm {EPC.SourceCodePath}/{server_filename}') - cmd.run(f'{cn_iperf_prefix} iperf -s {udpSwitch} -t {iperf_time * 1.5} {port} &> {EPC.SourceCodePath}/{server_filename} &') - cmd.close() - - cmd = cls_cmd.getConnection(ue.getHost()) - cmd.run(f'rm /tmp/{client_filename}') - cmd.run(f'{ue.getCmdPrefix()} {iperf_ue} -B {ueIP} -c {cn_target_ip} {iperf_opt} {port} &> /tmp/{client_filename}', timeout=iperf_time*1.5) - cmd.copyin(f'/tmp/{client_filename}', client_filename) - cmd.close() - - cmd = cls_cmd.getConnection(EPC.IPAddress) - cmd.copyin(f'{EPC.SourceCodePath}/{server_filename}', server_filename) - cmd.close() - - if udpIperf: - status, msg = self.Iperf_analyzeV2Server(iperf_opt, server_filename, 1) - else: - cmd = cls_cmd.getConnection(ue.getHost()) - status, msg = self.Iperf_analyzeV2TCPOutput(cmd, f"/tmp/{client_filename}") - cmd.close() - - elif self.iperf_direction=="BIDIR": - logging.debug("Bi-directional iperf requested") - cmd = cls_cmd.getConnection(EPC.IPAddress) - cmd.run(f'rm {EPC.SourceCodePath}/{server_filename}') - cmd.run(f'{cn_iperf_prefix} iperf3 -s -i 1 -1 {port} &> {EPC.SourceCodePath}/{server_filename} &') - cmd.close() - - cmd = cls_cmd.getConnection(ue.getHost()) - cmd.run(f'rm /tmp/{client_filename}') - cmd.run(f'iperf3 -B {ueIP} -c {cn_target_ip} {iperf_opt} {port} &> /tmp/{client_filename}', timeout=iperf_time*1.5) - cmd.copyin(f'/tmp/{client_filename}', client_filename) - cmd.close() - - cmd = cls_cmd.getConnection(EPC.IPAddress) - cmd.copyin(f'{EPC.SourceCodePath}/{server_filename}', server_filename) - cmd.close() - - status, msg = self.Iperf_analyzeV2BIDIR(server_filename, client_filename) - - elif self.iperf_direction == "IPERF3": - cmd = cls_cmd.getConnection(ue.getHost()) - cmd.run(f'rm /tmp/{server_filename}', reportNonZero=False) - port = f'{5002+idx}' - cmd.run(f'{ue.getCmdPrefix()} iperf3 -B {ueIP} -c {cn_target_ip} -p {port} {iperf_opt} --get-server-output &> /tmp/{server_filename}', timeout=iperf_time*1.5) - cmd.copyin(f'/tmp/{server_filename}', server_filename) - cmd.close() - if udpIperf: - status, msg = self.Iperf_analyzeV2Server(iperf_opt, server_filename, 1) - else: - cmd = cls_cmd.getConnection(EPC.IPAddress) - status, msg = self.Iperf_analyzeV2TCPOutput(cmd, f'/tmp/{server_filename}') - cmd.close() - - else : - raise Exception("Incorrect or missing IPERF direction in XML") + status, msg = Iperf_analyzeV3TCPJson(client_filename, self.iperf_tcp_rate_target) logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m') for l in msg.split('\n'): @@ -1086,13 +1023,16 @@ class OaiCiTest(): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') - logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_direction "{self.iperf_direction}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"') + logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"') + + if self.ue_ids == [] or self.svr_id == None: + raise Exception("no module names in self.ue_ids or/and self.svr_id provided") ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids] svr = cls_module.Module_UE(self.svr_id) logging.debug(ues) with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [executor.submit(self.Iperf_Module, EPC, ue, RAN, i, len(ues)) for i, ue in enumerate(ues)] + futures = [executor.submit(self.Iperf_Module, EPC, ue, svr, RAN, i, len(ues), CONTAINERS) for i, ue in enumerate(ues)] results = [f.result() for f in futures] # each result in results is a tuple, first member goes to successes, second to messages successes, messages = map(list, zip(*results)) @@ -1503,7 +1443,7 @@ class OaiCiTest(): time.sleep(self.idle_sleep_time) HTML.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', CONST.ALL_PROCESSES_OK) - def X2_Status(self, idx, fileName): + def X2_Status(self, idx, fileName, EPC): cmd = "curl --silent http://" + EPC.IPAddress + ":9999/stats | jq '.' > " + fileName message = cmd + '\n' logging.debug(cmd)