From 8d16ff845975dfea19ccc73dd2d815828745ccdd Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 24 May 2024 14:20:02 +0200 Subject: [PATCH] Use timeout command to kill hangling iperf3 In some pipelines, it happens that iperf3 is hanging on 100% CPU load. In these cases, a simple TERM signal does not seem to be enough to kill it; KILL is necessary. While e.g., RemoteCmd's driver paramiko has a timeout command, the documentation [1] is not really clear how it is trying to stop commands. Since it effectively does not stop, we have to assume it is only sending TERM. To remedy this, use the timeout command to start iperf3. We use option -v for diagnose timeouts on stderr, and send KILL after an additional 3 seconds (-k3) if TERM does not succeed. The timeout is sent to the same duration as the overall iperf timeout. [1] https://docs.paramiko.org/en/3.4/api/channel.html#paramiko.channel.Channel.settimeout --- ci-scripts/cls_oaicitest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 928e6bbfbd..db3cec5e35 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -818,10 +818,11 @@ class OaiCiTest(): with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(EPC.IPAddress) as cmd_svr: port = 5002 + idx # note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False` + t = iperf_time * 1.5 if runIperf3Server: - cmd_svr.run(f'{svr.getCmdPrefix()} nohup iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} &', timeout=iperf_time*1.5) + cmd_svr.run(f'{svr.getCmdPrefix()} nohup timeout -vk3 {t} iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} &', timeout=t) cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False) - cmd_ue.run(f'{ue.getCmdPrefix()} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=iperf_time*1.5) + cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=t) if svr.getHost() == 'localhost': cmd_ue.run(f'mkdir -p {logPath}') cmd_ue.run(f'cp /tmp/{client_filename} {logPath}/{client_filename}')