maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement runScript in cls_cmd for LocalCmd/RemoteCmd, add test

Browse Source 
A lot of CI code is python mixed with bash, e.g.,

    ssh = getConnection(host)
    ssh.run('ls')
    ssh.run('echo')

At least some of this CI code would benefit if it was written in a
simple bash script, returning error codes and potentially other
information either through stdout/stderr or files, to the calling Python
code:

    ssh = runScript(host, script)
    # script does: ls; echo

This commit introduces the possibility to run entire scripts. The idea
is that the executor has a script (on localhost), which is either
executed locally or on a remote host. For the remote host, the script is
not copied but piped into a remotely executed bash. In both cases,
output is either returned like the Cmd.run() function with returncode
and mixed stdout/stderr, or optionally redirected into a file on the
(remote) host, which can be treated further by the Python code in later
steps.
This commit is contained in:
Robert Schmidt
2024-09-28 12:52:59 +02:00
parent fc0bc2357c
commit e541aee49e
2 changed files with 104 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_cmd.py
+61 -4
View File
@@ -36,13 +36,22 @@ import time
SSHTIMEOUT=7
def is_local(host):
return host is None or host.lower() in ["", "none", "localhost"]
# helper that returns either LocalCmd or RemoteCmd based on passed host name
def getConnection(host, d=None):
if host is None or host.lower() in ["", "none", "localhost"]:
if is_local(host):
return LocalCmd(d=d)
else:
return RemoteCmd(host, d=d)
def runScript(host, path, timeout, parameters=None, redirect=None, silent=False):
if is_local(host):
return LocalCmd.exec_script(path, timeout, parameters, redirect, silent)
else:
return RemoteCmd.exec_script(host, path, timeout, parameters, redirect, silent)
# provides a partial interface for the legacy SSHconnection class (getBefore(), command())
class Cmd(metaclass=abc.ABCMeta):
def cd(self, d, silent=False):
@@ -58,6 +67,10 @@ class Cmd(metaclass=abc.ABCMeta):
if not silent:
logging.debug(f'cd {self.cwd}')
@abc.abstractmethod
def exec_script(path, timeout, parameters=None, redirect=None, silent=False):
return
@abc.abstractmethod
def run(self, line, timeout=300, silent=False):
return
@@ -99,6 +112,22 @@ class LocalCmd(Cmd):
logging.debug(f'Working dir is {self.cwd}')
self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
def exec_script(path, timeout, parameters=None, redirect=None, silent=False):
if redirect and not redirect.startswith("/"):
raise ValueError(f"redirect must be absolute, but is {redirect}")
c = f"{path} {parameters}" if parameters else path
if not redirect:
ret = sp.run(c, shell=True, timeout=timeout, stdout=sp.PIPE, stderr=sp.STDOUT)
ret.stdout = ret.stdout.decode('utf-8').strip()
else:
with open(redirect, "w") as f:
ret = sp.run(c, shell=True, timeout=timeout, stdout=f, stderr=f)
ret.args += f" &> {redirect}"
ret.stdout = ""
if not silent:
logging.info(f"local> {ret.args}")
return ret
def run(self, line, timeout=300, silent=False, reportNonZero=True):
if not silent:
logging.info(f"local> {line}")
@@ -169,9 +198,7 @@ class RemoteCmd(Cmd):
def __init__(self, hostname, d=None):
cIdx = 0
self.hostname = hostname
logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko
self.client = paramiko.SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.client = RemoteCmd._ssh_init()
cfg = RemoteCmd._lookup_ssh_config(hostname)
self.cwd = d
self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
@@ -184,6 +211,12 @@ class RemoteCmd(Cmd):
cIdx +=1
raise Exception ("Error: max retries, did not connect to host")
def _ssh_init():
logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
return client
def _lookup_ssh_config(hostname):
ssh_config = paramiko.SSHConfig()
user_config_file = os.path.expanduser("~/.ssh/config")
@@ -204,6 +237,30 @@ class RemoteCmd(Cmd):
cfg['sock'] = paramiko.ProxyCommand(ucfg['proxycommand'])
return cfg
def exec_script(host, path, timeout, parameters=None, redirect=None, silent=False):
if redirect and not redirect.startswith("/"):
raise ValueError(f"redirect must be absolute, but is {redirect}")
p = parameters if parameters else ""
r = f"> {redirect}" if redirect else ""
if not silent:
logging.info(f"local> ssh {host} bash -s {p} < {path} {r} # {path} from localhost")
client = RemoteCmd._ssh_init()
cfg = RemoteCmd._lookup_ssh_config(host)
client.connect(**cfg)
bash_opt = 'BASH_XTRACEFD=1' # write bash set -x output to stdout, see bash(1)
stdin, stdout, stderr = client.exec_command(f"{bash_opt} bash -s {p} {r}", timeout=timeout)
# open() the file f at path, read() it and write() it into the stdin of the bash -s cmd
with open(path) as f:
stdin.write(f.read())
stdin.close()
cmd = path
if parameters: cmd += f" {parameters}"
if redirect: cmd += f" &> {redirect}"
ret = sp.CompletedProcess(args=cmd, returncode=stdout.channel.recv_exit_status(), stdout=stdout.read(size=None) + stderr.read(size=None))
ret.stdout = ret.stdout.decode('utf-8').strip()
client.close()
return ret
def run(self, line, timeout=300, silent=False, reportNonZero=True):
if not silent:
logging.info(f"ssh[{self.hostname}]> {line}")
ci-scripts/tests/cmd.py
+43
View File
@@ -35,6 +35,27 @@ class TestCmd(unittest.TestCase):
self.assertEqual(ret.returncode, 0)
self.assertEqual(ret.stdout, "test")
def test_local_script(self):
ret = cls_cmd.runScript("localhost", "tests/scripts/hello-world.sh", 1)
self.assertEqual(ret.args, "tests/scripts/hello-world.sh")
self.assertEqual(ret.returncode, 0)
self.assertEqual(ret.stdout, "+ echo hello, world\nhello, world")
ret = cls_cmd.runScript("localhost", "tests/scripts/hello-fail.sh", 1, "TESTFAIL")
self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL")
self.assertEqual(ret.returncode, 1)
self.assertEqual(ret.stdout, "TESTFAIL")
ret = cls_cmd.runScript("localhost", "tests/scripts/hello-fail.sh", 1, "TESTFAIL2", "/tmp/result")
self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL2 &> /tmp/result")
self.assertEqual(ret.returncode, 1)
self.assertEqual(ret.stdout, "")
with cls_cmd.getConnection("localhost") as ssh:
ret = ssh.run("cat /tmp/result")
self.assertEqual(ret.args, "cat /tmp/result")
self.assertEqual(ret.returncode, 0)
self.assertEqual(ret.stdout, "TESTFAIL2")
@unittest.skip("need to be able to passwordlessly SSH to localhost, also disable stty -ixon")
def test_remote_cmd(self):
with cls_cmd.getConnection("127.0.0.1") as cmd:
@@ -52,5 +73,27 @@ class TestCmd(unittest.TestCase):
self.assertEqual(ret.returncode, 0)
self.assertEqual(ret.stdout, "test")
@unittest.skip("need to be able to passwordlessly SSH to localhost, also disable stty -ixon")
def test_remote_script(self):
ret = cls_cmd.runScript("127.0.0.1", "tests/scripts/hello-world.sh", 1)
self.assertEqual(ret.args, "tests/scripts/hello-world.sh")
self.assertEqual(ret.returncode, 0)
self.assertEqual(ret.stdout, "+ echo hello, world\nhello, world")
ret = cls_cmd.runScript("127.0.0.1", "tests/scripts/hello-fail.sh", 1, "TESTFAIL")
self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL")
self.assertEqual(ret.returncode, 1)
self.assertEqual(ret.stdout, "TESTFAIL")
ret = cls_cmd.runScript("127.0.0.1", "tests/scripts/hello-fail.sh", 1, "TESTFAIL2", "/tmp/result")
self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL2 &> /tmp/result")
self.assertEqual(ret.returncode, 1)
self.assertEqual(ret.stdout, "")
with cls_cmd.getConnection("localhost") as ssh:
ret = ssh.run("cat /tmp/result")
self.assertEqual(ret.args, "cat /tmp/result")
self.assertEqual(ret.returncode, 0)
self.assertEqual(ret.stdout, "TESTFAIL2")
if __name__ == '__main__':
unittest.main()