maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/ci-log-collection-modif' into integration_2024_w09

Browse Source
This commit is contained in:
Robert Schmidt
2024-03-05 15:14:58 +01:00
parent fc97e197fa fca1561bf7
commit 543250e933
2 changed files with 86 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_cmd.py
+57 -21
View File
@@ -86,6 +86,12 @@ class Cmd(metaclass=abc.ABCMeta):
return
class LocalCmd(Cmd):
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
self.close()
def __init__(self, d = None):
self.cwd = d
if self.cwd is not None:
@@ -122,12 +128,40 @@ class LocalCmd(Cmd):
if src[0] != '/' or tgt[0] != '/':
raise Exception('support only absolute file paths!')
opt = '-r' if recursive else ''
self.run(f'cp {opt} {src} {tgt}')
return self.run(f'cp {opt} {src} {tgt}').returncode == 0
def copyout(self, src, tgt, recursive=False):
self.copyin(src, tgt, recursive)
return self.copyin(src, tgt, recursive)
def PutFile(client, src, tgt):
success = True
sftp = client.open_sftp()
try:
sftp.put(src, tgt)
except FileNotFoundError as error:
logging.error(f"error while putting {src}: {error}")
success = False
sftp.close()
return success
def GetFile(client, src, tgt):
success = True
sftp = client.open_sftp()
try:
sftp.get(src, tgt)
except FileNotFoundError as error:
logging.error(f"error while getting {src}: {error}")
success = False
sftp.close()
return success
class RemoteCmd(Cmd):
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
self.close()
def __init__(self, hostname, d=None):
cIdx = 0
logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko
@@ -193,36 +227,38 @@ class RemoteCmd(Cmd):
def getBefore(self):
return self.cp.stdout
# if recursive is True, tgt must be a directory (and src is file or directory)
# if recursive is False, tgt and src must be a file name
def copyout(self, src, tgt, recursive=False):
logging.debug(f"copyout: local:{src} -> remote:{tgt}")
if recursive:
tmpfile = f"{uuid.uuid4()}.tar"
abstmpfile = f"/tmp/{tmpfile}"
cmd = LocalCmd()
cmd.run(f"tar -cf {abstmpfile} {src}")
sftp = self.client.open_sftp()
sftp.put(abstmpfile, abstmpfile)
sftp.close()
cmd.run(f"rm {abstmpfile}")
self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
with LocalCmd() as cmd:
if cmd.run(f"tar -cf {abstmpfile} {src}").returncode != 0:
return False
if not PutFile(self.client, abstmpfile, abstmpfile):
return False
cmd.run(f"rm {abstmpfile}")
ret = self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
return ret.returncode == 0
else:
sftp = self.client.open_sftp()
sftp.put(src, tgt)
sftp.close()
return PutFile(self.client, src, tgt)
# if recursive is True, tgt must be a directory (and src is file or directory)
# if recursive is False, tgt and src must be a file name
def copyin(self, src, tgt, recursive=False):
logging.debug(f"copyin: remote:{src} -> local:{tgt}")
if recursive:
tmpfile = f"{uuid.uuid4()}.tar"
abstmpfile = f"/tmp/{tmpfile}"
self.run(f"tar -cf {abstmpfile} {src}")
sftp = self.client.open_sftp()
sftp.get(abstmpfile, abstmpfile)
sftp.close()
if self.run(f"tar -cf {abstmpfile} {src}").returncode != 0:
return False
if not GetFile(self.client, abstmpfile, abstmpfile):
return False
self.run(f"rm {abstmpfile}")
cmd = LocalCmd()
cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
with LocalCmd() as cmd:
ret = cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
return ret.returncode == 0
else:
sftp = self.client.open_sftp()
sftp.get(src, tgt)
sftp.close()
return GetFile(self.client, src, tgt)
ci-scripts/cls_containerize.py
+29 -28
View File
@@ -955,12 +955,12 @@ class Containerize():
if svcName == '':
logging.warning('no service name given: starting all services in ci-docker-compose.yml!')
mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30)
mySSH.command(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30)
# Checking Status
grep = ''
if svcName != '': grep = f' | grep -A3 {svcName}'
mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5)
mySSH.command(f'docker compose --file ci-docker-compose.yml config {grep}', '\$', 5)
result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
unhealthyNb = 0
healthyNb = 0
@@ -1062,55 +1062,55 @@ class Containerize():
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m')
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
mySSH = cls_cmd.getConnection(lIpAddr)
yamlDir = f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}'
mySSH.run(f'cd {yamlDir}')
svcName = self.services[self.eNB_instance]
forceDown = False
if svcName != '':
logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!')
svcName = ''
mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5)
ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml config --services')
if ret.returncode != 0:
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services")
self.exitStatus = 1
return
# first line has command, last line has next command prompt
allServices = mySSH.getBefore().split('\r\n')[1:-1]
allServices = ret.stdout.splitlines()
services = []
for s in allServices:
mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False)
running = mySSH.getBefore().split('\r\n')[2:-1]
# outputs the hash if the container is running
ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml ps --all --quiet -- {s}')
running = ret.stdout.splitlines()
logging.debug(f'running services: {running}')
if len(running) > 0: # something is running for that service
if ret.stdout != "" and ret.returncode == 0: # something is running for that service
services.append(s)
logging.info(f'stopping services {services}')
mySSH.command(f'docker-compose -f ci-docker-compose.yml stop -t3', '\$', 30)
time.sleep(5) # give some time to running containers to stop
mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml stop -t3')
copyin_res = True
for svcName in services:
# head -n -1 suppresses the final "X exited with status code Y"
filename = f'{svcName}-{HTML.testCase_id}.log'
mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 120)
mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}')
copyin_res = mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', f'{filename}') and copyin_res
# when nv-cubb container is available, copy L1 pcap, OAI Aerial pipeline
if 'nv-cubb' in allServices:
mySSH.run(f'cp {lSourcePath}/cmake_targets/share/gnb_nvipc.pcap {lSourcePath}/cmake_targets/gnb_nvipc.pcap')
mySSH.command('docker-compose -f ci-docker-compose.yml down -v', '\$', 5)
mySSH.close()
mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml down -v')
# Analyzing log file!
files = ','.join([f'{s}-{HTML.testCase_id}' for s in services])
if len(services) > 1:
files = '{' + files + '}'
copyin_res = 0
if len(services) > 0:
copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.')
if copyin_res == -1:
HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!'
if not copyin_res:
HTML.htmleNBFailureMsg='Could not copy logfile(s) to analyze it!'
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
self.exitStatus = 1
# use function for UE log analysis, when oai-nr-ue container is used
elif 'oai-nr-ue' in services or 'lte_ue0' in services:
self.exitStatus == 0
logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m')
logging.debug(f'Analyzing UE logfile {filename}')
logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN)
if (logStatus < 0):
fullStatus = False
@@ -1129,11 +1129,12 @@ class Containerize():
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
# all the xNB run logs shall be on the server 0 for logCollecting
if self.eNB_serverId[self.eNB_instance] != '0':
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/')
mySSH.copyout(f'./*.log', f'{lSourcePath}/cmake_targets/', recursive=True)
if self.exitStatus == 0:
logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
else:
logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
mySSH.close()
def DeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0