maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Modifs of cls_containerize - UndeployObject()

Browse Source 
- use new ssh class
- do proper logging from multiple services
- use yaml dir
- Use splitlines(), minor modifs of syntax
- check that services are running
- Check all copyin() operations, not only last
- switch to docker compose:
  with the use of the new ssh class, we cannot (easily) use the old
  docker-compose anymore (which was an alias to docker compose). Switch to
  "docker compose".

Co-authored-by: Robert Schmidt <robert.schmidt@openairinterface.org>
This commit is contained in:
Jaroslava Fiedlerova
2024-03-05 15:09:25 +01:00
committed by Robert Schmidt
co-authored by Robert Schmidt
parent 3c1ed92b80
commit fca1561bf7
1 changed files with 29 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+29 -28
View File
@@ -951,12 +951,12 @@ class Containerize():
if svcName == '': if svcName == '':
logging.warning('no service name given: starting all services in ci-docker-compose.yml!') logging.warning('no service name given: starting all services in ci-docker-compose.yml!')
mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) mySSH.command(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30)
# Checking Status # Checking Status
grep = '' grep = ''
if svcName != '': grep = f' | grep -A3 {svcName}' if svcName != '': grep = f' | grep -A3 {svcName}'
mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5) mySSH.command(f'docker compose --file ci-docker-compose.yml config {grep}', '\$', 5)
result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
unhealthyNb = 0 unhealthyNb = 0
healthyNb = 0 healthyNb = 0
@@ -1058,55 +1058,55 @@ class Containerize():
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
HELP.GenericHelp(CONST.Version) HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter') sys.exit('Insufficient Parameter')
logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m') logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
mySSH = SSH.SSHConnection() mySSH = cls_cmd.getConnection(lIpAddr)
mySSH.open(lIpAddr, lUserName, lPassWord) yamlDir = f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}'
mySSH.run(f'cd {yamlDir}')
mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
svcName = self.services[self.eNB_instance] svcName = self.services[self.eNB_instance]
forceDown = False forceDown = False
if svcName != '': if svcName != '':
logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!') logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!')
svcName = '' svcName = ''
mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5) ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml config --services')
if ret.returncode != 0:
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services")
self.exitStatus = 1
return
# first line has command, last line has next command prompt # first line has command, last line has next command prompt
allServices = mySSH.getBefore().split('\r\n')[1:-1] allServices = ret.stdout.splitlines()
services = [] services = []
for s in allServices: for s in allServices:
mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False) # outputs the hash if the container is running
running = mySSH.getBefore().split('\r\n')[2:-1] ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml ps --all --quiet -- {s}')
running = ret.stdout.splitlines()
logging.debug(f'running services: {running}') logging.debug(f'running services: {running}')
if len(running) > 0: # something is running for that service if ret.stdout != "" and ret.returncode == 0: # something is running for that service
services.append(s) services.append(s)
logging.info(f'stopping services {services}') logging.info(f'stopping services {services}')
mySSH.command(f'docker-compose -f ci-docker-compose.yml stop -t3', '\$', 30) mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml stop -t3')
time.sleep(5) # give some time to running containers to stop copyin_res = True
for svcName in services: for svcName in services:
# head -n -1 suppresses the final "X exited with status code Y" # head -n -1 suppresses the final "X exited with status code Y"
filename = f'{svcName}-{HTML.testCase_id}.log' filename = f'{svcName}-{HTML.testCase_id}.log'
mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 120) mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}')
copyin_res = mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', f'{filename}') and copyin_res
# when nv-cubb container is available, copy L1 pcap, OAI Aerial pipeline
if 'nv-cubb' in allServices:
mySSH.run(f'cp {lSourcePath}/cmake_targets/share/gnb_nvipc.pcap {lSourcePath}/cmake_targets/gnb_nvipc.pcap')
mySSH.command('docker-compose -f ci-docker-compose.yml down -v', '\$', 5) mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml down -v')
mySSH.close()
# Analyzing log file! # Analyzing log file!
files = ','.join([f'{s}-{HTML.testCase_id}' for s in services]) if not copyin_res:
if len(services) > 1: HTML.htmleNBFailureMsg='Could not copy logfile(s) to analyze it!'
files = '{' + files + '}'
copyin_res = 0
if len(services) > 0:
copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.')
if copyin_res == -1:
HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!'
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
self.exitStatus = 1 self.exitStatus = 1
# use function for UE log analysis, when oai-nr-ue container is used # use function for UE log analysis, when oai-nr-ue container is used
elif 'oai-nr-ue' in services or 'lte_ue0' in services: elif 'oai-nr-ue' in services or 'lte_ue0' in services:
self.exitStatus == 0 self.exitStatus == 0
logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m') logging.debug(f'Analyzing UE logfile {filename}')
logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN) logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN)
if (logStatus < 0): if (logStatus < 0):
fullStatus = False fullStatus = False
@@ -1125,11 +1125,12 @@ class Containerize():
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
# all the xNB run logs shall be on the server 0 for logCollecting # all the xNB run logs shall be on the server 0 for logCollecting
if self.eNB_serverId[self.eNB_instance] != '0': if self.eNB_serverId[self.eNB_instance] != '0':
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/') mySSH.copyout(f'./*.log', f'{lSourcePath}/cmake_targets/', recursive=True)
if self.exitStatus == 0: if self.exitStatus == 0:
logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
else: else:
logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m') logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
mySSH.close()
def DeployGenObject(self, HTML, RAN, UE): def DeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0 self.exitStatus = 0