diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index d933c1e143..184bb7bdb3 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -773,22 +773,19 @@ class Containerize(): mySSH.command(f'sed -i -e "s#image: {image}:latest#image: {imageToUse}#" ci-docker-compose.yml', '\$', 2) localMmeIpAddr = EPC.MmeIPAddress mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2) -# if self.flexranCtrlDeployed: -# mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: \'yes\'/" ci-docker-compose.yml', '\$', 2) -# mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2) -# else: -# mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \'no\'/" ci-docker-compose.yml', '\$', 2) -# mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2) + # Currently support only one mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 15) - mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@" 2>&1', '\$', 10) - result = re.search('service=(?P[a-zA-Z0-9\_]+)', mySSH.getBefore()) - if result is not None: - svcName = result.group('svc_name') - mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 15) + svcName = self.services[self.eNB_instance] + if svcName == '': + logging.warning('no service name given: starting all services in ci-docker-compose.yml!') + + mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30) # Checking Status - mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) + grep = '' + if svcName != '': grep = f' | grep -A3 {svcName}' + mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5) result = re.search('container_name: (?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) unhealthyNb = 0 healthyNb = 0 @@ -848,8 +845,8 @@ class Containerize(): else: # containers are unhealthy, so we won't start. However, logs are stored at the end # in UndeployObject so we here store the logs of the unhealthy container to report it - logfilename = f'{lSourcePath}/cmake_targets/{self.eNB_logFile[self.eNB_instance]}' - mySSH.command('docker logs {containerName} > {logfilename}', '\$', 30) + logfilename = f'{lSourcePath}/cmake_targets/log/{self.eNB_logFile[self.eNB_instance]}' + mySSH.command(f'docker logs {containerName} > {logfilename}', '\$', 30) mySSH.copyin(lIpAddr, lUserName, lPassWord, logfilename, '.') mySSH.close() @@ -896,58 +893,67 @@ class Containerize(): mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) - # Currently support only one - mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) - containerName = '' - containerToKill = False - result = re.search('container_name: (?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) - if self.eNB_logFile[self.eNB_instance] == '': - self.eNB_logFile[self.eNB_instance] = 'enb_' + HTML.testCase_id + '.log' - if result is not None: - containerName = result.group('container_name') - containerToKill = True - if containerToKill: - mySSH.command('docker inspect ' + containerName, '\$', 30) - result = re.search('Error: No such object: ' + containerName, mySSH.getBefore()) - if result is not None: - containerToKill = False - if containerToKill: - mySSH.command('docker kill --signal INT ' + containerName, '\$', 30) - time.sleep(5) - mySSH.command('docker kill --signal KILL ' + containerName, '\$', 30) - time.sleep(5) - mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) - mySSH.command('docker rm -f ' + containerName, '\$', 30) - # Forcing the down now to remove the networks and any artifacts - mySSH.command('docker-compose --file ci-docker-compose.yml down', '\$', 5) + + svcName = self.services[self.eNB_instance] + forceDown = False + if svcName != '': + logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!') + svcName = '' + + mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5) + # first line has command, last line has next command prompt + allServices = mySSH.getBefore().split('\r\n')[1:-1] + services = [] + for s in allServices: + mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False) + running = mySSH.getBefore().split('\r\n')[3:-1] + #logging.debug(f'running services: {running}') + if len(running) > 0: # something is running for that service + services.append(s) + logging.info(f'stopping services {services}') + + mySSH.command(f'docker-compose -f ci-docker-compose.yml stop', '\$', 30) + time.sleep(5) # give some time to running containers to stop + for svcName in services: + # head -n -1 suppresses the final "X exited with status code Y" + filename = f'{svcName}-{HTML.testCase_id}.log' + #mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} | head -n -1 &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 30) + mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 30) + + mySSH.command('docker-compose -f ci-docker-compose.yml down', '\$', 5) # Cleaning any created tmp volume - mySSH.command('docker volume prune --force || true', '\$', 20) + mySSH.command('docker volume prune --force', '\$', 20) mySSH.close() # Analyzing log file! - if containerToKill: - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.') - else: - copyin_res = 0 - nodeB_prefix = 'e' - if (copyin_res == -1): - HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' + files = ','.join([f'{s}-{HTML.testCase_id}' for s in services]) + if len(services) > 1: + files = '{' + files + '}' + copyin_res = 0 + if len(services) > 0: + copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.') + if copyin_res == -1: + HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) + self.exitStatus = 1 else: - if containerToKill: - logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance]) - logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML, self.ran_checkers) - else: - logStatus = 0 - if (logStatus < 0): - HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) - else: - HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + for svcName in services: + filename = f'{svcName}-{HTML.testCase_id}.log' + logging.debug(f'\u001B[1m Analyzing logfile {filename}\u001B[0m') + logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) + if (logStatus < 0): + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) + self.exitStatus = 1 + else: + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) # all the xNB run logs shall be on the server 0 for logCollecting - if containerToKill and self.eNB_serverId[self.eNB_instance] != '0': - mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/') - logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') + if self.eNB_serverId[self.eNB_instance] != '0': + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/') + if self.exitStatus == 0: + logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') + else: + logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m') def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0 diff --git a/ci-scripts/main.py b/ci-scripts/main.py index d60477427e..f1ec41a1e7 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -423,6 +423,9 @@ def GetParametersFromXML(action): string_field=test.findtext('u_retx_th') if (string_field is not None): CONTAINERS.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] + string_field = test.findtext('services') + if string_field is not None: + CONTAINERS.services[CONTAINERS.eNB_instance] = string_field elif action == 'DeployGenObject' or action == 'UndeployGenObject' or action == 'StatsFromGenObject': string_field=test.findtext('yaml_path') @@ -986,6 +989,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re RAN.prematureExit = True elif action == 'Undeploy_Object': CONTAINERS.UndeployObject(HTML, RAN) + if CONTAINERS.exitStatus == 1: + CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS) + RAN.prematureExit = True elif action == 'Cppcheck_Analysis': SCA.CppCheckAnalysis(HTML) elif action == 'LicenceAndFormattingCheck':