Improved DeployObject cohesion
- Introduced helper functions: GetCredentials, GetContainerName, GetImageInfo, GetContainerHealth, ReTagImages, DeployServices.
This commit is contained in:
Bipin Adhikari
committed by
Raphael Defosseux
Raphael Defosseux
parent
ab1564aa1c
commit
bc950aaaec
+95
-352
@@ -171,6 +171,86 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus):
|
||||
collectInfo[image] = files
|
||||
return collectInfo
|
||||
|
||||
def GetCredentials(instance):
|
||||
server_id = instance.eNB_serverId[instance.eNB_instance]
|
||||
if server_id == '0':
|
||||
return (instance.eNBIPAddress, instance.eNBUserName, instance.eNBPassword, instance.eNBSourceCodePath)
|
||||
elif server_id == '1':
|
||||
return (instance.eNB1IPAddress, instance.eNB1UserName, instance.eNB1Password, instance.eNB1SourceCodePath)
|
||||
elif server_id == '2':
|
||||
return (instance.eNB2IPAddress, instance.eNB2UserName, instance.eNB2Password, instance.eNB2SourceCodePath)
|
||||
else:
|
||||
raise Exception ("Only supports maximum of 3 servers")
|
||||
|
||||
def GetContainerName(mySSH, svcName):
|
||||
ret = mySSH.run(f"docker compose -f docker-compose.y*ml config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'")
|
||||
containerName = ret.stdout
|
||||
return containerName
|
||||
|
||||
def GetImageInfo(mySSH, containerName):
|
||||
usedImage = ''
|
||||
ret = mySSH.run('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName)
|
||||
if ret.stdout.count('ImageUsed: oai-ci'):
|
||||
usedImage = ret.stdout.replace('ImageUsed: oai-ci', 'oai-ci').strip()
|
||||
logging.debug('Used image is ' + usedImage)
|
||||
if usedImage != '':
|
||||
ret = mySSH.run('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage)
|
||||
imageInfo = f"Used image is {usedImage}\n{ret.stdout}"
|
||||
return imageInfo
|
||||
else:
|
||||
return "Could not retrieve used image info!\n"
|
||||
|
||||
def GetContainerHealth(mySSH, containerName):
|
||||
if containerName is None:
|
||||
return 0, 0
|
||||
unhealthyNb = 0
|
||||
healthyNb = 0
|
||||
time.sleep(5)
|
||||
for _ in range(3):
|
||||
if containerName != 'db_init':
|
||||
result = mySSH.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}')
|
||||
unhealthyNb = result.stdout.count('unhealthy')
|
||||
healthyNb = result.stdout.count('healthy') - unhealthyNb
|
||||
if healthyNb == 1:
|
||||
break
|
||||
else:
|
||||
time.sleep(10)
|
||||
else:
|
||||
unhealthyNb = 0
|
||||
healthyNb = 1
|
||||
break
|
||||
return healthyNb, unhealthyNb
|
||||
|
||||
def ReTagImages(mySSH,IMAGES,ranCommitID,ranBranch,ranAllowMerge,displayedNewTags):
|
||||
mySSH.run('cp docker-compose.y*ml ci-docker-compose.yml', 5)
|
||||
for image in IMAGES:
|
||||
imageTag = ImageTagToUse(image, ranCommitID, ranBranch, ranAllowMerge)
|
||||
if image == 'oai-gnb' or image == 'oai-nr-ue' or image == 'oai-nr-cuup':
|
||||
ret = mySSH.run(f'docker image inspect oai-ci/{imageTag}', reportNonZero=False, silent=False)
|
||||
if ret.returncode != 0:
|
||||
imageTag = imageTag.replace('oai-gnb', 'oai-gnb-asan')
|
||||
imageTag = imageTag.replace('oai-nr-ue', 'oai-nr-ue-asan')
|
||||
imageTag = imageTag.replace('oai-nr-cuup', 'oai-nr-cuup-asan')
|
||||
if not displayedNewTags:
|
||||
logging.debug(f'\u001B[1m Using sanitized version of {image} with {imageTag}\u001B[0m')
|
||||
mySSH.run(f'sed -i -e "s@oaisoftwarealliance/{image}:develop@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True)
|
||||
mySSH.run(f'sed -i -e "s@{image}:latest@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True) # temporary solution for not using the new branch
|
||||
|
||||
def DeployServices(mySSH,svcName):
|
||||
allServices = []
|
||||
if svcName == '':
|
||||
logging.warning('No service name given: starting all services in ci-docker-compose.yml!')
|
||||
ret = mySSH.run(f'docker compose -f ci-docker-compose.yml config --services')
|
||||
allServices = ret.stdout.splitlines()
|
||||
deployStatus = mySSH.run(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', 50)
|
||||
return deployStatus.returncode,allServices
|
||||
|
||||
def CopyFailedDeployLog(mySSH,lSourcePath,ymlPath,containerName,filename):
|
||||
logging.warning(f'Deployment Failed, trying to copy the log from {filename}')
|
||||
logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[2]}'
|
||||
os.system(f'mkdir -p {logPath}')
|
||||
mySSH.run(f'docker logs {containerName} > {lSourcePath}/cmake_targets/log/{filename} 2>&1')
|
||||
mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', os.path.join(logPath, filename))
|
||||
# pyshark livecapture launches 2 processes:
|
||||
# * One using dumpcap -i lIfs -w - (ie redirecting the packets to STDOUT)
|
||||
# * One using tshark -i - -w loFile (ie capturing from STDIN from previous process)
|
||||
@@ -884,136 +964,38 @@ class Containerize():
|
||||
CreateWorkspace(sshSession, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
|
||||
def DeployObject(self, HTML, EPC):
|
||||
if self.eNB_serverId[self.eNB_instance] == '0':
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lUserName = self.eNBUserName
|
||||
lPassWord = self.eNBPassword
|
||||
lSourcePath = self.eNBSourceCodePath
|
||||
elif self.eNB_serverId[self.eNB_instance] == '1':
|
||||
lIpAddr = self.eNB1IPAddress
|
||||
lUserName = self.eNB1UserName
|
||||
lPassWord = self.eNB1Password
|
||||
lSourcePath = self.eNB1SourceCodePath
|
||||
elif self.eNB_serverId[self.eNB_instance] == '2':
|
||||
lIpAddr = self.eNB2IPAddress
|
||||
lUserName = self.eNB2UserName
|
||||
lPassWord = self.eNB2Password
|
||||
lSourcePath = self.eNB2SourceCodePath
|
||||
lIpAddr, lUserName, lPassWord, lSourcePath = GetCredentials(self)
|
||||
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter')
|
||||
logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
|
||||
self.deployKind[self.eNB_instance] = True
|
||||
|
||||
mySSH = cls_cmd.getConnection(lIpAddr)
|
||||
|
||||
mySSH.cd(lSourcePath + '/' + self.yamlPath[self.eNB_instance])
|
||||
mySSH.run('cp docker-compose.y*ml ci-docker-compose.yml', 5)
|
||||
for image in IMAGES:
|
||||
imageTag = ImageTagToUse(image, self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
||||
if image == 'oai-gnb' or image == 'oai-nr-ue' or image == 'oai-nr-cuup':
|
||||
ret = mySSH.run(f'docker image inspect oai-ci/{imageTag}', reportNonZero=False, silent=self.displayedNewTags)
|
||||
if ret.returncode != 0:
|
||||
imageTag = imageTag.replace('oai-gnb', 'oai-gnb-asan')
|
||||
imageTag = imageTag.replace('oai-nr-ue', 'oai-nr-ue-asan')
|
||||
imageTag = imageTag.replace('oai-nr-cuup', 'oai-nr-cuup-asan')
|
||||
if not self.displayedNewTags:
|
||||
logging.debug(f'\u001B[1m Using sanitized version of {image} with {imageTag}\u001B[0m')
|
||||
mySSH.run(f'sed -i -e "s@oaisoftwarealliance/{image}:develop@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True)
|
||||
mySSH.run(f'sed -i -e "s@{image}:latest@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True) # temporary solution for not using the new branch
|
||||
|
||||
svcName = self.services[self.eNB_instance]
|
||||
if svcName == '':
|
||||
logging.warning('no service name giv en: starting all services in ci-docker-compose.yml!')
|
||||
ret = mySSH.run(f'docker compose -f {lSourcePath}/{self.yamlPath[self.eNB_instance]}/ci-docker-compose.yml config --services')
|
||||
if ret.returncode != 0:
|
||||
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services")
|
||||
self.exitStatus = 1
|
||||
return
|
||||
allServices = ret.stdout.splitlines()
|
||||
|
||||
deployStatus = mySSH.run(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', 50)
|
||||
if deployStatus.returncode != 0:
|
||||
mySSH = cls_cmd.getConnection(lIpAddr, f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}')
|
||||
ReTagImages(mySSH,IMAGES,self.ranCommitID, self.ranBranch, self.ranAllowMerge, self.displayedNewTags)
|
||||
deployStatus,allServices = DeployServices(mySSH,self.services[self.eNB_instance])
|
||||
if deployStatus != 0:
|
||||
mySSH.close()
|
||||
self.exitStatus = 1
|
||||
logging.error('Could not deploy')
|
||||
HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
return
|
||||
# Checking Status
|
||||
services_list = self.services[self.eNB_instance].split()
|
||||
if services_list == []: services_list=allServices #handing empty services
|
||||
services_list = allServices if self.services[self.eNB_instance].split() == [] else self.services[self.eNB_instance].split()
|
||||
status = True
|
||||
for svcName in services_list:
|
||||
logging.debug('current service in the service list: \033[4m%s\033[0m', svcName)
|
||||
mySSH.run(f"docker compose -f docker-compose.yaml config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'")
|
||||
result = mySSH.getBefore()
|
||||
unhealthyNb = 0
|
||||
healthyNb = 0
|
||||
containerName = ''
|
||||
usedImage = ''
|
||||
imageInfo = ''
|
||||
if result is not None:
|
||||
containerName = result.group('container_name')
|
||||
time.sleep(5)
|
||||
cnt = 0
|
||||
while (cnt < 3):
|
||||
if svcName != 'db_init':
|
||||
mySSH.run('docker inspect --format="{{.State.Health.Status}}" ' + containerName, 5)
|
||||
unhealthyNb = mySSH.getBefore().count('unhealthy')
|
||||
healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb
|
||||
if healthyNb == 1:
|
||||
cnt = 10
|
||||
else:
|
||||
time.sleep(10)
|
||||
cnt += 1
|
||||
else:
|
||||
# faking health of db_init
|
||||
unhealthyNb = 0
|
||||
healthyNb = 1
|
||||
break
|
||||
mySSH.run('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName)
|
||||
for stdoutLine in mySSH.getBefore().split('\n'):
|
||||
if stdoutLine.count('ImageUsed: oai-ci'):
|
||||
usedImage = stdoutLine.replace('ImageUsed: oai-ci', 'oai-ci').strip()
|
||||
logging.debug('Used image is ' + usedImage)
|
||||
if usedImage != '':
|
||||
ret = mySSH.run('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage)
|
||||
imageInfo = ret.stdout
|
||||
logging.debug(f' -- for service {svcName} containers: {str(healthyNb)} healthy, {str(healthyNb)} unhealthy')
|
||||
|
||||
containerName = GetContainerName(mySSH, svcName)
|
||||
healthyNb,unhealthyNb = GetContainerHealth(mySSH,containerName)
|
||||
self.testCase_id = HTML.testCase_id
|
||||
self.eNB_logFile[self.eNB_instance] = f'{svcName}-{self.testCase_id}.log'
|
||||
|
||||
status = False
|
||||
if healthyNb == 1:
|
||||
status = True
|
||||
logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m')
|
||||
else:
|
||||
# containers are unhealthy, so we won't start. However, logs are stored at the end
|
||||
# in UndeployObject so we here store the logs of the unhealthy container to report it
|
||||
logfilename = f'{lSourcePath}/cmake_targets/log/{self.eNB_logFile[self.eNB_instance]}'
|
||||
mySSH.run(f'docker logs {containerName} > {logfilename}', 30)
|
||||
cwd = os.getcwd()
|
||||
logging.warning(f'Deployment Failed, trying to copy the log from {logfilename} at {cwd}')
|
||||
mySSH.copyin(logfilename,cwd,True)
|
||||
|
||||
if unhealthyNb:
|
||||
CopyFailedDeployLog(mySSH,lSourcePath,self.yamlPath[0].split('/'),containerName,self.eNB_logFile[self.eNB_instance])
|
||||
status = False
|
||||
logging.debug(GetImageInfo(mySSH, containerName))
|
||||
mySSH.close()
|
||||
|
||||
message = ''
|
||||
if usedImage != '':
|
||||
message += f'Used Image = {usedImage} :\n'
|
||||
message += imageInfo
|
||||
else:
|
||||
message += 'Could not retrieve used image info!\n'
|
||||
if status:
|
||||
message += '\nHealthy deployment!\n'
|
||||
else:
|
||||
message += '\nUnhealthy deployment! -- Check logs for reason!\n'
|
||||
if status:
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'OK', [message])
|
||||
HTML.CreateHtmlTestRowQueue('N/A', '0K', '\nHealthy deployment!\n')
|
||||
else:
|
||||
self.exitStatus = 1
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [message])
|
||||
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'KO', '\nUnhealthy deployment! -- Check logs for reason!\n')
|
||||
|
||||
def UndeployObject(self, HTML, RAN):
|
||||
if self.eNB_serverId[self.eNB_instance] == '0':
|
||||
@@ -1111,135 +1093,6 @@ class Containerize():
|
||||
logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
|
||||
mySSH.close()
|
||||
|
||||
def DeployGenObject(self, HTML, RAN, UE):
|
||||
self.exitStatus = 0
|
||||
logging.debug('\u001B[1m Checking Services to deploy\u001B[0m')
|
||||
# Implicitly we are running locally
|
||||
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
||||
self.deployKind[0] = False
|
||||
cmd = 'docker-compose config --services'
|
||||
listServices = myCmd.run(cmd)
|
||||
if listServices.returncode != 0:
|
||||
myCmd.close()
|
||||
self.exitStatus = 1
|
||||
HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
return
|
||||
for reqSvc in self.services[0].split(' '):
|
||||
res = re.search(reqSvc, listServices.stdout)
|
||||
if res is None:
|
||||
logging.error(reqSvc + ' not found in specified docker-compose')
|
||||
self.exitStatus = 1
|
||||
if (self.exitStatus == 1):
|
||||
myCmd.close()
|
||||
HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
return
|
||||
cmd = 'cp docker-compose.y*ml docker-compose-ci.yml'
|
||||
myCmd.run(cmd, silent=self.displayedNewTags)
|
||||
imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru', 'oai-nr-cuup']
|
||||
for image in imageNames:
|
||||
tagToUse = ImageTagToUse(image, self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
||||
# In the scenario, for 5G images, we have the choice of either pulling normal images
|
||||
# or -asan images. We need to detect which kind we did pull.
|
||||
if image == 'oai-gnb' or image == 'oai-nr-ue' or image == 'oai-nr-cuup':
|
||||
ret = myCmd.run(f'docker image inspect oai-ci/{tagToUse}', reportNonZero=False, silent=self.displayedNewTags)
|
||||
if ret.returncode != 0:
|
||||
tagToUse = tagToUse.replace('oai-gnb', 'oai-gnb-asan')
|
||||
tagToUse = tagToUse.replace('oai-nr-ue', 'oai-nr-ue-asan')
|
||||
tagToUse = tagToUse.replace('oai-nr-cuup', 'oai-nr-cuup-asan')
|
||||
if not self.displayedNewTags:
|
||||
logging.debug(f'\u001B[1m Using sanitized version of {image} with {tagToUse}\u001B[0m')
|
||||
cmd = f'sed -i -e "s@oaisoftwarealliance/{image}:develop@oai-ci/{tagToUse}@" docker-compose-ci.yml'
|
||||
myCmd.run(cmd, silent=self.displayedNewTags)
|
||||
self.displayedNewTags = True
|
||||
|
||||
cmd = f'docker-compose -f docker-compose-ci.yml up -d {self.services[0]}'
|
||||
deployStatus = myCmd.run(cmd, timeout=100)
|
||||
if deployStatus.returncode != 0:
|
||||
myCmd.close()
|
||||
self.exitStatus = 1
|
||||
logging.error('Could not deploy')
|
||||
HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
return
|
||||
|
||||
logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m')
|
||||
cmd = 'docker-compose -f docker-compose-ci.yml ps -a'
|
||||
count = 0
|
||||
healthy = 0
|
||||
restarting = 0
|
||||
newContainers = []
|
||||
while (count < 10):
|
||||
count += 1
|
||||
containerStatus = []
|
||||
deployStatus = myCmd.run(cmd, 30, silent=True)
|
||||
healthy = 0
|
||||
restarting = 0
|
||||
for state in deployStatus.stdout.split('\n'):
|
||||
state = state.strip()
|
||||
res = re.search('Name|NAME|----------', state)
|
||||
if res is not None:
|
||||
continue
|
||||
if len(state) == 0:
|
||||
continue
|
||||
res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
|
||||
if res is not None:
|
||||
cName = res.group('container_name')
|
||||
found = False
|
||||
for alreadyDeployed in self.deployedContainers:
|
||||
if cName == alreadyDeployed:
|
||||
found = True
|
||||
if not found:
|
||||
newContainers.append(cName)
|
||||
self.deployedContainers.append(cName)
|
||||
if re.search('\(healthy\)', state) is not None:
|
||||
healthy += 1
|
||||
if re.search('rfsim4g-db-init.*Exit 0', state) is not None or re.search('rfsim4g-db-init.*Exited \(0\)', state) is not None:
|
||||
myCmd.run('docker rm -f rfsim4g-db-init', timeout=30, silent=True, reportNonZero=False)
|
||||
if re.search('l2sim4g-db-init.*Exit 0', state) is not None or re.search('l2sim4g-db-init.*Exited \(0\)', state) is not None:
|
||||
myCmd.run('docker rm -f l2sim4g-db-init', timeout=30, silent=True, reportNonZero=False)
|
||||
if re.search('Restarting', state) is None:
|
||||
containerStatus.append(state)
|
||||
else:
|
||||
restarting += 1
|
||||
if healthy == self.nb_healthy[0] and restarting == 0:
|
||||
count = 100
|
||||
else:
|
||||
time.sleep(10)
|
||||
|
||||
html_cell = ''
|
||||
for newCont in newContainers:
|
||||
if newCont == 'rfsim4g-db-init':
|
||||
continue
|
||||
cmd = 'docker inspect -f "{{.Config.Image}}" ' + newCont
|
||||
imageInspect = myCmd.run(cmd, timeout=30, silent=True)
|
||||
imageName = str(imageInspect.stdout).strip()
|
||||
cmd = 'docker image inspect --format \'{{.RepoTags}}\t{{.Size}} bytes\t{{index (split .Created ".") 0}}\n{{.Id}}\' ' + imageName
|
||||
imageInspect = myCmd.run(cmd, 30, silent=True)
|
||||
html_cell += imageInspect.stdout + '\n'
|
||||
myCmd.close()
|
||||
|
||||
for cState in containerStatus:
|
||||
html_cell += cState + '\n'
|
||||
if count == 100 and healthy == self.nb_healthy[0]:
|
||||
if self.tsharkStarted == False:
|
||||
logging.debug('Starting tshark on public network')
|
||||
self.CaptureOnDockerNetworks()
|
||||
HTML.CreateHtmlTestRowQueue('n/a', 'OK', [html_cell])
|
||||
for cState in containerStatus:
|
||||
logging.debug(cState)
|
||||
logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m')
|
||||
else:
|
||||
HTML.CreateHtmlTestRowQueue('Could not deploy in time', 'KO', [html_cell])
|
||||
for cState in containerStatus:
|
||||
logging.debug(cState)
|
||||
logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m')
|
||||
HTML.testCase_id = 'AUTO-UNDEPLOY'
|
||||
UE.testCase_id = 'AUTO-UNDEPLOY'
|
||||
HTML.desc = 'Automatic Undeployment'
|
||||
UE.desc = 'Automatic Undeployment'
|
||||
UE.ShowTestID()
|
||||
self.UndeployGenObject(HTML, RAN, UE)
|
||||
self.exitStatus = 1
|
||||
|
||||
def CaptureOnDockerNetworks(self):
|
||||
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
||||
cmd = 'docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
|
||||
@@ -1268,116 +1121,6 @@ class Containerize():
|
||||
x.daemon = True
|
||||
x.start()
|
||||
|
||||
def UndeployGenObject(self, HTML, RAN, UE):
|
||||
self.exitStatus = 0
|
||||
# Implicitly we are running locally
|
||||
ymlPath = self.yamlPath[0].split('/')
|
||||
logPath = '../cmake_targets/log/' + ymlPath[1]
|
||||
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
||||
cmd = 'cp docker-compose.y*ml docker-compose-ci.yml'
|
||||
myCmd.run(cmd)
|
||||
|
||||
# check which containers are running for log recovery later
|
||||
cmd = 'docker-compose -f docker-compose-ci.yml ps --all'
|
||||
deployStatusLogs = myCmd.run(cmd, timeout=30)
|
||||
|
||||
# Stop the containers to shut down objects
|
||||
logging.debug('\u001B[1m Stopping containers \u001B[0m')
|
||||
cmd = 'docker-compose -f docker-compose-ci.yml stop -t3'
|
||||
deployStatus = myCmd.run(cmd, timeout=100)
|
||||
if deployStatus.returncode != 0:
|
||||
myCmd.close()
|
||||
self.exitStatus = 1
|
||||
logging.error('Could not stop containers')
|
||||
HTML.CreateHtmlTestRow('Could not stop', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
logging.error('\u001B[1m Undeploying OAI Object(s) FAILED\u001B[0m')
|
||||
return
|
||||
|
||||
anyLogs = False
|
||||
logging.debug('Working dir is now . (ie ci-scripts)')
|
||||
myCmd2 = cls_cmd.LocalCmd()
|
||||
myCmd2.run(f'mkdir -p {logPath}')
|
||||
myCmd2.cd(logPath)
|
||||
for state in deployStatusLogs.stdout.split('\n'):
|
||||
res = re.search('Name|NAME|----------', state)
|
||||
if res is not None:
|
||||
continue
|
||||
if len(state) == 0:
|
||||
continue
|
||||
res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
|
||||
if res is not None:
|
||||
anyLogs = True
|
||||
cName = res.group('container_name')
|
||||
cmd = f'docker logs {cName} > {cName}.log 2>&1'
|
||||
myCmd2.run(cmd, timeout=30, reportNonZero=False)
|
||||
if re.search('magma-mme', cName) is not None:
|
||||
cmd = f'docker cp -L {cName}:/var/log/mme.log {cName}-full.log'
|
||||
myCmd2.run(cmd, timeout=30, reportNonZero=False)
|
||||
fullStatus = True
|
||||
if anyLogs:
|
||||
# Analyzing log file(s)!
|
||||
listOfPossibleRanContainers = ['enb*', 'gnb*', 'cu*', 'du*']
|
||||
for container in listOfPossibleRanContainers:
|
||||
filenames = './*-oai-' + container + '.log'
|
||||
cmd = f'ls {filenames}'
|
||||
lsStatus = myCmd2.run(cmd, silent=True, reportNonZero=False)
|
||||
if lsStatus.returncode != 0:
|
||||
continue
|
||||
filenames = str(lsStatus.stdout).strip()
|
||||
|
||||
for filename in filenames.split('\n'):
|
||||
logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
|
||||
logStatus = RAN.AnalyzeLogFile_eNB(f'{logPath}/{filename}', HTML, self.ran_checkers)
|
||||
if (logStatus < 0):
|
||||
fullStatus = False
|
||||
self.exitStatus = 1
|
||||
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
|
||||
else:
|
||||
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
||||
|
||||
listOfPossibleUeContainers = ['lte-ue*', 'nr-ue*']
|
||||
for container in listOfPossibleUeContainers:
|
||||
filenames = './*-oai-' + container + '.log'
|
||||
cmd = f'ls {filenames}'
|
||||
lsStatus = myCmd2.run(cmd, silent=True, reportNonZero=False)
|
||||
if lsStatus.returncode != 0:
|
||||
continue
|
||||
filenames = str(lsStatus.stdout).strip()
|
||||
|
||||
for filename in filenames.split('\n'):
|
||||
logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
|
||||
logStatus = UE.AnalyzeLogFile_UE(f'{logPath}/{filename}', HTML, RAN)
|
||||
if (logStatus < 0):
|
||||
fullStatus = False
|
||||
self.exitStatus = 1
|
||||
HTML.CreateHtmlTestRow('UE log Analysis', 'KO', logStatus)
|
||||
else:
|
||||
HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
myCmd2.close()
|
||||
if self.tsharkStarted:
|
||||
self.tsharkStarted = StopPySharkCapture(ymlPath[1])
|
||||
logging.debug('\u001B[1m Undeploying \u001B[0m')
|
||||
logging.debug(f'Working dir is back {self.yamlPath[0]}')
|
||||
cmd = 'docker-compose -f docker-compose-ci.yml down -v'
|
||||
deployStatus = myCmd.run(cmd, timeout=100)
|
||||
if deployStatus.returncode != 0:
|
||||
myCmd.close()
|
||||
self.exitStatus = 1
|
||||
logging.error('Could not undeploy')
|
||||
HTML.CreateHtmlTestRow('Could not undeploy', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
logging.error('\u001B[1m Undeploying OAI Object(s) FAILED\u001B[0m')
|
||||
return
|
||||
|
||||
self.deployedContainers = []
|
||||
myCmd.close()
|
||||
|
||||
if fullStatus:
|
||||
HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m')
|
||||
else:
|
||||
HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
logging.error('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m')
|
||||
|
||||
def StatsFromGenObject(self, HTML):
|
||||
self.exitStatus = 0
|
||||
ymlPath = self.yamlPath[0].split('/')
|
||||
|
||||
Reference in New Issue
Block a user