Improved DeployObject cohesion
- Introduced helper functions: GetCredentials, GetContainerName, GetImageInfo, GetContainerHealth, ReTagImages, DeployServices.
This commit is contained in:
committed by
Raphael Defosseux
parent
ab1564aa1c
commit
bc950aaaec
+94
-351
@@ -171,6 +171,86 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus):
|
|||||||
collectInfo[image] = files
|
collectInfo[image] = files
|
||||||
return collectInfo
|
return collectInfo
|
||||||
|
|
||||||
|
def GetCredentials(instance):
|
||||||
|
server_id = instance.eNB_serverId[instance.eNB_instance]
|
||||||
|
if server_id == '0':
|
||||||
|
return (instance.eNBIPAddress, instance.eNBUserName, instance.eNBPassword, instance.eNBSourceCodePath)
|
||||||
|
elif server_id == '1':
|
||||||
|
return (instance.eNB1IPAddress, instance.eNB1UserName, instance.eNB1Password, instance.eNB1SourceCodePath)
|
||||||
|
elif server_id == '2':
|
||||||
|
return (instance.eNB2IPAddress, instance.eNB2UserName, instance.eNB2Password, instance.eNB2SourceCodePath)
|
||||||
|
else:
|
||||||
|
raise Exception ("Only supports maximum of 3 servers")
|
||||||
|
|
||||||
|
def GetContainerName(mySSH, svcName):
|
||||||
|
ret = mySSH.run(f"docker compose -f docker-compose.y*ml config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'")
|
||||||
|
containerName = ret.stdout
|
||||||
|
return containerName
|
||||||
|
|
||||||
|
def GetImageInfo(mySSH, containerName):
|
||||||
|
usedImage = ''
|
||||||
|
ret = mySSH.run('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName)
|
||||||
|
if ret.stdout.count('ImageUsed: oai-ci'):
|
||||||
|
usedImage = ret.stdout.replace('ImageUsed: oai-ci', 'oai-ci').strip()
|
||||||
|
logging.debug('Used image is ' + usedImage)
|
||||||
|
if usedImage != '':
|
||||||
|
ret = mySSH.run('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage)
|
||||||
|
imageInfo = f"Used image is {usedImage}\n{ret.stdout}"
|
||||||
|
return imageInfo
|
||||||
|
else:
|
||||||
|
return "Could not retrieve used image info!\n"
|
||||||
|
|
||||||
|
def GetContainerHealth(mySSH, containerName):
|
||||||
|
if containerName is None:
|
||||||
|
return 0, 0
|
||||||
|
unhealthyNb = 0
|
||||||
|
healthyNb = 0
|
||||||
|
time.sleep(5)
|
||||||
|
for _ in range(3):
|
||||||
|
if containerName != 'db_init':
|
||||||
|
result = mySSH.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}')
|
||||||
|
unhealthyNb = result.stdout.count('unhealthy')
|
||||||
|
healthyNb = result.stdout.count('healthy') - unhealthyNb
|
||||||
|
if healthyNb == 1:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
time.sleep(10)
|
||||||
|
else:
|
||||||
|
unhealthyNb = 0
|
||||||
|
healthyNb = 1
|
||||||
|
break
|
||||||
|
return healthyNb, unhealthyNb
|
||||||
|
|
||||||
|
def ReTagImages(mySSH,IMAGES,ranCommitID,ranBranch,ranAllowMerge,displayedNewTags):
|
||||||
|
mySSH.run('cp docker-compose.y*ml ci-docker-compose.yml', 5)
|
||||||
|
for image in IMAGES:
|
||||||
|
imageTag = ImageTagToUse(image, ranCommitID, ranBranch, ranAllowMerge)
|
||||||
|
if image == 'oai-gnb' or image == 'oai-nr-ue' or image == 'oai-nr-cuup':
|
||||||
|
ret = mySSH.run(f'docker image inspect oai-ci/{imageTag}', reportNonZero=False, silent=False)
|
||||||
|
if ret.returncode != 0:
|
||||||
|
imageTag = imageTag.replace('oai-gnb', 'oai-gnb-asan')
|
||||||
|
imageTag = imageTag.replace('oai-nr-ue', 'oai-nr-ue-asan')
|
||||||
|
imageTag = imageTag.replace('oai-nr-cuup', 'oai-nr-cuup-asan')
|
||||||
|
if not displayedNewTags:
|
||||||
|
logging.debug(f'\u001B[1m Using sanitized version of {image} with {imageTag}\u001B[0m')
|
||||||
|
mySSH.run(f'sed -i -e "s@oaisoftwarealliance/{image}:develop@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True)
|
||||||
|
mySSH.run(f'sed -i -e "s@{image}:latest@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True) # temporary solution for not using the new branch
|
||||||
|
|
||||||
|
def DeployServices(mySSH,svcName):
|
||||||
|
allServices = []
|
||||||
|
if svcName == '':
|
||||||
|
logging.warning('No service name given: starting all services in ci-docker-compose.yml!')
|
||||||
|
ret = mySSH.run(f'docker compose -f ci-docker-compose.yml config --services')
|
||||||
|
allServices = ret.stdout.splitlines()
|
||||||
|
deployStatus = mySSH.run(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', 50)
|
||||||
|
return deployStatus.returncode,allServices
|
||||||
|
|
||||||
|
def CopyFailedDeployLog(mySSH,lSourcePath,ymlPath,containerName,filename):
|
||||||
|
logging.warning(f'Deployment Failed, trying to copy the log from {filename}')
|
||||||
|
logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[2]}'
|
||||||
|
os.system(f'mkdir -p {logPath}')
|
||||||
|
mySSH.run(f'docker logs {containerName} > {lSourcePath}/cmake_targets/log/{filename} 2>&1')
|
||||||
|
mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', os.path.join(logPath, filename))
|
||||||
# pyshark livecapture launches 2 processes:
|
# pyshark livecapture launches 2 processes:
|
||||||
# * One using dumpcap -i lIfs -w - (ie redirecting the packets to STDOUT)
|
# * One using dumpcap -i lIfs -w - (ie redirecting the packets to STDOUT)
|
||||||
# * One using tshark -i - -w loFile (ie capturing from STDIN from previous process)
|
# * One using tshark -i - -w loFile (ie capturing from STDIN from previous process)
|
||||||
@@ -884,136 +964,38 @@ class Containerize():
|
|||||||
CreateWorkspace(sshSession, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
CreateWorkspace(sshSession, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||||
|
|
||||||
def DeployObject(self, HTML, EPC):
|
def DeployObject(self, HTML, EPC):
|
||||||
if self.eNB_serverId[self.eNB_instance] == '0':
|
lIpAddr, lUserName, lPassWord, lSourcePath = GetCredentials(self)
|
||||||
lIpAddr = self.eNBIPAddress
|
|
||||||
lUserName = self.eNBUserName
|
|
||||||
lPassWord = self.eNBPassword
|
|
||||||
lSourcePath = self.eNBSourceCodePath
|
|
||||||
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
||||||
lIpAddr = self.eNB1IPAddress
|
|
||||||
lUserName = self.eNB1UserName
|
|
||||||
lPassWord = self.eNB1Password
|
|
||||||
lSourcePath = self.eNB1SourceCodePath
|
|
||||||
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
||||||
lIpAddr = self.eNB2IPAddress
|
|
||||||
lUserName = self.eNB2UserName
|
|
||||||
lPassWord = self.eNB2Password
|
|
||||||
lSourcePath = self.eNB2SourceCodePath
|
|
||||||
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
||||||
HELP.GenericHelp(CONST.Version)
|
HELP.GenericHelp(CONST.Version)
|
||||||
sys.exit('Insufficient Parameter')
|
sys.exit('Insufficient Parameter')
|
||||||
logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
|
logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
|
||||||
self.deployKind[self.eNB_instance] = True
|
self.deployKind[self.eNB_instance] = True
|
||||||
|
mySSH = cls_cmd.getConnection(lIpAddr, f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}')
|
||||||
mySSH = cls_cmd.getConnection(lIpAddr)
|
ReTagImages(mySSH,IMAGES,self.ranCommitID, self.ranBranch, self.ranAllowMerge, self.displayedNewTags)
|
||||||
|
deployStatus,allServices = DeployServices(mySSH,self.services[self.eNB_instance])
|
||||||
mySSH.cd(lSourcePath + '/' + self.yamlPath[self.eNB_instance])
|
if deployStatus != 0:
|
||||||
mySSH.run('cp docker-compose.y*ml ci-docker-compose.yml', 5)
|
|
||||||
for image in IMAGES:
|
|
||||||
imageTag = ImageTagToUse(image, self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
|
||||||
if image == 'oai-gnb' or image == 'oai-nr-ue' or image == 'oai-nr-cuup':
|
|
||||||
ret = mySSH.run(f'docker image inspect oai-ci/{imageTag}', reportNonZero=False, silent=self.displayedNewTags)
|
|
||||||
if ret.returncode != 0:
|
|
||||||
imageTag = imageTag.replace('oai-gnb', 'oai-gnb-asan')
|
|
||||||
imageTag = imageTag.replace('oai-nr-ue', 'oai-nr-ue-asan')
|
|
||||||
imageTag = imageTag.replace('oai-nr-cuup', 'oai-nr-cuup-asan')
|
|
||||||
if not self.displayedNewTags:
|
|
||||||
logging.debug(f'\u001B[1m Using sanitized version of {image} with {imageTag}\u001B[0m')
|
|
||||||
mySSH.run(f'sed -i -e "s@oaisoftwarealliance/{image}:develop@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True)
|
|
||||||
mySSH.run(f'sed -i -e "s@{image}:latest@oai-ci/{imageTag}@" ci-docker-compose.yml',silent=True) # temporary solution for not using the new branch
|
|
||||||
|
|
||||||
svcName = self.services[self.eNB_instance]
|
|
||||||
if svcName == '':
|
|
||||||
logging.warning('no service name giv en: starting all services in ci-docker-compose.yml!')
|
|
||||||
ret = mySSH.run(f'docker compose -f {lSourcePath}/{self.yamlPath[self.eNB_instance]}/ci-docker-compose.yml config --services')
|
|
||||||
if ret.returncode != 0:
|
|
||||||
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services")
|
|
||||||
self.exitStatus = 1
|
|
||||||
return
|
|
||||||
allServices = ret.stdout.splitlines()
|
|
||||||
|
|
||||||
deployStatus = mySSH.run(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', 50)
|
|
||||||
if deployStatus.returncode != 0:
|
|
||||||
mySSH.close()
|
mySSH.close()
|
||||||
self.exitStatus = 1
|
self.exitStatus = 1
|
||||||
logging.error('Could not deploy')
|
logging.error('Could not deploy')
|
||||||
HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
|
HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
|
||||||
return
|
return
|
||||||
# Checking Status
|
services_list = allServices if self.services[self.eNB_instance].split() == [] else self.services[self.eNB_instance].split()
|
||||||
services_list = self.services[self.eNB_instance].split()
|
status = True
|
||||||
if services_list == []: services_list=allServices #handing empty services
|
|
||||||
for svcName in services_list:
|
for svcName in services_list:
|
||||||
logging.debug('current service in the service list: \033[4m%s\033[0m', svcName)
|
containerName = GetContainerName(mySSH, svcName)
|
||||||
mySSH.run(f"docker compose -f docker-compose.yaml config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'")
|
healthyNb,unhealthyNb = GetContainerHealth(mySSH,containerName)
|
||||||
result = mySSH.getBefore()
|
|
||||||
unhealthyNb = 0
|
|
||||||
healthyNb = 0
|
|
||||||
containerName = ''
|
|
||||||
usedImage = ''
|
|
||||||
imageInfo = ''
|
|
||||||
if result is not None:
|
|
||||||
containerName = result.group('container_name')
|
|
||||||
time.sleep(5)
|
|
||||||
cnt = 0
|
|
||||||
while (cnt < 3):
|
|
||||||
if svcName != 'db_init':
|
|
||||||
mySSH.run('docker inspect --format="{{.State.Health.Status}}" ' + containerName, 5)
|
|
||||||
unhealthyNb = mySSH.getBefore().count('unhealthy')
|
|
||||||
healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb
|
|
||||||
if healthyNb == 1:
|
|
||||||
cnt = 10
|
|
||||||
else:
|
|
||||||
time.sleep(10)
|
|
||||||
cnt += 1
|
|
||||||
else:
|
|
||||||
# faking health of db_init
|
|
||||||
unhealthyNb = 0
|
|
||||||
healthyNb = 1
|
|
||||||
break
|
|
||||||
mySSH.run('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName)
|
|
||||||
for stdoutLine in mySSH.getBefore().split('\n'):
|
|
||||||
if stdoutLine.count('ImageUsed: oai-ci'):
|
|
||||||
usedImage = stdoutLine.replace('ImageUsed: oai-ci', 'oai-ci').strip()
|
|
||||||
logging.debug('Used image is ' + usedImage)
|
|
||||||
if usedImage != '':
|
|
||||||
ret = mySSH.run('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage)
|
|
||||||
imageInfo = ret.stdout
|
|
||||||
logging.debug(f' -- for service {svcName} containers: {str(healthyNb)} healthy, {str(healthyNb)} unhealthy')
|
|
||||||
|
|
||||||
self.testCase_id = HTML.testCase_id
|
self.testCase_id = HTML.testCase_id
|
||||||
self.eNB_logFile[self.eNB_instance] = f'{svcName}-{self.testCase_id}.log'
|
self.eNB_logFile[self.eNB_instance] = f'{svcName}-{self.testCase_id}.log'
|
||||||
|
if unhealthyNb:
|
||||||
|
CopyFailedDeployLog(mySSH,lSourcePath,self.yamlPath[0].split('/'),containerName,self.eNB_logFile[self.eNB_instance])
|
||||||
status = False
|
status = False
|
||||||
if healthyNb == 1:
|
logging.debug(GetImageInfo(mySSH, containerName))
|
||||||
status = True
|
|
||||||
logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m')
|
|
||||||
else:
|
|
||||||
# containers are unhealthy, so we won't start. However, logs are stored at the end
|
|
||||||
# in UndeployObject so we here store the logs of the unhealthy container to report it
|
|
||||||
logfilename = f'{lSourcePath}/cmake_targets/log/{self.eNB_logFile[self.eNB_instance]}'
|
|
||||||
mySSH.run(f'docker logs {containerName} > {logfilename}', 30)
|
|
||||||
cwd = os.getcwd()
|
|
||||||
logging.warning(f'Deployment Failed, trying to copy the log from {logfilename} at {cwd}')
|
|
||||||
mySSH.copyin(logfilename,cwd,True)
|
|
||||||
|
|
||||||
mySSH.close()
|
mySSH.close()
|
||||||
|
|
||||||
message = ''
|
|
||||||
if usedImage != '':
|
|
||||||
message += f'Used Image = {usedImage} :\n'
|
|
||||||
message += imageInfo
|
|
||||||
else:
|
|
||||||
message += 'Could not retrieve used image info!\n'
|
|
||||||
if status:
|
if status:
|
||||||
message += '\nHealthy deployment!\n'
|
HTML.CreateHtmlTestRowQueue('N/A', '0K', '\nHealthy deployment!\n')
|
||||||
else:
|
|
||||||
message += '\nUnhealthy deployment! -- Check logs for reason!\n'
|
|
||||||
if status:
|
|
||||||
HTML.CreateHtmlTestRowQueue('N/A', 'OK', [message])
|
|
||||||
else:
|
else:
|
||||||
self.exitStatus = 1
|
self.exitStatus = 1
|
||||||
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [message])
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', '\nUnhealthy deployment! -- Check logs for reason!\n')
|
||||||
|
|
||||||
|
|
||||||
def UndeployObject(self, HTML, RAN):
|
def UndeployObject(self, HTML, RAN):
|
||||||
if self.eNB_serverId[self.eNB_instance] == '0':
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
||||||
@@ -1111,135 +1093,6 @@ class Containerize():
|
|||||||
logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
|
logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
|
||||||
mySSH.close()
|
mySSH.close()
|
||||||
|
|
||||||
def DeployGenObject(self, HTML, RAN, UE):
|
|
||||||
self.exitStatus = 0
|
|
||||||
logging.debug('\u001B[1m Checking Services to deploy\u001B[0m')
|
|
||||||
# Implicitly we are running locally
|
|
||||||
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
|
||||||
self.deployKind[0] = False
|
|
||||||
cmd = 'docker-compose config --services'
|
|
||||||
listServices = myCmd.run(cmd)
|
|
||||||
if listServices.returncode != 0:
|
|
||||||
myCmd.close()
|
|
||||||
self.exitStatus = 1
|
|
||||||
HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
return
|
|
||||||
for reqSvc in self.services[0].split(' '):
|
|
||||||
res = re.search(reqSvc, listServices.stdout)
|
|
||||||
if res is None:
|
|
||||||
logging.error(reqSvc + ' not found in specified docker-compose')
|
|
||||||
self.exitStatus = 1
|
|
||||||
if (self.exitStatus == 1):
|
|
||||||
myCmd.close()
|
|
||||||
HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
return
|
|
||||||
cmd = 'cp docker-compose.y*ml docker-compose-ci.yml'
|
|
||||||
myCmd.run(cmd, silent=self.displayedNewTags)
|
|
||||||
imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru', 'oai-nr-cuup']
|
|
||||||
for image in imageNames:
|
|
||||||
tagToUse = ImageTagToUse(image, self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
|
||||||
# In the scenario, for 5G images, we have the choice of either pulling normal images
|
|
||||||
# or -asan images. We need to detect which kind we did pull.
|
|
||||||
if image == 'oai-gnb' or image == 'oai-nr-ue' or image == 'oai-nr-cuup':
|
|
||||||
ret = myCmd.run(f'docker image inspect oai-ci/{tagToUse}', reportNonZero=False, silent=self.displayedNewTags)
|
|
||||||
if ret.returncode != 0:
|
|
||||||
tagToUse = tagToUse.replace('oai-gnb', 'oai-gnb-asan')
|
|
||||||
tagToUse = tagToUse.replace('oai-nr-ue', 'oai-nr-ue-asan')
|
|
||||||
tagToUse = tagToUse.replace('oai-nr-cuup', 'oai-nr-cuup-asan')
|
|
||||||
if not self.displayedNewTags:
|
|
||||||
logging.debug(f'\u001B[1m Using sanitized version of {image} with {tagToUse}\u001B[0m')
|
|
||||||
cmd = f'sed -i -e "s@oaisoftwarealliance/{image}:develop@oai-ci/{tagToUse}@" docker-compose-ci.yml'
|
|
||||||
myCmd.run(cmd, silent=self.displayedNewTags)
|
|
||||||
self.displayedNewTags = True
|
|
||||||
|
|
||||||
cmd = f'docker-compose -f docker-compose-ci.yml up -d {self.services[0]}'
|
|
||||||
deployStatus = myCmd.run(cmd, timeout=100)
|
|
||||||
if deployStatus.returncode != 0:
|
|
||||||
myCmd.close()
|
|
||||||
self.exitStatus = 1
|
|
||||||
logging.error('Could not deploy')
|
|
||||||
HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
return
|
|
||||||
|
|
||||||
logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m')
|
|
||||||
cmd = 'docker-compose -f docker-compose-ci.yml ps -a'
|
|
||||||
count = 0
|
|
||||||
healthy = 0
|
|
||||||
restarting = 0
|
|
||||||
newContainers = []
|
|
||||||
while (count < 10):
|
|
||||||
count += 1
|
|
||||||
containerStatus = []
|
|
||||||
deployStatus = myCmd.run(cmd, 30, silent=True)
|
|
||||||
healthy = 0
|
|
||||||
restarting = 0
|
|
||||||
for state in deployStatus.stdout.split('\n'):
|
|
||||||
state = state.strip()
|
|
||||||
res = re.search('Name|NAME|----------', state)
|
|
||||||
if res is not None:
|
|
||||||
continue
|
|
||||||
if len(state) == 0:
|
|
||||||
continue
|
|
||||||
res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
|
|
||||||
if res is not None:
|
|
||||||
cName = res.group('container_name')
|
|
||||||
found = False
|
|
||||||
for alreadyDeployed in self.deployedContainers:
|
|
||||||
if cName == alreadyDeployed:
|
|
||||||
found = True
|
|
||||||
if not found:
|
|
||||||
newContainers.append(cName)
|
|
||||||
self.deployedContainers.append(cName)
|
|
||||||
if re.search('\(healthy\)', state) is not None:
|
|
||||||
healthy += 1
|
|
||||||
if re.search('rfsim4g-db-init.*Exit 0', state) is not None or re.search('rfsim4g-db-init.*Exited \(0\)', state) is not None:
|
|
||||||
myCmd.run('docker rm -f rfsim4g-db-init', timeout=30, silent=True, reportNonZero=False)
|
|
||||||
if re.search('l2sim4g-db-init.*Exit 0', state) is not None or re.search('l2sim4g-db-init.*Exited \(0\)', state) is not None:
|
|
||||||
myCmd.run('docker rm -f l2sim4g-db-init', timeout=30, silent=True, reportNonZero=False)
|
|
||||||
if re.search('Restarting', state) is None:
|
|
||||||
containerStatus.append(state)
|
|
||||||
else:
|
|
||||||
restarting += 1
|
|
||||||
if healthy == self.nb_healthy[0] and restarting == 0:
|
|
||||||
count = 100
|
|
||||||
else:
|
|
||||||
time.sleep(10)
|
|
||||||
|
|
||||||
html_cell = ''
|
|
||||||
for newCont in newContainers:
|
|
||||||
if newCont == 'rfsim4g-db-init':
|
|
||||||
continue
|
|
||||||
cmd = 'docker inspect -f "{{.Config.Image}}" ' + newCont
|
|
||||||
imageInspect = myCmd.run(cmd, timeout=30, silent=True)
|
|
||||||
imageName = str(imageInspect.stdout).strip()
|
|
||||||
cmd = 'docker image inspect --format \'{{.RepoTags}}\t{{.Size}} bytes\t{{index (split .Created ".") 0}}\n{{.Id}}\' ' + imageName
|
|
||||||
imageInspect = myCmd.run(cmd, 30, silent=True)
|
|
||||||
html_cell += imageInspect.stdout + '\n'
|
|
||||||
myCmd.close()
|
|
||||||
|
|
||||||
for cState in containerStatus:
|
|
||||||
html_cell += cState + '\n'
|
|
||||||
if count == 100 and healthy == self.nb_healthy[0]:
|
|
||||||
if self.tsharkStarted == False:
|
|
||||||
logging.debug('Starting tshark on public network')
|
|
||||||
self.CaptureOnDockerNetworks()
|
|
||||||
HTML.CreateHtmlTestRowQueue('n/a', 'OK', [html_cell])
|
|
||||||
for cState in containerStatus:
|
|
||||||
logging.debug(cState)
|
|
||||||
logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m')
|
|
||||||
else:
|
|
||||||
HTML.CreateHtmlTestRowQueue('Could not deploy in time', 'KO', [html_cell])
|
|
||||||
for cState in containerStatus:
|
|
||||||
logging.debug(cState)
|
|
||||||
logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m')
|
|
||||||
HTML.testCase_id = 'AUTO-UNDEPLOY'
|
|
||||||
UE.testCase_id = 'AUTO-UNDEPLOY'
|
|
||||||
HTML.desc = 'Automatic Undeployment'
|
|
||||||
UE.desc = 'Automatic Undeployment'
|
|
||||||
UE.ShowTestID()
|
|
||||||
self.UndeployGenObject(HTML, RAN, UE)
|
|
||||||
self.exitStatus = 1
|
|
||||||
|
|
||||||
def CaptureOnDockerNetworks(self):
|
def CaptureOnDockerNetworks(self):
|
||||||
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
||||||
cmd = 'docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
|
cmd = 'docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
|
||||||
@@ -1268,116 +1121,6 @@ class Containerize():
|
|||||||
x.daemon = True
|
x.daemon = True
|
||||||
x.start()
|
x.start()
|
||||||
|
|
||||||
def UndeployGenObject(self, HTML, RAN, UE):
|
|
||||||
self.exitStatus = 0
|
|
||||||
# Implicitly we are running locally
|
|
||||||
ymlPath = self.yamlPath[0].split('/')
|
|
||||||
logPath = '../cmake_targets/log/' + ymlPath[1]
|
|
||||||
myCmd = cls_cmd.LocalCmd(d = self.yamlPath[0])
|
|
||||||
cmd = 'cp docker-compose.y*ml docker-compose-ci.yml'
|
|
||||||
myCmd.run(cmd)
|
|
||||||
|
|
||||||
# check which containers are running for log recovery later
|
|
||||||
cmd = 'docker-compose -f docker-compose-ci.yml ps --all'
|
|
||||||
deployStatusLogs = myCmd.run(cmd, timeout=30)
|
|
||||||
|
|
||||||
# Stop the containers to shut down objects
|
|
||||||
logging.debug('\u001B[1m Stopping containers \u001B[0m')
|
|
||||||
cmd = 'docker-compose -f docker-compose-ci.yml stop -t3'
|
|
||||||
deployStatus = myCmd.run(cmd, timeout=100)
|
|
||||||
if deployStatus.returncode != 0:
|
|
||||||
myCmd.close()
|
|
||||||
self.exitStatus = 1
|
|
||||||
logging.error('Could not stop containers')
|
|
||||||
HTML.CreateHtmlTestRow('Could not stop', 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
logging.error('\u001B[1m Undeploying OAI Object(s) FAILED\u001B[0m')
|
|
||||||
return
|
|
||||||
|
|
||||||
anyLogs = False
|
|
||||||
logging.debug('Working dir is now . (ie ci-scripts)')
|
|
||||||
myCmd2 = cls_cmd.LocalCmd()
|
|
||||||
myCmd2.run(f'mkdir -p {logPath}')
|
|
||||||
myCmd2.cd(logPath)
|
|
||||||
for state in deployStatusLogs.stdout.split('\n'):
|
|
||||||
res = re.search('Name|NAME|----------', state)
|
|
||||||
if res is not None:
|
|
||||||
continue
|
|
||||||
if len(state) == 0:
|
|
||||||
continue
|
|
||||||
res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
|
|
||||||
if res is not None:
|
|
||||||
anyLogs = True
|
|
||||||
cName = res.group('container_name')
|
|
||||||
cmd = f'docker logs {cName} > {cName}.log 2>&1'
|
|
||||||
myCmd2.run(cmd, timeout=30, reportNonZero=False)
|
|
||||||
if re.search('magma-mme', cName) is not None:
|
|
||||||
cmd = f'docker cp -L {cName}:/var/log/mme.log {cName}-full.log'
|
|
||||||
myCmd2.run(cmd, timeout=30, reportNonZero=False)
|
|
||||||
fullStatus = True
|
|
||||||
if anyLogs:
|
|
||||||
# Analyzing log file(s)!
|
|
||||||
listOfPossibleRanContainers = ['enb*', 'gnb*', 'cu*', 'du*']
|
|
||||||
for container in listOfPossibleRanContainers:
|
|
||||||
filenames = './*-oai-' + container + '.log'
|
|
||||||
cmd = f'ls {filenames}'
|
|
||||||
lsStatus = myCmd2.run(cmd, silent=True, reportNonZero=False)
|
|
||||||
if lsStatus.returncode != 0:
|
|
||||||
continue
|
|
||||||
filenames = str(lsStatus.stdout).strip()
|
|
||||||
|
|
||||||
for filename in filenames.split('\n'):
|
|
||||||
logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
|
|
||||||
logStatus = RAN.AnalyzeLogFile_eNB(f'{logPath}/{filename}', HTML, self.ran_checkers)
|
|
||||||
if (logStatus < 0):
|
|
||||||
fullStatus = False
|
|
||||||
self.exitStatus = 1
|
|
||||||
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
|
|
||||||
else:
|
|
||||||
HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
||||||
|
|
||||||
listOfPossibleUeContainers = ['lte-ue*', 'nr-ue*']
|
|
||||||
for container in listOfPossibleUeContainers:
|
|
||||||
filenames = './*-oai-' + container + '.log'
|
|
||||||
cmd = f'ls {filenames}'
|
|
||||||
lsStatus = myCmd2.run(cmd, silent=True, reportNonZero=False)
|
|
||||||
if lsStatus.returncode != 0:
|
|
||||||
continue
|
|
||||||
filenames = str(lsStatus.stdout).strip()
|
|
||||||
|
|
||||||
for filename in filenames.split('\n'):
|
|
||||||
logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
|
|
||||||
logStatus = UE.AnalyzeLogFile_UE(f'{logPath}/{filename}', HTML, RAN)
|
|
||||||
if (logStatus < 0):
|
|
||||||
fullStatus = False
|
|
||||||
self.exitStatus = 1
|
|
||||||
HTML.CreateHtmlTestRow('UE log Analysis', 'KO', logStatus)
|
|
||||||
else:
|
|
||||||
HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK)
|
|
||||||
myCmd2.close()
|
|
||||||
if self.tsharkStarted:
|
|
||||||
self.tsharkStarted = StopPySharkCapture(ymlPath[1])
|
|
||||||
logging.debug('\u001B[1m Undeploying \u001B[0m')
|
|
||||||
logging.debug(f'Working dir is back {self.yamlPath[0]}')
|
|
||||||
cmd = 'docker-compose -f docker-compose-ci.yml down -v'
|
|
||||||
deployStatus = myCmd.run(cmd, timeout=100)
|
|
||||||
if deployStatus.returncode != 0:
|
|
||||||
myCmd.close()
|
|
||||||
self.exitStatus = 1
|
|
||||||
logging.error('Could not undeploy')
|
|
||||||
HTML.CreateHtmlTestRow('Could not undeploy', 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
logging.error('\u001B[1m Undeploying OAI Object(s) FAILED\u001B[0m')
|
|
||||||
return
|
|
||||||
|
|
||||||
self.deployedContainers = []
|
|
||||||
myCmd.close()
|
|
||||||
|
|
||||||
if fullStatus:
|
|
||||||
HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
|
|
||||||
logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m')
|
|
||||||
else:
|
|
||||||
HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK)
|
|
||||||
logging.error('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m')
|
|
||||||
|
|
||||||
def StatsFromGenObject(self, HTML):
|
def StatsFromGenObject(self, HTML):
|
||||||
self.exitStatus = 0
|
self.exitStatus = 0
|
||||||
ymlPath = self.yamlPath[0].split('/')
|
ymlPath = self.yamlPath[0].split('/')
|
||||||
|
|||||||
Reference in New Issue
Block a user