Robert Schmidt
5201a5c37a
This reimplements the CI deployment functions, and makes them a bit more robust. Concretely: - Introduce a generic "deployment tag" option that can be used to override the tag to use for specific images. By default, it is the current "branch-commitID[0:8]" tag name (requires the change from function ImageTagToUse() to CreateTag(), impacting also pull/push image functions) - Avoid sed for image names, and use an .env file that docker-compose picks up automatically; the deployment analyzes a potentially existing .env file and updates instead of simply overriding. For instance, some pipelines might use -asan images for the gNB and "normal" (non-asan) images for UE, and a simple overwriting might make the -asan image name tag disappear for the gNB, resulting in deployment failures). Finally, undeployment removes the .env file, so that there are no modifications in the repository after undeployment. - Redo the previous behavior of autodetecting asan, and use that (the current function always uses asan, no matter what) - Remove deployKind/displayedNewTags globals, as they are not necessary - Make the usedImage output in HTML slimmer - On undeployment, print service names next to undeploy analysis, and return success/failure. - Make the functions generally less verbose and easier to read Note that as of and only in this commit, deployment does not work, as all the YAML files have not been updated to work with this updated version. The next commit adds tests for the new deployment, and updates one YAML file (also being used in the tests). The follow-up commit then modifies all YAML files.
1213 lines
53 KiB
Python
1213 lines
53 KiB
Python
#/*
|
|
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
|
# * contributor license agreements. See the NOTICE file distributed with
|
|
# * this work for additional information regarding copyright ownership.
|
|
# * The OpenAirInterface Software Alliance licenses this file to You under
|
|
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
|
# * except in compliance with the License.
|
|
# * You may obtain a copy of the License at
|
|
# *
|
|
# * http://www.openairinterface.org/?page_id=698
|
|
# *
|
|
# * Unless required by applicable law or agreed to in writing, software
|
|
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# * See the License for the specific language governing permissions and
|
|
# * limitations under the License.
|
|
# *-------------------------------------------------------------------------------
|
|
# * For more information about the OpenAirInterface (OAI) Software Alliance:
|
|
# * contact@openairinterface.org
|
|
# */
|
|
#---------------------------------------------------------------------
|
|
# Python for CI of OAI-eNB + COTS-UE
|
|
#
|
|
# Required Python Version
|
|
# Python 3.x
|
|
#
|
|
# Required Python Package
|
|
# pexpect
|
|
#---------------------------------------------------------------------
|
|
|
|
#-----------------------------------------------------------
|
|
# Import
|
|
#-----------------------------------------------------------
|
|
import sys # arg
|
|
import re # reg
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import pyshark
|
|
import threading
|
|
import cls_cmd
|
|
from multiprocessing import Process, Lock, SimpleQueue
|
|
from zipfile import ZipFile
|
|
|
|
#-----------------------------------------------------------
|
|
# OAI Testing modules
|
|
#-----------------------------------------------------------
|
|
import cls_cluster as OC
|
|
import cls_cmd
|
|
import sshconnection as SSH
|
|
import helpreadme as HELP
|
|
import constants as CONST
|
|
import cls_oaicitest
|
|
|
|
#-----------------------------------------------------------
|
|
# Helper functions used here and in other classes
|
|
# (e.g., cls_cluster.py)
|
|
#-----------------------------------------------------------
|
|
IMAGES = ['oai-enb', 'oai-lte-ru', 'oai-lte-ue', 'oai-gnb', 'oai-nr-cuup', 'oai-gnb-aw2s', 'oai-nr-ue', 'oai-gnb-asan', 'oai-nr-ue-asan', 'oai-nr-cuup-asan', 'oai-gnb-aerial', 'oai-gnb-fhi72']
|
|
|
|
def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTargetBranch, ranAllowMerge):
|
|
if ranCommitID == '':
|
|
logging.error('need ranCommitID in CreateWorkspace()')
|
|
sys.exit('Insufficient Parameter in CreateWorkspace()')
|
|
|
|
sshSession.command(f'rm -rf {sourcePath}', '\$', 10)
|
|
sshSession.command('mkdir -p ' + sourcePath, '\$', 5)
|
|
(sshSession.cd(sourcePath) if isinstance(sshSession, cls_cmd.Cmd) else sshSession.command('cd ' + sourcePath, '\$', 5))
|
|
# Recent version of git (>2.20?) should handle missing .git extension # without problems
|
|
if ranTargetBranch == 'null':
|
|
ranTargetBranch = 'develop'
|
|
baseBranch = re.sub('origin/', '', ranTargetBranch)
|
|
sshSession.command(f'git clone --filter=blob:none -n -b {baseBranch} {ranRepository} .', '\$', 60)
|
|
if sshSession.getBefore().count('error') > 0 or sshSession.getBefore().count('error') > 0:
|
|
sys.exit('error during clone')
|
|
sshSession.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
|
sshSession.command('git config user.name "OAI Jenkins"', '\$', 5)
|
|
|
|
sshSession.command('mkdir -p cmake_targets/log', '\$', 5)
|
|
# if the commit ID is provided use it to point to it
|
|
sshSession.command(f'git checkout -f {ranCommitID}', '\$', 30)
|
|
if sshSession.getBefore().count(f'HEAD is now at {ranCommitID[:6]}') != 1:
|
|
sshSession.command('git log --oneline | head -n5', '\$', 5)
|
|
logging.error(f'problems during checkout, is at: {sshSession.getBefore()}')
|
|
return False
|
|
else:
|
|
logging.debug('successful checkout')
|
|
# if the branch is not develop, then it is a merge request and we need to do
|
|
# the potential merge. Note that merge conflicts should already been checked earlier
|
|
if ranAllowMerge:
|
|
if ranTargetBranch == '':
|
|
ranTargetBranch = 'develop'
|
|
logging.debug(f'Merging with the target branch: {ranTargetBranch}')
|
|
sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30)
|
|
return True
|
|
|
|
def CreateTag(ranCommitID, ranBranch, ranAllowMerge):
|
|
shortCommit = ranCommitID[0:8]
|
|
if ranAllowMerge:
|
|
# Allowing contributor to have a name/branchName format
|
|
branchName = ranBranch.replace('/','-')
|
|
tagToUse = f'{branchName}-{shortCommit}'
|
|
else:
|
|
tagToUse = f'develop-{shortCommit}'
|
|
return tagToUse
|
|
|
|
def CopyLogsToExecutor(cmd, sourcePath, log_name):
|
|
cmd.cd(f'{sourcePath}/cmake_targets')
|
|
cmd.run(f'rm -f {log_name}.zip')
|
|
cmd.run(f'mkdir -p {log_name}')
|
|
cmd.run(f'mv log/* {log_name}')
|
|
cmd.run(f'zip -r -qq {log_name}.zip {log_name}')
|
|
|
|
# copy zip to executor for analysis
|
|
if (os.path.isfile(f'./{log_name}.zip')):
|
|
os.remove(f'./{log_name}.zip')
|
|
if (os.path.isdir(f'./{log_name}')):
|
|
shutil.rmtree(f'./{log_name}')
|
|
cmd.copyin(f'{sourcePath}/cmake_targets/{log_name}.zip', f'./{log_name}.zip')
|
|
cmd.run(f'rm -f {log_name}.zip')
|
|
ZipFile(f'{log_name}.zip').extractall('.')
|
|
|
|
def AnalyzeBuildLogs(buildRoot, images, globalStatus):
|
|
collectInfo = {}
|
|
for image in images:
|
|
files = {}
|
|
file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')]
|
|
# Analyze the "sub-logs" of every target image
|
|
for fil in file_list:
|
|
errorandwarnings = {}
|
|
warningsNo = 0
|
|
errorsNo = 0
|
|
with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile:
|
|
for line in inputfile:
|
|
result = re.search(' ERROR ', str(line))
|
|
if result is not None:
|
|
errorsNo += 1
|
|
result = re.search(' error:', str(line))
|
|
if result is not None:
|
|
errorsNo += 1
|
|
result = re.search(' WARNING ', str(line))
|
|
if result is not None:
|
|
warningsNo += 1
|
|
result = re.search(' warning:', str(line))
|
|
if result is not None:
|
|
warningsNo += 1
|
|
errorandwarnings['errors'] = errorsNo
|
|
errorandwarnings['warnings'] = warningsNo
|
|
errorandwarnings['status'] = globalStatus
|
|
files[fil] = errorandwarnings
|
|
# Analyze the target image
|
|
if os.path.isfile(f'{buildRoot}/{image}.log'):
|
|
errorandwarnings = {}
|
|
committed = False
|
|
tagged = False
|
|
with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
|
|
for line in inputfile:
|
|
lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
|
|
lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
|
|
tagged = tagged or lineHasTag or lineHasTag2
|
|
# the OpenShift Cluster builder prepends image registry URL
|
|
lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
|
|
committed = committed or lineHasCommit
|
|
errorandwarnings['errors'] = 0 if committed or tagged else 1
|
|
errorandwarnings['warnings'] = 0
|
|
errorandwarnings['status'] = committed or tagged
|
|
files['Target Image Creation'] = errorandwarnings
|
|
collectInfo[image] = files
|
|
return collectInfo
|
|
|
|
def GetCredentials(instance):
|
|
server_id = instance.eNB_serverId[instance.eNB_instance]
|
|
if server_id == '0':
|
|
return (instance.eNBIPAddress, instance.eNBUserName, instance.eNBPassword, instance.eNBSourceCodePath)
|
|
elif server_id == '1':
|
|
return (instance.eNB1IPAddress, instance.eNB1UserName, instance.eNB1Password, instance.eNB1SourceCodePath)
|
|
elif server_id == '2':
|
|
return (instance.eNB2IPAddress, instance.eNB2UserName, instance.eNB2Password, instance.eNB2SourceCodePath)
|
|
else:
|
|
raise Exception ("Only supports maximum of 3 servers")
|
|
|
|
def GetContainerName(ssh, svcName, file):
|
|
ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'", silent=True)
|
|
return ret.stdout
|
|
|
|
def GetImageName(ssh, svcName, file):
|
|
ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".image'", silent=True)
|
|
if ret.returncode != 0:
|
|
return f"cannot retrieve image info for {containerName}: {ret.stdout}"
|
|
else:
|
|
return ret.stdout.strip()
|
|
|
|
def GetContainerHealth(ssh, containerName):
|
|
if containerName is None:
|
|
return False
|
|
if 'db_init' in containerName or 'db-init' in containerName: # exits with 0, there cannot be healthy
|
|
return True
|
|
time.sleep(5)
|
|
for _ in range(3):
|
|
result = ssh.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}', silent=True)
|
|
if result.stdout == 'healthy':
|
|
return True
|
|
time.sleep(10)
|
|
return False
|
|
|
|
def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'):
|
|
ret = ssh.run(f'cat {wd}/.env', silent=True)
|
|
if ret.returncode != 0:
|
|
return False
|
|
env_vars = ret.stdout.strip().splitlines()
|
|
logging.info(f'{prompt} {wd}/.env: {env_vars}')
|
|
return True
|
|
|
|
def WriteEnvFile(ssh, services, wd, tag):
|
|
ret = ssh.run(f'cat {wd}/.env', silent=True)
|
|
registry = "oai-ci" # pull_images() gives us this registry path
|
|
envs = {"REGISTRY":registry, "TAG": tag}
|
|
if ret.returncode == 0: # it exists, we have to update
|
|
# transforms env file to dictionary
|
|
old_envs = {}
|
|
for l in ret.stdout.strip().splitlines():
|
|
var, val = l.split('=', 1)
|
|
old_envs[var] = val.strip('"')
|
|
# will retain the old environment variables
|
|
envs = {**envs, **old_envs}
|
|
for svc in services.split():
|
|
# In some scenarios we have the choice of either pulling normal images
|
|
# or -asan images. We need to detect which kind we did pull.
|
|
fullImageName = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml")
|
|
image = fullImageName.split("/")[-1].split(":")[0]
|
|
checkimg = f"{registry}/{image}-asan:{tag}"
|
|
ret = ssh.run(f'docker image inspect {checkimg}', reportNonZero=False)
|
|
if ret.returncode == 0:
|
|
logging.info(f"detected pulled image {checkimg}")
|
|
if "oai-gnb" in image: envs["GNB_IMG"] = "oai-gnb-asan"
|
|
elif "oai-nr-ue" in image: envs["NRUE_IMG"] = "oai-nr-ue-asan"
|
|
elif "oai-nr-cuup" in image: envs["NRCUUP_IMG"] = "oai-nr-cuup-asan"
|
|
else: logging.warning("undetected image format {image}, cannot use asan")
|
|
env_string = "\n".join([f"{var}=\"{val}\"" for var,val in envs.items()])
|
|
ssh.run(f'echo -e \'{env_string}\' > {wd}/.env', silent=True)
|
|
ExistEnvFilePrint(ssh, wd, prompt='New env vars in file')
|
|
|
|
def GetServices(ssh, requested, file):
|
|
if requested == [] or requested is None or requested == "":
|
|
logging.warning('No service name given: starting all services in docker-compose.yml!')
|
|
ret = ssh.run(f'docker compose -f {file} config --services')
|
|
if ret.returncode != 0:
|
|
return ""
|
|
else:
|
|
return ' '.join(ret.stdout.splitlines())
|
|
else:
|
|
return requested
|
|
|
|
def CopyinContainerLog(ssh, lSourcePath, yaml, containerName, filename):
|
|
remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}"
|
|
ssh.run(f'docker logs {containerName} &> {remote_filename}')
|
|
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}"
|
|
os.system(f'mkdir -p {local_dir}')
|
|
local_filename = f"{local_dir}/{filename}"
|
|
return ssh.copyin(remote_filename, local_filename)
|
|
|
|
def GetRunningServices(ssh, file):
|
|
ret = ssh.run(f'docker compose -f {file} config --services')
|
|
if ret.returncode != 0:
|
|
return None
|
|
allServices = ret.stdout.splitlines()
|
|
running_services = []
|
|
for s in allServices:
|
|
# outputs the hash if the container is running
|
|
ret = ssh.run(f'docker compose -f {file} ps --all --quiet -- {s}')
|
|
if ret.returncode != 0:
|
|
logging.info(f"service {s}: {ret.stdout}")
|
|
elif ret.stdout == "":
|
|
logging.warning(f"could not retrieve information for service {s}")
|
|
else:
|
|
c = ret.stdout
|
|
logging.debug(f'running service {s} with container id {c}')
|
|
running_services.append((s, c))
|
|
logging.info(f'stopping services: {running_services}')
|
|
return running_services
|
|
|
|
def CheckLogs(self, yaml, service_name, HTML, RAN):
|
|
logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}'
|
|
filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log'
|
|
success = True
|
|
if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
|
|
logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
|
|
logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN)
|
|
opt = f"UE log analysis for service {service_name}"
|
|
# usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files
|
|
# abuse HTML to store their reports, and we here want to put custom options,
|
|
# which is not possible with CreateHtmlTestRow
|
|
# solution: use HTML templates, where we don't need different HTML write funcs
|
|
if (logStatus < 0):
|
|
HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmlUEFailureMsg])
|
|
success = False
|
|
else:
|
|
HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg])
|
|
HTML.htmlUEFailureMsg = ""
|
|
elif service_name == 'nv-cubb':
|
|
msg = 'Undeploy PNF/Nvidia CUBB'
|
|
HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK)
|
|
elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])):
|
|
logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m')
|
|
logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
|
|
opt = f"xNB log analysis for service {service_name}"
|
|
if (logStatus < 0):
|
|
HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg])
|
|
success = False
|
|
else:
|
|
HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg])
|
|
HTML.htmlUEFailureMsg = ""
|
|
else:
|
|
logging.info(f'Skipping to analyze log for service name {service_name}')
|
|
HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"])
|
|
logging.debug(f"log check: service {service_name} passed analysis {success}")
|
|
return success
|
|
|
|
#-----------------------------------------------------------
|
|
# Class Declaration
|
|
#-----------------------------------------------------------
|
|
class Containerize():
|
|
|
|
def __init__(self):
|
|
|
|
self.ranRepository = ''
|
|
self.ranBranch = ''
|
|
self.ranAllowMerge = False
|
|
self.ranCommitID = ''
|
|
self.ranTargetBranch = ''
|
|
self.eNBIPAddress = ''
|
|
self.eNBUserName = ''
|
|
self.eNBPassword = ''
|
|
self.eNBSourceCodePath = ''
|
|
self.eNB1IPAddress = ''
|
|
self.eNB1UserName = ''
|
|
self.eNB1Password = ''
|
|
self.eNB1SourceCodePath = ''
|
|
self.eNB2IPAddress = ''
|
|
self.eNB2UserName = ''
|
|
self.eNB2Password = ''
|
|
self.eNB2SourceCodePath = ''
|
|
self.forcedWorkspaceCleanup = False
|
|
self.imageKind = ''
|
|
self.proxyCommit = None
|
|
self.eNB_instance = 0
|
|
self.eNB_serverId = ['', '', '']
|
|
self.yamlPath = ['', '', '']
|
|
self.services = ['', '', '']
|
|
self.deploymentTag = ''
|
|
self.eNB_logFile = ['', '', '']
|
|
|
|
self.testCase_id = ''
|
|
|
|
self.cli = ''
|
|
self.cliBuildOptions = ''
|
|
self.dockerfileprefix = ''
|
|
self.host = ''
|
|
|
|
self.deployedContainers = []
|
|
self.tsharkStarted = False
|
|
self.pingContName = ''
|
|
self.pingOptions = ''
|
|
self.pingLossThreshold = ''
|
|
self.svrContName = ''
|
|
self.svrOptions = ''
|
|
self.cliContName = ''
|
|
self.cliOptions = ''
|
|
|
|
self.imageToCopy = ''
|
|
self.registrySvrId = ''
|
|
self.testSvrId = ''
|
|
self.imageToPull = []
|
|
#checkers from xml
|
|
self.ran_checkers={}
|
|
|
|
#-----------------------------------------------------------
|
|
# Container management functions
|
|
#-----------------------------------------------------------
|
|
|
|
def BuildImage(self, HTML):
|
|
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
logging.debug('Building on server: ' + lIpAddr)
|
|
cmd = cls_cmd.RemoteCmd(lIpAddr)
|
|
|
|
# Checking the hostname to get adapted on cli and dockerfileprefixes
|
|
cmd.run('hostnamectl')
|
|
result = re.search('Ubuntu|Red Hat', cmd.getBefore())
|
|
self.host = result.group(0)
|
|
if self.host == 'Ubuntu':
|
|
self.cli = 'docker'
|
|
self.dockerfileprefix = '.ubuntu22'
|
|
self.cliBuildOptions = ''
|
|
elif self.host == 'Red Hat':
|
|
self.cli = 'sudo podman'
|
|
self.dockerfileprefix = '.rhel9'
|
|
self.cliBuildOptions = '--disable-compression'
|
|
|
|
# we always build the ran-build image with all targets
|
|
# Creating a tupple with the imageName, the DockerFile prefix pattern, targetName and sanitized option
|
|
imageNames = [('ran-build', 'build', 'ran-build', '')]
|
|
result = re.search('eNB', self.imageKind)
|
|
if result is not None:
|
|
imageNames.append(('oai-enb', 'eNB', 'oai-enb', ''))
|
|
result = re.search('gNB', self.imageKind)
|
|
if result is not None:
|
|
imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
|
|
result = re.search('all', self.imageKind)
|
|
if result is not None:
|
|
imageNames.append(('oai-enb', 'eNB', 'oai-enb', ''))
|
|
imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
|
|
imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup', ''))
|
|
imageNames.append(('oai-lte-ue', 'lteUE', 'oai-lte-ue', ''))
|
|
imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
|
|
if self.host == 'Red Hat':
|
|
imageNames.append(('oai-physim', 'phySim', 'oai-physim', ''))
|
|
if self.host == 'Ubuntu':
|
|
imageNames.append(('oai-lte-ru', 'lteRU', 'oai-lte-ru', ''))
|
|
imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', ''))
|
|
# Building again the 5G images with Address Sanitizer
|
|
imageNames.append(('ran-build', 'build', 'ran-build-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
|
|
imageNames.append(('oai-gnb', 'gNB', 'oai-gnb-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
|
|
imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
|
|
imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
|
|
imageNames.append(('ran-build-fhi72', 'build.fhi72', 'ran-build-fhi72', ''))
|
|
imageNames.append(('oai-gnb', 'gNB.fhi72', 'oai-gnb-fhi72', ''))
|
|
result = re.search('build_cross_arm64', self.imageKind)
|
|
if result is not None:
|
|
self.dockerfileprefix = '.ubuntu22.cross-arm64'
|
|
|
|
self.testCase_id = HTML.testCase_id
|
|
cmd.cd(lSourcePath)
|
|
# if asterix, copy the entitlement and subscription manager configurations
|
|
if self.host == 'Red Hat':
|
|
cmd.run('mkdir -p ./etc-pki-entitlement')
|
|
cmd.run('cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/')
|
|
|
|
baseImage = 'ran-base'
|
|
baseTag = 'develop'
|
|
forceBaseImageBuild = False
|
|
imageTag = 'develop'
|
|
if (self.ranAllowMerge):
|
|
imageTag = 'ci-temp'
|
|
if self.ranTargetBranch == 'develop':
|
|
cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX')
|
|
result = re.search('index', cmd.getBefore())
|
|
if result is not None:
|
|
forceBaseImageBuild = True
|
|
baseTag = 'ci-temp'
|
|
# if the branch name contains integration_20xx_wyy, let rebuild ran-base
|
|
result = re.search('integration_20([0-9]{2})_w([0-9]{2})', self.ranBranch)
|
|
if not forceBaseImageBuild and result is not None:
|
|
forceBaseImageBuild = True
|
|
baseTag = 'ci-temp'
|
|
else:
|
|
forceBaseImageBuild = True
|
|
|
|
# Let's remove any previous run artifacts if still there
|
|
cmd.run(f"{self.cli} image prune --force")
|
|
for image,pattern,name,option in imageNames:
|
|
cmd.run(f"{self.cli} image rm {name}:{imageTag}")
|
|
|
|
# Build the base image only on Push Events (not on Merge Requests)
|
|
# On when the base image docker file is being modified.
|
|
if forceBaseImageBuild:
|
|
cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
|
|
cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600)
|
|
# First verify if the base image was properly created.
|
|
ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
|
|
allImagesSize = {}
|
|
if ret.returncode != 0:
|
|
logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
|
|
# Recover the name of the failed container?
|
|
cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs --no-run-if-empty {self.cli} rm -f")
|
|
cmd.run(f"{self.cli} image prune --force")
|
|
cmd.close()
|
|
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
|
|
HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
else:
|
|
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
|
if result is not None:
|
|
size = float(result.group("size")) / 1000000
|
|
imageSizeStr = f'{size:.1f}'
|
|
logging.debug(f'\u001B[1m ran-base size is {imageSizeStr} Mbytes\u001B[0m')
|
|
allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
|
|
else:
|
|
logging.debug('ran-base size is unknown')
|
|
|
|
# Recover build logs, for the moment only possible when build is successful
|
|
cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
|
|
cmd.run("mkdir -p cmake_targets/log/ran-base")
|
|
cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base")
|
|
cmd.run(f"{self.cli} rm -f test")
|
|
|
|
# Build the target image(s)
|
|
status = True
|
|
attemptedImages = ['ran-base']
|
|
for image,pattern,name,option in imageNames:
|
|
attemptedImages += [name]
|
|
# the archived Dockerfiles have "ran-base:latest" as base image
|
|
# we need to update them with proper tag
|
|
cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
|
cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
|
# target images should use the proper ran-build image
|
|
if image != 'ran-build' and "-asan" in name:
|
|
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build-asan:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
|
elif "fhi72" in name:
|
|
cmd.run(f'sed -i -e "s#ran-build-fhi72:latest#ran-build-fhi72:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
|
elif image != 'ran-build':
|
|
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
|
if image == 'oai-gnb-aerial':
|
|
cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.2024.05.23.tar.gz .')
|
|
ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200)
|
|
if image == 'oai-gnb-aerial':
|
|
cmd.run('rm -f nvipc_src.2024.05.23.tar.gz')
|
|
if image == 'ran-build' and ret.returncode == 0:
|
|
cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true")
|
|
cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/")
|
|
cmd.run(f"docker rm -f test-log")
|
|
else:
|
|
cmd.run(f"mkdir -p cmake_targets/log/{name}")
|
|
# check the status of the build
|
|
ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}")
|
|
if ret.returncode != 0:
|
|
logging.error('\u001B[1m Could not build properly ' + name + '\u001B[0m')
|
|
status = False
|
|
# Here we should check if the last container corresponds to a failed command and destroy it
|
|
cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs --no-run-if-empty {self.cli} rm -f")
|
|
allImagesSize[name] = 'N/A -- Build Failed'
|
|
break
|
|
else:
|
|
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
|
if result is not None:
|
|
size = float(result.group("size")) / 1000000 # convert to MB
|
|
imageSizeStr = f'{size:.1f}'
|
|
logging.debug(f'\u001B[1m {name} size is {imageSizeStr} Mbytes\u001B[0m')
|
|
allImagesSize[name] = f'{imageSizeStr} Mbytes'
|
|
else:
|
|
logging.debug(f'{name} size is unknown')
|
|
allImagesSize[name] = 'unknown'
|
|
# Now pruning dangling images in between target builds
|
|
cmd.run(f"{self.cli} image prune --force")
|
|
|
|
# Remove all intermediate build images and clean up
|
|
cmd.run(f"{self.cli} image rm ran-build:{imageTag} ran-build-asan:{imageTag} ran-build-fhi72:{imageTag} || true")
|
|
cmd.run(f"{self.cli} volume prune --force")
|
|
|
|
# Remove some cached artifacts to prevent out of diskspace problem
|
|
logging.debug(cmd.run("df -h").stdout)
|
|
logging.debug(cmd.run("docker system df").stdout)
|
|
cmd.run(f"{self.cli} buildx prune --filter until=1h --force")
|
|
logging.debug(cmd.run("df -h").stdout)
|
|
logging.debug(cmd.run("docker system df").stdout)
|
|
|
|
# create a zip with all logs
|
|
build_log_name = f'build_log_{self.testCase_id}'
|
|
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
|
cmd.close()
|
|
|
|
# Analyze the logs
|
|
collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
|
|
|
if status:
|
|
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
|
|
HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
|
|
return True
|
|
else:
|
|
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
|
|
HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
|
|
def BuildProxy(self, HTML):
|
|
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
if self.proxyCommit is None:
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter (need proxyCommit for proxy build)')
|
|
logging.debug('Building on server: ' + lIpAddr)
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
|
|
# Check that we are on Ubuntu
|
|
mySSH.command('hostnamectl', '\$', 5)
|
|
result = re.search('Ubuntu', mySSH.getBefore())
|
|
self.host = result.group(0)
|
|
if self.host != 'Ubuntu':
|
|
logging.error('\u001B[1m Can build proxy only on Ubuntu server\u001B[0m')
|
|
mySSH.close()
|
|
sys.exit(1)
|
|
|
|
self.cli = 'docker'
|
|
self.cliBuildOptions = ''
|
|
|
|
# Workaround for some servers, we need to erase completely the workspace
|
|
if self.forcedWorkspaceCleanup:
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
|
|
|
|
oldRanCommidID = self.ranCommitID
|
|
oldRanRepository = self.ranRepository
|
|
oldRanAllowMerge = self.ranAllowMerge
|
|
oldRanTargetBranch = self.ranTargetBranch
|
|
self.ranCommitID = self.proxyCommit
|
|
self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git'
|
|
self.ranAllowMerge = False
|
|
self.ranTargetBranch = 'master'
|
|
mySSH.command('cd ' +lSourcePath, '\$', 3)
|
|
# to prevent accidentally overwriting data that might be used later
|
|
self.ranCommitID = oldRanCommidID
|
|
self.ranRepository = oldRanRepository
|
|
self.ranAllowMerge = oldRanAllowMerge
|
|
self.ranTargetBranch = oldRanTargetBranch
|
|
|
|
# Let's remove any previous run artifacts if still there
|
|
mySSH.command(self.cli + ' image prune --force', '\$', 30)
|
|
# Remove any previous proxy image
|
|
mySSH.command(self.cli + ' image rm oai-lte-multi-ue-proxy:latest || true', '\$', 30)
|
|
|
|
tag = self.proxyCommit
|
|
logging.debug('building L2sim proxy image for tag ' + tag)
|
|
# check if the corresponding proxy image with tag exists. If not, build it
|
|
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
|
|
buildProxy = mySSH.getBefore().count('o such image') != 0
|
|
if buildProxy:
|
|
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target oai-lte-multi-ue-proxy --tag proxy:' + tag + ' --file docker/Dockerfile.ubuntu18.04 . > cmake_targets/log/proxy-build.log 2>&1', '\$', 180)
|
|
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
|
|
mySSH.command(self.cli + ' image prune --force || true','\$', 15)
|
|
if mySSH.getBefore().count('o such image') != 0:
|
|
logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
|
|
mySSH.close()
|
|
HTML.CreateHtmlTestRow('commit ' + tag, 'KO', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
else:
|
|
logging.debug('L2sim proxy image for tag ' + tag + ' already exists, skipping build')
|
|
|
|
# retag the build images to that we pick it up later
|
|
mySSH.command('docker image tag proxy:' + tag + ' oai-lte-multi-ue-proxy:latest', '\$', 5)
|
|
|
|
# no merge: is a push to develop, tag the image so we can push it to the registry
|
|
if not self.ranAllowMerge:
|
|
mySSH.command('docker image tag proxy:' + tag + ' proxy:develop', '\$', 5)
|
|
|
|
# we assume that the host on which this is built will also run the proxy. The proxy
|
|
# currently requires the following command, and the docker-compose up mechanism of
|
|
# the CI does not allow to run arbitrary commands. Note that the following actually
|
|
# belongs to the deployment, not the build of the proxy...
|
|
logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!')
|
|
mySSH.command('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up', '\$', 5)
|
|
|
|
# Analyzing the logs
|
|
if buildProxy:
|
|
self.testCase_id = HTML.testCase_id
|
|
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
|
mySSH.command('mkdir -p proxy_build_log_' + self.testCase_id, '\$', 5)
|
|
mySSH.command('mv log/* ' + 'proxy_build_log_' + self.testCase_id, '\$', 5)
|
|
if (os.path.isfile('./proxy_build_log_' + self.testCase_id + '.zip')):
|
|
os.remove('./proxy_build_log_' + self.testCase_id + '.zip')
|
|
if (os.path.isdir('./proxy_build_log_' + self.testCase_id)):
|
|
shutil.rmtree('./proxy_build_log_' + self.testCase_id)
|
|
mySSH.command('zip -r -qq proxy_build_log_' + self.testCase_id + '.zip proxy_build_log_' + self.testCase_id, '\$', 5)
|
|
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '.zip', '.')
|
|
# don't delete such that we might recover the zips
|
|
#mySSH.command('rm -f build_log_' + self.testCase_id + '.zip','\$', 5)
|
|
|
|
# we do not analyze the logs (we assume the proxy builds fine at this stage),
|
|
# but need to have the following information to correctly display the HTML
|
|
files = {}
|
|
errorandwarnings = {}
|
|
errorandwarnings['errors'] = 0
|
|
errorandwarnings['warnings'] = 0
|
|
errorandwarnings['status'] = True
|
|
files['Target Image Creation'] = errorandwarnings
|
|
collectInfo = {}
|
|
collectInfo['proxy'] = files
|
|
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
|
|
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
|
# Cleaning any created tmp volume
|
|
mySSH.command(self.cli + ' volume prune --force || true','\$', 15)
|
|
mySSH.close()
|
|
|
|
allImagesSize = {}
|
|
if result is not None:
|
|
imageSize = float(result.group('size')) / 1000000
|
|
logging.debug('\u001B[1m proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
|
|
allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes'
|
|
logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m')
|
|
HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
|
|
return True
|
|
else:
|
|
logging.error('proxy size is unknown')
|
|
allImagesSize['proxy'] = 'unknown'
|
|
logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
|
|
HTML.CreateHtmlTestRow('commit ' + tag, 'KO', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
|
|
def BuildRunTests(self, HTML):
|
|
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
logging.debug('Building on server: ' + lIpAddr)
|
|
cmd = cls_cmd.RemoteCmd(lIpAddr)
|
|
cmd.cd(lSourcePath)
|
|
|
|
ret = cmd.run('hostnamectl')
|
|
result = re.search('Ubuntu', ret.stdout)
|
|
host = result.group(0)
|
|
if host != 'Ubuntu':
|
|
cmd.close()
|
|
raise Exception("Can build unit tests only on Ubuntu server")
|
|
logging.debug('running on Ubuntu as expected')
|
|
|
|
if self.forcedWorkspaceCleanup:
|
|
cmd.run(f'sudo -S rm -Rf {lSourcePath}')
|
|
self.testCase_id = HTML.testCase_id
|
|
|
|
# check that ran-base image exists as we expect it
|
|
baseImage = 'ran-base'
|
|
baseTag = 'develop'
|
|
if self.ranAllowMerge:
|
|
if self.ranTargetBranch == 'develop':
|
|
cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX')
|
|
result = re.search('index', cmd.getBefore())
|
|
if result is not None:
|
|
baseTag = 'ci-temp'
|
|
ret = cmd.run(f"docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
|
|
if ret.returncode != 0:
|
|
logging.error(f'No {baseImage} image present, cannot build tests')
|
|
HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
|
|
# build ran-unittests image
|
|
dockerfile = "ci-scripts/docker/Dockerfile.unittest.ubuntu22"
|
|
ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {lSourcePath}/cmake_targets/log/unittest-build.log')
|
|
if ret.returncode != 0:
|
|
build_log_name = f'build_log_{self.testCase_id}'
|
|
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
|
logging.error(f'Cannot build unit tests')
|
|
HTML.CreateHtmlTestRow("Unit test build failed", 'KO', [dockerfile])
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
|
|
HTML.CreateHtmlTestRowQueue("Build unit tests", 'OK', [dockerfile])
|
|
|
|
# it worked, build and execute tests, and close connection
|
|
ret = cmd.run(f'docker run -a STDOUT --rm ran-unittests:{baseTag} ctest --output-on-failure --no-label-summary -j$(nproc)')
|
|
cmd.run(f'docker rmi ran-unittests:{baseTag}')
|
|
build_log_name = f'build_log_{self.testCase_id}'
|
|
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
|
cmd.close()
|
|
|
|
if ret.returncode == 0:
|
|
HTML.CreateHtmlTestRowQueue('Unit tests succeeded', 'OK', [ret.stdout])
|
|
HTML.CreateHtmlTabFooter(True)
|
|
return True
|
|
else:
|
|
HTML.CreateHtmlTestRowQueue('Unit tests failed (see also doc/UnitTests.md)', 'KO', [ret.stdout])
|
|
HTML.CreateHtmlTabFooter(False)
|
|
return False
|
|
|
|
def Push_Image_to_Local_Registry(self, HTML):
|
|
if self.registrySvrId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.registrySvrId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.registrySvrId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
logging.debug('Pushing images from server: ' + lIpAddr)
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
imagePrefix = 'porcepix.sboai.cs.eurecom.fr'
|
|
mySSH.command(f'docker login -u oaicicd -p oaicicd {imagePrefix}', '\$', 5)
|
|
if re.search('Login Succeeded', mySSH.getBefore()) is None:
|
|
msg = 'Could not log into local registry'
|
|
logging.error(msg)
|
|
mySSH.close()
|
|
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
|
|
return False
|
|
|
|
orgTag = 'develop'
|
|
if self.ranAllowMerge:
|
|
orgTag = 'ci-temp'
|
|
for image in IMAGES:
|
|
tagToUse = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
|
imageTag = f"{image}:{tagToUse}"
|
|
mySSH.command(f'docker image tag {image}:{orgTag} {imagePrefix}/{imageTag}', '\$', 5)
|
|
if re.search('Error response from daemon: No such image:', mySSH.getBefore()) is not None:
|
|
continue
|
|
mySSH.command(f'docker push {imagePrefix}/{imageTag}', '\$', 120)
|
|
if re.search(': digest:', mySSH.getBefore()) is None:
|
|
logging.debug(mySSH.getBefore())
|
|
msg = f'Could not push {image} to local registry : {imageTag}'
|
|
logging.error(msg)
|
|
mySSH.close()
|
|
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
|
|
return False
|
|
mySSH.command(f'docker rmi {imagePrefix}/{imageTag} {image}:{orgTag}', '\$', 30)
|
|
|
|
mySSH.command(f'docker logout {imagePrefix}', '\$', 5)
|
|
if re.search('Removing login credentials', mySSH.getBefore()) is None:
|
|
msg = 'Could not log off from local registry'
|
|
logging.error(msg)
|
|
mySSH.close()
|
|
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
|
|
return False
|
|
|
|
mySSH.close()
|
|
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
|
return True
|
|
|
|
def Pull_Image_from_Local_Registry(self, HTML):
|
|
# This method can be called either onto a remote server (different from python executor)
|
|
# or directly on the python executor (ie lIpAddr == 'none')
|
|
if self.testSvrId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.testSvrId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.testSvrId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
logging.debug('\u001B[1m Pulling image(s) on server: ' + lIpAddr + '\u001B[0m')
|
|
myCmd = cls_cmd.getConnection(lIpAddr)
|
|
imagePrefix = 'porcepix.sboai.cs.eurecom.fr'
|
|
response = myCmd.run(f'docker login -u oaicicd -p oaicicd {imagePrefix}')
|
|
if response.returncode != 0:
|
|
msg = 'Could not log into local registry'
|
|
logging.error(msg)
|
|
myCmd.close()
|
|
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
|
|
return False
|
|
for image in self.imageToPull:
|
|
tagToUse = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
|
imageTag = f"{image}:{tagToUse}"
|
|
cmd = f'docker pull {imagePrefix}/{imageTag}'
|
|
response = myCmd.run(cmd, timeout=120)
|
|
if response.returncode != 0:
|
|
logging.debug(response)
|
|
msg = f'Could not pull {image} from local registry : {imageTag}'
|
|
logging.error(msg)
|
|
myCmd.close()
|
|
HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK)
|
|
return False
|
|
myCmd.run(f'docker tag {imagePrefix}/{imageTag} oai-ci/{imageTag}')
|
|
myCmd.run(f'docker rmi {imagePrefix}/{imageTag}')
|
|
response = myCmd.run(f'docker logout {imagePrefix}')
|
|
if response.returncode != 0:
|
|
msg = 'Could not log off from local registry'
|
|
logging.error(msg)
|
|
myCmd.close()
|
|
HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
|
|
return False
|
|
myCmd.close()
|
|
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
|
return True
|
|
|
|
def Clean_Test_Server_Images(self, HTML):
|
|
# This method can be called either onto a remote server (different from python executor)
|
|
# or directly on the python executor (ie lIpAddr == 'none')
|
|
if self.testSvrId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.testSvrId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.testSvrId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
if lIpAddr != 'none':
|
|
logging.debug('Removing test images from server: ' + lIpAddr)
|
|
myCmd = cls_cmd.RemoteCmd(lIpAddr)
|
|
else:
|
|
logging.debug('Removing test images locally')
|
|
myCmd = cls_cmd.LocalCmd()
|
|
|
|
for image in IMAGES:
|
|
tag = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
|
imageTag = f"{image}:{tag}"
|
|
cmd = f'docker rmi oai-ci/{imageTag}'
|
|
myCmd.run(cmd, reportNonZero=False)
|
|
|
|
myCmd.close()
|
|
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
|
return True
|
|
|
|
def Create_Workspace(self,HTML):
|
|
if self.eNB_serverId[self.eNB_instance] == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId[self.eNB_instance] == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
|
|
logging.info(f"Running on server {lIpAddr}")
|
|
sshSession = cls_cmd.getConnection(lIpAddr)
|
|
|
|
success = CreateWorkspace(sshSession, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
|
if success:
|
|
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
|
else:
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"])
|
|
return success
|
|
|
|
def DeployObject(self, HTML):
|
|
lIpAddr, lUserName, lPassWord, lSourcePath = GetCredentials(self)
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
|
|
yaml = self.yamlPath[self.eNB_instance].strip('/')
|
|
wd = f'{lSourcePath}/{yaml}'
|
|
|
|
with cls_cmd.getConnection(lIpAddr) as ssh:
|
|
services = GetServices(ssh, self.services[self.eNB_instance], f"{wd}/docker-compose.y*ml")
|
|
if services == [] or services == ' ' or services == None:
|
|
msg = "Cannot determine services to start"
|
|
logging.error(msg)
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
|
|
return False
|
|
|
|
ExistEnvFilePrint(ssh, wd)
|
|
WriteEnvFile(ssh, services, wd, self.deploymentTag)
|
|
|
|
logging.info(f"will start services {services}")
|
|
status = ssh.run(f'docker compose -f {wd}/docker-compose.y*ml up -d -- {services}')
|
|
if status.returncode != 0:
|
|
msg = f"cannot deploy services {services}: {status.stdout}"
|
|
logging.error(msg)
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
|
|
return False
|
|
|
|
imagesInfo = []
|
|
fstatus = True
|
|
for svc in services.split():
|
|
containerName = GetContainerName(ssh, svc, f"{wd}/docker-compose.y*ml")
|
|
healthy = GetContainerHealth(ssh, containerName)
|
|
if not healthy:
|
|
tgtlogfile = f'{svc}-{HTML.testCase_id}.log'
|
|
logging.warning(f"Deployment Failed: Trying to copy container logs to {tgtlogfile}")
|
|
yaml_dir = yaml.split('/')[-1]
|
|
CopyinContainerLog(ssh, lSourcePath, yaml_dir, containerName, tgtlogfile)
|
|
imagesInfo += [f"Failed to deploy: service {svc}"]
|
|
fstatus = False
|
|
else:
|
|
image = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml")
|
|
logging.info(f"service {svc} healthy, container {containerName}, image {image}")
|
|
imagesInfo += [f"service {svc} healthy, image {image}"]
|
|
if fstatus:
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)])
|
|
else:
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
|
|
return fstatus
|
|
|
|
def UndeployObject(self, HTML, RAN):
|
|
lIpAddr, lUserName, lPassWord, lSourcePath = GetCredentials(self)
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
|
|
yaml = self.yamlPath[self.eNB_instance].strip('/')
|
|
wd = f'{lSourcePath}/{yaml}'
|
|
yaml_dir = yaml.split('/')[-1]
|
|
with cls_cmd.getConnection(lIpAddr) as ssh:
|
|
ExistEnvFilePrint(ssh, wd)
|
|
services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml")
|
|
copyin_res = None
|
|
if services is not None:
|
|
all_serv = " ".join([s for s, _ in services])
|
|
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
|
|
copyin_res = all(CopyinContainerLog(ssh, lSourcePath, yaml_dir, c, f'{s}-{HTML.testCase_id}.log') for s, c in services)
|
|
else:
|
|
logging.warning('could not identify services to stop => no log file')
|
|
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
|
|
ssh.run(f'rm {wd}/.env')
|
|
if not copyin_res:
|
|
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)'])
|
|
return False
|
|
else:
|
|
log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services]
|
|
success = all(log_results)
|
|
if success:
|
|
logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
|
|
else:
|
|
logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
|
|
return success
|
|
|
|
def CheckAndAddRoute(self, svrName, ipAddr, userName, password):
|
|
logging.debug('Checking IP routing on ' + svrName)
|
|
mySSH = SSH.SSHConnection()
|
|
if svrName == 'porcepix':
|
|
mySSH.open(ipAddr, userName, password)
|
|
# Check if route to asterix gnb exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.64/26"', '\$', 10)
|
|
result = re.search('172.21.16.127', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.64/26 via 172.21.16.127 dev eno1', '\$', 10)
|
|
# Check if route to obelix enb exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
|
|
result = re.search('172.21.16.128', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 172.21.16.128 dev eno1', '\$', 10)
|
|
# Check if forwarding is enabled
|
|
mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
|
|
result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
|
|
# Check if iptables forwarding is accepted
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
|
|
result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
|
|
mySSH.close()
|
|
if svrName == 'asterix':
|
|
mySSH.open(ipAddr, userName, password)
|
|
# Check if route to porcepix epc exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
|
|
result = re.search('172.21.16.136', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 172.21.16.136 dev em1', '\$', 10)
|
|
# Check if route to porcepix cn5g exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.70.128/26"', '\$', 10)
|
|
result = re.search('172.21.16.136', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.70.128/26 via 172.21.16.136 dev em1', '\$', 10)
|
|
# Check if X2 route to obelix enb exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
|
|
result = re.search('172.21.16.128', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 172.21.16.128 dev em1', '\$', 10)
|
|
# Check if forwarding is enabled
|
|
mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
|
|
result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
|
|
# Check if iptables forwarding is accepted
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
|
|
result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
|
|
mySSH.close()
|
|
if svrName == 'obelix':
|
|
mySSH.open(ipAddr, userName, password)
|
|
# Check if route to porcepix epc exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
|
|
result = re.search('172.21.16.136', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 172.21.16.136 dev eno1', '\$', 10)
|
|
# Check if X2 route to asterix gnb exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.64/26"', '\$', 10)
|
|
result = re.search('172.21.16.127', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.64/26 via 172.21.16.127 dev eno1', '\$', 10)
|
|
# Check if X2 route to nepes gnb exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.192/26"', '\$', 10)
|
|
result = re.search('172.21.16.137', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.192/26 via 172.21.16.137 dev eno1', '\$', 10)
|
|
# Check if forwarding is enabled
|
|
mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
|
|
result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
|
|
# Check if iptables forwarding is accepted
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
|
|
result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
|
|
mySSH.close()
|
|
if svrName == 'nepes':
|
|
mySSH.open(ipAddr, userName, password)
|
|
# Check if route to ofqot gnb exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.192/26"', '\$', 10)
|
|
result = re.search('172.21.16.109', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.192/26 via 172.21.16.109 dev enp0s31f6', '\$', 10)
|
|
mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
|
|
result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
|
|
# Check if iptables forwarding is accepted
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
|
|
result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
|
|
mySSH.close()
|
|
if svrName == 'ofqot':
|
|
mySSH.open(ipAddr, userName, password)
|
|
# Check if X2 route to nepes enb/epc exists
|
|
mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
|
|
result = re.search('172.21.16.137', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 172.21.16.137 dev enp2s0', '\$', 10)
|
|
# Check if forwarding is enabled
|
|
mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
|
|
result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
|
|
# Check if iptables forwarding is accepted
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
|
|
result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
|
|
if result is None:
|
|
mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
|
|
mySSH.close()
|