maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Repair and refactor proxy build function

Browse Source 
Recent refactoring replaced the CreateWorkspace function call with a
dedicated XML step (for clarity, and because it avoids needless
recreation of workspaces). In the proxy function, the same
CreateWorkspace xml step has been added, cloning the OAI RAN repo, when
we actually need the repository for the proxy. Hence, fix this by
calling the CreateWorkspace() function manually, but only when
necessary, to build the proxy. (It would only be a problem if the
pipeline would rebuild the proxy, which did not happen because not much
work is happening on the proxy repo).

Further, refactoring and simplication of the proxy build function were
done:

- never tag proxy with develop, that makes no sense
- overwrite CreateWorkspace() info after having created and used the
  workspace (the "if" to build the proxy)
- simplify copying of logs
- add simple unit test
- remove check for ubuntu; the dependcy is "docker". Simplify the code
  to not obscure with a class variable.
This commit is contained in:
Robert Schmidt
2024-11-12 17:20:14 +01:00
parent cf576fc255
commit 6a36db4569
3 changed files with 70 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+29 -48
View File
@@ -601,21 +601,7 @@ class Containerize():
logging.debug('Building on server: ' + lIpAddr)
ssh = cls_cmd.getConnection(lIpAddr)
# Check that we are on Ubuntu
ret = ssh.run('hostnamectl')
result = re.search('Ubuntu', ret.stdout)
self.host = result.group(0)
if self.host != 'Ubuntu':
ssh.close()
raise Exception('Can build proxy only on Ubuntu server')
self.cli = 'docker'
self.cliBuildOptions = ''
# Workaround for some servers, we need to erase completely the workspace
if self.forcedWorkspaceCleanup:
ssh.run(f'echo {lPassWord} | sudo -S rm -Rf {lSourcePath}')
self.testCase_id = HTML.testCase_id
oldRanCommidID = self.ranCommitID
oldRanRepository = self.ranRepository
oldRanAllowMerge = self.ranAllowMerge
@@ -624,26 +610,35 @@ class Containerize():
self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git'
self.ranAllowMerge = False
self.ranTargetBranch = 'master'
ssh.cd(lSourcePath)
# to prevent accidentally overwriting data that might be used later
self.ranCommitID = oldRanCommidID
self.ranRepository = oldRanRepository
self.ranAllowMerge = oldRanAllowMerge
self.ranTargetBranch = oldRanTargetBranch
# Let's remove any previous run artifacts if still there
ssh.run(f'{self.cli} image prune --force')
ssh.run('docker image prune --force')
# Remove any previous proxy image
ssh.run(f'{self.cli} image rm oai-lte-multi-ue-proxy:latest')
ssh.run('docker image rm oai-lte-multi-ue-proxy:latest')
tag = self.proxyCommit
logging.debug('building L2sim proxy image for tag ' + tag)
# check if the corresponding proxy image with tag exists. If not, build it
ret = ssh.run(f'{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
if ret.returncode != 0:
ssh.run(f'{self.cli} build {self.cliBuildOptions} --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file docker/Dockerfile.ubuntu18.04 . > cmake_targets/log/proxy-build.log 2>&1')
ssh.run(f'{self.cli} image prune --force')
ret = ssh.run(f'{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
buildProxy = ret.returncode != 0 # if no image, build new proxy
if buildProxy:
ssh.run(f'rm -Rf {lSourcePath}')
success = CreateWorkspace(lIpAddr, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
if not success:
raise Exception("could not clone proxy repository")
filename = f'build_log_{self.testCase_id}'
fullpath = f'{lSourcePath}/{filename}'
ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1')
ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}')
local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip"
ssh.copyin(f'{fullpath}.zip', local_file)
# don't delete such that we might recover the zips
#ssh.run(f'rm -f {fullpath}.zip')
ssh.run('docker image prune --force')
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
if ret.returncode != 0:
logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
ssh.close()
@@ -656,10 +651,6 @@ class Containerize():
# retag the build images to that we pick it up later
ssh.run(f'docker image tag proxy:{tag} oai-lte-multi-ue-proxy:latest')
# no merge: is a push to develop, tag the image so we can push it to the registry
if not self.ranAllowMerge:
ssh.run('docker image tag proxy:{tag} proxy:develop')
# we assume that the host on which this is built will also run the proxy. The proxy
# currently requires the following command, and the docker-compose up mechanism of
# the CI does not allow to run arbitrary commands. Note that the following actually
@@ -667,21 +658,11 @@ class Containerize():
logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!')
ssh.run('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up')
# Analyzing the logs
if buildProxy:
self.testCase_id = HTML.testCase_id
ssh.cd(f'{lSourcePath}/cmake_targets')
ssh.run(f'mkdir -p proxy_build_log_{self.testCase_id}')
ssh.run(f'mv log/* proxy_build_log_{self.testCase_id}')
if (os.path.isfile('./proxy_build_log_' + self.testCase_id + '.zip')):
os.remove('./proxy_build_log_' + self.testCase_id + '.zip')
if (os.path.isdir('./proxy_build_log_' + self.testCase_id)):
shutil.rmtree('./proxy_build_log_' + self.testCase_id)
ssh.run(f'zip -r -qq proxy_build_log_{self.testCase_id}.zip proxy_build_log_{self.testCase_id}')
filename = f'build_log_{self.testCase_id}.zip'
ssh.copyin(f'{lSourcePath}/cmake_targets/{filename}', filename)
# don't delete such that we might recover the zips
#ssh.run(f'rm -f build_log_{self.testCase_id}.zip')
# to prevent accidentally overwriting data that might be used later
self.ranCommitID = oldRanCommidID
self.ranRepository = oldRanRepository
self.ranAllowMerge = oldRanAllowMerge
self.ranTargetBranch = oldRanTargetBranch
# we do not analyze the logs (we assume the proxy builds fine at this stage),
# but need to have the following information to correctly display the HTML
@@ -696,7 +677,7 @@ class Containerize():
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
# Cleaning any created tmp volume
ssh.run(f'{self.cli} volume prune --force')
ssh.run('docker volume prune --force')
ssh.close()
allImagesSize = {}
ci-scripts/tests/README.md
+1
View File
@@ -6,6 +6,7 @@ scripts.
There are some unit tests. From the parent directory, i.e., `ci-scripts/`,
start with
python tests/build.py -v
python tests/cmd.py -v
python tests/deployment.py -v
python tests/iperf-analysis.py -v
ci-scripts/tests/build.py
+40
View File
@@ -0,0 +1,40 @@
import sys
import logging
logging.basicConfig(
level=logging.DEBUG,
stream=sys.stdout,
format="[%(asctime)s] %(levelname)8s: %(message)s"
)
import unittest
import tempfile
sys.path.append('./') # to find OAI imports below
import cls_oai_html
import cls_containerize
import cls_cmd
class TestBuild(unittest.TestCase):
def setUp(self):
self.html = cls_oai_html.HTMLManagement()
self.html.testCase_id = "000000"
self.cont = cls_containerize.Containerize()
self.cont.eNB_serverId[0] = '0'
self.cont.eNBIPAddress = 'localhost'
self.cont.eNBUserName = None
self.cont.eNBPassword = None
self._d = tempfile.mkdtemp()
logging.warning(f"temporary directory: {self._d}")
self.cont.eNBSourceCodePath = self._d
def tearDown(self):
logging.warning(f"removing directory contents")
with cls_cmd.getConnection(None) as cmd:
cmd.run(f"rm -rf {self._d}")
def test_build_proxy(self):
self.cont.proxyCommit = "b64d9bce986b38ca59e8582864ade3fcdd05c0dc"
success = self.cont.BuildProxy(self.html)
self.assertTrue(success)
if __name__ == '__main__':
unittest.main()