From 6a36db456960b3d9e818a62793c176db525641a8 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 6 Nov 2024 08:29:33 +0100 Subject: [PATCH] Repair and refactor proxy build function Recent refactoring replaced the CreateWorkspace function call with a dedicated XML step (for clarity, and because it avoids needless recreation of workspaces). In the proxy function, the same CreateWorkspace xml step has been added, cloning the OAI RAN repo, when we actually need the repository for the proxy. Hence, fix this by calling the CreateWorkspace() function manually, but only when necessary, to build the proxy. (It would only be a problem if the pipeline would rebuild the proxy, which did not happen because not much work is happening on the proxy repo). Further, refactoring and simplication of the proxy build function were done: - never tag proxy with develop, that makes no sense - overwrite CreateWorkspace() info after having created and used the workspace (the "if" to build the proxy) - simplify copying of logs - add simple unit test - remove check for ubuntu; the dependcy is "docker". Simplify the code to not obscure with a class variable. --- ci-scripts/cls_containerize.py | 77 +++++++++++++--------------------- ci-scripts/tests/README.md | 1 + ci-scripts/tests/build.py | 40 ++++++++++++++++++ 3 files changed, 70 insertions(+), 48 deletions(-) create mode 100644 ci-scripts/tests/build.py diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index e3fdae9c0f..bac87de124 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -601,21 +601,7 @@ class Containerize(): logging.debug('Building on server: ' + lIpAddr) ssh = cls_cmd.getConnection(lIpAddr) - # Check that we are on Ubuntu - ret = ssh.run('hostnamectl') - result = re.search('Ubuntu', ret.stdout) - self.host = result.group(0) - if self.host != 'Ubuntu': - ssh.close() - raise Exception('Can build proxy only on Ubuntu server') - - self.cli = 'docker' - self.cliBuildOptions = '' - - # Workaround for some servers, we need to erase completely the workspace - if self.forcedWorkspaceCleanup: - ssh.run(f'echo {lPassWord} | sudo -S rm -Rf {lSourcePath}') - + self.testCase_id = HTML.testCase_id oldRanCommidID = self.ranCommitID oldRanRepository = self.ranRepository oldRanAllowMerge = self.ranAllowMerge @@ -624,26 +610,35 @@ class Containerize(): self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git' self.ranAllowMerge = False self.ranTargetBranch = 'master' - ssh.cd(lSourcePath) - # to prevent accidentally overwriting data that might be used later - self.ranCommitID = oldRanCommidID - self.ranRepository = oldRanRepository - self.ranAllowMerge = oldRanAllowMerge - self.ranTargetBranch = oldRanTargetBranch # Let's remove any previous run artifacts if still there - ssh.run(f'{self.cli} image prune --force') + ssh.run('docker image prune --force') # Remove any previous proxy image - ssh.run(f'{self.cli} image rm oai-lte-multi-ue-proxy:latest') + ssh.run('docker image rm oai-lte-multi-ue-proxy:latest') tag = self.proxyCommit logging.debug('building L2sim proxy image for tag ' + tag) # check if the corresponding proxy image with tag exists. If not, build it - ret = ssh.run(f'{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') - if ret.returncode != 0: - ssh.run(f'{self.cli} build {self.cliBuildOptions} --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file docker/Dockerfile.ubuntu18.04 . > cmake_targets/log/proxy-build.log 2>&1') - ssh.run(f'{self.cli} image prune --force') - ret = ssh.run(f'{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') + ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') + buildProxy = ret.returncode != 0 # if no image, build new proxy + if buildProxy: + ssh.run(f'rm -Rf {lSourcePath}') + success = CreateWorkspace(lIpAddr, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) + if not success: + raise Exception("could not clone proxy repository") + + filename = f'build_log_{self.testCase_id}' + fullpath = f'{lSourcePath}/{filename}' + + ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1') + ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}') + local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip" + ssh.copyin(f'{fullpath}.zip', local_file) + # don't delete such that we might recover the zips + #ssh.run(f'rm -f {fullpath}.zip') + + ssh.run('docker image prune --force') + ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') if ret.returncode != 0: logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m') ssh.close() @@ -656,10 +651,6 @@ class Containerize(): # retag the build images to that we pick it up later ssh.run(f'docker image tag proxy:{tag} oai-lte-multi-ue-proxy:latest') - # no merge: is a push to develop, tag the image so we can push it to the registry - if not self.ranAllowMerge: - ssh.run('docker image tag proxy:{tag} proxy:develop') - # we assume that the host on which this is built will also run the proxy. The proxy # currently requires the following command, and the docker-compose up mechanism of # the CI does not allow to run arbitrary commands. Note that the following actually @@ -667,21 +658,11 @@ class Containerize(): logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!') ssh.run('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up') - # Analyzing the logs - if buildProxy: - self.testCase_id = HTML.testCase_id - ssh.cd(f'{lSourcePath}/cmake_targets') - ssh.run(f'mkdir -p proxy_build_log_{self.testCase_id}') - ssh.run(f'mv log/* proxy_build_log_{self.testCase_id}') - if (os.path.isfile('./proxy_build_log_' + self.testCase_id + '.zip')): - os.remove('./proxy_build_log_' + self.testCase_id + '.zip') - if (os.path.isdir('./proxy_build_log_' + self.testCase_id)): - shutil.rmtree('./proxy_build_log_' + self.testCase_id) - ssh.run(f'zip -r -qq proxy_build_log_{self.testCase_id}.zip proxy_build_log_{self.testCase_id}') - filename = f'build_log_{self.testCase_id}.zip' - ssh.copyin(f'{lSourcePath}/cmake_targets/{filename}', filename) - # don't delete such that we might recover the zips - #ssh.run(f'rm -f build_log_{self.testCase_id}.zip') + # to prevent accidentally overwriting data that might be used later + self.ranCommitID = oldRanCommidID + self.ranRepository = oldRanRepository + self.ranAllowMerge = oldRanAllowMerge + self.ranTargetBranch = oldRanTargetBranch # we do not analyze the logs (we assume the proxy builds fine at this stage), # but need to have the following information to correctly display the HTML @@ -696,7 +677,7 @@ class Containerize(): ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') result = re.search('Size *= *(?P[0-9\-]+) *bytes', ret.stdout) # Cleaning any created tmp volume - ssh.run(f'{self.cli} volume prune --force') + ssh.run('docker volume prune --force') ssh.close() allImagesSize = {} diff --git a/ci-scripts/tests/README.md b/ci-scripts/tests/README.md index dc8f9ea80a..9a19bacb67 100644 --- a/ci-scripts/tests/README.md +++ b/ci-scripts/tests/README.md @@ -6,6 +6,7 @@ scripts. There are some unit tests. From the parent directory, i.e., `ci-scripts/`, start with + python tests/build.py -v python tests/cmd.py -v python tests/deployment.py -v python tests/iperf-analysis.py -v diff --git a/ci-scripts/tests/build.py b/ci-scripts/tests/build.py new file mode 100644 index 0000000000..669e6711b4 --- /dev/null +++ b/ci-scripts/tests/build.py @@ -0,0 +1,40 @@ +import sys +import logging +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import unittest +import tempfile + +sys.path.append('./') # to find OAI imports below +import cls_oai_html +import cls_containerize +import cls_cmd + +class TestBuild(unittest.TestCase): + def setUp(self): + self.html = cls_oai_html.HTMLManagement() + self.html.testCase_id = "000000" + self.cont = cls_containerize.Containerize() + self.cont.eNB_serverId[0] = '0' + self.cont.eNBIPAddress = 'localhost' + self.cont.eNBUserName = None + self.cont.eNBPassword = None + self._d = tempfile.mkdtemp() + logging.warning(f"temporary directory: {self._d}") + self.cont.eNBSourceCodePath = self._d + + def tearDown(self): + logging.warning(f"removing directory contents") + with cls_cmd.getConnection(None) as cmd: + cmd.run(f"rm -rf {self._d}") + + def test_build_proxy(self): + self.cont.proxyCommit = "b64d9bce986b38ca59e8582864ade3fcdd05c0dc" + success = self.cont.BuildProxy(self.html) + self.assertTrue(success) + +if __name__ == '__main__': + unittest.main()