From d81abdaf7b6fc03e93eb5ddb943c83ad92fd0b4f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 25 Sep 2024 16:37:57 +0200 Subject: [PATCH] Reimplement Native Build (CI step Build_eNB) The BuildeNB steps either does things that are already implemented (created workspace), or things that we don't use (separate build_wait), and is generally quite convoluted when all it does is calling build_oai. Reimplement by reusing Create_Workspace, and make build_oai in a single step. To clean up, make a new module in which this function does not need a class instantiation. Instead, we give all necessary parameters directly. --- ci-scripts/cls_native.py | 64 +++++++++ ci-scripts/main.py | 3 +- ci-scripts/ran.py | 179 ------------------------ ci-scripts/xml_files/gnb_usrp_build.xml | 8 +- 4 files changed, 73 insertions(+), 181 deletions(-) create mode 100644 ci-scripts/cls_native.py diff --git a/ci-scripts/cls_native.py b/ci-scripts/cls_native.py new file mode 100644 index 0000000000..d9c281907a --- /dev/null +++ b/ci-scripts/cls_native.py @@ -0,0 +1,64 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- + +import logging +import re + +import cls_cmd +import cls_oai_html +import constants as CONST + +class Native(): + + def Build(test_case, HTML, host, directory, options): + logging.debug(f'Building on server: {host}') + HTML.testCase_id = test_case + + with cls_cmd.getConnection(host) as ssh: + base = f"{directory}/cmake_targets" + ret = ssh.run(f"{base}/build_oai {options} > {base}/log/build_oai.log", timeout=900) + success = ret.returncode == 0 + logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True) + logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}") + + # create log directory, and copy build logs + target = f"{base}/build_log_{test_case}/" + ssh.run(f"mkdir -p {target}") + ssh.run(f"mv {base}/log/* {target}") + + # check if build artifacts are there + # NOTE: build_oai should fail with exit code if it could not build, but it does not + build_dir = f"{base}/ran_build/build" + build_gnb = re.search('--gNB', options) is not None + if build_gnb: + success = success and ssh.run(f"ls {build_dir}/nr-softmodem").returncode == 0 + build_enb = re.search('--eNB', options) is not None + if build_enb: + success = success and ssh.run(f"ls {build_dir}/lte-softmodem").returncode == 0 + + if success: + logging.info('\u001B[1m Building OAI Pass\u001B[0m') + HTML.CreateHtmlTestRow(options, 'OK', CONST.ALL_PROCESSES_OK) + else: + logging.error('\u001B[1m Building OAI Failed\u001B[0m') + HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK) + return success diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 9c6a20d935..dba19d0aa2 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -44,6 +44,7 @@ import cls_containerize #class Containerize for all container-based operations import cls_static_code_analysis #class for static code analysis import cls_physim1 #class PhySim for physical simulators deploy and run import cls_cluster # class for building/deploying on cluster +import cls_native # class for all native/source-based operations import sshconnection import epc @@ -149,7 +150,7 @@ def ExecuteActionWithParam(action): if proxy_commit is not None: CONTAINERS.proxyCommit = proxy_commit if action == 'Build_eNB': - success = RAN.BuildeNB(HTML) + success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) elif action == 'Build_Image': success = CONTAINERS.BuildImage(HTML) elif action == 'Build_Proxy': diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 9fc62eadd4..4c56c5e8ac 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -105,185 +105,6 @@ class RANManagement(): # RAN management functions #----------------------------------------------------------- - def BuildeNB(self, HTML): - if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - if self.eNB_serverId[self.eNB_instance] == '0': - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - lSourcePath = self.eNBSourceCodePath - elif self.eNB_serverId[self.eNB_instance] == '1': - lIpAddr = self.eNB1IPAddress - lUserName = self.eNB1UserName - lPassWord = self.eNB1Password - lSourcePath = self.eNB1SourceCodePath - elif self.eNB_serverId[self.eNB_instance] == '2': - lIpAddr = self.eNB2IPAddress - lUserName = self.eNB2UserName - lPassWord = self.eNB2Password - lSourcePath = self.eNB2SourceCodePath - if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - logging.debug('Building on server: ' + lIpAddr) - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - - # Check if we build an 5G-NR gNB or an LTE eNB - result = re.search('--RU', self.Build_eNB_args) - if result is not None: - self.air_interface[self.eNB_instance] = 'oairu' - else: - result = re.search('--gNB', self.Build_eNB_args) - if result is not None: - self.air_interface[self.eNB_instance] = 'nr-softmodem' - else: - self.air_interface[self.eNB_instance] = 'lte-softmodem' - - # Worakround for some servers, we need to erase completely the workspace - if self.Build_eNB_forced_workspace_cleanup: - mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) - self.testCase_id = HTML.testCase_id - # on RedHat/CentOS .git extension is mandatory - result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) - if result is not None: - full_ran_repo_name = self.ranRepository.replace('git/', 'git') - else: - full_ran_repo_name = self.ranRepository + '.git' - mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) - mySSH.command('cd ' + lSourcePath, '\$', 5) - mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) - # Raphael: here add a check if git clone or git fetch went smoothly - mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) - mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) - mySSH.command('git advice.detachedHead false', '\$', 5) - # Checking the BUILD INFO file - if not self.backgroundBuild: - mySSH.command('ls *.txt', '\$', 5) - result = re.search('LAST_BUILD_INFO', mySSH.getBefore()) - if result is not None: - mismatch = False - mySSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2) - result = re.search(self.ranCommitID, mySSH.getBefore()) - if result is None: - mismatch = True - mySSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) - if (self.ranAllowMerge): - result = re.search('YES', mySSH.getBefore()) - if result is None: - mismatch = True - mySSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) - if self.ranTargetBranch == '': - result = re.search('develop', mySSH.getBefore()) - else: - result = re.search(self.ranTargetBranch, mySSH.getBefore()) - if result is None: - mismatch = True - else: - result = re.search('NO', mySSH.getBefore()) - if result is None: - mismatch = True - if not mismatch: - mySSH.close() - HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK) - return True - - mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) - # if the commit ID is provided use it to point to it - if self.ranCommitID != '': - mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30) - # if the branch is not develop, then it is a merge request and we need to do - # the potential merge. Note that merge conflicts should already been checked earlier - if (self.ranAllowMerge): - if self.ranTargetBranch == '': - if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) - else: - logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) - logging.debug(mySSH.getBefore()) # print what git said when merging/checking out - mySSH.command('source oaienv', '\$', 5) - mySSH.command('cd cmake_targets', '\$', 5) - mySSH.command('mkdir -p log', '\$', 5) - mySSH.command('chmod 777 log', '\$', 5) - # no need to remove in log (git clean did the trick) - if self.backgroundBuild: - mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5) - mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5) - mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5) - logging.debug(mySSH.getBefore()) # print current directory contents for verification - mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5) - mySSH.close() - HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK) - self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id - return True - mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'BUILD SHOULD BE SUCCESSFUL|build have failed', 900) - mySSH.close() - return self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id, HTML) - - def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML): - HTML.testCase_id=testcaseId - - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3) - mySSH.command('ls ran_build/build', '\$', 3) - mySSH.command('ls ran_build/build', '\$', 3) - - #check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem) - logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath) - result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore()) - if result is None: - buildStatus = False #if not, build failed - else: - buildStatus = True - # Generating a BUILD INFO file - mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2) - mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2) - if (self.ranAllowMerge): - mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2) - if self.ranTargetBranch == '': - mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2) - else: - mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2) - else: - mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2) - - - mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5) - mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5) - mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5) - if self.eNB_serverId[self.eNB_instance] != '0': - mySSH.command('cd cmake_targets', '\$', 5) - mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5) - mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5) - mySSH.close() - if (os.path.isfile('./tmp_build' + testcaseId + '.zip')): - os.remove('./tmp_build' + testcaseId + '.zip') - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.') - if (os.path.isfile('./tmp_build' + testcaseId + '.zip')): - mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.') - os.remove('./tmp_build' + testcaseId + '.zip') - mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) - mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5) - #-qq quiet / -u update orcreate files - mySSH.command('unzip -o -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5) - mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5) - mySSH.close() - else: - mySSH.close() - - #generate logging info depending on buildStatus and air interface - if buildStatus: - logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m') - HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK) - else: - logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m') - HTML.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK) - return buildStatus - def InitializeeNB(self, HTML, EPC): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress diff --git a/ci-scripts/xml_files/gnb_usrp_build.xml b/ci-scripts/xml_files/gnb_usrp_build.xml index cabf602bf9..da0c321956 100644 --- a/ci-scripts/xml_files/gnb_usrp_build.xml +++ b/ci-scripts/xml_files/gnb_usrp_build.xml @@ -26,15 +26,21 @@ wrench 100000 +000000 010101 + + Create_Workspace + Creating new Workspace + + TesteNB Build_eNB Build gNB (USRP) - --gNB -w USRP --ninja -c -P --build-lib "ldpc_cuda ldpc_t2" --cmake-opt -DASN1C_EXEC=/opt/asn1c/bin/asn1c + --gNB -w USRP --ninja -c -P --build-lib ldpc_cuda --build-lib ldpc_t2 --cmake-opt -DASN1C_EXEC=/opt/asn1c/bin/asn1c True