From e0b7ebd3758cd30be43a025c24f15af5144cb0f7 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 14 Sep 2023 09:38:28 +0200 Subject: [PATCH 1/5] chore(ci): remove the useless verbose-ci option Signed-off-by: Raphael Defosseux --- docker/Dockerfile.build.ubuntu20 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.build.ubuntu20 b/docker/Dockerfile.build.ubuntu20 index 8186ca06f4..807b7d258a 100644 --- a/docker/Dockerfile.build.ubuntu20 +++ b/docker/Dockerfile.build.ubuntu20 @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope nrqtscope" -w USRP -t Ethernet --verbose-ci --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope nrqtscope" -w USRP -t Ethernet --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror From eedcf0708f2378e78b52b1fae8b55ed589c887bd Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 14 Sep 2023 11:35:03 +0200 Subject: [PATCH 2/5] fix(ci): docker_log_split is useless with current build scheme. Just copying all txt files from ran-build image Signed-off-by: Raphael Defosseux --- ci-scripts/cls_cluster.py | 1 - ci-scripts/cls_containerize.py | 25 +++++---- ci-scripts/docker_log_split.py | 92 ---------------------------------- 3 files changed, 12 insertions(+), 106 deletions(-) delete mode 100644 ci-scripts/docker_log_split.py diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 24b0c8902e..d4d9601d17 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -412,7 +412,6 @@ class Cluster: imageSize = {} for image in attemptedImages: self.cmd.run(f'mkdir -p cmake_targets/log/{image}') - self.cmd.run(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log') tag = imageTag if image != 'ran-base' else baseTag size = self._get_image_size(image, tag) if size <= 0: diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 88de9d135d..dc4be1f93b 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -153,17 +153,13 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus): committed = False tagged = False with open(f'{buildRoot}/{image}.log', mode='r') as inputfile: - startOfTargetImageCreation = False # check for tagged/committed only after image created for line in inputfile: - result = re.search(f'FROM .* [aA][sS] {image}$', str(line)) - if result is not None: - startOfTargetImageCreation = True - if startOfTargetImageCreation: - lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None - tagged = tagged or lineHasTag - # the OpenShift Cluster builder prepends image registry URL - lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None - committed = committed or lineHasCommit + lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None + lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None + tagged = tagged or lineHasTag or lineHasTag2 + # the OpenShift Cluster builder prepends image registry URL + lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None + committed = committed or lineHasCommit errorandwarnings['errors'] = 0 if committed or tagged else 1 errorandwarnings['warnings'] = 0 errorandwarnings['status'] = committed or tagged @@ -468,9 +464,12 @@ class Containerize(): if image != 'ran-build': cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', timeout=1200) - # split the log - cmd.run(f"mkdir -p cmake_targets/log/{image}") - cmd.run(f"python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log") + if image == 'ran-build': + cmd.run(f"docker run --name test-log -d {image}:{imageTag} /bin/true") + cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{image}/") + cmd.run(f"docker rm -f test-log") + else: + cmd.run(f"mkdir -p cmake_targets/log/{image}") # check the status of the build ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {image}:{imageTag}") if ret.returncode != 0: diff --git a/ci-scripts/docker_log_split.py b/ci-scripts/docker_log_split.py deleted file mode 100644 index 8c3c9e41dc..0000000000 --- a/ci-scripts/docker_log_split.py +++ /dev/null @@ -1,92 +0,0 @@ -#/* -# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The OpenAirInterface Software Alliance licenses this file to You under -# * the OAI Public License, Version 1.1 (the "License"); you may not use this file -# * except in compliance with the License. -# * You may obtain a copy of the License at -# * -# * http://www.openairinterface.org/?page_id=698 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# *------------------------------------------------------------------------------- -# * For more information about the OpenAirInterface (OAI) Software Alliance: -# * contact@openairinterface.org -# */ -#--------------------------------------------------------------------- -# Python for CI of OAI-eNB + COTS-UE -# -# Required Python Version -# Python 3.x -# -# Required Python Package -# pexpect -#--------------------------------------------------------------------- - - -#----------------------------------------------------------- -# Import Libs -#----------------------------------------------------------- -import sys # arg -import re # reg -import os -import subprocess - - -class SplitReport(): - def __init__(self): - self.logfilename = '' - self.destinationFolder = '' - - def split(self): - self.destinationFolder = self.logfilename.replace(".log","") - if os.path.isfile(self.logfilename): - newImageLog = open(self.logfilename + '.new', 'w') - copyFlag = True - with open(self.logfilename, 'r') as imageLog: - for line in imageLog: - header = False - ret = re.search('====== Start of log for ([0-9\.A-Za-z\-\_]+) ======', line) - if ret is not None: - copyFlag = False - header = True - detailedLogFile = open(self.destinationFolder + '/' + ret.group(1), 'w') - if copyFlag: - newImageLog.write(line) - ret = re.search('====== End of log for ([0-9\.A-Za-z\-\_]+) ======', line) - if ret is not None: - copyFlag = True - detailedLogFile.close() - elif not copyFlag and not header: - detailedLogFile.write(line) - imageLog.close() - newImageLog.close() - os.rename(self.logfilename + '.new', self.logfilename) - else: - print('Cannot split unfound file') - -#-------------------------------------------------------------------------------------------------------- -# -# Start of main -# -#-------------------------------------------------------------------------------------------------------- - -argvs = sys.argv -argc = len(argvs) - -SP = SplitReport() - -while len(argvs) > 1: - myArgv = argvs.pop(1) - if re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE): - matchReg = re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE) - SP.logfilename = matchReg.group(1) - -SP.split() - -sys.exit(0) From ec6e32f3e5222286b1a5d86b1a7071d2c14a7b81 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 14 Sep 2023 12:11:28 +0200 Subject: [PATCH 3/5] fix(ci): allowing the branch name to be in contributorName/branchName format Signed-off-by: Raphael Defosseux --- ci-scripts/cls_cluster.py | 3 ++- ci-scripts/cls_containerize.py | 4 +++- ci-scripts/cls_physim1.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index d4d9601d17..d8b1835ca8 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -266,7 +266,8 @@ class Cluster: baseTag = 'develop' forceBaseImageBuild = False if self.ranAllowMerge: # merging MR branch into develop -> temporary image - imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}' + branchName = self.ranBranch.replace('/','-') + imageTag = f'{branchName}-{self.ranCommitID[0:8]}' if self.ranTargetBranch == 'develop': ret = self.cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel9 | grep --colour=never -i INDEX') result = re.search('index', ret.stdout) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index dc4be1f93b..fa7b2d337b 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -97,7 +97,9 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge def ImageTagToUse(imageName, ranCommitID, ranBranch, ranAllowMerge): shortCommit = ranCommitID[0:8] if ranAllowMerge: - tagToUse = f'{ranBranch}-{shortCommit}' + # Allowing contributor to have a name/branchName format + branchName = ranBranch.replace('/','-') + tagToUse = f'{branchName}-{shortCommit}' else: tagToUse = f'develop-{shortCommit}' fullTag = f'{imageName}:{tagToUse}' diff --git a/ci-scripts/cls_physim1.py b/ci-scripts/cls_physim1.py index 2306db513f..b0c9012d7d 100644 --- a/ci-scripts/cls_physim1.py +++ b/ci-scripts/cls_physim1.py @@ -106,7 +106,8 @@ class PhySim: if self.ranCommitID != '': mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30) if self.ranAllowMerge: - imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}' + branchName = self.ranBranch.replace('/','-') + imageTag = f'{branchName}-{self.ranCommitID[0:8]}' if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) From 24efc76de012c9b6b4d08b6c5f48fed8f80e63e7 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 14 Sep 2023 12:15:44 +0200 Subject: [PATCH 4/5] chore(ci): removing verbose-ci useless option Signed-off-by: Raphael Defosseux --- cmake_targets/build_oai | 7 ------- cmake_targets/tools/build_helper | 5 ----- docker/Dockerfile.build.rhel9 | 2 +- docker/Dockerfile.build.rocky | 2 +- docker/Dockerfile.clang.rhel9 | 2 +- docker/Dockerfile.phySim.rhel9 | 2 +- 6 files changed, 4 insertions(+), 16 deletions(-) diff --git a/cmake_targets/build_oai b/cmake_targets/build_oai index 246c45dcda..dbceb680a2 100755 --- a/cmake_targets/build_oai +++ b/cmake_targets/build_oai @@ -38,7 +38,6 @@ gen_nvram_path=$OPENAIR_DIR/cmake_targets/ran_build/build conf_nvram_path=$OPENAIR_DIR/openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf HW="" -VERBOSE_CI=0 VERBOSE_COMPILE=0 RUN_GROUP=0 TEST_CASE_GROUP="" @@ -110,8 +109,6 @@ Options: Adds a debgging facility to the binary files: GUI with major internal synchronization events -x | --xforms Will compile with software oscilloscope features ---verbose-ci - Compile with verbose instructions in CI Docker env --verbose-compile Shows detailed compilation instructions in makefile --build-doxygen @@ -336,10 +333,6 @@ function main() { -x | --xforms) echo_info "Will generate the software oscilloscope features" shift;; - --verbose-ci) - VERBOSE_CI=1 - echo_info "Will compile with verbose instructions in CI Docker env" - shift;; --verbose-compile) VERBOSE_COMPILE=1 echo_info "Will compile with verbose instructions" diff --git a/cmake_targets/tools/build_helper b/cmake_targets/tools/build_helper index 4f1ea858a4..bacd781c43 100755 --- a/cmake_targets/tools/build_helper +++ b/cmake_targets/tools/build_helper @@ -233,11 +233,6 @@ compilations() { $CMAKE --build . $verbose --target $targets -- -j$(nproc) ret=$? } > $dlog/$logfile 2>&1 - if [ "$VERBOSE_CI" == "1" ]; then - echo "====== Start of log for $logfile ======" - cat $dlog/$logfile - echo "====== End of log for $logfile ======" - fi check_warnings "$dlog/$logfile" if [[ $ret -eq 0 ]]; then echo_success "$targets compiled" diff --git a/docker/Dockerfile.build.rhel9 b/docker/Dockerfile.build.rhel9 index 1a12843ea4..ff9c312f8c 100644 --- a/docker/Dockerfile.build.rhel9 +++ b/docker/Dockerfile.build.rhel9 @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --verbose-ci --noavx512 -c + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --noavx512 -c diff --git a/docker/Dockerfile.build.rocky b/docker/Dockerfile.build.rocky index 468869b277..10b04d90c0 100644 --- a/docker/Dockerfile.build.rocky +++ b/docker/Dockerfile.build.rocky @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --verbose-ci --noavx512 -c + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --noavx512 -c diff --git a/docker/Dockerfile.clang.rhel9 b/docker/Dockerfile.clang.rhel9 index 654bae8539..b5cb74b0ce 100644 --- a/docker/Dockerfile.clang.rhel9 +++ b/docker/Dockerfile.clang.rhel9 @@ -40,4 +40,4 @@ RUN /bin/sh oaienv && \ mkdir -p log && \ export CC=/usr/bin/clang && \ export CXX=/usr/bin/clang++ && \ - ./build_oai --phy_simulators --gNB --eNB --nrUE --UE --ninja --verbose-ci --noavx512 --disable-T-Tracer -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror + ./build_oai --phy_simulators --gNB --eNB --nrUE --UE --ninja --noavx512 --disable-T-Tracer -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror diff --git a/docker/Dockerfile.phySim.rhel9 b/docker/Dockerfile.phySim.rhel9 index 223a5bb68a..9935d61979 100644 --- a/docker/Dockerfile.phySim.rhel9 +++ b/docker/Dockerfile.phySim.rhel9 @@ -38,7 +38,7 @@ RUN dnf install -y libasan libubsan RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --phy_simulators --ninja --verbose-ci --sanitize --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror + ./build_oai --phy_simulators --ninja --sanitize --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror #start from scratch for target executable FROM registry.access.redhat.com/ubi9/ubi:latest as oai-physim From bd0457e40ebbcbe89c2be1e0a4949c306b8ab59a Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 14 Sep 2023 15:34:59 +0200 Subject: [PATCH 5/5] doc(ci): restriction on branch naming no longer appply Signed-off-by: Raphael Defosseux --- CONTRIBUTING.md | 2 -- ci-scripts/Jenkinsfile-GitLab-Container | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d0b5c4f30..73886f3f7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,8 +16,6 @@ Please refer to the steps described on our website: [How to contribute to OAI](h * This decision was made for the license reasons. * The Continuous Integration will reject your merge request. - All merge requests SHALL have `develop` branch as target branch. - - All merge requests SHALL have source branch names that SHALL not contain the `/` character. - * We are using a `docker` registry scheme and the image names are based on the source branch name. ## Coding Styles ## diff --git a/ci-scripts/Jenkinsfile-GitLab-Container b/ci-scripts/Jenkinsfile-GitLab-Container index 93b3d38d75..2f94a3e318 100644 --- a/ci-scripts/Jenkinsfile-GitLab-Container +++ b/ci-scripts/Jenkinsfile-GitLab-Container @@ -589,6 +589,7 @@ def triggerCN5GSlaveJob (jobName, gitlabStatusName) { if ("MERGE".equals(env.gitlabActionType)) { shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" --abbrev=8 ' + env.gitlabMergeRequestLastCommit shortenShaOne = shortenShaOne.trim() + branchName = env.gitlabSourceBranch.replaceAll("/", "-").trim() fullRanTag = 'porcepix.sboai.cs.eurecom.fr/oai-gnb:' + env.gitlabSourceBranch + '-' + shortenShaOne } else { shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" --abbrev=8 ' + env.GIT_COMMIT