Merge remote-tracking branch 'origin/raphael/ci-fix-ubuntu-build-log-scheme' into integration_2023_w37

CONTRIBUTING.md

@@ -16,8 +16,6 @@ Please refer to the steps described on our website: [How to contribute to OAI](h
       * This decision was made for the license reasons.
       * The Continuous Integration will reject your merge request.
    - All merge requests SHALL have `develop` branch as target branch.
-   - All merge requests SHALL have source branch names that SHALL not contain the `/` character.
-      * We are using a `docker` registry scheme and the image names are based on the source branch name.
 
 ## Coding Styles ##
 

ci-scripts/Jenkinsfile-GitLab-Container

@@ -589,6 +589,7 @@ def triggerCN5GSlaveJob (jobName, gitlabStatusName) {
   if ("MERGE".equals(env.gitlabActionType)) {
     shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" --abbrev=8 ' + env.gitlabMergeRequestLastCommit
     shortenShaOne = shortenShaOne.trim()
+    branchName = env.gitlabSourceBranch.replaceAll("/", "-").trim()
     fullRanTag = 'porcepix.sboai.cs.eurecom.fr/oai-gnb:' + env.gitlabSourceBranch + '-' + shortenShaOne
   } else {
     shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" --abbrev=8 ' + env.GIT_COMMIT

ci-scripts/cls_cluster.py

@@ -266,7 +266,8 @@ class Cluster:
 		baseTag = 'develop'
 		forceBaseImageBuild = False
 		if self.ranAllowMerge: # merging MR branch into develop -> temporary image
-			imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}'
+			branchName = self.ranBranch.replace('/','-')
+			imageTag = f'{branchName}-{self.ranCommitID[0:8]}'
 			if self.ranTargetBranch == 'develop':
 				ret = self.cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel9 | grep --colour=never -i INDEX')
 				result = re.search('index', ret.stdout)
@@ -412,7 +413,6 @@ class Cluster:
 		imageSize = {}
 		for image in attemptedImages:
 			self.cmd.run(f'mkdir -p cmake_targets/log/{image}')
-			self.cmd.run(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log')
 			tag = imageTag if image != 'ran-base' else baseTag
 			size = self._get_image_size(image, tag)
 			if size <= 0:

ci-scripts/cls_containerize.py

@@ -97,7 +97,9 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge
 def ImageTagToUse(imageName, ranCommitID, ranBranch, ranAllowMerge):
 	shortCommit = ranCommitID[0:8]
 	if ranAllowMerge:
-		tagToUse = f'{ranBranch}-{shortCommit}'
+		# Allowing contributor to have a name/branchName format
+		branchName = ranBranch.replace('/','-')
+		tagToUse = f'{branchName}-{shortCommit}'
 	else:
 		tagToUse = f'develop-{shortCommit}'
 	fullTag = f'{imageName}:{tagToUse}'
@@ -153,17 +155,13 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus):
 			committed = False
 			tagged = False
 			with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
-				startOfTargetImageCreation = False # check for tagged/committed only after image created
 				for line in inputfile:
-					result = re.search(f'FROM .* [aA][sS] {image}$', str(line))
+					lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
-					if result is not None:
+					lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
-						startOfTargetImageCreation = True
+					tagged = tagged or lineHasTag or lineHasTag2
-					if startOfTargetImageCreation:
+					# the OpenShift Cluster builder prepends image registry URL
-						lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
+					lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
-						tagged = tagged or lineHasTag
+					committed = committed or lineHasCommit
-						# the OpenShift Cluster builder prepends image registry URL
-						lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
-						committed = committed or lineHasCommit
 			errorandwarnings['errors'] = 0 if committed or tagged else 1
 			errorandwarnings['warnings'] = 0
 			errorandwarnings['status'] = committed or tagged
@@ -469,9 +467,12 @@ class Containerize():
 			if image != 'ran-build':
 				cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
 			cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', timeout=1200)
-			# split the log
+			if image == 'ran-build':
-			cmd.run(f"mkdir -p cmake_targets/log/{image}")
+				cmd.run(f"docker run --name test-log -d {image}:{imageTag} /bin/true")
-			cmd.run(f"python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log")
+				cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{image}/")
+				cmd.run(f"docker rm -f test-log")
+			else:
+				cmd.run(f"mkdir -p cmake_targets/log/{image}")
 			# check the status of the build
 			ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {image}:{imageTag}")
 			if ret.returncode != 0:

ci-scripts/cls_physim1.py

@@ -106,7 +106,8 @@ class PhySim:
 		if self.ranCommitID != '':
 			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
 		if self.ranAllowMerge:
-			imageTag = f'{self.ranBranch}-{self.ranCommitID[0:8]}'
+			branchName = self.ranBranch.replace('/','-')
+			imageTag = f'{branchName}-{self.ranCommitID[0:8]}'
 			if self.ranTargetBranch == '':
 				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
 					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30)

ci-scripts/docker_log_split.py

@@ -1,92 +0,0 @@
-#/*
-# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
-# * contributor license agreements.  See the NOTICE file distributed with
-# * this work for additional information regarding copyright ownership.
-# * The OpenAirInterface Software Alliance licenses this file to You under
-# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
-# * except in compliance with the License.
-# * You may obtain a copy of the License at
-# *
-# *      http://www.openairinterface.org/?page_id=698
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# *-------------------------------------------------------------------------------
-# * For more information about the OpenAirInterface (OAI) Software Alliance:
-# *      contact@openairinterface.org
-# */
-#---------------------------------------------------------------------
-# Python for CI of OAI-eNB + COTS-UE
-#
-#   Required Python Version
-#     Python 3.x
-#
-#   Required Python Package
-#     pexpect
-#---------------------------------------------------------------------
-
-
-#-----------------------------------------------------------
-# Import Libs
-#-----------------------------------------------------------
-import sys		# arg
-import re		# reg
-import os
-import subprocess
-
-
-class SplitReport():
-	def __init__(self):
-		self.logfilename = ''
-		self.destinationFolder = ''
-
-	def split(self):
-		self.destinationFolder = self.logfilename.replace(".log","")
-		if os.path.isfile(self.logfilename):
-			newImageLog = open(self.logfilename + '.new', 'w')
-			copyFlag = True
-			with open(self.logfilename, 'r') as imageLog:
-				for line in imageLog:
-					header = False
-					ret = re.search('====== Start of log for ([0-9\.A-Za-z\-\_]+) ======', line)
-					if ret is not None:
-						copyFlag = False
-						header = True
-						detailedLogFile = open(self.destinationFolder + '/' + ret.group(1), 'w')
-					if copyFlag:
-						newImageLog.write(line)
-					ret = re.search('====== End of log for ([0-9\.A-Za-z\-\_]+) ======', line)
-					if ret is not None:
-						copyFlag = True
-						detailedLogFile.close()
-					elif not copyFlag and not header:
-						detailedLogFile.write(line)
-			imageLog.close()
-			newImageLog.close()
-			os.rename(self.logfilename + '.new', self.logfilename)
-		else:
-			print('Cannot split unfound file')
-
-#--------------------------------------------------------------------------------------------------------
-#
-# Start of main
-#
-#--------------------------------------------------------------------------------------------------------
-
-argvs = sys.argv
-argc = len(argvs)
-
-SP = SplitReport()
-
-while len(argvs) > 1:
-	myArgv = argvs.pop(1)
-	if re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE):
-		matchReg = re.match('^\-\-logfilename=(.+)$', myArgv, re.IGNORECASE)
-		SP.logfilename = matchReg.group(1)
-
-SP.split()
-
-sys.exit(0)

cmake_targets/build_oai

@@ -38,7 +38,6 @@ gen_nvram_path=$OPENAIR_DIR/cmake_targets/ran_build/build
 conf_nvram_path=$OPENAIR_DIR/openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf
 
 HW=""
-VERBOSE_CI=0
 VERBOSE_COMPILE=0
 RUN_GROUP=0
 TEST_CASE_GROUP=""
@@ -110,8 +109,6 @@ Options:
    Adds a debgging facility to the binary files: GUI with major internal synchronization events
 -x | --xforms
    Will compile with software oscilloscope features
---verbose-ci
-   Compile with verbose instructions in CI Docker env
 --verbose-compile
    Shows detailed compilation instructions in makefile
 --build-doxygen
@@ -336,10 +333,6 @@ function main() {
        -x | --xforms)
             echo_info "Will generate the software oscilloscope features"
             shift;;
-       --verbose-ci)
-            VERBOSE_CI=1
-            echo_info "Will compile with verbose instructions in CI Docker env"
-            shift;;
        --verbose-compile)
             VERBOSE_COMPILE=1
             echo_info "Will compile with verbose instructions"

cmake_targets/tools/build_helper

@@ -233,11 +233,6 @@ compilations() {
     $CMAKE --build . $verbose --target $targets -- -j$(nproc)
     ret=$?
   } > $dlog/$logfile 2>&1
-  if [ "$VERBOSE_CI" == "1" ]; then
-     echo "====== Start of log for $logfile ======"
-     cat $dlog/$logfile
-     echo "====== End of log for $logfile ======"
-  fi
   check_warnings "$dlog/$logfile"
   if [[ $ret -eq 0 ]]; then
      echo_success "$targets compiled"

docker/Dockerfile.build.rhel9

@@ -35,4 +35,4 @@ COPY . .
 RUN /bin/sh oaienv && \ 
     cd cmake_targets && \
     mkdir -p log && \
-    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --verbose-ci --noavx512 -c
+    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --noavx512 -c

docker/Dockerfile.build.rocky

@@ -35,4 +35,4 @@ COPY . .
 RUN /bin/sh oaienv && \ 
     cd cmake_targets && \
     mkdir -p log && \
-    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --verbose-ci --noavx512 -c
+    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope" -w USRP -t Ethernet --noavx512 -c

docker/Dockerfile.build.ubuntu20

@@ -35,4 +35,4 @@ COPY . .
 RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
-    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope nrqtscope" -w USRP -t Ethernet --verbose-ci --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror
+    ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib "telnetsrv enbscope uescope nrscope nrqtscope" -w USRP -t Ethernet --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror

docker/Dockerfile.clang.rhel9

@@ -40,4 +40,4 @@ RUN /bin/sh oaienv && \
     mkdir -p log && \
     export CC=/usr/bin/clang && \
     export CXX=/usr/bin/clang++ && \
-    ./build_oai --phy_simulators --gNB --eNB --nrUE --UE --ninja --verbose-ci --noavx512 --disable-T-Tracer -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror
+    ./build_oai --phy_simulators --gNB --eNB --nrUE --UE --ninja --noavx512 --disable-T-Tracer -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror

docker/Dockerfile.phySim.rhel9

@@ -38,7 +38,7 @@ RUN dnf install -y libasan libubsan
 RUN /bin/sh oaienv && \ 
     cd cmake_targets && \
     mkdir -p log && \
-    ./build_oai --phy_simulators --ninja --verbose-ci --sanitize --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror
+    ./build_oai --phy_simulators --ninja --sanitize --noavx512 -c --cmake-opt -DCMAKE_C_FLAGS=-Werror --cmake-opt -DCMAKE_CXX_FLAGS=-Werror
 
 #start from scratch for target executable
 FROM registry.access.redhat.com/ubi9/ubi:latest as oai-physim