maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/arm-build-pipeline' into integration_2025_w12 (!3286)

Browse Source 
Add support for ARM build pipeline

Modify the existing python code to be able to build images where the
image tag can be prepended with a prefix, here arm_. This is preparatory
work to create the ARM build image pipeline, and reuse the existing
internal registry on porcepix to have x86 and ARM images coexist.

Fix various bugs in the build system to allow to build on a system with
as many cores as gracehopper.

Also, fix two programs (usim, nvram) to work correctly under ARM.
This commit is contained in:
Robert Schmidt
2025-03-24 14:45:12 +01:00
parent 99ecbad530 5b07283003
commit bbae3b24ad
21 changed files with 228 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/Jenkinsfile-GitLab-Container
+22
View File
@@ -153,6 +153,28 @@ pipeline {
}
}
}
stage ("Ubuntu-ARM-Image-Builder") {
steps {
script {
triggerSlaveJob ('RAN-Ubuntu-ARM-Image-Builder', 'Ubuntu-ARM-Image-Builder')
}
}
post {
always {
script {
// Using a unique variable name for each test stage to avoid overwriting on a global variable
// due to parallel-time concurrency
ubuntuArmBuildStatus = finalizeSlaveJob('RAN-Ubuntu-ARM-Image-Builder')
}
}
failure {
script {
currentBuild.result = 'FAILURE'
failingStages += ubuntuArmBuildStatus
}
}
}
}
stage ("RHEL-Cluster-Image-Builder") {
steps {
script {
ci-scripts/cls_cluster.py
+2 -2
View File
@@ -226,7 +226,7 @@ class Cluster:
def _undeploy_pod(self, filename):
self.cmd.run(f'oc delete -f {filename}')
def PullClusterImage(self, HTML, node, images):
def PullClusterImage(self, HTML, node, images, tag_prefix):
logging.debug(f'Pull OC image {images} to server {node}')
self.testCase_id = HTML.testCase_id
with cls_cmd.getConnection(node) as cmd:
@@ -242,7 +242,7 @@ class Cluster:
return False
tag = cls_containerize.CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
registry = f'{self.OCRegistry}/{CI_OC_RAN_NAMESPACE}'
success, msg = cls_containerize.Containerize.Pull_Image(cmd, images, tag, registry, None, None)
success, msg = cls_containerize.Containerize.Pull_Image(cmd, images, tag, tag_prefix, registry, None, None)
OC_logout(cmd)
param = f"on node {node}"
if success:
ci-scripts/cls_containerize.py
+25 -17
View File
@@ -93,7 +93,7 @@ def CopyLogsToExecutor(cmd, sourcePath, log_name):
os.remove(f'./{log_name}.zip')
if (os.path.isdir(f'./{log_name}')):
shutil.rmtree(f'./{log_name}')
cmd.copyin(f'{sourcePath}/cmake_targets/{log_name}.zip', f'./{log_name}.zip')
cmd.copyin(src=f'{sourcePath}/cmake_targets/{log_name}.zip', tgt=f'{os.getcwd()}/{log_name}.zip')
cmd.run(f'rm -f {log_name}.zip')
ZipFile(f'{log_name}.zip').extractall('.')
@@ -359,7 +359,7 @@ class Containerize():
svr = self.eNB_serverId[self.eNB_instance]
lIpAddr, lSourcePath = self.GetCredentials(svr)
logging.debug('Building on server: ' + lIpAddr)
cmd = cls_cmd.RemoteCmd(lIpAddr)
cmd = cls_cmd.getConnection(lIpAddr)
# Checking the hostname to get adapted on cli and dockerfileprefixes
cmd.run('hostnamectl')
@@ -407,6 +407,12 @@ class Containerize():
result = re.search('build_cross_arm64', self.imageKind)
if result is not None:
self.dockerfileprefix = '.ubuntu22.cross-arm64'
result = re.search('native_arm', self.imageKind)
if result is not None:
imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup', ''))
imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', ''))
self.testCase_id = HTML.testCase_id
cmd.cd(lSourcePath)
@@ -491,10 +497,10 @@ class Containerize():
elif image != 'ran-build':
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
if image == 'oai-gnb-aerial':
cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.2024.05.23.tar.gz .')
cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.*.tar.gz .')
ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200)
if image == 'oai-gnb-aerial':
cmd.run('rm -f nvipc_src.2024.05.23.tar.gz')
cmd.run('rm -f nvipc_src.*.tar.gz')
if image == 'ran-build' and ret.returncode == 0:
cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true")
cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/")
@@ -720,9 +726,9 @@ class Containerize():
HTML.CreateHtmlTabFooter(False)
return False
def Push_Image_to_Local_Registry(self, HTML, svr_id):
def Push_Image_to_Local_Registry(self, HTML, svr_id, tag_prefix=""):
lIpAddr, lSourcePath = self.GetCredentials(svr_id)
logging.debug('Pushing images from server: ' + lIpAddr)
logging.debug('Pushing images to server: ' + lIpAddr)
ssh = cls_cmd.getConnection(lIpAddr)
imagePrefix = 'porcepix.sboai.cs.eurecom.fr'
ret = ssh.run(f'docker login -u oaicicd -p oaicicd {imagePrefix}')
@@ -737,7 +743,7 @@ class Containerize():
if self.ranAllowMerge:
orgTag = 'ci-temp'
for image in IMAGES:
tagToUse = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
tagToUse = tag_prefix + CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
imageTag = f"{image}:{tagToUse}"
ret = ssh.run(f'docker image tag {image}:{orgTag} {imagePrefix}/{imageTag}')
if ret.returncode != 0:
@@ -751,9 +757,10 @@ class Containerize():
return False
# Creating a develop tag on the local private registry
if not self.ranAllowMerge:
ssh.run(f'docker image tag {image}:{orgTag} {imagePrefix}/{image}:develop')
ssh.run(f'docker push {imagePrefix}/{image}:develop')
ssh.run(f'docker rmi {imagePrefix}/{image}:develop')
devTag = f"{tag_prefix}develop"
ssh.run(f'docker image tag {image}:{orgTag} {imagePrefix}/{image}:{devTag}')
ssh.run(f'docker push {imagePrefix}/{image}:{devTag}')
ssh.run(f'docker rmi {imagePrefix}/{image}:{devTag}')
ssh.run(f'docker rmi {imagePrefix}/{imageTag} {image}:{orgTag}')
ret = ssh.run(f'docker logout {imagePrefix}')
@@ -768,7 +775,7 @@ class Containerize():
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
return True
def Pull_Image(cmd, images, tag, registry, username, password):
def Pull_Image(cmd, images, tag, tag_prefix, registry, username, password):
if username is not None and password is not None:
logging.info(f"logging into registry {username}@{registry}")
response = cmd.run(f'docker login -u {username} -p {password} {registry}', silent=True, reportNonZero=False)
@@ -778,14 +785,15 @@ class Containerize():
return False, msg
pulled_images = []
for image in images:
imagePrefTag = f"{image}:{tag_prefix}{tag}"
imageTag = f"{image}:{tag}"
response = cmd.run(f'docker pull {registry}/{imageTag}')
response = cmd.run(f'docker pull {registry}/{imagePrefTag}')
if response.returncode != 0:
msg = f'Could not pull {image} from local registry: {imageTag}'
msg = f'Could not pull {image} from local registry: {imagePrefTag}'
logging.error(msg)
return False, msg
cmd.run(f'docker tag {registry}/{imageTag} oai-ci/{imageTag}')
cmd.run(f'docker rmi {registry}/{imageTag}')
cmd.run(f'docker tag {registry}/{imagePrefTag} oai-ci/{imageTag}')
cmd.run(f'docker rmi {registry}/{imagePrefTag}')
pulled_images += [f"oai-ci/{imageTag}"]
if username is not None and password is not None:
response = cmd.run(f'docker logout {registry}')
@@ -793,13 +801,13 @@ class Containerize():
msg = "Pulled Images:\n" + '\n'.join(pulled_images)
return True, msg
def Pull_Image_from_Registry(self, HTML, svr_id, images, tag=None, registry="porcepix.sboai.cs.eurecom.fr", username="oaicicd", password="oaicicd"):
def Pull_Image_from_Registry(self, HTML, svr_id, images, tag=None, tag_prefix="", registry="porcepix.sboai.cs.eurecom.fr", username="oaicicd", password="oaicicd"):
lIpAddr, lSourcePath = self.GetCredentials(svr_id)
logging.debug('\u001B[1m Pulling image(s) on server: ' + lIpAddr + '\u001B[0m')
if not tag:
tag = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
with cls_cmd.getConnection(lIpAddr) as cmd:
success, msg = Containerize.Pull_Image(cmd, images, tag, registry, username, password)
success, msg = Containerize.Pull_Image(cmd, images, tag, tag_prefix, registry, username, password)
param = f"on node {lIpAddr}"
if success:
HTML.CreateHtmlTestRowQueue(param, 'OK', [msg])
ci-scripts/main.py
+6 -3
View File
@@ -387,20 +387,22 @@ def ExecuteActionWithParam(action):
elif action == 'Push_Local_Registry':
svr_id = test.findtext('svr_id')
success = CONTAINERS.Push_Image_to_Local_Registry(HTML, svr_id)
tag_prefix = test.findtext('tag_prefix') or ""
success = CONTAINERS.Push_Image_to_Local_Registry(HTML, svr_id, tag_prefix)
elif action == 'Pull_Local_Registry' or action == 'Clean_Test_Server_Images':
if force_local:
# Do not pull or remove images when running locally. User is supposed to handle image creation & cleanup
return True
svr_id = test.findtext('svr_id')
tag_prefix = test.findtext('tag_prefix') or ""
images = test.findtext('images').split()
# hack: for FlexRIC, we need to overwrite the tag to use
tag = None
if len(images) == 1 and images[0] == "oai-flexric":
tag = CONTAINERS.flexricTag
if action == "Pull_Local_Registry":
success = CONTAINERS.Pull_Image_from_Registry(HTML, svr_id, images, tag=tag)
success = CONTAINERS.Pull_Image_from_Registry(HTML, svr_id, images, tag=tag, tag_prefix=tag_prefix)
if action == "Clean_Test_Server_Images":
success = CONTAINERS.Clean_Test_Server_Images(HTML, svr_id, images, tag=tag)
@@ -420,9 +422,10 @@ def ExecuteActionWithParam(action):
success = cls_oaicitest.Custom_Script(HTML, node, script, command_fail)
elif action == 'Pull_Cluster_Image':
tag_prefix = test.findtext('tag_prefix') or ""
images = test.findtext('images').split()
node = test.findtext('node')
success = CLUSTER.PullClusterImage(HTML, node, images)
success = CLUSTER.PullClusterImage(HTML, node, images, tag_prefix=tag_prefix)
else:
logging.warning(f"unknown action {action}, skip step")
ci-scripts/xml_files/container_image_build_arm.xml
+55
View File
@@ -0,0 +1,55 @@
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>build-tab</htmlTabRef>
<htmlTabName>Build Container Images for ARM</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
800813
000001
000010
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="800813">
<class>Create_Workspace</class>
<desc>Create new Workspace for server 0</desc>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
</testCase>
<testCase id="000001">
<class>Build_Image</class>
<desc>Build all Images</desc>
<kind>native_arm</kind>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
</testCase>
<testCase id="000010">
<class>Push_Local_Registry</class>
<desc>Push Images to Local Registry</desc>
<svr_id>0</svr_id>
<tag_prefix>arm_</tag_prefix>
</testCase>
</testCaseList>
ci-scripts/xml_files/container_sa_gh_aerial_quectel.xml
+49
View File
@@ -0,0 +1,49 @@
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>TEST-GH-AERIAL-SA</htmlTabRef>
<htmlTabName>Gracehopper AERIAL 100 MHz TDD SA</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<TestCaseRequestedList>
111111
333333
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="111111">
<class>Pull_Local_Registry</class>
<desc>Pull Images from Local Registry</desc>
<svr_id>0</svr_id>
<images>oai-gnb-aerial</images>
<tag_prefix>arm_</tag_prefix>
</testCase>
<testCase id="333333">
<class>Clean_Test_Server_Images</class>
<always_exec>true</always_exec>
<desc>Clean Test Images on Test Server</desc>
<svr_id>0</svr_id>
<images>oai-gnb-aerial</images>
</testCase>
</testCaseList>