From bd2b833c5858568df22cd43d44587f5886c93b42 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 19 Mar 2025 08:51:47 +0100 Subject: [PATCH] CI: deploy OpenShift-based 5GC through CoreNetwork, use in Ping/Iperf - Use scripts to deploy OpenShift-based 5GC in ci_infra.yaml - Use for oc-cn5g, oc-cn5g-20897, and oc-cn5g-20897-aerial2 for (un)deployment, ping, iperf in relevant XML files - Convert hardcoded IP addresses to %cn_ip% --- ci-scripts/ci_infra.yaml | 15 +++--- ci-scripts/cls_cluster.py | 46 ------------------- ci-scripts/epc.py | 39 +++------------- ci-scripts/scripts/oc-cn5g-deploy.sh | 14 ++++++ ci-scripts/scripts/oc-cn5g-logcollect.sh | 21 +++++++++ ci-scripts/scripts/oc-cn5g-undeploy.sh | 14 ++++++ .../xml_files/container_sa_aerial_quectel.xml | 7 +-- .../xml_files/container_sa_aw2s_asue.xml | 2 + .../xml_files/container_sa_aw2s_asue_2x2.xml | 2 + .../container_sa_fhi72_metanoia_up2_2x2.xml | 3 +- .../xml_files/container_sa_fhi72_vvdn_up2.xml | 3 +- .../xml_files/container_sa_n310_nrue.xml | 2 + .../xml_files/sa_cn5g_20897_closure.xml | 2 +- ci-scripts/xml_files/sa_cn5g_20897_start.xml | 2 +- ci-scripts/xml_files/sa_cn5g_closure.xml | 2 +- ci-scripts/xml_files/sa_cn5g_start.xml | 2 +- ci-scripts/xml_files/sa_oaiue_cn5g_deploy.xml | 2 +- .../xml_files/sa_oaiue_cn5g_undeploy.xml | 2 +- 18 files changed, 84 insertions(+), 96 deletions(-) create mode 100755 ci-scripts/scripts/oc-cn5g-deploy.sh create mode 100755 ci-scripts/scripts/oc-cn5g-logcollect.sh create mode 100755 ci-scripts/scripts/oc-cn5g-undeploy.sh diff --git a/ci-scripts/ci_infra.yaml b/ci-scripts/ci_infra.yaml index 06e42bd5e6..f1c7d4b294 100644 --- a/ci-scripts/ci_infra.yaml +++ b/ci-scripts/ci_infra.yaml @@ -63,24 +63,27 @@ adb_ue_2: oc-cn5g: Host: avra - Namespace: "oaicicd-core-for-ci-ran" - CNPath: "/opt/oai-cn5g-fed-develop-2025-jan" NetworkScript: echo "inet 172.21.6.102" RunIperf3Server: False + Deploy: "! scripts/oc-cn5g-deploy.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-ci-ran" + Undeploy: "! scripts/oc-cn5g-undeploy.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-ci-ran" + LogCollect: "! scripts/oc-cn5g-logcollect.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-ci-ran %%log_dir%%" oc-cn5g-20897: Host: cacofonix - Namespace: "oaicicd-core-for-fhi72" - CNPath: "/opt/oai-cn5g-fed-develop-2025-jan" NetworkScript: echo "inet 172.21.6.105" RunIperf3Server: False + Deploy: "! scripts/oc-cn5g-deploy.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-fhi72" + Undeploy: "! scripts/oc-cn5g-undeploy.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-fhi72" + LogCollect: "! scripts/oc-cn5g-logcollect.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-fhi72 %%log_dir%%" oc-cn5g-20897-aerial2: Host: aerial2 - Namespace: "oaicicd-core-for-nvidia-aerial" - CNPath: "/opt/oai-cn5g-fed-develop-2025-jan" NetworkScript: echo "inet 172.21.6.105" RunIperf3Server: False + Deploy: "! scripts/oc-cn5g-deploy.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-nvidia-aerial" + Undeploy: "! scripts/oc-cn5g-undeploy.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-nvidia-aerial" + LogCollect: "! scripts/oc-cn5g-logcollect.sh /opt/oai-cn5g-fed-develop-2025-jan oaicicd-core-for-nvidia-aerial %%log_dir%%" matix-cn5g: Host: matix diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index bfa315b28e..67e6253dcc 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -43,9 +43,6 @@ NAMESPACE = "oaicicd-ran" OCUrl = "https://api.oai.cs.eurecom.fr:6443" OCRegistry = "default-route-openshift-image-registry.apps.oai.cs.eurecom.fr" CI_OC_RAN_NAMESPACE = "oaicicd-ran" -CN_IMAGES = ["mysql", "oai-nrf", "oai-amf", "oai-smf", "oai-upf", "oai-ausf", "oai-udm", "oai-udr", "oai-traffic-server"] -CN_CONTAINERS = ["", "-c nrf", "-c amf", "-c smf", "-c upf", "-c ausf", "-c udm", "-c udr", ""] - def OC_login(cmd, ocUserName, ocPassword, ocProjectName): if ocUserName == '' or ocPassword == '' or ocProjectName == '': @@ -67,49 +64,6 @@ def OC_login(cmd, ocUserName, ocPassword, ocProjectName): def OC_logout(cmd): cmd.run(f'oc logout') -def OC_deploy_CN(cmd, ocUserName, ocPassword, ocNamespace, path): - logging.debug(f'OC OAI CN5G: Deploying OAI CN5G on Openshift Cluster: {ocNamespace}') - succeeded = OC_login(cmd, ocUserName, ocPassword, ocNamespace) - if not succeeded: - return False, CONST.OC_LOGIN_FAIL - cmd.run(f'helm list -aq -n {ocNamespace} | xargs -r helm uninstall -n {ocNamespace} --wait') - ret = cmd.run(f'helm install --wait oai5gcn {path}/ci-scripts/charts/oai-5g-basic/.') - if ret.returncode != 0: - logging.error('OC OAI CN5G: Deployment failed') - OC_logout(cmd) - return False, CONST.OC_PROJECT_FAIL - report = cmd.run('oc get pods') - OC_logout(cmd) - return True, report - -def OC_undeploy_CN(cmd, ocUserName, ocPassword, ocNamespace, path): - logging.debug(f'OC OAI CN5G: Terminating CN on Openshift Cluster: {ocNamespace}') - succeeded = OC_login(cmd, ocUserName, ocPassword, ocNamespace) - if not succeeded: - return False, CONST.OC_LOGIN_FAIL - cmd.run(f'rm -Rf {path}/logs') - cmd.run(f'mkdir -p {path}/logs') - logging.debug('OC OAI CN5G: Collecting log files to workspace') - cmd.run(f'oc describe pod &> {path}/logs/describe-pods-post-test.log') - cmd.run(f'oc get pods.metrics.k8s &> {path}/logs/nf-resource-consumption.log') - for ii, ci in zip(CN_IMAGES, CN_CONTAINERS): - podName = cmd.run(f"oc get pods | grep {ii} | awk \'{{print $1}}\'").stdout.strip() - if not podName: - logging.debug(f'{ii} pod not found!') - else: - cmd.run(f'oc logs -f {podName} {ci} &> {path}/logs/{ii}.log &') - cmd.run(f'cd {path}/logs && zip -r -qq test_logs_CN.zip *.log') - cmd.copyin(f'{path}/logs/test_logs_CN.zip','test_logs_CN.zip') - ret = cmd.run(f'helm list -aq -n {ocNamespace} | xargs -r helm uninstall -n {ocNamespace} --wait') - if ret.returncode != 0: - logging.error('OC OAI CN5G: Undeployment failed') - cmd.run(f'helm list -aq -n {ocNamespace} | xargs -r helm uninstall -n {ocNamespace} --wait') - OC_logout(cmd) - return False, CONST.OC_PROJECT_FAIL - report = cmd.run('oc get pods') - OC_logout(cmd) - return True, report - class Cluster: def __init__(self): self.eNBIPAddress = "" diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index e10c9ef485..c4585b98b3 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -293,20 +293,7 @@ class EPCManagement(): html_cell += '(' + res4.group('date') + ')' html_cell += '\n' elif re.match('OC-OAI-CN5G', self.Type, re.IGNORECASE): - cn = cls_module.Module_UE(self.cnID) - succeeded, report = OC.OC_deploy_CN(mySSH, self.OCUserName, self.OCPassword, cn.getNamespace(), cn.getCNPath()) - if not succeeded: - HTML.CreateHtmlTestRow('N/A', 'KO', report) - HTML.CreateHtmlTabFooter(False) - mySSH.close() - logging.error("OC OAI CN5G: CN deployment failed!") - return False - for line in report.stdout.split('\n')[1:]: - columns = line.strip().split() - name = columns[0] - status = columns[2] - html_cell += status + ' ' + name - html_cell += '\n' + raise NotImplemented("use cls_corenetwork.py") else: logging.error('This option should not occur!') mySSH.close() @@ -334,9 +321,7 @@ class EPCManagement(): logging.error('no container with name oai-amf found, could not retrieve AMF IP address') mySSH.close() elif re.match('OC-OAI-CN5G', self.Type, re.IGNORECASE): - mySSH = SSH.SSHConnection() - mySSH.open(self.IPAddress, self.UserName, self.Password) - response=mySSH.command3('oc pods ls -f name=oai-amf', 10) + raise NotImplemented("use cls_corenetwork.py") def TerminateHSS(self, HTML): mySSH = SSH.SSHConnection() @@ -479,15 +464,7 @@ class EPCManagement(): mySSH.copyin(f'{self.SourceCodePath}/logs/test_logs_CN.zip','test_logs_CN.zip') logging.debug(message) elif re.match('OC-OAI-CN5G', self.Type, re.IGNORECASE): - cn = cls_module.Module_UE(self.cnID) - succeeded, report = OC.OC_undeploy_CN(mySSH, self.OCUserName, self.OCPassword, cn.getNamespace(), cn.getCNPath()) - if not succeeded: - HTML.CreateHtmlTestRow('N/A', 'KO', report) - HTML.CreateHtmlTabFooter(False) - logging.error("OC OAI CN5G: CN undeployment failed!") - return False - else: - message = report.stdout + raise NotImplemented("use cls_corenetwork.py") else: logging.error('This should not happen!') mySSH.close() @@ -737,7 +714,7 @@ class EPCManagement(): elif re.match('OAICN5G', self.Type, re.IGNORECASE): logging.debug('LogCollect is bypassed for that variant') elif re.match('OC-OAI-CN5G', self.Type, re.IGNORECASE): - logging.debug('LogCollect is bypassed for that variant') + raise NotImplemented("use cls_corenetwork.py") elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip hss.log.zip hss*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm hss*.log', '\$', 5) @@ -777,9 +754,7 @@ class EPCManagement(): mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g*.pcap','\$', 30) mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OC-OAI-CN5G', self.Type, re.IGNORECASE): - mySSH.run('cd ' + self.SourceCodePath + '/logs') - mySSH.run('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g*.pcap') - mySSH.run('mv mme.log.zip ' + self.SourceCodePath + '/scripts') + raise NotImplemented("use cls_corenetwork.py") elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip mme.log.zip mme*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm mme*.log', '\$', 5) @@ -814,9 +789,7 @@ class EPCManagement(): mySSH.command('zip spgw.log.zip oai-smf.log oai-spgwu.log','\$', 30) mySSH.command('mv spgw.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OC-OAI-CN5G', self.Type, re.IGNORECASE): - mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5) - mySSH.command('zip spgw.log.zip oai-smf.log oai-spgwu.log','\$', 30) - mySSH.command('mv spgw.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) + raise NotImplemented("use cls_corenetwork.py") elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip spgw.log.zip spgw*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm spgw*.log', '\$', 5) diff --git a/ci-scripts/scripts/oc-cn5g-deploy.sh b/ci-scripts/scripts/oc-cn5g-deploy.sh new file mode 100755 index 0000000000..99ba768898 --- /dev/null +++ b/ci-scripts/scripts/oc-cn5g-deploy.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +function die() { echo $@; exit 1; } +[ $# -eq 2 ] || die "usage: $0 " + +OC_DIR=${1} +OC_NS=${2} + +cat ${OC_DIR}/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null +oc project ${OC_NS} > /dev/null +helm install --wait oai5gcn ${OC_DIR}/ci-scripts/charts/oai-5g-basic/. +oc logout > /dev/null diff --git a/ci-scripts/scripts/oc-cn5g-logcollect.sh b/ci-scripts/scripts/oc-cn5g-logcollect.sh new file mode 100755 index 0000000000..52cc5a8f8d --- /dev/null +++ b/ci-scripts/scripts/oc-cn5g-logcollect.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +function die() { echo $@; exit 1; } +[ $# -eq 3 ] || die "usage: $0 " + +OC_DIR=${1} +OC_NS=${2} +DIR=${3} + +cat ${OC_DIR}/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null +oc project ${OC_NS} > /dev/null +oc describe pod > ${DIR}/describe-pods-post-test.log +oc get pods.metrics.k8s &> ${DIR}/nf-resource-consumption.log +for p in nrf amf smf upf ausf udm udr; do + oc logs -l app.kubernetes.io/name=oai-${p} -c ${p} --tail=-1 > ${DIR}/oai-${p}.log +done +oc logs -l app.kubernetes.io/name=mysql --tail=-1 > ${DIR}/mysql.log +oc logs -l app.kubernetes.io/name=oai-traffic-server --tail=-1 > ${DIR}/oai-traffic-server.log +oc logout > /dev/null diff --git a/ci-scripts/scripts/oc-cn5g-undeploy.sh b/ci-scripts/scripts/oc-cn5g-undeploy.sh new file mode 100755 index 0000000000..e9cf421f6b --- /dev/null +++ b/ci-scripts/scripts/oc-cn5g-undeploy.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +function die() { echo $@; exit 1; } +[ $# -eq 2 ] || die "usage: $0 " + +OC_DIR=${1} +OC_NS=${2} + +cat ${OC_DIR}/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null +oc project ${OC_NS} > /dev/null +helm uninstall oai5gcn --wait +oc logout > /dev/null diff --git a/ci-scripts/xml_files/container_sa_aerial_quectel.xml b/ci-scripts/xml_files/container_sa_aerial_quectel.xml index 0b110cab3b..1d91dbd482 100644 --- a/ci-scripts/xml_files/container_sa_aerial_quectel.xml +++ b/ci-scripts/xml_files/container_sa_aerial_quectel.xml @@ -42,7 +42,7 @@ - Initialize_5GCN + DeployCoreNetwork Initialize 5G Core oc-cn5g-20897-aerial2 @@ -89,7 +89,8 @@ Ping Ping: 100pings in 20sec up2-aerial - -c 100 -i 0.2 172.21.6.104 + oc-cn5g-20897-aerial2 + -c 100 -i 0.2 %cn_ip% 5 @@ -140,7 +141,7 @@ - Terminate_5GCN + UndeployCoreNetwork true Terminate 5G Core oc-cn5g-20897-aerial2 diff --git a/ci-scripts/xml_files/container_sa_aw2s_asue.xml b/ci-scripts/xml_files/container_sa_aw2s_asue.xml index 701d4c23d5..29c864e8d1 100644 --- a/ci-scripts/xml_files/container_sa_aw2s_asue.xml +++ b/ci-scripts/xml_files/container_sa_aw2s_asue.xml @@ -225,6 +225,7 @@ Ping Ping: 50 pings in 10sec, multi-ue amarisoft_ue_1 amarisoft_ue_2 amarisoft_ue_3 amarisoft_ue_4 amarisoft_ue_5 amarisoft_ue_6 amarisoft_ue_7 amarisoft_ue_8 amarisoft_ue_9 amarisoft_ue_10 amarisoft_ue_11 amarisoft_ue_12 amarisoft_ue_13 amarisoft_ue_14 amarisoft_ue_15 + oc-cn5g -c 50 -i 0.2 %cn_ip% 5 25 @@ -240,6 +241,7 @@ Ping Ping: 50pings in 10sec, multi-ue amarisoft_ue_16 + oc-cn5g -c 50 -i 0.2 %cn_ip% 5 30 diff --git a/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml b/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml index 9f6a373eca..87031898ba 100644 --- a/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml +++ b/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml @@ -222,6 +222,7 @@ Ping Ping: 20pings in 20sec, multi-ue amarisoft_ue_1 amarisoft_ue_2 amarisoft_ue_3 amarisoft_ue_4 amarisoft_ue_5 amarisoft_ue_6 amarisoft_ue_7 amarisoft_ue_8 amarisoft_ue_9 amarisoft_ue_10 amarisoft_ue_11 amarisoft_ue_12 amarisoft_ue_13 amarisoft_ue_14 amarisoft_ue_15 + oc-cn5g -c 20 %cn_ip% 5 25 @@ -237,6 +238,7 @@ Ping Ping: 50pings in 10sec, multi-ue amarisoft_ue_16 + oc-cn5g -c 50 -i 0.2 %cn_ip% 5 30 diff --git a/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml b/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml index 197a75e6b2..9c408a81f5 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml @@ -84,7 +84,8 @@ Ping Ping: 100 pings in 10 sec up2-fhi72 - -c 100 -i 0.1 172.21.6.104 + oc-cn5g-20897 + -c 100 -i 0.1 %cn_ip% 1 15 diff --git a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml index 927fe1a3e7..5374d216c8 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml @@ -83,7 +83,8 @@ Ping Ping: 100 pings in 10 sec up2-fhi72 - -c 100 -i 0.1 172.21.6.104 + oc-cn5g-20897 + -c 100 -i 0.1 %cn_ip% 1 15 diff --git a/ci-scripts/xml_files/container_sa_n310_nrue.xml b/ci-scripts/xml_files/container_sa_n310_nrue.xml index e6b1ba1643..00b4dfa583 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue.xml @@ -127,6 +127,7 @@ Ping Ping: 20pings in 20sec oai_ue_caracal + oc-cn5g -c 20 %cn_ip% 1 25 @@ -136,6 +137,7 @@ Ping Ping: 100pings in 20sec oai_ue_caracal + oc-cn5g -c 100 -i 0.2 %cn_ip% 1 30 diff --git a/ci-scripts/xml_files/sa_cn5g_20897_closure.xml b/ci-scripts/xml_files/sa_cn5g_20897_closure.xml index 685775c1a4..eeab1b32dc 100644 --- a/ci-scripts/xml_files/sa_cn5g_20897_closure.xml +++ b/ci-scripts/xml_files/sa_cn5g_20897_closure.xml @@ -30,7 +30,7 @@ - Terminate_5GCN + UndeployCoreNetwork Terminate 5G Core oc-cn5g-20897 diff --git a/ci-scripts/xml_files/sa_cn5g_20897_start.xml b/ci-scripts/xml_files/sa_cn5g_20897_start.xml index 2d60baedc1..9c29b83563 100644 --- a/ci-scripts/xml_files/sa_cn5g_20897_start.xml +++ b/ci-scripts/xml_files/sa_cn5g_20897_start.xml @@ -31,7 +31,7 @@ - Initialize_5GCN + DeployCoreNetwork Initialize 5G Core oc-cn5g-20897 diff --git a/ci-scripts/xml_files/sa_cn5g_closure.xml b/ci-scripts/xml_files/sa_cn5g_closure.xml index 727e208d4c..beb490dbed 100644 --- a/ci-scripts/xml_files/sa_cn5g_closure.xml +++ b/ci-scripts/xml_files/sa_cn5g_closure.xml @@ -30,7 +30,7 @@ - Terminate_5GCN + UndeployCoreNetwork Terminate 5G Core oc-cn5g diff --git a/ci-scripts/xml_files/sa_cn5g_start.xml b/ci-scripts/xml_files/sa_cn5g_start.xml index 3de615330a..a49a2cda18 100644 --- a/ci-scripts/xml_files/sa_cn5g_start.xml +++ b/ci-scripts/xml_files/sa_cn5g_start.xml @@ -31,7 +31,7 @@ - Initialize_5GCN + DeployCoreNetwork Initialize 5G Core oc-cn5g diff --git a/ci-scripts/xml_files/sa_oaiue_cn5g_deploy.xml b/ci-scripts/xml_files/sa_oaiue_cn5g_deploy.xml index 3de615330a..a49a2cda18 100644 --- a/ci-scripts/xml_files/sa_oaiue_cn5g_deploy.xml +++ b/ci-scripts/xml_files/sa_oaiue_cn5g_deploy.xml @@ -31,7 +31,7 @@ - Initialize_5GCN + DeployCoreNetwork Initialize 5G Core oc-cn5g diff --git a/ci-scripts/xml_files/sa_oaiue_cn5g_undeploy.xml b/ci-scripts/xml_files/sa_oaiue_cn5g_undeploy.xml index 727e208d4c..beb490dbed 100644 --- a/ci-scripts/xml_files/sa_oaiue_cn5g_undeploy.xml +++ b/ci-scripts/xml_files/sa_oaiue_cn5g_undeploy.xml @@ -30,7 +30,7 @@ - Terminate_5GCN + UndeployCoreNetwork Terminate 5G Core oc-cn5g