From a4f9a7a6d4048a888fd39ccb49e249c8fc48322e Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Thu, 30 Jan 2025 18:12:13 +0100 Subject: [PATCH 1/6] CI: tuning gNB parameters of SABOX-B200 --- .../conf_files/gnb-du.sa.band1.52prb.usrpb210.conf | 4 ++-- .../conf_files/gnb-du.sa.band78.106prb.usrpb200.conf | 6 +++--- .../gnb.sa.band78.106prb.usrpb200.sc-fdma.conf | 11 ++++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ci-scripts/conf_files/gnb-du.sa.band1.52prb.usrpb210.conf b/ci-scripts/conf_files/gnb-du.sa.band1.52prb.usrpb210.conf index e3ba0b881e..2af3185ffc 100644 --- a/ci-scripts/conf_files/gnb-du.sa.band1.52prb.usrpb210.conf +++ b/ci-scripts/conf_files/gnb-du.sa.band1.52prb.usrpb210.conf @@ -189,11 +189,11 @@ RUs = ( nb_rx = 1 # The higher att for TX than RX is because we use a circulator (as for TDD), # while we should use a duplexer in the FDD case. However, it uses the same setup. - att_tx = 18 + att_tx = 16 att_rx = 0; bands = [1]; max_pdschReferenceSignalPower = -27; - max_rxgain = 108; + max_rxgain = 110; eNB_instances = [0]; clock_src = "internal"; } diff --git a/ci-scripts/conf_files/gnb-du.sa.band78.106prb.usrpb200.conf b/ci-scripts/conf_files/gnb-du.sa.band78.106prb.usrpb200.conf index abce64a0c6..3626cec227 100644 --- a/ci-scripts/conf_files/gnb-du.sa.band78.106prb.usrpb200.conf +++ b/ci-scripts/conf_files/gnb-du.sa.band78.106prb.usrpb200.conf @@ -194,11 +194,11 @@ RUs = ( local_rf = "yes" nb_tx = 1 nb_rx = 1 - att_tx = 10; - att_rx = 10; + att_tx = 4; + att_rx = 4; bands = [78]; max_pdschReferenceSignalPower = -27; - max_rxgain = 114; + max_rxgain = 106; eNB_instances = [0]; clock_src = "internal"; } diff --git a/ci-scripts/conf_files/gnb.sa.band78.106prb.usrpb200.sc-fdma.conf b/ci-scripts/conf_files/gnb.sa.band78.106prb.usrpb200.sc-fdma.conf index c2960b3aa7..92b0ad794c 100644 --- a/ci-scripts/conf_files/gnb.sa.band78.106prb.usrpb200.sc-fdma.conf +++ b/ci-scripts/conf_files/gnb.sa.band78.106prb.usrpb200.sc-fdma.conf @@ -83,7 +83,7 @@ gNBs = ra_ResponseWindow = 4; #ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR #1=oneeighth,2=onefourth,3=half,4=one,5=two,6=four,7=eight,8=sixteen - ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR = 4; + ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR = 2; #one (0..15) 4,8,12,16,...60,64 ssb_perRACH_OccasionAndCB_PreamblesPerSSB = 14; #ra_ContentionResolutionTimer @@ -192,8 +192,9 @@ L1s = ( { num_cc = 1; tr_n_preference = "local_mac"; - prach_dtx_threshold = 120; + prach_dtx_threshold = 150; pucch0_dtx_threshold = 30; + max_ldpc_iterations = 15; ofdm_offset_divisor = 8; #set this to UINT_MAX for offset 0 } ); @@ -203,11 +204,11 @@ RUs = ( local_rf = "yes" nb_tx = 1 nb_rx = 1 - att_tx = 10; - att_rx = 10; + att_tx = 6; + att_rx = 6; bands = [78]; max_pdschReferenceSignalPower = -27; - max_rxgain = 114; + max_rxgain = 110; eNB_instances = [0]; clock_src = "internal"; } From b3328e96a41e08b9f9b88f4d2b71d6a2c04f0167 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Thu, 30 Jan 2025 21:44:19 +0100 Subject: [PATCH 2/6] CI: increase healthcheck interval for VNF-PNF deployment The healthcheck interval sets when the first health check occurs after the container starts. Increasing the healthcheck interval helps avoid marking the container as "healthy" during the initialization phase of PNF. In particular, USRP initialization of PNF service may take longer, and deployment failures can occur even after the 10 seconds interval. Extending the healtcheck interval provides more time for the service before assessing container health. --- ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml index c349ac0406..0ecba5f39e 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml @@ -16,7 +16,7 @@ services: - ../../conf_files/gnb-vnf.sa.band77.162prb.nfapi.4x4.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + interval: 15s timeout: 5s retries: 5 oai-pnf: @@ -36,6 +36,6 @@ services: - ../../conf_files/gnb-pnf.band77.usrpn310.4x4.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + interval: 15s timeout: 5s retries: 5 From 955cad5741ae5d1acf211ee8ac6d258e2420ddce Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Thu, 30 Jan 2025 22:34:09 +0100 Subject: [PATCH 3/6] CI: add warning log to HTML when service is restarted --- ci-scripts/cls_containerize.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index dc461dfe7e..6562c15f28 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -889,7 +889,10 @@ class Containerize(): if deployed: break elif (attempt < num_attempts - 1): - logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}') + warning_msg = f'Restart services {services}' + logging.warning(warning_msg) + imagesInfo.append(warning_msg) + HTML.CreateHtmlTestRowQueue('N/A', 'NOK', ['\n'.join(imagesInfo)]) for svc in services.split(): CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') ssh.run(f'docker compose -f {wd_yaml} down -- {services}') From 7417687f50495f63eb32aaee407a009bb7f364e4 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Mon, 24 Mar 2025 13:34:13 +0100 Subject: [PATCH 4/6] CI: update TESTBenches doc - Cleanup and update of "Machines" table: - remove asterix - not used in CI - remove nokiabox - not used in CI - remove orion - not used in CI - update of "Lockable Resource" for individual machines - synch with https://jenkins-oai.eurecom.fr/lockable-resources/ - remove column of "IP Address" - Update information about build pipelines --- doc/TESTBenches.md | 50 ++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/doc/TESTBenches.md b/doc/TESTBenches.md index 2ab379c88c..3a7a1f8f67 100644 --- a/doc/TESTBenches.md +++ b/doc/TESTBenches.md @@ -4,28 +4,25 @@ ## Machines -| Machine | IP address | Lockable Resource | Function | Connected devices | -| ------------- | --------------- | --------------------- | ------------------ | ----------------------------------------------------- | -| asterix | 172.21.16.127 | CI-Asterix-Usage | *unused* | 172.21.19.14 | -| obelix | 172.21.16.128 | CI-Obelix-Usage | eNB (n40, n78), nrUE | 172.21.19.13, X300 (192.168.60.2) | -| porcepix | 172.21.16.136 | CI-Porcepix | Executor, EPC, 5GC | -- | -| up2 | 172.21.19.68 | CI-UP2-Usage | COTS UE | Quectel RM520N | -| nepes | 172.21.16.137 | CI-Nepes | gNB (n78), EPC/5GC | B200mini (30C51EB) | -| ofqot | 172.21.16.109 | CI-Ofqot | gNB (n78) | B200mini (30C51D4) | -| idefix | 172.21.16.135 | CI-Idefix | COTS UE | Quectel RM500Q | -| caracal | 172.21.16.132 | CI-Caracal | gNB/phytest | N300 (192.168.10.2) | -| amariue | 172.21.16.144 | CI-Amarisoft-UE-Usage | nrUE | Amarisoft UE simulator | -| nano | 172.21.18.48 | CI-Nano-Legacy-EPC | Executor, EPC, adb | 2x COTS (adb) | -| hutch | 172.21.18.46 | CI-Hutch-Legacy-FDD-eNB | eNB (B7) | B200mini (30C5239) | -| starsky | 172.21.18.45 | CI-Starsky-Legacy-TDD-eNB | eNB (B40) | b200mini (30A3E3C) | -| carabe | 172.21.18.47 | CI-Carabe-Legacy-FDD-OAI-LTE-UE | UE (B7UE) | B200mini (30AE8C9) | -| nokiabox | 172.21.19.39 | _None_ | gNB (Nokia), 5GC | _Nokia RF integrated_ | -| avra | 172.21.16.124 | CI-Avra-Usage | gNB (n78) | AW2S Jaguar (192.168.80.239) | -| orion | 172.21.16.134 | CI-Orion-Build-Sanity-Check-Deploy-Test, CI-Orion-DsTester-Deploy-Test | Build | | -| aerial2 | 172.21.16.131 | CI-Aerial2-Usage | gNB (PNF/Nvidia CUBB + VNF) | Foxconn RU, _Nvidia Aerial SDK integrated_ | -| cacofonix | 172.21.16.150 | CI-Cacofonix-Usage | gNB (n78, FHI7.2) | | -| matix | 172.21.19.58 | CI-Matix-Usage | gNB (n77) | N310 | -| gracehopper1-oai | -- | Gracehopper1 | build, gNB/Aerial | _Nvidia Aerial SDK integrated_ | +| Machine | Lockable Resource | Function | Connected devices | +| ------------- | --------------------- | -------------------- | ----------------------------------------------------- | +| obelix | Obelix | eNB (n40, n78), nrUE | 172.21.19.13, X300 (192.168.60.2) | +| porcepix | Porcepix | Executor, EPC, 5GC | -- | +| up2 | UP2 | COTS UE | Quectel RM520N | +| nepes | Nepes | gNB (n78), EPC/5GC | B200mini (30C51EB) | +| ofqot | Ofqot | gNB (n78) | B200mini (30C51D4) | +| idefix | Idefix | COTS UE | Quectel RM500Q | +| caracal | Caracal | gNB/phytest, OAI UE | N300 (192.168.10.2), _AMD T2 card_ | +| amariue | Amarisoft-UE | nrUE | Amarisoft UE simulator | +| nano | Nano_EPC/Nano_5G_COTS_UE | Executor, EPC, adb | 2x COTS (adb) | +| hutch | Hutch | eNB (B7) | B200mini (30C5239) | +| starsky | Starsky | eNB (B40) | b200mini (30A3E3C) | +| carabe | Carabe | UE (B7UE) | B200mini (30AE8C9) | +| avra | Avra | gNB (n78) | AW2S Jaguar (192.168.80.239) | +| aerial2 | Aerial2 | gNB (PNF/Nvidia CUBB + VNF) | Foxconn RU, _Nvidia Aerial SDK integrated_ | +| cacofonix | Cacofonix | gNB (n78, FHI7.2) | -- | +| matix | Matix | gNB (n77) | N310 | +| gracehopper1-oai | Gracehopper1 | build, gNB/Aerial | _Nvidia Aerial SDK integrated_ | Note: The available resources, and their current usage, is indicated here: - [Lockable resources of jenkins-oai](https://jenkins-oai.eurecom.fr/lockable-resources/): @@ -107,6 +104,8 @@ information on how the images are built. - target image from `Dockerfile.nr-cuup.rhel9` - target image from `Dockerfile.lteUE.rhel9` - target image from `Dockerfile.nrUE.rhel9` + - build image from `Dockerfile.build.fhi72.rhel9`, followed by + - target image from `Dockerfile.gNB.fhi72.rhel9` - build image from `Dockerfile.phySim.rhel9` (creates as direct target physical simulator image) - build image from `Dockerfile.clang.rhel9` (compilation only, artifacts not used currently) @@ -122,6 +121,9 @@ information on how the images are built. - target image from `Dockerfile.nrUE.ubuntu22` - target image from `Dockerfile.lteUE.ubuntu22` - target image from `Dockerfile.lteRU.ubuntu22` + - target image from `Dockerfile.gNB.aerial.ubuntu22` + - build image from `Dockerfile.build.fhi72.ubuntu22`, followed by + - target image from `Dockerfile.gNB.fhi72.ubuntu22` - build unit tests from `ci-scripts/docker/Dockerfile.unittest.ubuntu22`, and run them - [RAN-Ubuntu-ARM-Image-Builder](https://jenkins-oai.eurecom.fr/job/RAN-Ubuntu-ARM-Image-Builder/) ~BUILD-ONLY ~4G-LTE ~5G-NR @@ -137,7 +139,7 @@ information on how the images are built. - [OAI-CN5G-COTS-UE-Test](https://jenkins-oai.eurecom.fr/job/OAI-CN5G-COTS-UE-Test/) ~5G-NR - - using 5GC bench (resources `CI-Cetautomatix-OC-oaicicd-session`, `CI-Dogmatix-CN5G-gNB`): Attach/Detach of UE with multiple PDU sessions + - using 5GC bench (resources `Cetautomatix`, `Dogmatix`): Attach/Detach of UE with multiple PDU sessions - [OAI-FLEXRIC-RAN-Integration-Test](https://jenkins-oai.eurecom.fr/job/OAI-FLEXRIC-RAN-Integration-Test/) ~5G-NR ~nrUE - selfix (gNB, nrUE, OAI 5GC, FlexRIC) - uses RFsimulator, tests FlexRIC/E2 interface and xApps @@ -171,7 +173,7 @@ information on how the images are built. - basic NSA test - [RAN-PhySim-Cluster](https://jenkins-oai.eurecom.fr/job/RAN-PhySim-Cluster/) ~4G-LTE ~5G-NR ~nrUE - - cluster (`Asterix-OC-oaicicd-session` resource), tests in OpenShift Cluster + - cluster (`RAN_OC` resource), tests in OpenShift Cluster - unitary simulators (`nr_dlsim`, etc.) - see [`./physical-simulators.md`](./physical-simulators.md) for an overview - [RAN-RF-Sim-Test-4G](https://jenkins-oai.eurecom.fr/job/RAN-RF-Sim-Test-4G/) From 1e9e0b6dc78e907309065f3d8864aa3dda9f804f Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Tue, 25 Mar 2025 10:46:21 +0100 Subject: [PATCH 5/6] CI: temporarily disable DRB integrity protection in B200 pipeline Temporarily deactivate DRB integrity protection in SABOX-B200 E1 testcase to avoid CI failure due to low measured throughput. Our investigation suggests that the low throughput is likely due to limitation of the UE (Quectel RM500Q), which appears to restrict the maximum data rate when integrity protection is enabled. This limitation is not observed with Quectel RM520N. --- ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml index 5a55ade3c8..c20a43c0e1 100644 --- a/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml @@ -9,7 +9,7 @@ services: environment: USE_ADDITIONAL_OPTIONS: --telnetsrv --telnetsrv.listenport 9090 --telnetsrv.shrmod ci --log_config.global_log_options level,nocolor,time,line_num,function - --security.drb_integrity yes + --security.drb_integrity no volumes: - ../../conf_files/gnb-cucp.sa.f1.quectel.conf:/opt/oai-gnb/etc/gnb.conf # for performance reasons, we use host mode: in bridge mode, we have From a88f540081124b7a813bdeddcd938ced48b51bde Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Wed, 26 Mar 2025 23:17:33 +0100 Subject: [PATCH 6/6] CI: ping after reestablishment in sabox-b200 --- ci-scripts/xml_files/container_sa_b200_quectel.xml | 9 ++++++--- ci-scripts/xml_files/container_sa_e1_b200_quectel.xml | 9 ++++++--- ci-scripts/xml_files/container_sa_f1_b200_quectel.xml | 9 ++++++--- ci-scripts/xml_files/container_sa_sc_b200_quectel.xml | 9 ++++++--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/ci-scripts/xml_files/container_sa_b200_quectel.xml b/ci-scripts/xml_files/container_sa_b200_quectel.xml index f4f415acdf..9f3559aff1 100644 --- a/ci-scripts/xml_files/container_sa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_b200_quectel.xml @@ -120,9 +120,12 @@ - IdleSleep - Sleep - 10 + Ping + Ping: 20pings + idefix + sabox-nepes + -c 20 -i0.5 -w25 + 80 diff --git a/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml b/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml index 1718eb3152..dcbcb9a45f 100644 --- a/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml @@ -199,9 +199,12 @@ - IdleSleep - Sleep - 10 + Ping + Ping: 20pings + idefix + sabox-nepes + -c 20 -i0.5 -w25 + 80 diff --git a/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml b/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml index 614b9ac57e..9622b2cf13 100644 --- a/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml @@ -134,9 +134,12 @@ - IdleSleep - Sleep - 10 + Ping + Ping: 20pings + idefix + sabox-nepes + -c 20 -i0.5 -w25 + 80 diff --git a/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml b/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml index f72dafee06..69201ebcc8 100644 --- a/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml @@ -114,9 +114,12 @@ - IdleSleep - Sleep - 10 + Ping + Ping: 20pings + idefix + sabox-nepes + -c 20 -i0.5 -w25 + 80