From e845a9c3b7e6210237b20c28acc191418157fb5e Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 29 Feb 2024 18:56:31 +0100 Subject: [PATCH 1/5] CI: Make RFsim start-up faster Make RFsim start up faster by starting core and RAN start at the same time. This speeds up because we do not wait for "healthy" containers serially, but parallely. For the RAN, it is safe to start gNB and UE at the same time in SA mode (as the UE will only attach once the radio, and hence the full gNB, is ready). We do not start in parallel in do-ra and phytest, as in these modes the gNB first has to write a UE-specific configuration file. If we start both gNB and UE at the same time, the UE might try to read the file before it is written, leading to a failure. --- .../xml_files/container_5g_fdd_rfsim.xml | 24 +++---------------- .../xml_files/container_5g_rfsim_24prb.xml | 24 +++---------------- .../xml_files/container_5g_rfsim_2x2.xml | 24 +++---------------- .../container_5g_rfsim_f1_accelleran.xml | 13 ++-------- .../xml_files/container_5g_rfsim_u0_25prb.xml | 24 +++---------------- 5 files changed, 14 insertions(+), 95 deletions(-) diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml index cfcf40e69e..463cb3f2e8 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml @@ -28,10 +28,8 @@ 111111 100011 - 000010 000011 000012 - 000013 020011 020012 030011 @@ -47,35 +45,19 @@ oai-gnb-asan oai-nr-ue-asan - - DeployGenObject - Deploy MySql Database - yaml_files/5g_fdd_rfsimulator - mysql - 1 - - DeployGenObject Deploy OAI 5G CoreNetwork yaml_files/5g_fdd_rfsimulator - oai-amf oai-smf oai-upf oai-ext-dn + mysql oai-amf oai-smf oai-upf oai-ext-dn 5 DeployGenObject - Deploy OAI 5G gNB RF sim SA + Deploy OAI 5G gNB+UE RF sim SA yaml_files/5g_fdd_rfsimulator - oai-gnb - 6 - - - - DeployGenObject - Deploy OAI 5G NR-UE RF sim SA - yaml_files/5g_fdd_rfsimulator - oai-nr-ue + oai-gnb oai-nr-ue 7 diff --git a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml index 41f25476c3..4f838658ec 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml @@ -28,10 +28,8 @@ 111111 100001 - 000000 000001 000002 - 000003 020001 020002 030001 @@ -47,35 +45,19 @@ oai-gnb-asan oai-nr-ue-asan - - DeployGenObject - Deploy MySql Database - yaml_files/5g_rfsimulator_24prb - mysql - 1 - - DeployGenObject Deploy OAI 5G CoreNetwork yaml_files/5g_rfsimulator_24prb - oai-amf oai-smf oai-upf oai-ext-dn + mysql oai-amf oai-smf oai-upf oai-ext-dn 5 DeployGenObject - Deploy OAI 5G gNB RF sim SA + Deploy OAI 5G gNB+UE RFsim SA yaml_files/5g_rfsimulator_24prb - oai-gnb - 6 - - - - DeployGenObject - Deploy OAI 5G NR-UE RF sim SA - yaml_files/5g_rfsimulator_24prb - oai-nr-ue + oai-gnb oai-nr-ue 7 diff --git a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml index defe15a1df..45c311fd97 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml @@ -28,10 +28,8 @@ 111111 100001 - 000000 000001 000002 - 000003 020001 020002 030001 @@ -47,35 +45,19 @@ oai-gnb-asan oai-nr-ue-asan - - DeployGenObject - Deploy MySql Database - yaml_files/5g_rfsimulator_2x2 - mysql - 1 - - DeployGenObject Deploy OAI 5G CoreNetwork yaml_files/5g_rfsimulator_2x2 - oai-amf oai-smf oai-upf oai-ext-dn + mysql oai-amf oai-smf oai-upf oai-ext-dn 5 DeployGenObject - Deploy OAI 5G gNB RF sim SA + Deploy OAI 5G gNB+UE RF sim SA yaml_files/5g_rfsimulator_2x2 - oai-gnb - 6 - - - - DeployGenObject - Deploy OAI 5G NR-UE RF sim SA - yaml_files/5g_rfsimulator_2x2 - oai-nr-ue + oai-gnb oai-nr-ue 7 diff --git a/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml b/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml index d1433db738..5baa950549 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml @@ -27,7 +27,6 @@ 111111 000020 - 000021 020021 100021 100022 @@ -43,17 +42,9 @@ DeployGenObject - Deploy OAI-DU + Deploy OAI-DU+nrUE yaml_files/5g_rfsimulator_accelleran - oai-du - 1 - - - - DeployGenObject - Deploy nrUE - yaml_files/5g_rfsimulator_accelleran - oai-nr-ue + oai-du oai-nr-ue 2 diff --git a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml index f059d69614..c31d6c1843 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml @@ -28,10 +28,8 @@ 111111 100001 - 000000 000001 000002 - 000003 020001 020002 030001 @@ -47,35 +45,19 @@ oai-gnb-asan oai-nr-ue-asan - - DeployGenObject - Deploy MySql Database - yaml_files/5g_rfsimulator_u0_25prb - mysql - 1 - - DeployGenObject Deploy OAI 5G CoreNetwork yaml_files/5g_rfsimulator_u0_25prb - oai-amf oai-smf oai-upf oai-ext-dn + mysql oai-amf oai-smf oai-upf oai-ext-dn 5 DeployGenObject - Deploy OAI 5G gNB RF sim SA + Deploy OAI 5G gNB+nrUE RF sim SA yaml_files/5g_rfsimulator_u0_25prb - oai-gnb - 6 - - - - DeployGenObject - Deploy OAI 5G NR-UE RF sim SA - yaml_files/5g_rfsimulator_u0_25prb - oai-nr-ue + oai-gnb oai-nr-ue 7 From 9af386c761d29e664f66e5464f6dc9dbfe8c7bc5 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 29 Feb 2024 18:57:43 +0100 Subject: [PATCH 2/5] CI: RFsim tests: Wait for IP Ensure that the UE has an IP address before continuing the test. Otherwise, it might happen we ping before the UE is connected. Do not do this in do-ra and phytest, as they use --noS1, which implies statically assigned IP address, so IP address check always succeeds(!). --- ci-scripts/xml_files/container_5g_e1_rfsim.xml | 7 +++++++ ci-scripts/xml_files/container_5g_f1_rfsim.xml | 7 +++++++ ci-scripts/xml_files/container_5g_fdd_rfsim.xml | 7 +++++++ ci-scripts/xml_files/container_5g_rfsim_24prb.xml | 7 +++++++ ci-scripts/xml_files/container_5g_rfsim_2x2.xml | 7 +++++++ ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml | 7 +++++++ ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml | 7 +++++++ 7 files changed, 49 insertions(+) diff --git a/ci-scripts/xml_files/container_5g_e1_rfsim.xml b/ci-scripts/xml_files/container_5g_e1_rfsim.xml index e23cffd0dd..e4ddb9e4d7 100644 --- a/ci-scripts/xml_files/container_5g_e1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_e1_rfsim.xml @@ -30,6 +30,7 @@ 000021 000022 000023 + 000024 020021 100021 @@ -66,6 +67,12 @@ 14 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue rfsim5g_ue2 rfsim5g_ue3 + + Ping Ping ext-dn from all NR-UEs diff --git a/ci-scripts/xml_files/container_5g_f1_rfsim.xml b/ci-scripts/xml_files/container_5g_f1_rfsim.xml index 769de69e35..09b69730a3 100644 --- a/ci-scripts/xml_files/container_5g_f1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_f1_rfsim.xml @@ -29,6 +29,7 @@ 100021 000021 000022 + 000023 020021 030021 030022 @@ -59,6 +60,12 @@ 8 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + + Ping Ping ext-dn from all NR-UEs diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml index 463cb3f2e8..59be0e3738 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml @@ -30,6 +30,7 @@ 100011 000011 000012 + 000013 020011 020012 030011 @@ -61,6 +62,12 @@ 7 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + + Ping Ping ext-dn from NR-UE diff --git a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml index 4f838658ec..e90cbc309b 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml @@ -30,6 +30,7 @@ 100001 000001 000002 + 000003 020001 020002 030001 @@ -61,6 +62,12 @@ 7 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + + Ping Ping ext-dn from NR-UE diff --git a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml index 45c311fd97..2b3f3413db 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml @@ -30,6 +30,7 @@ 100001 000001 000002 + 000003 020001 020002 030001 @@ -61,6 +62,12 @@ 7 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + + Ping Ping ext-dn from NR-UE diff --git a/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml b/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml index 5baa950549..fe02da00a0 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_f1_accelleran.xml @@ -27,6 +27,7 @@ 111111 000020 + 000021 020021 100021 100022 @@ -48,6 +49,12 @@ 2 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + + Ping Test Connectivity (ping) diff --git a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml index c31d6c1843..f2f4d5e78e 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml @@ -30,6 +30,7 @@ 100001 000001 000002 + 000003 020001 020002 030001 @@ -61,6 +62,12 @@ 7 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + + Ping Ping ext-dn from NR-UE From b7a1b4b405d788aa867091c2fcf45b98e399fcd8 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 1 Mar 2024 12:09:40 +0100 Subject: [PATCH 3/5] Always stop Quectel on XML terminate --- .../xml_files/container_nsa_b200_terminate.xml | 13 +++++++++++++ .../xml_files/container_sa_b200_terminate.xml | 13 +++++++++++++ .../xml_files/container_sa_e1_b200_terminate.xml | 13 +++++++++++++ .../xml_files/container_sa_f1_b200_terminate.xml | 13 +++++++++++++ .../xml_files/container_sa_sc_b200_terminate.xml | 13 +++++++++++++ 5 files changed, 65 insertions(+) diff --git a/ci-scripts/xml_files/container_nsa_b200_terminate.xml b/ci-scripts/xml_files/container_nsa_b200_terminate.xml index a766fb7bcb..c76e3c5891 100644 --- a/ci-scripts/xml_files/container_nsa_b200_terminate.xml +++ b/ci-scripts/xml_files/container_nsa_b200_terminate.xml @@ -26,6 +26,8 @@ tasks 1 + 010001 + 010002 030202 030201 222220 @@ -33,6 +35,17 @@ + + Detach_UE + Detach UE + idefix + + + Terminate_UE + Terminate Quectel + idefix + + Undeploy_Object Undeploy eNB diff --git a/ci-scripts/xml_files/container_sa_b200_terminate.xml b/ci-scripts/xml_files/container_sa_b200_terminate.xml index bddc94aedf..26836fb096 100644 --- a/ci-scripts/xml_files/container_sa_b200_terminate.xml +++ b/ci-scripts/xml_files/container_sa_b200_terminate.xml @@ -26,10 +26,23 @@ tasks 1 + 010001 + 010002 030201 + + Detach_UE + Detach UE + idefix + + + Terminate_UE + Terminate Quectel + idefix + + Undeploy_Object Undeploy gNB diff --git a/ci-scripts/xml_files/container_sa_e1_b200_terminate.xml b/ci-scripts/xml_files/container_sa_e1_b200_terminate.xml index ad902e838a..ae61c1b6ce 100644 --- a/ci-scripts/xml_files/container_sa_e1_b200_terminate.xml +++ b/ci-scripts/xml_files/container_sa_e1_b200_terminate.xml @@ -26,11 +26,24 @@ tasks 1 + 010001 + 010002 330201 222222 + + Detach_UE + Detach UE + idefix + + + Terminate_UE + Terminate Quectel + idefix + + Undeploy_Object Undeploy CU-CP/CU-UP/DU diff --git a/ci-scripts/xml_files/container_sa_f1_b200_terminate.xml b/ci-scripts/xml_files/container_sa_f1_b200_terminate.xml index f5b13beaed..df02f63b7f 100644 --- a/ci-scripts/xml_files/container_sa_f1_b200_terminate.xml +++ b/ci-scripts/xml_files/container_sa_f1_b200_terminate.xml @@ -26,10 +26,23 @@ tasks 1 + 010001 + 010002 230201 + + Detach_UE + Detach UE + idefix + + + Terminate_UE + Terminate Quectel + idefix + + Undeploy_Object Undeploy CU-DU diff --git a/ci-scripts/xml_files/container_sa_sc_b200_terminate.xml b/ci-scripts/xml_files/container_sa_sc_b200_terminate.xml index 9a041fed7a..f47c16258f 100644 --- a/ci-scripts/xml_files/container_sa_sc_b200_terminate.xml +++ b/ci-scripts/xml_files/container_sa_sc_b200_terminate.xml @@ -26,10 +26,23 @@ tasks 1 + 010001 + 010002 030201 + + Detach_UE + Detach UE + idefix + + + Terminate_UE + Terminate Quectel + idefix + + Undeploy_Object Undeploy gNB From 961981f45686c8dfc9858f0a29a7eb78530960cc Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 1 Mar 2024 13:01:04 +0100 Subject: [PATCH 4/5] CI: make docker volume always succeed in startup Some CI users (OAI developers) expressed that the warning of the `docker volume prune` before starting RFsim is confusing. To avoid the warning in the HTML, force the removal; if the volume does not exist, no warning is emitted. To still catch possible problems, do not do this in the `terminate.xml` so we might still see if there are other problems. --- ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml | 2 +- ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml | 2 +- ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml b/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml index 7eba071e16..6ed13ceb6e 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml @@ -50,7 +50,7 @@ Custom_Command Clean-Up any residual volume localhost - docker volume rm 5g_rfsimulator_fdd_phytest_rrc.config + docker volume rm 5g_rfsimulator_fdd_phytest_rrc.config -f diff --git a/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml b/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml index 25ffb4b7b2..452f2c33f9 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml @@ -48,7 +48,7 @@ Custom_Command Clean-Up any residual volume localhost - docker volume rm 5g_rfsimulator_fr2_32prb_rrc.config + docker volume rm 5g_rfsimulator_fr2_32prb_rrc.config -f diff --git a/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml b/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml index 9f0c3bc93a..306b8e4e05 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml @@ -48,7 +48,7 @@ Custom_Command Clean-Up any residual volume localhost - docker volume rm 5g_rfsimulator_tdd_dora_rrc.config + docker volume rm 5g_rfsimulator_tdd_dora_rrc.config -f From e9f0541ad04d8260abb03c32a871404eba8eabb9 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 1 Mar 2024 14:27:14 +0100 Subject: [PATCH 5/5] BuildProxy: report success, correctly print HTML --- ci-scripts/cls_containerize.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 708c2a6f3c..20ce23ce35 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -662,22 +662,26 @@ class Containerize(): collectInfo['proxy'] = files mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5) result = re.search('Size *= *(?P[0-9\-]+) *bytes', mySSH.getBefore()) + # Cleaning any created tmp volume + mySSH.command(self.cli + ' volume prune --force || true','\$', 15) + mySSH.close() + allImagesSize = {} if result is not None: imageSize = float(result.group('size')) / 1000000 logging.debug('\u001B[1m proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes' + logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m') + HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) + return True else: - logging.debug('proxy size is unknown') + logging.error('proxy size is unknown') allImagesSize['proxy'] = 'unknown' - - # Cleaning any created tmp volume - mySSH.command(self.cli + ' volume prune --force || true','\$', 15) - mySSH.close() - - logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m') - HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) - HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) + logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m') + HTML.CreateHtmlTestRow('commit ' + tag, 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTabFooter(False) + return False def BuildRunTests(self, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':