From f7b3e04cf9bc56a0909a4c8e24c4c35af03ddcf6 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Fri, 4 Apr 2025 15:45:44 +0200 Subject: [PATCH] CI: new healthcheck condition for OTA docker deployments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This modification aims to improve health status detection by using PRACH IO as an indicator of the gNB's status (container is marked as "healthy" when the PRACH IO value in gNB nrL1_stats.log exceeds 0.0 dB.). The new condition accounts for scenarios where the container may not exit or produce error codes (e.g., “sleep” logs or "error code overflow" messages). --- .../yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml | 6 ++++-- .../yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml | 6 ++++-- .../yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml | 8 +++++--- ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml | 8 +++++--- .../sa_fhi_7.2_benetel550_gnb/docker-compose.yml | 8 +++++--- .../sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml | 8 +++++--- .../yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml | 9 ++++++--- docker/Dockerfile.gNB.fhi72.rhel9 | 1 + docker/Dockerfile.gNB.fhi72.rocky | 1 + docker/Dockerfile.gNB.fhi72.ubuntu22 | 1 + docker/Dockerfile.gNB.rhel9 | 1 + docker/Dockerfile.gNB.rocky | 1 + docker/Dockerfile.gNB.ubuntu22 | 1 + docker/scripts/check-prach-io.sh | 5 +++++ 14 files changed, 45 insertions(+), 19 deletions(-) create mode 100755 docker/scripts/check-prach-io.sh diff --git a/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml index 6d597d3a86..11ecb20f08 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml @@ -14,7 +14,9 @@ services: volumes: - ../../conf_files/gnb.sa.band77.273prb.usrpn310.2x2.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml index 43211d7381..332fb23503 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml @@ -14,7 +14,9 @@ services: volumes: - ../../conf_files/gnb.sa.band77.162prb.usrpn310.2x2.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml index 0ecba5f39e..d3d2ddf582 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml @@ -16,7 +16,7 @@ services: - ../../conf_files/gnb-vnf.sa.band77.162prb.nfapi.4x4.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" - interval: 15s + interval: 5s timeout: 5s retries: 5 oai-pnf: @@ -35,7 +35,9 @@ services: volumes: - ../../conf_files/gnb-pnf.band77.usrpn310.4x4.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 15s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml index e93152bd37..562b72561a 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml @@ -16,7 +16,9 @@ services: volumes: - ../../conf_files/gnb.sa.band78.106prb.n310.7ds2u.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml index a0bc415730..b70229ac94 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml @@ -22,10 +22,12 @@ services: oai-net: ipv4_address: 172.21.18.20 healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 networks: oai-net: driver: macvlan diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml index 0dd0a6c99d..06273747bf 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml @@ -21,10 +21,12 @@ services: oai-net: ipv4_address: 172.21.18.20 healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 networks: oai-net: driver: macvlan diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml index a74b583e2b..ff06893fa0 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml @@ -21,10 +21,13 @@ services: oai-net: ipv4_address: 172.21.18.20 healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 + networks: oai-net: driver: macvlan diff --git a/docker/Dockerfile.gNB.fhi72.rhel9 b/docker/Dockerfile.gNB.fhi72.rhel9 index 5f68599873..122b3b668d 100644 --- a/docker/Dockerfile.gNB.fhi72.rhel9 +++ b/docker/Dockerfile.gNB.fhi72.rhel9 @@ -71,6 +71,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \ diff --git a/docker/Dockerfile.gNB.fhi72.rocky b/docker/Dockerfile.gNB.fhi72.rocky index 3fdc2f97af..f95310059d 100644 --- a/docker/Dockerfile.gNB.fhi72.rocky +++ b/docker/Dockerfile.gNB.fhi72.rocky @@ -59,6 +59,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \ diff --git a/docker/Dockerfile.gNB.fhi72.ubuntu22 b/docker/Dockerfile.gNB.fhi72.ubuntu22 index 069055d63b..eb781281ec 100644 --- a/docker/Dockerfile.gNB.fhi72.ubuntu22 +++ b/docker/Dockerfile.gNB.fhi72.ubuntu22 @@ -71,6 +71,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \ diff --git a/docker/Dockerfile.gNB.rhel9 b/docker/Dockerfile.gNB.rhel9 index 824e3b3833..d02b6907d7 100644 --- a/docker/Dockerfile.gNB.rhel9 +++ b/docker/Dockerfile.gNB.rhel9 @@ -60,6 +60,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \ diff --git a/docker/Dockerfile.gNB.rocky b/docker/Dockerfile.gNB.rocky index 225432fdab..912de17ede 100644 --- a/docker/Dockerfile.gNB.rocky +++ b/docker/Dockerfile.gNB.rocky @@ -57,6 +57,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \ diff --git a/docker/Dockerfile.gNB.ubuntu22 b/docker/Dockerfile.gNB.ubuntu22 index 68c903450a..ea7b538c41 100644 --- a/docker/Dockerfile.gNB.ubuntu22 +++ b/docker/Dockerfile.gNB.ubuntu22 @@ -94,6 +94,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \ diff --git a/docker/scripts/check-prach-io.sh b/docker/scripts/check-prach-io.sh new file mode 100755 index 0000000000..1e1544dadf --- /dev/null +++ b/docker/scripts/check-prach-io.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eo pipefail + +test -f /opt/oai-gnb/nrL1_stats.log && grep -oP 'PRACH I0 = \K[0-9.]*' /opt/oai-gnb/nrL1_stats.log | awk '{if ($1 > 0.0) exit 0; else exit 1}'