From ebe1b711ac3e14628c9656f0104d0a2dfa117ed3 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 4 Mar 2024 12:17:02 +0100 Subject: [PATCH 1/4] Print core_pattern in all entrypoint scripts --- docker/scripts/enb_entrypoint.sh | 3 +++ docker/scripts/gnb-aw2s_entrypoint.sh | 3 +++ docker/scripts/gnb_entrypoint.sh | 2 ++ docker/scripts/lte_ru_entrypoint.sh | 3 +++ docker/scripts/lte_ue_entrypoint.sh | 3 +++ docker/scripts/nr_ue_entrypoint.sh | 3 +++ 6 files changed, 17 insertions(+) diff --git a/docker/scripts/enb_entrypoint.sh b/docker/scripts/enb_entrypoint.sh index cda5793ab3..42a9bb44cc 100755 --- a/docker/scripts/enb_entrypoint.sh +++ b/docker/scripts/enb_entrypoint.sh @@ -5,6 +5,9 @@ set -uo pipefail PREFIX=/opt/oai-enb CONFIGFILE=$PREFIX/etc/enb.conf +echo "==================================" +echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)" + if [ ! -f $CONFIGFILE ]; then echo "No configuration file found: please mount at $CONFIGFILE" exit 255 diff --git a/docker/scripts/gnb-aw2s_entrypoint.sh b/docker/scripts/gnb-aw2s_entrypoint.sh index 752e3005ed..7085800d13 100755 --- a/docker/scripts/gnb-aw2s_entrypoint.sh +++ b/docker/scripts/gnb-aw2s_entrypoint.sh @@ -5,6 +5,9 @@ set -uo pipefail PREFIX=/opt/oai-gnb-aw2s CONFIGFILE=$PREFIX/etc/gnb.conf +echo "==================================" +echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)" + if [ ! -f $CONFIGFILE ]; then echo "No configuration file found: please mount at $CONFIGFILE" exit 255 diff --git a/docker/scripts/gnb_entrypoint.sh b/docker/scripts/gnb_entrypoint.sh index 230f3f0f7d..9253d4a3e1 100755 --- a/docker/scripts/gnb_entrypoint.sh +++ b/docker/scripts/gnb_entrypoint.sh @@ -5,6 +5,8 @@ set -uo pipefail PREFIX=/opt/oai-gnb CONFIGFILE=$PREFIX/etc/gnb.conf +echo "==================================" +echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)" if [ ! -f $CONFIGFILE ]; then echo "No configuration file found: please mount at $CONFIGFILE" diff --git a/docker/scripts/lte_ru_entrypoint.sh b/docker/scripts/lte_ru_entrypoint.sh index 0305cf27ff..85d99419c0 100755 --- a/docker/scripts/lte_ru_entrypoint.sh +++ b/docker/scripts/lte_ru_entrypoint.sh @@ -5,6 +5,9 @@ set -uo pipefail PREFIX=/opt/oai-lte-ru CONFIGFILE=$PREFIX/etc/rru.conf +echo "==================================" +echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)" + if [ ! -f $CONFIGFILE ]; then echo "No configuration file found: please mount at $CONFIGFILE" exit 255 diff --git a/docker/scripts/lte_ue_entrypoint.sh b/docker/scripts/lte_ue_entrypoint.sh index f88692a957..bf3a72b0d2 100755 --- a/docker/scripts/lte_ue_entrypoint.sh +++ b/docker/scripts/lte_ue_entrypoint.sh @@ -5,6 +5,9 @@ set -uo pipefail PREFIX=/opt/oai-lte-ue USIM_CONFIGFILE=$PREFIX/etc/ue_usim.conf +echo "==================================" +echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)" + if [ ! -f $USIM_CONFIGFILE ]; then echo "No ue_usim.conf configuration file found: please mount at $USIM_CONFIGFILE" exit 255 diff --git a/docker/scripts/nr_ue_entrypoint.sh b/docker/scripts/nr_ue_entrypoint.sh index e84386276a..31382a9b07 100755 --- a/docker/scripts/nr_ue_entrypoint.sh +++ b/docker/scripts/nr_ue_entrypoint.sh @@ -5,6 +5,9 @@ set -uo pipefail PREFIX=/opt/oai-nr-ue CONFIGFILE=$PREFIX/etc/nr-ue.conf +echo "==================================" +echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)" + if [ ! -f $CONFIGFILE ]; then echo "No configuration file found: please mount at $CONFIGFILE" exit 255 From d7c3806f13503baa7c5061a02b9b11c29fc3e08c Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 4 Mar 2024 12:01:15 +0100 Subject: [PATCH 2/4] Use tini as init in all containers - Print problems, e.g., segfault - Handle signals and zombie processes properly --- docker/Dockerfile.base.rhel9 | 6 ++++++ docker/Dockerfile.base.rocky | 6 ++++++ docker/Dockerfile.base.ubuntu20 | 6 ++++++ docker/Dockerfile.base.ubuntu20.cross-arm64 | 6 ++++++ docker/Dockerfile.eNB.rhel9 | 3 ++- docker/Dockerfile.eNB.rocky | 3 ++- docker/Dockerfile.eNB.ubuntu20 | 3 ++- docker/Dockerfile.gNB.aw2s.rhel9 | 3 ++- docker/Dockerfile.gNB.aw2s.rocky | 3 ++- docker/Dockerfile.gNB.aw2s.ubuntu20 | 3 ++- docker/Dockerfile.gNB.rhel9 | 3 ++- docker/Dockerfile.gNB.rocky | 3 ++- docker/Dockerfile.gNB.ubuntu20 | 3 ++- docker/Dockerfile.lteRU.rhel9 | 3 ++- docker/Dockerfile.lteRU.ubuntu20 | 3 ++- docker/Dockerfile.lteUE.rhel9 | 3 ++- docker/Dockerfile.lteUE.rocky | 3 ++- docker/Dockerfile.lteUE.ubuntu20 | 3 ++- docker/Dockerfile.nr-cuup.rhel9 | 3 ++- docker/Dockerfile.nr-cuup.rocky | 3 ++- docker/Dockerfile.nr-cuup.ubuntu20 | 3 ++- docker/Dockerfile.nrUE.rhel9 | 3 ++- docker/Dockerfile.nrUE.rocky | 3 ++- docker/Dockerfile.nrUE.ubuntu20 | 3 ++- 24 files changed, 64 insertions(+), 20 deletions(-) diff --git a/docker/Dockerfile.base.rhel9 b/docker/Dockerfile.base.rhel9 index 56b1ce7a23..7142b721fa 100644 --- a/docker/Dockerfile.base.rhel9 +++ b/docker/Dockerfile.base.rhel9 @@ -58,6 +58,12 @@ RUN rm -f /etc/rhsm-host && \ echo "/usr/local/lib" > /etc/ld.so.conf.d/local-lib.conf && \ echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local-lib.conf +# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini +# it will be copied into target containers, to print exit numbers and handle signals properly +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + # In some network environments, GIT proxy is required RUN /bin/bash -c "if [[ -v NEEDED_GIT_PROXY ]]; then git config --global http.proxy $NEEDED_GIT_PROXY; fi" diff --git a/docker/Dockerfile.base.rocky b/docker/Dockerfile.base.rocky index 42267f5d59..41210177b8 100644 --- a/docker/Dockerfile.base.rocky +++ b/docker/Dockerfile.base.rocky @@ -46,6 +46,12 @@ RUN dnf install 'dnf-command(config-manager)' -y && \ python3-pip && \ pip3 install --ignore-installed pyyaml +# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini +# it will be copied into target containers, to print exit numbers and handle signals properly +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + #create the WORKDIR WORKDIR /oai-ran COPY . . diff --git a/docker/Dockerfile.base.ubuntu20 b/docker/Dockerfile.base.ubuntu20 index 73fe0905d9..3434bdaa4b 100644 --- a/docker/Dockerfile.base.ubuntu20 +++ b/docker/Dockerfile.base.ubuntu20 @@ -46,6 +46,12 @@ RUN apt-get update && \ python3-pip && \ pip3 install --ignore-installed pyyaml +# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini +# it will be copied into target containers, to print exit numbers and handle signals properly +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + # In some network environments, GIT proxy is required RUN /bin/bash -c "if [[ -v NEEDED_GIT_PROXY ]]; then git config --global http.proxy $NEEDED_GIT_PROXY; fi" diff --git a/docker/Dockerfile.base.ubuntu20.cross-arm64 b/docker/Dockerfile.base.ubuntu20.cross-arm64 index 3f83b751af..bdd6c41774 100644 --- a/docker/Dockerfile.base.ubuntu20.cross-arm64 +++ b/docker/Dockerfile.base.ubuntu20.cross-arm64 @@ -73,6 +73,12 @@ RUN apt-get update && \ g++-9-aarch64-linux-gnu && \ apt-get clean +# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini +# it will be copied into target containers, to print exit numbers and handle signals properly +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + # create the WORKDIR WORKDIR /oai-ran COPY . . diff --git a/docker/Dockerfile.eNB.rhel9 b/docker/Dockerfile.eNB.rhel9 index 7883f8560a..191c8eb2b5 100644 --- a/docker/Dockerfile.eNB.rhel9 +++ b/docker/Dockerfile.eNB.rhel9 @@ -112,5 +112,6 @@ WORKDIR /opt/oai-enb # 36422 --> X2C, SCTP/UDP EXPOSE 2152/udp 36412/udp 36422/udp -ENTRYPOINT ["/opt/oai-enb/bin/entrypoint.sh"] +COPY --from=enb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-enb/bin/entrypoint.sh"] CMD ["/opt/oai-enb/bin/lte-softmodem", "-O", "/opt/oai-enb/etc/enb.conf"] diff --git a/docker/Dockerfile.eNB.rocky b/docker/Dockerfile.eNB.rocky index 5bd3aa8458..51705f0e7b 100644 --- a/docker/Dockerfile.eNB.rocky +++ b/docker/Dockerfile.eNB.rocky @@ -122,5 +122,6 @@ WORKDIR /opt/oai-enb # 36422 --> X2C, SCTP/UDP EXPOSE 2152/udp 36412/udp 36422/udp -ENTRYPOINT ["/opt/oai-enb/bin/entrypoint.sh"] +COPY --from=enb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-enb/bin/entrypoint.sh"] CMD ["/opt/oai-enb/bin/lte-softmodem", "-O", "/opt/oai-enb/etc/enb.conf"] diff --git a/docker/Dockerfile.eNB.ubuntu20 b/docker/Dockerfile.eNB.ubuntu20 index 2f10bda9ef..73a96dac62 100644 --- a/docker/Dockerfile.eNB.ubuntu20 +++ b/docker/Dockerfile.eNB.ubuntu20 @@ -112,5 +112,6 @@ EXPOSE 2152/udp 36412/udp 36422/udp #EXPOSE 50000/udp # IF5 / ORI (control) #EXPOSE 50001/udp # IF5 / ECPRI (data) -ENTRYPOINT ["/opt/oai-enb/bin/entrypoint.sh"] +COPY --from=enb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-enb/bin/entrypoint.sh"] CMD ["/opt/oai-enb/bin/lte-softmodem", "-O", "/opt/oai-enb/etc/enb.conf"] diff --git a/docker/Dockerfile.gNB.aw2s.rhel9 b/docker/Dockerfile.gNB.aw2s.rhel9 index 863f126be6..0885a9962d 100644 --- a/docker/Dockerfile.gNB.aw2s.rhel9 +++ b/docker/Dockerfile.gNB.aw2s.rhel9 @@ -86,5 +86,6 @@ RUN ln -s /usr/local/lib/libaw2sori_transpro.so /usr/local/lib/libthirdparty_tra WORKDIR /opt/oai-gnb-aw2s -ENTRYPOINT ["/opt/oai-gnb-aw2s/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb-aw2s/bin/entrypoint.sh"] CMD ["/opt/oai-gnb-aw2s/bin/nr-softmodem", "-O", "/opt/oai-gnb-aw2s/etc/gnb.conf"] diff --git a/docker/Dockerfile.gNB.aw2s.rocky b/docker/Dockerfile.gNB.aw2s.rocky index 1897bf2720..57b6e8c1b5 100644 --- a/docker/Dockerfile.gNB.aw2s.rocky +++ b/docker/Dockerfile.gNB.aw2s.rocky @@ -95,5 +95,6 @@ RUN ln -s /usr/local/lib/libaw2sori_transpro.so /usr/local/lib/libthirdparty_tra WORKDIR /opt/oai-gnb-aw2s -ENTRYPOINT ["/opt/oai-gnb-aw2s/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb-aw2s/bin/entrypoint.sh"] CMD ["/opt/oai-gnb-aw2s/bin/nr-softmodem", "-O", "/opt/oai-gnb-aw2s/etc/gnb.conf"] diff --git a/docker/Dockerfile.gNB.aw2s.ubuntu20 b/docker/Dockerfile.gNB.aw2s.ubuntu20 index 321876638d..4d28c38707 100644 --- a/docker/Dockerfile.gNB.aw2s.ubuntu20 +++ b/docker/Dockerfile.gNB.aw2s.ubuntu20 @@ -81,5 +81,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/libaw2sori_transpro.so /usr/local/lib/lib WORKDIR /opt/oai-gnb-aw2s -ENTRYPOINT ["/opt/oai-gnb-aw2s/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb-aw2s/bin/entrypoint.sh"] CMD ["/opt/oai-gnb-aw2s/bin/nr-softmodem", "-O", "/opt/oai-gnb-aw2s/etc/gnb.conf"] diff --git a/docker/Dockerfile.gNB.rhel9 b/docker/Dockerfile.gNB.rhel9 index fc6a6c5975..358782519f 100644 --- a/docker/Dockerfile.gNB.rhel9 +++ b/docker/Dockerfile.gNB.rhel9 @@ -111,5 +111,6 @@ WORKDIR /opt/oai-gnb #EXPOSE 50000/udp # IF5 / ORI (control) #EXPOSE 50001/udp # IF5 / ECPRI (data) -ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"] CMD ["/opt/oai-gnb/bin/nr-softmodem", "-O", "/opt/oai-gnb/etc/gnb.conf"] diff --git a/docker/Dockerfile.gNB.rocky b/docker/Dockerfile.gNB.rocky index 035f1122a5..d4c8504bb8 100644 --- a/docker/Dockerfile.gNB.rocky +++ b/docker/Dockerfile.gNB.rocky @@ -121,5 +121,6 @@ WORKDIR /opt/oai-gnb #EXPOSE 50000/udp # IF5 / ORI (control) #EXPOSE 50001/udp # IF5 / ECPRI (data) -ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"] CMD ["/opt/oai-gnb/bin/nr-softmodem", "-O", "/opt/oai-gnb/etc/gnb.conf"] diff --git a/docker/Dockerfile.gNB.ubuntu20 b/docker/Dockerfile.gNB.ubuntu20 index 6cc0af8965..16e3795fe8 100644 --- a/docker/Dockerfile.gNB.ubuntu20 +++ b/docker/Dockerfile.gNB.ubuntu20 @@ -108,5 +108,6 @@ WORKDIR /opt/oai-gnb #EXPOSE 50000/udp # IF5 / ORI (control) #EXPOSE 50001/udp # IF5 / ECPRI (data) -ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"] CMD ["/opt/oai-gnb/bin/nr-softmodem", "-O", "/opt/oai-gnb/etc/gnb.conf"] diff --git a/docker/Dockerfile.lteRU.rhel9 b/docker/Dockerfile.lteRU.rhel9 index 81741441ab..30e996600c 100644 --- a/docker/Dockerfile.lteRU.rhel9 +++ b/docker/Dockerfile.lteRU.rhel9 @@ -95,5 +95,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai WORKDIR /opt/oai-lte-ru -ENTRYPOINT ["/opt/oai-lte-ru/bin/entrypoint.sh"] +COPY --from=ru-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ru/bin/entrypoint.sh"] CMD ["/opt/oai-lte-ru/bin/oairu", "-O", "/opt/oai-lte-ru/etc/rru.conf"] diff --git a/docker/Dockerfile.lteRU.ubuntu20 b/docker/Dockerfile.lteRU.ubuntu20 index 1a476c5733..6fa2a167b7 100644 --- a/docker/Dockerfile.lteRU.ubuntu20 +++ b/docker/Dockerfile.lteRU.ubuntu20 @@ -88,5 +88,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai WORKDIR /opt/oai-lte-ru -ENTRYPOINT ["/opt/oai-lte-ru/bin/entrypoint.sh"] +COPY --from=ru-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ru/bin/entrypoint.sh"] CMD ["/opt/oai-lte-ru/bin/oairu", "-O", "/opt/oai-lte-ru/etc/rru.conf"] diff --git a/docker/Dockerfile.lteUE.rhel9 b/docker/Dockerfile.lteUE.rhel9 index 1ba07477c1..a9a95e23a6 100644 --- a/docker/Dockerfile.lteUE.rhel9 +++ b/docker/Dockerfile.lteUE.rhel9 @@ -104,5 +104,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai ldconfig WORKDIR /opt/oai-lte-ue +COPY --from=lte-ue-base /tini /tini CMD ["/opt/oai-lte-ue/bin/lte-uesoftmodem"] -ENTRYPOINT ["/opt/oai-lte-ue/bin/entrypoint.sh"] +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ue/bin/entrypoint.sh"] diff --git a/docker/Dockerfile.lteUE.rocky b/docker/Dockerfile.lteUE.rocky index 9e4043fb9b..95845a0fd4 100644 --- a/docker/Dockerfile.lteUE.rocky +++ b/docker/Dockerfile.lteUE.rocky @@ -116,5 +116,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai echo "ldd on libtelnetsrv.so" && ldd /usr/local/lib/libtelnetsrv.so WORKDIR /opt/oai-lte-ue +COPY --from=lte-ue-base /tini /tini CMD ["/opt/oai-lte-ue/bin/lte-uesoftmodem"] -ENTRYPOINT ["/opt/oai-lte-ue/bin/entrypoint.sh"] +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ue/bin/entrypoint.sh"] diff --git a/docker/Dockerfile.lteUE.ubuntu20 b/docker/Dockerfile.lteUE.ubuntu20 index 89dbd505d5..3320c36bf8 100644 --- a/docker/Dockerfile.lteUE.ubuntu20 +++ b/docker/Dockerfile.lteUE.ubuntu20 @@ -104,5 +104,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai ldd /opt/oai-lte-ue/bin/lte-uesoftmodem WORKDIR /opt/oai-lte-ue +COPY --from=lte-ue-base /tini /tini CMD ["/opt/oai-lte-ue/bin/lte-uesoftmodem"] -ENTRYPOINT ["/opt/oai-lte-ue/bin/entrypoint.sh"] +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ue/bin/entrypoint.sh"] diff --git a/docker/Dockerfile.nr-cuup.rhel9 b/docker/Dockerfile.nr-cuup.rhel9 index 5ae63a0e48..00d6236bfe 100644 --- a/docker/Dockerfile.nr-cuup.rhel9 +++ b/docker/Dockerfile.nr-cuup.rhel9 @@ -75,5 +75,6 @@ WORKDIR /opt/oai-gnb/etc WORKDIR /opt/oai-gnb -ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"] CMD ["/opt/oai-gnb/bin/nr-cuup", "-O", "/opt/oai-gnb/etc/gnb.conf"] diff --git a/docker/Dockerfile.nr-cuup.rocky b/docker/Dockerfile.nr-cuup.rocky index adeaf053d9..e3dc1bf5be 100644 --- a/docker/Dockerfile.nr-cuup.rocky +++ b/docker/Dockerfile.nr-cuup.rocky @@ -76,5 +76,6 @@ WORKDIR /opt/oai-gnb/etc WORKDIR /opt/oai-gnb -ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"] CMD ["/opt/oai-gnb/bin/nr-cuup", "-O", "/opt/oai-gnb/etc/gnb.conf"] diff --git a/docker/Dockerfile.nr-cuup.ubuntu20 b/docker/Dockerfile.nr-cuup.ubuntu20 index 365e48a4b8..0a371e669c 100644 --- a/docker/Dockerfile.nr-cuup.ubuntu20 +++ b/docker/Dockerfile.nr-cuup.ubuntu20 @@ -66,5 +66,6 @@ RUN ldconfig && ldd /opt/oai-gnb/bin/nr-cuup WORKDIR /opt/oai-gnb/etc WORKDIR /opt/oai-gnb -ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"] +COPY --from=gnb-base /tini /tini +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"] CMD ["/opt/oai-gnb/bin/nr-cuup", "-O", "/opt/oai-gnb/etc/gnb.conf"] diff --git a/docker/Dockerfile.nrUE.rhel9 b/docker/Dockerfile.nrUE.rhel9 index 90d7499dd0..14d9f6c7f7 100644 --- a/docker/Dockerfile.nrUE.rhel9 +++ b/docker/Dockerfile.nrUE.rhel9 @@ -108,5 +108,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai WORKDIR /opt/oai-nr-ue +COPY --from=nr-ue-base /tini /tini CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem", "-O", "/opt/oai-nr-ue/etc/nr-ue.conf"] -ENTRYPOINT ["/opt/oai-nr-ue/bin/entrypoint.sh"] +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"] diff --git a/docker/Dockerfile.nrUE.rocky b/docker/Dockerfile.nrUE.rocky index 638b4406c4..1ffccd60d5 100644 --- a/docker/Dockerfile.nrUE.rocky +++ b/docker/Dockerfile.nrUE.rocky @@ -117,5 +117,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai echo "ldd on libtelnetsrv_5Gue.so" && ldd /usr/local/lib/libtelnetsrv_5Gue.so WORKDIR /opt/oai-nr-ue +COPY --from=nr-ue-base /tini /tini CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem", "-O", "/opt/oai-nr-ue/etc/nr-ue.conf"] -ENTRYPOINT ["/opt/oai-nr-ue/bin/entrypoint.sh"] +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"] diff --git a/docker/Dockerfile.nrUE.ubuntu20 b/docker/Dockerfile.nrUE.ubuntu20 index 30d32b2f31..7418c6ccb9 100644 --- a/docker/Dockerfile.nrUE.ubuntu20 +++ b/docker/Dockerfile.nrUE.ubuntu20 @@ -104,5 +104,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai ldd /opt/oai-nr-ue/bin/nr-uesoftmodem WORKDIR /opt/oai-nr-ue +COPY --from=nr-ue-base /tini /tini CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem", "-O", "/opt/oai-nr-ue/etc/nr-ue.conf"] -ENTRYPOINT ["/opt/oai-nr-ue/bin/entrypoint.sh"] +ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"] From adbb051bb72d9ee8051d33f5b26ddd1451728d38 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 4 Mar 2024 13:21:18 +0100 Subject: [PATCH 3/4] Set ulimit for core to -1 Set ulimit of core to -1 enable core dumps for file-based core dumping. Note that it is not necessary for e.g., systemd-coredumpd, but put it for completeness. --- ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml | 2 ++ ci-scripts/yaml_files/5g_sa_n310_nrue/docker-compose.yml | 2 ++ ci-scripts/yaml_files/nsa_b200_enb/docker-compose.yml | 2 ++ ci-scripts/yaml_files/nsa_b200_gnb/docker-compose.yml | 2 ++ ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml | 2 ++ ci-scripts/yaml_files/sa_b200_gnb/docker-compose.yml | 2 ++ ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml | 6 ++++++ ci-scripts/yaml_files/sa_f1_b200_gnb/docker-compose.yml | 4 ++++ ci-scripts/yaml_files/sa_sc_b200_gnb/docker-compose.yml | 2 ++ 9 files changed, 24 insertions(+) diff --git a/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml index c19400b109..6c67479aec 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: privileged: true network_mode: "host" container_name: oai-gnb + ulimits: + core: -1 # for core dumps environment: TZ: Europe/Paris USE_ADDITIONAL_OPTIONS: --sa --tune-offset 20000000 -A 45 --log_config.global_log_options level,nocolor,time diff --git a/ci-scripts/yaml_files/5g_sa_n310_nrue/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_nrue/docker-compose.yml index 85230d467c..20c6f81b1b 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_nrue/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_nrue/docker-compose.yml @@ -5,6 +5,8 @@ services: privileged: true network_mode: host container_name: oai-nr-ue + ulimits: + core: -1 # for core dumps #entrypoint: /bin/bash -c "sleep infinity" environment: TZ: Europe/Paris diff --git a/ci-scripts/yaml_files/nsa_b200_enb/docker-compose.yml b/ci-scripts/yaml_files/nsa_b200_enb/docker-compose.yml index f6e7214659..9959100271 100644 --- a/ci-scripts/yaml_files/nsa_b200_enb/docker-compose.yml +++ b/ci-scripts/yaml_files/nsa_b200_enb/docker-compose.yml @@ -5,6 +5,8 @@ services: image: oai-enb:latest privileged: true container_name: nsa-b200-enb + ulimits: + core: -1 # for core dumps environment: USE_B2XX: 'yes' USE_ADDITIONAL_OPTIONS: --log_config.global_log_options level,nocolor,time,line_num,function diff --git a/ci-scripts/yaml_files/nsa_b200_gnb/docker-compose.yml b/ci-scripts/yaml_files/nsa_b200_gnb/docker-compose.yml index 67d240d796..530e9c7c25 100644 --- a/ci-scripts/yaml_files/nsa_b200_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/nsa_b200_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: image: oai-gnb:latest privileged: true container_name: nsa-b200-gnb + ulimits: + core: -1 # for core dumps environment: USE_B2XX: 'yes' USE_ADDITIONAL_OPTIONS: -E -q --RUs.[0].sdr_addrs serial=30C51D4 --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function diff --git a/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml index d3875aa5bc..eefcfac3e5 100644 --- a/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: privileged: true network_mode: "host" container_name: oai-gnb-aw2s + ulimits: + core: -1 # for core dumps environment: TZ: Europe/Paris USE_ADDITIONAL_OPTIONS: --sa diff --git a/ci-scripts/yaml_files/sa_b200_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_b200_gnb/docker-compose.yml index f46d7b1b0a..72a9ac2831 100644 --- a/ci-scripts/yaml_files/sa_b200_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_b200_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: image: oai-gnb:latest privileged: true container_name: sa-b200-gnb + ulimits: + core: -1 # for core dumps environment: USE_B2XX: 'yes' USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function diff --git a/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml index 8e353b945e..224c28c297 100644 --- a/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_e1_b200_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: image: oai-gnb:latest privileged: true container_name: sa-cucp-gnb + ulimits: + core: -1 # for core dumps environment: USE_ADDITIONAL_OPTIONS: --sa --log_config.global_log_options level,nocolor,time,line_num,function volumes: @@ -23,6 +25,8 @@ services: image: oai-nr-cuup:latest privileged: true container_name: sa-cuup-gnb + ulimits: + core: -1 # for core dumps environment: USE_ADDITIONAL_OPTIONS: --sa --log_config.global_log_options level,nocolor,time,line_num,function volumes: @@ -41,6 +45,8 @@ services: image: oai-gnb:latest privileged: true container_name: sa-du-b200-gnb + ulimits: + core: -1 # for core dumps environment: USE_B2XX: 'yes' USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --continuous-tx -E --log_config.global_log_options level,nocolor,time,line_num,function --gNBs.[0].min_rxtxtime 2 --gNBs.[0].do_CSIRS 1 --gNBs.[0].do_SRS 1 --RUs.[0].att_rx 18 --RUs.[0].att_tx 18 diff --git a/ci-scripts/yaml_files/sa_f1_b200_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_f1_b200_gnb/docker-compose.yml index 072461af5c..c4a4d57ff9 100644 --- a/ci-scripts/yaml_files/sa_f1_b200_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_f1_b200_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: image: oai-gnb:latest privileged: true container_name: sa-cu-gnb + ulimits: + core: -1 # for core dumps environment: USE_ADDITIONAL_OPTIONS: --sa --telnetsrv --telnetsrv.shrmod ci --log_config.global_log_options level,nocolor,time,line_num,function volumes: @@ -24,6 +26,8 @@ services: image: oai-gnb:latest privileged: true container_name: sa-du-b200-gnb + ulimits: + core: -1 # for core dumps environment: USE_B2XX: 'yes' USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --log_config.global_log_options level,nocolor,time,line_num,function --gNBs.[0].min_rxtxtime 2 --gNBs.[0].do_CSIRS 1 --gNBs.[0].do_SRS 0 --MACRLCs.[0].ul_max_mcs 28 --L1s.[0].max_ldpc_iterations 20 diff --git a/ci-scripts/yaml_files/sa_sc_b200_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_sc_b200_gnb/docker-compose.yml index 90f8bf2271..60cbf1ccd9 100644 --- a/ci-scripts/yaml_files/sa_sc_b200_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_sc_b200_gnb/docker-compose.yml @@ -5,6 +5,8 @@ services: image: oai-gnb:latest privileged: true container_name: sa-b200-gnb + ulimits: + core: -1 # for core dumps environment: USE_B2XX: 'yes' USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function -E From 56acd1d6d342f6ca6dc45072fba1c3397be6608a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 4 Mar 2024 13:47:01 +0100 Subject: [PATCH 4/4] Add doc on how to retrieve core dumps --- doc/TESTBenches.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/doc/TESTBenches.md b/doc/TESTBenches.md index 40f410e9c5..fa488a42f7 100644 --- a/doc/TESTBenches.md +++ b/doc/TESTBenches.md @@ -249,3 +249,71 @@ steps](../docker/README.md) and then use the docker-compose file directly. Some tests are run from source (e.g. `ci-scripts/xml_files/gnb_phytest_usrp_run.xml`), which directly give the options they are run with. + +## How to retrieve core dumps (for CI team members) + +The entrypoint scripts of all containers print the core pattern that is used on +the running machine. Search for `core_pattern` at the start of the container +logs to retrieve the possible location. Possible locations might be: + +- a path: the corresponding directory must be mounted in the container to be + writable +- systemd-coredumpd: see [documentation](https://systemd.io/COREDUMP/) +- abrt: see [documentation](https://abrt.readthedocs.io/en/latest/usage.html) +- apport: see [documentation](https://wiki.ubuntu.com/Apport) + +You furthermore have to extract the executable that caused the core dump. +Download the container image, and extract, e.g.: + +``` +docker create --name c1 porcepix.sboai.cs.eurecom.fr/oai-gnb:develop-c99db698 +docker cp c1:/opt/oai-gnb/bin/nr-softmodem /tmp +docker rm c1 +``` + +### Core dump in a file + +**This is not recommended, as files could pile up and fill the system disk +completely!** Prefer systemd or abrt instead. + +If the core pattern is a path: it should at least include the time in the +pattern name (suggested pattern: `/tmp/core.%e.%p.%t`) to correlate the time +the segfault occurred with the CI logs. If you identified the core dump, +copy the core dump from that machine; if identification is difficult, consider +rerunning the pipeline. + +### Core dump via systemd + +Run this command to list all core dumps: + +``` +sudo coredumpctl list +``` + +Scroll to the end and find the core dump of interest (it lists the executables +in the last column; use the time to correlate the segfault and the CI run). +Take the PID of the executable (first column after the time). Dump the core +dump to a location of your choice: + +``` +sudo coredumpctl dump > /tmp/coredump +``` + +### Core dump via abrt (automatic bug reporting tool) + +TBD: use the documentation page for the moment. + +### Core dump via apport + +On Ubuntu machines, apport first needs to be enabled to collect core dumps: + +``` +sudo systemctl enable apport.service +``` + +and [needs to be enabled](https://wiki.ubuntu.com/Apport#How_to_enable_apport). +Then, show a list of core dumps using + +``` +sudo apport-cli +```