Add T tracer to record configured events to database

This extends the functionality provided by record in that it allows
collection of data into a Clickhouse database, allowing aggregation of
data from gNB and UE for functions like channel estimation for PUSCH
transmissions that did not decode successfully at the gNB.

It is intended to be used in conjunction with Aerial Data Lake but is
not limited to it. Data access is simpler than in !3462, but could be
improved using the metadata approach from !3462.

The integration uses logfmt for fast logging, and a clickhouse DB
client, both integrated using CPM.

This change set also adds a docker-compose file for a UE on 100MHz, to
be used with Aerial configurations, and builds the tracer for Aerial and
nrUE docker files.

ci-scripts/yaml_files/sa_gnb_aerial/docker-compose-ue.yaml

@@ -0,0 +1,61 @@
+services:
+    oai-nr-ue:
+        image: ${REGISTRY-oaisoftwarealliance/}${NRUE_IMG:-oai-nr-ue}:${UE_TAG:-${TAG:-develop}}
+        logging:
+           options:
+              max-file: "1"
+        network_mode: host
+        container_name: oai-nr-ue
+        privileged: true
+        cpuset: 4,6,8,10,12,14,16,18,20,22,24,26,28
+        cap_drop:
+            - ALL
+        cap_add:
+            - NET_ADMIN  # for interface bringup
+            - NET_RAW    # for ping
+            - SYS_NICE   # for performance
+            - IPC_LOCK   # for memory locking
+            - SYS_PTRACE # for debugging
+        devices:
+             - /dev/net/tun:/dev/net/tun
+        ulimits:
+          core: -1 # for core dumps
+        #entrypoint: /bin/bash -c "sleep infinity"
+        environment:
+            TZ: UTC
+            #DISPLAY: $DISPLAY #Uncomment for GUI
+            USE_ADDITIONAL_OPTIONS: |
+               --thread-pool 4,6,8,10,12,14,16,18,20,22,24,26
+               --usrp-args type=x4xx,addr=192.168.10.2,clock_source=external,time_source=external
+               --ssb 1478 -r 273 --numerology 1 --band 78 -C 3750000000
+               --num-dl-actors 20
+               --ue-nb-ant-tx 1
+               --ue-nb-ant-rx 1
+               --ue-rxgain 25
+               --ue-txgain 13
+               --uicc0.imsi 001010000000003
+               --ue-fo-compensation -A 90
+               --usrp-tx-thread-config 1
+               --log_config.global_log_options level,nocolor,utc_time
+           # Unused options:
+           #   --T_stdout 2
+           #   --imscope
+           #   --ssb 1478 -r 273 --numerology 1 --band 78 -C 3750000000
+           #   --usrp-tx-thread-config 1
+           # --log_config.mac_log_level debug
+           #  --log_config.nr_mac_log_level debug
+           # --usrp-tx-thread-config 1 -d
+           # --ssb 194 -r 51 --numerology 1 --band 78 -C 3750000000 #20MHz
+           # --ssb 236 -r 106 --numerology 1 --band 78 -C 3750000000 #40MHz
+        volumes:
+            - ../../conf_files/ue.sa.conf:/opt/oai-nr-ue/etc/nr-ue.conf
+            # UL throughput is worse with 256-qam table
+            - ../../../targets/PROJECTS/GENERIC-NR-5GC/CONF/uecap_ports1.xml:/opt/oai-nr-ue/uecap_ports1.xml
+            #For GUI
+            - ${HOME}/.Xauthority:/root/.Xauthority:rw
+            - /tmp/.X11-unix:/tmp/.X11-unix
+        healthcheck:
+            test: /bin/bash -c "pgrep nr-uesoftmodem"
+            interval: 10s
+            timeout: 5s
+            retries: 5

common/utils/T/T_messages.txt

@@ -1404,7 +1404,7 @@ ID = UE_PHY_PDCCH_ENERGY
 ID = UE_PHY_PDSCH_IQ
     DESC = UE PDSCH received IQ data
     GROUP = ALL:PHY:GRAPHIC:HEAVY:UE
-    FORMAT = int,eNB_ID : int,frame : int,subframe : int,nb_rb : int,N_RB_UL : int,symbols_per_tti : buffer,pusch_comp
+    FORMAT = int,eNB_ID : int,frame : int,subframe : int,nb_rb : int,N_RB_UL : int,symbols_per_tti : buffer,pdsch_comp
 ID = UE_PHY_PDSCH_ENERGY
     DESC = UE PDSCH 1 energy and threshold
     GROUP = ALL:PHY:GRAPHIC:HEAVY:UE

common/utils/T/tracer/CMakeLists.txt

@@ -37,11 +37,65 @@ add_subdirectory(filter)
 add_subdirectory(logger)
 add_subdirectory(view)
 
+# Option to enable record_db tracer tool with ClickHouse database support
+add_boolean_option(T_RECORD_DB OFF "Compile record_db tracer tool with ClickHouse support" OFF)
+
+if(T_RECORD_DB)
+  # Save current BUILD_SHARED_LIBS setting
+  set(SAVED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+
+  # Configure options for clickhouse-cpp
+  set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
+
+  # Save current compiler flags and disable -Werror for zstd
+  set(SAVED_CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+
+  # Check if system zstd library is available
+  find_library(ZSTD_LIBRARY NAMES zstd)
+  if(ZSTD_LIBRARY)
+    # Use system zstd library to avoid building legacy zstd code with warnings
+    set(WITH_SYSTEM_ZSTD ON CACHE BOOL "" FORCE)
+  else()
+    # Fall back to bundled zstd and suppress warnings from legacy code
+    # https://github.com/ClickHouse/clickhouse-cpp/blob/master/contrib/zstd/zstd/legacy/zstd_v01.c#L1935
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-maybe-uninitialized -Wno-uninitialized")
+  endif()
+
+  CPMAddPackage(
+    NAME clickhouse-cpp
+    GITHUB_REPOSITORY ClickHouse/clickhouse-cpp
+    GIT_TAG master
+  )
+
+  # Restore original compiler flags and BUILD_SHARED_LIBS
+  set(CMAKE_C_FLAGS "${SAVED_CMAKE_C_FLAGS}")
+  set(BUILD_SHARED_LIBS ${SAVED_BUILD_SHARED_LIBS} CACHE BOOL "" FORCE)
+
+  CPMAddPackage(
+    NAME fmtlog
+    GITHUB_REPOSITORY MengRao/fmtlog
+    GIT_TAG main
+    GIT_SHALLOW FALSE
+  )
+
+  add_executable(record_db record_db.cpp)
+
+  target_include_directories(record_db PRIVATE ${fmtlog_SOURCE_DIR} ${fmtlog_SOURCE_DIR}/fmt/include)
+  target_link_libraries(record_db PRIVATE tracer_utils clickhouse-cpp-lib fmtlog-shared fmt)
+
+endif() # T_RECORD_DB
+
 add_custom_target(T_tools)
 add_dependencies(T_tools
                  record replay extract_config extract_input_subframe
                  extract_output_subframe extract macpdu2wireshark multi
                  textlog csv)
+
+# Add record_db to T_tools if it was built
+if(TARGET record_db)
+  add_dependencies(T_tools record_db)
+endif()
+
 add_dependencies(nr-softmodem T_tools)
 add_dependencies(nr-uesoftmodem T_tools)
 add_dependencies(lte-softmodem T_tools)

common/utils/T/tracer/Makefile

@@ -1,18 +1,23 @@
 CC=gcc
+CXX=g++
 CFLAGS=-Wall -g -pthread -DT_TRACER -I. -I../incgen
+CXXFLAGS=-Wall -g -pthread -DT_TRACER -I. -I../incgen -std=c++11
 
 #CFLAGS += -O3 -ffast-math -fomit-frame-pointer
 
 LIBS=-lm
 XLIBS=-lX11 -lpng -lXft
 
-all: record replay extract_config textlog enb ue vcd macpdu2wireshark \
+all: record record_db replay extract_config textlog enb ue vcd macpdu2wireshark \
      extract_input_subframe extract_output_subframe to_vcd extract multi \
      gnb t_tracer_app_gnb t_tracer_app_ue gnb_mac csv
 
 record: utils.o record.o database.o configuration.o
 	$(CC) $(CFLAGS) -o record $^ $(LIBS)
 
+record_db: utils.o record_db.o database.o configuration.o
+	$(CXX) $(CXXFLAGS) -o record_db $^ $(LIBS) -lclickhouse-cpp-lib $(FMTLOG_LIBS)
+
 replay: utils.o replay.o
 	$(CC) $(CFLAGS) -o replay $^ $(LIBS)
 
@@ -102,8 +107,11 @@ filter/filter.a:
 %.o: %.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 
+%.o: %.cpp
+	$(CXX) $(CXXFLAGS) -c -o $@ $<
+
 clean:
-	rm -f *.o core tracer_remote textlog enb ue vcd record replay
+	rm -f *.o core tracer_remote textlog enb ue vcd record record_db replay
 	rm -f extract_config macpdu2wireshark extract_input_subframe
 	rm -f extract_output_subframe to_vcd extract multi gnb
 	rm -f t_tracer_app_gnb t_tracer_app_ue gnb_mac csv

docker/Dockerfile.base.ubuntu

@@ -49,6 +49,8 @@ RUN apt-get update && \
        bison \
        flex \
        m4 \
+       # libzstd-dev for ClickHouse C++ library
+       libzstd-dev \
        # python3-pip for conf template generation
        python3-pip \
        wget \

docker/Dockerfile.build.fhi72.native_arm.ubuntu

@@ -81,8 +81,7 @@ RUN /bin/sh oaienv && \
       --ninja --gNB \
       --build-lib "telnetsrv enbscope uescope nrscope" \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
-      --build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
-      $BUILD_OPTION && \
+      --build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION && \
     # Mainly to see if the sanitize option was perfectly executed
     ldd ran_build/build/nr-softmodem && \
     ldd ran_build/build/liboran_fhlib_5g.so && \

docker/Dockerfile.build.fhi72.ubuntu

@@ -71,8 +71,7 @@ RUN /bin/sh oaienv && \
       --ninja --gNB --nrRU \
       --build-lib "telnetsrv enbscope uescope nrscope" \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
-      --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
-      $BUILD_OPTION && \
+      --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION && \
     # Mainly to see if the sanitize option was perfectly executed
     ldd ran_build/build/nr-softmodem && \
     ldd ran_build/build/liboran_fhlib_5g.so && \

docker/Dockerfile.build.ubuntu

@@ -42,15 +42,23 @@ RUN \
    # Mount CPM package cache
    --mount=type=cache,target=/root/.cache/cpm/ \
     /bin/sh oaienv && \
-    mkdir -p cmake_targets/ran_build/build && \
-    cd cmake_targets/ran_build/build && \
-    cmake ../../../ -GNinja \
-        -DENABLE_TELNETSRV=ON -DENABLE_ENBSCOPE=ON -DENABLE_UESCOPE=ON -DENABLE_NRSCOPE=ON \
-        -DOAI_USRP=ON -DOAI_BLADERF=ON -DOAI_ETHERNET=ON -DOAI_VRTSIM_TAPS_CLIENT=ON \
-        -DE2_AGENT=ON -DKPM_VERSION=KPM_V3_00 -DE2AP_VERSION=E2AP_V3 \
-        -DAVX512=OFF && \
-    # -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror && \
-    ninja
+    cd cmake_targets && \
+    mkdir -p log && \
+    ./build_oai -c --ninja \
+      --eNB --gNB --RU --UE --nrUE \
+      --build-lib "telnetsrv enbscope uescope nrscope" \
+      -w USRP -t Ethernet -w BLADERF \
+      --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
+      --cmake-opt -DOAI_VRTSIM_TAPS_CLIENT=ON \
+      --noavx512 \
+      --build-tool-opt -k10 \
+      --cmake-opt -DT_RECORD_DB=ON \
+      --cmake-opt -DCMAKE_C_FLAGS="-Werror" --cmake-opt -DCMAKE_CXX_FLAGS="-Werror" $BUILD_OPTION && \
+    # Mainly to see if the sanitize option was perfectly executed
+    echo "---- ldd on executables ----" && \
+    ldd ran_build/build/*softmodem* ran_build/build/nr-cuup ran_build/build/oairu && \
+    echo "---- ldd on shared libraries ----" && \
+    ldd ran_build/build/*.so
 
 ## Build FlexRIC for SM Models
 RUN \

docker/Dockerfile.gNB.aerial.ubuntu

@@ -55,13 +55,15 @@ RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
     ./build_oai -c -w AERIAL --gNB --ninja --build-tool-opt -k10 \
-      --cmake-opt -DCMAKE_C_FLAGS="-Werror" --cmake-opt -DCMAKE_CXX_FLAGS="-Werror"
+      --cmake-opt -DCMAKE_C_FLAGS="-Werror" --cmake-opt -DCMAKE_CXX_FLAGS="-Werror" \
+      --cmake-opt -DT_RECORD_DB=ON
 
 
 #start from scratch for target executable
 FROM ubuntu:noble AS oai-gnb-aerial
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=Europe
+ENV PATH="/opt/oai-gnb/bin:$PATH"
 
 RUN apt-get update && \
     apt-get upgrade --yes && \
@@ -85,17 +87,20 @@ RUN apt-get update && \
 
 WORKDIR /opt/oai-gnb/bin
 COPY --from=ran-build \
-    /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./
+    /oai-ran/cmake_targets/ran_build/build/nr-softmodem \
+    /oai-ran/cmake_targets/ran_build/build/common/utils/T/tracer/record* \
+    ./
+
 COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh
 
 COPY --from=ran-build \
   /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
   /oai-ran/cmake_targets/ran_build/build/libparams_yaml.so \
+  /oai-ran/cmake_targets/ran_build/build/_deps*/fmtlog-build*/libfmtlog-shared*.so \
+  /oai-ran/cmake_targets/ran_build/build/_deps*/clickhouse-cpp-build*/clickhouse/libclickhouse*.so \
   /usr/local/lib/
 
-#copy files from nvIPC
 COPY --from=ran-build \
-    /usr/local/lib/libfmtlog-shared.so \
     /usr/local/lib/libnvlog.so \
     /usr/local/lib/libnvipc.so \
     /usr/local/lib/
@@ -113,5 +118,6 @@ WORKDIR /opt/oai-gnb
 
 COPY --from=ran-build /usr/local/bin/yq /usr/local/bin/yq
 COPY --from=ran-build /tini /tini
+COPY --from=ran-build /oai-ran/common/utils/T/T_messages.txt ./
 ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
 CMD ["/opt/oai-gnb/bin/nr-softmodem"]

docker/Dockerfile.nrORU.fhi72.ubuntu

@@ -25,7 +25,6 @@ FROM ran-build-fhi72:latest AS ran-build
 
 #start from scratch for target executable
 FROM ubuntu:noble AS oai-nr-oru
-ARG BUILD_OPTION
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=Europe
 
@@ -50,11 +49,6 @@ RUN apt-get update && \
         xz-utils \
         libnuma-dev \
         libyaml-cpp-dev && \
-# if the --sanitize option was used to build, additional packages are required
-    /bin/bash -c 'if [[ "$BUILD_OPTION" = "--sanitize" ]]; then DEBIAN_FRONTEND=noninteractive apt-get install --yes \
-        libasan8 \
-        libubsan1 \
-        liblapacke; fi' && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt/nr-oru/bin

docker/Dockerfile.nrUE.ubuntu

@@ -35,6 +35,7 @@ FROM ubuntu:noble AS oai-nr-ue
 ARG BUILD_OPTION
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=Europe/Paris
+ENV PATH="/opt/oai-nr-ue/bin:$PATH"
 
 RUN apt-get update && \
     apt-get upgrade --yes && \
@@ -67,6 +68,7 @@ RUN apt-get update && \
         iperf3 \
         iperf \
         libyaml-cpp-dev \
+        libzstd1 \
         libnanomsg5 && \
 # if the --sanitize option was used to build, additional packages are required
     /bin/bash -c 'if [[ "$BUILD_OPTION" = "--sanitize" ]]; then DEBIAN_FRONTEND=noninteractive apt-get install --yes \
@@ -78,6 +80,7 @@ RUN apt-get update && \
 WORKDIR /opt/oai-nr-ue/bin
 COPY --from=nr-ue-build \
     /oai-ran/cmake_targets/ran_build/build/nr-uesoftmodem \
+    /oai-ran/cmake_targets/ran_build/build/common/utils/T/tracer/record* \
     ./
 
 COPY ./docker/scripts/nr_ue_entrypoint.sh ./entrypoint.sh
@@ -94,10 +97,17 @@ COPY --from=nr-ue-build \
     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ciUE.so \
     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
+    /oai-ran/cmake_targets/ran_build/build/libnrscope.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
     /oai-ran/cmake_targets/ran_build/build/libparams_yaml.so \
     /oai-ran/cmake_targets/ran_build/build/libvrtsim.so \
+    # Wildcards so it doesn't fail if not present
+    /oai-ran/cmake_targets/ran_build/build/_deps*/fmtlog-build*/libfmtlog-shared*.so \
+    /oai-ran/cmake_targets/ran_build/build/_deps*/clickhouse-cpp-build*/clickhouse/libclickhouse*.so \
     /usr/local/lib/
 
+#    /oai-ran/cmake_targets/ran_build/build/libimscope.so \
+
 # Now we are copying from builder-image the UHD files.
 COPY --from=nr-ue-base /usr/local/bin/uhd_find_devices /usr/local/bin
 COPY --from=nr-ue-base /usr/local/lib/libuhd.so.4.8.0 /usr/local/lib
@@ -124,5 +134,6 @@ WORKDIR /opt/oai-nr-ue
 
 COPY --from=nr-ue-base /usr/local/bin/yq /usr/local/bin/yq
 COPY --from=nr-ue-base /tini /tini
+COPY --from=nr-ue-build /oai-ran/common/utils/T/T_messages.txt ./
 CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem"]
 ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"]