Merge remote-tracking branch 'origin/ci-rework-logs' into integration_2026_w09 (!3942)

CI: rework log analysis to simplify and allow new use cases The CI has some log analysis, but it is unmaintainable (huge functions that mix eNB/gNB or lteUE/nrUE, are not tested, most of it is not used or useful...) or limited to specific use cases (the timing-phytest time analysis is hardcoded to source builds instead of also handling analysis when running from docker images). This MR reimplements both functions. For the log analysis, the CI team decided to throw away the old analysis, and reimplement a mechanism that checks the couple of things we care for. - Allow to analyze phy-test timing analysis from docker files, and add a unit test - Rework existing log analysis to remove useless checks, and make it more composable * The undeployment steps (Undeploy_Object(), Terminate_eNB()) take an <analysis> XML object which then either has a <services> subelement with check that are whitespace delimited (similar to how other xml task steps have <services>). For checks that contain whitespace, they can use <service> (singular) that encodes a single check, such that e.g., arguments could include spaces. * Each check is of form service=check[=arguments], where service is the docker services, check is a checking function (implemented as a class with run(file, options) function inside cls_loganalysis.py, and options arguments after the second = that is to be read by run() (basically, options will contain arguments that can be freeform) * these definitions can occur multiple times, e.g., it could be serviceA=Check1 and serviceA=Check2=Args and it will execute both Check1 and Check2 * if a service is declared, it will execute alse the Default check which currently checks for an assertion * Implemented (and inspired from the existing CI analysis) are: - Default check to check for assertions (will be extended, to e.g., address/undefined behavior sanitizer analysis) - RetxCheck: retransmission check as used in the CI - LastLineContains to check the last line containes a specific string - ContainsString to check a specific string appears in logs - EndsWithBye: check that towards the end (but not necessarily the last line), there is a Bye. string (a workaround since we don't always terminate with 0) - There is a plan to use return codes after stopping in docker, but most executables do not end with 0 most of the time, such that we needed to abandon this plan. - Additional fixes (see commit messages) and delete unused code
2026-02-26 22:39:29 +01:00
parent 3eabd5363a da28708459
commit f12b8cb44d
105 changed files with 1461 additions and 1192 deletions
@@ -10,7 +10,7 @@ services:
            - IPC_LOCK
        environment:
            TZ: Europe/Paris
-            USE_ADDITIONAL_OPTIONS: --nfapi VNF
+            USE_ADDITIONAL_OPTIONS: --nfapi VNF --log_config.global_log_options level,nocolor,utc_time
            NFAPI_TRACE_LEVEL: info
        volumes:
            - ../../conf_files/gnb-vnf.sa.band77.162prb.nfapi.4x4.conf:/opt/oai-gnb/etc/gnb.conf
@@ -30,7 +30,7 @@ services:
            - IPC_LOCK
        environment:
            TZ: Europe/Paris
-            USE_ADDITIONAL_OPTIONS: --nfapi PNF
+            USE_ADDITIONAL_OPTIONS: --nfapi PNF --log_config.global_log_options level,nocolor,utc_time
            NFAPI_TRACE_LEVEL: info
        volumes:
            - ../../conf_files/gnb-pnf.band77.usrpn310.4x4.conf:/opt/oai-gnb/etc/gnb.conf
@@ -1,27 +0,0 @@
-services:
-    enb_mono_fdd:
-        image: ${REGISTRY-oaisoftwarealliance/}oai-enb:${TAG:-develop}
-        privileged: true
-        container_name: prod-enb-mono-fdd
-        environment:
-            USE_B2XX: 'yes'
-            USE_ADDITIONAL_OPTIONS: --RUs.[0].max_rxgain 115 --RUs.[0].max_pdschReferenceSignalPower -27 --eNBs.[0].component_carriers.[0].pucch_p0_Nominal -96 --log_config.global_log_options level,nocolor,time
-        volumes:
-            - ../../conf_files/enb.band7.25prb.usrpb200.conf:/opt/oai-enb/etc/enb.conf
-        networks:
-            public_net:
-                ipv4_address: 192.168.61.30
-        healthcheck:
-            # pgrep does NOT work
-            test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem"
-            interval: 10s
-            timeout: 5s
-            retries: 5
-
-networks:
-    public_net:
-        name: prod-oai-public-net
-        ipam:
-            config:
-                - subnet: 192.168.61.0/26
-
@@ -5,7 +5,7 @@ services:
        container_name: oai-enb
        environment:
            TZ: Europe/Paris
-            USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6
+            USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 --log_config.global_log_options level,nocolor,utc_time
        volumes:
            - ../../conf_files/enb.band38.lte_2x2.100prb.usrpn310.conf:/opt/oai-enb/etc/enb.conf
        network_mode: "host"
@@ -5,7 +5,7 @@ services:
        container_name: oai-enb
        environment:
            TZ: Europe/Paris
-            USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6
+            USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 --log_config.global_log_options level,nocolor,utc_time
        volumes:
            - ../../conf_files/enb.band38.lte_2x2_tm2.100prb.usrpn310.conf:/opt/oai-enb/etc/enb.conf
        network_mode: "host"
@@ -0,0 +1,22 @@
+services:
+    oai-gnb:
+        image: ${REGISTRY-oaisoftwarealliance/}${GNB_IMG:-oai-gnb-fhi72}:${TAG:-develop}
+        container_name: oai-gnb-phytest
+        cap_add:
+            - IPC_LOCK
+            - SYS_NICE
+        cap_drop:
+            - ALL
+        environment:
+            USE_ADDITIONAL_OPTIONS: --phy-test -q -M 273 -m 27 -l 4 -D 231 -T 273 -t 16 -L 2 -U 528 --log_config.global_log_options level,nocolor,time --thread-pool 16,17,18,19,20,21,22,23
+        devices:
+            - /dev/vfio:/dev/vfio/
+        volumes:
+            - ../../conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf:/opt/oai-gnb/etc/gnb.conf
+            - /dev/hugepages:/dev/hugepages
+        cpuset: "8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23"
+        healthcheck:
+            test: /bin/bash -c "pgrep nr-softmodem"
+            interval: 10s
+            timeout: 5s
+            retries: 5
@@ -0,0 +1,38 @@
+#!/bin/sh
+set -eu
+
+pci_addr()
+{
+	PF_IF=$1
+	VF_INDEX=$2
+	SYSFS_PATH="/sys/class/net/${PF_IF}/device/virtfn${VF_INDEX}"
+
+	if [ ! -e "$SYSFS_PATH" ]; then
+		    echo "VF $VF_INDEX not found for interface $PF_IF"
+		        exit 1
+	fi
+	PCI_ADDR=$(basename "$(readlink "$SYSFS_PATH")")
+	echo "$PCI_ADDR"
+}
+IF_NAME=ens1f0np0
+## O-DU C Plane MAC ADDR and VLAN
+C_U_PLANE_MAC_ADD=22:11:55:33:99:55
+C_U_PLANE_VLAN=123
+MTU=9000
+DPDK_DEVBIND_PREFIX=/home/oaicicd/test/dpdk-stable-22.11.7/bin
+NUM_VFs=1
+sudo ethtool -G $IF_NAME rx 8160 tx 8160
+sudo sh -c "echo 0 > /sys/class/net/$IF_NAME/device/sriov_numvfs"
+sudo sh -c "echo $NUM_VFs > /sys/class/net/$IF_NAME/device/sriov_numvfs"
+C_U_PLANE_PCI=$(pci_addr $IF_NAME 0)
+# this next 2 lines is for C/U planes
+sudo ip link set $IF_NAME vf 0 mac $C_U_PLANE_MAC_ADD vlan $C_U_PLANE_VLAN spoofchk off mtu $MTU
+sleep 1
+sudo modprobe iavf
+sudo ${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --unbind $C_U_PLANE_PCI
+sudo modprobe vfio-pci
+sudo ${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --bind vfio-pci $C_U_PLANE_PCI
+echo "Successfully configured C-PLANE and U-PLANE:
+  - C-PLANE MAC: $C_U_PLANE_MAC_ADD, VLAN: $C_U_PLANE_VLAN, PCI: $C_U_PLANE_PCI"
+exit 0
+
@@ -12,6 +12,7 @@ services:
          core: -1 # for core dumps
        environment:
            TZ: Europe/Paris
+            USE_ADDITIONAL_OPTIONS: --log_config.global_log_options level,nocolor,utc_time
        volumes:
            - ../../conf_files/gnb.sa.band78.51prb.aw2s.ddsuu.conf:/opt/oai-gnb-aw2s/etc/gnb.conf
        healthcheck: