Merge remote-tracking branch 'origin/fhi72-no-polling' into integration_2025_w07 (!3200)

FHI72: remove polling, fix memory leak

- FHI72 used polling to track the next slot. This is not necessary, a
  simple queue does the job and allows to save one core that was
  previously at 100%
- Remove a workaround to avoid a memory leak (fixes #812); allows to
  also run RFsim/USRP/otehr radios when compiling for FHI72

common/utils/threadPool/notified_fifo.h

@@ -23,6 +23,7 @@
 #define NOTIFIED_FIFO_H
 #include "pthread_utils.h"
 #include <stdint.h>
+#include <malloc.h>
 #include <pthread.h>
 #include "time_meas.h"
 #include <memory.h>

doc/ORAN_FHI7.2_Tutorial.md

@@ -475,6 +475,19 @@ cmake .. -GNinja -DOAI_FHI72=ON -Dxran_LOCATION=$HOME/phy/fhi_lib/lib
 ninja nr-softmodem oran_fhlib_5g params_libconfig
 ```
 
+Note that in tags 2025.w06 and prior, the FHI72 driver used polling to wait for
+the next slot. This is inefficient as it burns CPU time, and has been replaced
+with a more efficient mechanism. Nevertheless, if you experience problems that
+did not occur previously, it is possible to re-enable polling, either with
+`build_oai` like this
+
+    ./build_oai --gNB --ninja -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=$HOME/phy/fhi_lib/lib --cmake-opt -DOAI_FHI72_USE_POLLING=ON
+
+or with `cmake` like so
+
+    cmake .. -GNinja -DOAI_FHI72=ON -Dxran_LOCATION=$HOME/phy/fhi_lib/lib -DOAI_FHI72_USE_POLLING=ON
+    ninja oran_fhlib_5g
+
 # Configuration
 
 RU and DU configurations have a circular dependency: you have to configure DU MAC address in the RU configuration and the RU MAC address, VLAN and Timing advance parameters in the DU configuration.
@@ -1040,6 +1053,8 @@ In this case, you should reverify that `ptp4l` and `phc2sys` are working, e.g.,
 do not do any jumps (during the last hour). While an occasional jump is not
 necessarily problematic for the gNB, many such messages mean that the system is
 not working, and UEs might not be able to attach or reach good performance.
+Also, you can try to compile with polling (see [the build
+section](.#build-oai-gnb)) to see if it resolves the problem.
 
 # Operation with multiple RUs
 

executables/nr-gnb.c

@@ -219,7 +219,6 @@ static void rx_func(processingData_L1_t *info)
     gNB->if_inst->NR_UL_indication(&UL_INFO);
     stop_meas(&gNB->ul_indication_stats);
 
-#ifndef OAI_FHI72
     notifiedFIFO_elt_t *res = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->L1_rx_out, NULL);
     processingData_L1_t *syncMsg = NotifiedFifoData(res);
     syncMsg->gNB = gNB;
@@ -228,7 +227,6 @@ static void rx_func(processingData_L1_t *info)
     res->key = slot_rx;
     LOG_D(NR_PHY, "Signaling completion for %d.%d (mod_slot %d) on L1_rx_out\n", frame_rx, slot_rx, slot_rx % RU_RX_SLOT_DEPTH);
     pushNotifiedFIFO(&gNB->L1_rx_out, res);
-#endif
   }
 
 }

executables/nr-ru.c

@@ -1084,6 +1084,48 @@ void ru_tx_func(void *param)
   }//else  emulate_rf
 }
 
+/* @brief wait for the next RX TTI to be free
+ *
+ * Certain radios, e.g., RFsim, can run faster than real-time. This might
+ * create problems, e.g., if RX and TX get too far from each other. This
+ * function ensures that a maximum of 4 RX slots are processed at a time (and
+ * not more than those four are started).
+ *
+ * Through the queue L1_rx_out, we are informed about completed RX jobs.
+ * rx_tti_busy keeps track of individual slots that have been started; this
+ * function blocks until the current frame/slot is completed, signaled through
+ * a message.
+ *
+ * @param L1_rx_out the queue from which to read completed RX jobs
+ * @param rx_tti_busy array to mark RX job completion
+ * @param frame_rx the frame to wait for
+ * @param slot_rx the slot to wait for
+ */
+static bool wait_free_rx_tti(notifiedFIFO_t *L1_rx_out, bool rx_tti_busy[RU_RX_SLOT_DEPTH], int frame_rx, int slot_rx)
+{
+  int idx = slot_rx % RU_RX_SLOT_DEPTH;
+  if (rx_tti_busy[idx]) {
+    bool not_done = true;
+    LOG_D(NR_PHY, "%d.%d Waiting to access RX slot %d\n", frame_rx, slot_rx, idx);
+    // block and wait for frame_rx/slot_rx free from previous slot processing.
+    // as we can get other slots, we loop on the queue
+    while (not_done) {
+      notifiedFIFO_elt_t *res = pullNotifiedFIFO(L1_rx_out);
+      if (!res)
+        return false;
+      processingData_L1_t *info = NotifiedFifoData(res);
+      LOG_D(NR_PHY, "%d.%d Got access to RX slot %d.%d (%d)\n", frame_rx, slot_rx, info->frame_rx, info->slot_rx, idx);
+      rx_tti_busy[info->slot_rx % RU_RX_SLOT_DEPTH] = false;
+      if ((info->slot_rx % RU_RX_SLOT_DEPTH) == idx)
+        not_done = false;
+      delNotifiedFIFO_elt(res);
+    }
+  }
+  // set the tti to busy: the caller will process this slot now
+  rx_tti_busy[idx] = true;
+  return true;
+}
+
 void *ru_thread(void *param)
 {
   static int ru_thread_status;
@@ -1097,9 +1139,7 @@ void *ru_thread(void *param)
   char               threadname[40];
   int initial_wait = 0;
 
-#ifndef OAI_FHI72
   bool rx_tti_busy[RU_RX_SLOT_DEPTH] = {false};
-#endif
   // set default return value
   ru_thread_status = 0;
   // set default return value
@@ -1245,39 +1285,12 @@ void *ru_thread(void *param)
     if (ru->idx != 0)
       proc->frame_tx = (proc->frame_tx + proc->frame_offset) & 1023;
 
-#ifndef OAI_FHI72
     // do RX front-end processing (frequency-shift, dft) if needed
     int slot_type = nr_slot_select(&ru->config, proc->frame_rx, proc->tti_rx);
     if (slot_type == NR_UPLINK_SLOT || slot_type == NR_MIXED_SLOT) {
+      if (!wait_free_rx_tti(&gNB->L1_rx_out, rx_tti_busy, proc->frame_rx, proc->tti_rx))
+        break; // nothing to wait for: we have to stop
       if (ru->feprx) {
-        if (rx_tti_busy[proc->tti_rx % RU_RX_SLOT_DEPTH]) {
-          bool not_done = true;
-          LOG_D(NR_PHY, "%d.%d Waiting to access RX slot %d\n", proc->frame_rx, proc->tti_rx, proc->tti_rx % RU_RX_SLOT_DEPTH);
-          // now we block and wait our slot memory zone is freed from previous slot processing
-          // as we can get other slots ending, we loop on the queue
-          notifiedFIFO_elt_t *res = NULL;
-          while (not_done) {
-            res = pullNotifiedFIFO(&gNB->L1_rx_out);
-            if (!res)
-              break;
-            processingData_L1_t *info = (processingData_L1_t *)NotifiedFifoData(res);
-            LOG_D(NR_PHY,
-                  "%d.%d Got access to RX slot %d.%d (%d)\n",
-                  proc->frame_rx,
-                  proc->tti_rx,
-                  info->frame_rx,
-                  info->slot_rx,
-                  proc->tti_rx % RU_RX_SLOT_DEPTH);
-            rx_tti_busy[info->slot_rx % RU_RX_SLOT_DEPTH] = false;
-            if ((info->slot_rx % RU_RX_SLOT_DEPTH) == (proc->tti_rx % RU_RX_SLOT_DEPTH))
-              not_done = false;
-            delNotifiedFIFO_elt(res);
-          }
-          if (!res)
-            break;
-        }
-        // set the tti that was generated to busy
-        rx_tti_busy[proc->tti_rx % RU_RX_SLOT_DEPTH] = true;
         ru->feprx(ru,proc->tti_rx);
         LOG_D(NR_PHY, "Setting %d.%d (%d) to busy\n", proc->frame_rx, proc->tti_rx, proc->tti_rx % RU_RX_SLOT_DEPTH);
         //LOG_M("rxdata.m","rxs",ru->common.rxdata[0],1228800,1,1);
@@ -1315,7 +1328,6 @@ void *ru_thread(void *param)
         } // end if (prach_id > 0)
       } // end if (ru->feprx)
     } // end if (slot_type == NR_UPLINK_SLOT || slot_type == NR_MIXED_SLOT) {
-#endif
 
     notifiedFIFO_elt_t *resTx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, NULL);
     processingData_L1tx_t *syncMsgTx = NotifiedFifoData(resTx);

radio/fhi_72/CMakeLists.txt

@@ -39,10 +39,9 @@ target_link_libraries(oran_fhlib_5g PRIVATE ${T_LIB})
 # TODO: can be removed?
 target_include_directories(oran_fhlib_5g PRIVATE ${xran_INCLUDE_DIRS}/../src/)
 
+add_boolean_option(OAI_FHI72_USE_POLLING OFF "Enable polling in FHI72 driver" ON)
+
 set_target_properties(oran_fhlib_5g PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 add_custom_command(TARGET oran_fhlib_5g POST_BUILD
                    COMMAND ${CMAKE_COMMAND} -E create_symlink liboran_fhlib_5g.so liboai_transpro.so
                    WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-
-message(STATUS "Temporary added -DOAI_FHI72")
-target_compile_definitions(nr-softmodem PUBLIC OAI_FHI72)

radio/fhi_72/oaioran.c

@@ -37,26 +37,14 @@
 
 #include "oran-config.h" // for g_kbar
 
-#define USE_POLLING 1
+#include "common/utils/threadPool/notified_fifo.h"
+
+#if OAI_FHI72_USE_POLLING
+#define USE_POLLING
+#endif
+
 // Declare variable useful for the send buffer function
 volatile uint8_t first_call_set = 0;
-volatile uint8_t first_rx_set = 0;
-volatile int first_read_set = 0;
-
-// Variable declaration useful for fill IQ samples from file
-#define IQ_PLAYBACK_BUFFER_BYTES (XRAN_NUM_OF_SLOT_IN_TDD_LOOP * N_SYM_PER_SLOT * XRAN_MAX_PRBS * N_SC_PER_PRB * 4L)
-/*
-int rx_tti;
-int rx_sym;
-volatile uint32_t rx_cb_tti = 0;
-volatile uint32_t rx_cb_frame = 0;
-volatile uint32_t rx_cb_subframe = 0;
-volatile uint32_t rx_cb_slot = 0;
-*/
-
-#define GetFrameNum(tti, SFNatSecStart, numSubFramePerSystemFrame, numSlotPerSubFrame) \
-  ((((uint32_t)tti / ((uint32_t)numSubFramePerSystemFrame * (uint32_t)numSlotPerSubFrame)) + SFNatSecStart) & 0x3FF)
-#define GetSlotNum(tti, numSlotPerSfn) ((uint32_t)tti % ((uint32_t)numSlotPerSfn))
 
 int xran_is_prach_slot(uint8_t PortId, uint32_t subframe_id, uint32_t slot_id);
 #include "common/utils/LOG/log.h"
@@ -64,7 +52,7 @@ int xran_is_prach_slot(uint8_t PortId, uint32_t subframe_id, uint32_t slot_id);
 #ifndef USE_POLLING
 extern notifiedFIFO_t oran_sync_fifo;
 #else
-volatile oran_sync_info_t oran_sync_info;
+volatile oran_sync_info_t oran_sync_info = {0};
 #endif
 void oai_xran_fh_rx_callback(void *pCallbackTag, xran_status_t status)
 {
@@ -124,47 +112,42 @@ void oai_xran_fh_rx_callback(void *pCallbackTag, xran_status_t status)
       }
     }
 #endif
-    if (first_call_set) {
+    // if xran did not call xran_physide_dl_tti callback, it's not ready yet.
-      if (!first_rx_set) {
+    // wait till first callback to advance counters, because otherwise users
-        LOG_I(NR_PHY, "first_rx is set (num_ports %d), first_read_set %d\n", num_ports, first_read_set);
+    // would see periodic output with only "0" in stats counters
-      }
+    if (!first_call_set)
-      first_rx_set = 1;
+      return;
-      if (first_read_set == 1) {
+    slot2 = slot + (subframe * slots_per_subframe);
-        slot2 = slot + (subframe * slots_per_subframe);
+    rx_RU[ru_id][slot2] = 1;
-        rx_RU[ru_id][slot2] = 1;
+    if (last_frame > 0 && frame > 0
-        if (last_frame > 0 && frame > 0
+        && ((slot2 > 0 && last_frame != frame) || (slot2 == 0 && last_frame != ((1024 + frame - 1) & 1023))))
-            && ((slot2 > 0 && last_frame != frame) || (slot2 == 0 && last_frame != ((1024 + frame - 1) & 1023))))
+      LOG_E(PHY, "Jump in frame counter last_frame %d => %d, slot %d\n", last_frame, frame, slot2);
-          LOG_E(PHY, "Jump in frame counter last_frame %d => %d, slot %d\n", last_frame, frame, slot2);
+    for (int i = 0; i < num_ports; i++) {
-        for (int i = 0; i < num_ports; i++) {
+      if (rx_RU[i][slot2] == 0)
-          if (rx_RU[i][slot2] == 0)
+        return;
-            return;
+    }
-        }
+    for (int i = 0; i < num_ports; i++)
-        for (int i = 0; i < num_ports; i++)
+      rx_RU[i][slot2] = 0;
-          rx_RU[i][slot2] = 0;
 
-        if (last_slot == -1 || slot2 != last_slot) {
+    if (last_slot == -1 || slot2 != last_slot) {
 #ifndef USE_POLLING
-          notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(oran_sync_info_t), 0, &oran_sync_fifo, NULL);
+      notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(oran_sync_info_t), 0, &oran_sync_fifo, NULL);
-          oran_sync_info_t *info = (oran_sync_info_t *)NotifiedFifoData(req);
+      oran_sync_info_t *info = NotifiedFifoData(req);
-          info->sl = slot2;
+      info->tti = tti;
-          info->f = frame;
+      info->sl = slot2;
-          LOG_D(PHY, "Push %d.%d.%d (slot %d, subframe %d,last_slot %d)\n", frame, info->sl, slot, ru_id, subframe, last_slot);
+      info->f = frame;
+      LOG_D(PHY, "Push %d.%d.%d (slot %d, subframe %d,last_slot %d)\n", frame, info->sl, slot, ru_id, subframe, last_slot);
+      pushNotifiedFIFO(&oran_sync_fifo, req);
 #else
-          LOG_D(PHY, "Writing %d.%d.%d (slot %d, subframe %d,last_slot %d)\n", frame, slot2, ru_id, slot, subframe, last_slot);
+      LOG_D(PHY, "Writing %d.%d.%d (slot %d, subframe %d,last_slot %d)\n", frame, slot2, ru_id, slot, subframe, last_slot);
-          oran_sync_info.tti = tti;
+      oran_sync_info.tti = tti;
-          oran_sync_info.sl = slot2;
+      oran_sync_info.sl = slot2;
-          oran_sync_info.f = frame;
+      oran_sync_info.f = frame;
 #endif
-#ifndef USE_POLLING
+    } else
-          pushNotifiedFIFO(&oran_sync_fifo, req);
+      LOG_E(PHY, "Cannot Push %d.%d.%d (slot %d, subframe %d,last_slot %d)\n", frame, slot2, ru_id, slot, subframe, last_slot);
-#else
+    last_slot = slot2;
-#endif
+    last_frame = frame;
-        } else
-          LOG_E(PHY, "Cannot Push %d.%d.%d (slot %d, subframe %d,last_slot %d)\n", frame, slot2, ru_id, slot, subframe, last_slot);
-        last_slot = slot2;
-        last_frame = frame;
-      } // first_read_set == 1
-    } // first_call_set
   } // rx_sym == 7
 }
 void oai_xran_fh_srs_callback(void *pCallbackTag, xran_status_t status)
@@ -286,8 +269,6 @@ int xran_fh_rx_read_slot(ru_info_t *ru, int *frame, int *slot)
   void *ptr = NULL;
   int32_t *pos = NULL;
   int idx = 0;
-  static int last_slot = -1;
-  first_read_set = 1;
 
   static int64_t old_rx_counter[XRAN_PORTS_NUM] = {0};
   static int64_t old_tx_counter[XRAN_PORTS_NUM] = {0};
@@ -295,24 +276,28 @@ int xran_fh_rx_read_slot(ru_info_t *ru, int *frame, int *slot)
   static int outcnt = 0;
 #ifndef USE_POLLING
   // pull next even from oran_sync_fifo
-  notifiedFIFO_elt_t *res = pollNotifiedFIFO(&oran_sync_fifo);
+  notifiedFIFO_elt_t *res = pullNotifiedFIFO(&oran_sync_fifo);
-  while (res == NULL) {
+
-    res = pollNotifiedFIFO(&oran_sync_fifo);
+  notifiedFIFO_elt_t *f;
+  while ((f = pollNotifiedFIFO(&oran_sync_fifo)) != NULL) {
+    oran_sync_info_t *old_info = NotifiedFifoData(res);
+    oran_sync_info_t *new_info = NotifiedFifoData(f);
+    LOG_E(PHY, "Detected double sync message %d.%d => %d.%d\n", old_info->f, old_info->sl, new_info->f, new_info->sl);
+    delNotifiedFIFO_elt(res);
+    res = f;
   }
-  oran_sync_info_t *info = (oran_sync_info_t *)NotifiedFifoData(res);
+
+  oran_sync_info_t *info = NotifiedFifoData(res);
 
   *slot = info->sl;
   *frame = info->f;
   delNotifiedFIFO_elt(res);
 #else
-  LOG_D(PHY, "In  xran_fh_rx_read_slot, first_rx_set %d\n", first_rx_set);
-  while (first_rx_set == 0) {
-  }
-
   *slot = oran_sync_info.sl;
   *frame = oran_sync_info.f;
   uint32_t tti_in = oran_sync_info.tti;
 
+  static int last_slot = -1;
   LOG_D(PHY, "oran slot %d, last_slot %d\n", *slot, last_slot);
   int cnt = 0;
   // while (*slot == last_slot)  {