Introduce separate thread to handle gNB TX processing

This commit introduces a separate thread (l1_tx_thread) that processes
gNB DL/TX slots. It receives a message from the ru_thread when a new
slot started, and starts processing.

The DL part of the scheduler is run in the l1_tx_thread.  Therefore,
only call UL indication (for scheduler) in UL slots.  The UL indication
previously triggered the DL scheduler -- hence, it had to be called in
every slot. Now, since the DL scheduler is moved into the DL thread, we
don't have to call the scheduler in every slot anymore.

The reorder thread is removed, as reordering with this scheme is not
necessary anymore.

The main advantage of this version is that the TX can be scheduled
earlier (sl_ahead). Further, there can no longer be race conditions in
the TX path, since the scheduler/L2, TX L1 processing and the RU tx
function are all execute in the same thread (L1_tX_thread).

Also, the scheduler prepares PRACH/PUSCH/PUCCH FAPI PDUs. As of this
commit, the scheduler runs in the TX processing chain, whereas these
PDUs are destined for RX processing.  Thus, too avoid data races,
instead of triggering the RX processing at the same time as TX
processing in the RU thread, this commit changes the code to trigger RX
processing after the scheduler has been run in TX processing to avoid
data races.

Finally, we synchronize RU and L1 threads. This is important for
rfsimulator, since unlike most radios, rfsimulator can run slower or
faster, depending on I/O and CPU (it does not have a stable "tick").

Co-authored-by: kiran <saikiran@iitj.ac.in>

openair1/PHY/NR_TRANSPORT/nr_prach.c

@@ -76,8 +76,7 @@ void nr_fill_prach(PHY_VARS_gNB *gNB,
 		   nfapi_nr_prach_pdu_t *prach_pdu) {
 
   int prach_id = find_nr_prach(gNB,SFN,Slot,SEARCH_EXIST_OR_FREE);
-  AssertFatal( ((prach_id>=0) && (prach_id<NUMBER_OF_NR_PRACH_MAX)) || (prach_id < 0),
-              "illegal or no prach_id found!!! prach_id %d\n",prach_id);
+  AssertFatal(((prach_id >= 0) && (prach_id < NUMBER_OF_NR_PRACH_MAX)), "illegal or no prach_id found!!! prach_id %d\n", prach_id);
 
   gNB->prach_vars.list[prach_id].frame=SFN;
   gNB->prach_vars.list[prach_id].slot=Slot;

openair1/PHY/NR_TRANSPORT/nr_ulsch.c

@@ -74,8 +74,8 @@ void nr_fill_ulsch(PHY_VARS_gNB *gNB, int frame, int slot, nfapi_nr_pusch_pdu_t
   NR_UL_gNB_HARQ_t *harq = ulsch->harq_process;
   if (ulsch_pdu->pusch_data.new_data_indicator)
     harq->harq_to_be_cleared = true;
-  LOG_D(PHY,
-        "%d.%d RNTI %x HARQ PID %d new data indicator %d\n",
+  LOG_D(NR_PHY,
+        "NEW ULSCH %d.%d RNTI %x HARQ PID %d new data indicator %d\n",
         frame,
         slot,
         ulsch_pdu->rnti,
@@ -98,8 +98,17 @@ void reset_active_ulsch(PHY_VARS_gNB *gNB, int frame)
   // assuming UE disconnected or some other error occurred
   for (int i = 0; i < gNB->max_nb_pusch; i++) {
     NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[i];
-    if (ulsch->active && (((frame - ulsch->frame + 1024) % 1024) > NUMBER_FRAMES_PHY_UE_INACTIVE))
+    int diff = (frame - ulsch->frame + 1024) & 1023;
+    if (ulsch->active && diff > NUMBER_FRAMES_PHY_UE_INACTIVE && diff < 100) {
       ulsch->active = false;
+      LOG_D(NR_PHY,
+            "Frame %d: resetting ulsch %d harq %d (programmed in %d.%d)\n",
+            frame,
+            i,
+            ulsch->harq_pid,
+            ulsch->frame,
+            ulsch->slot);
+    }
   }
 }
 

openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c

@@ -1409,7 +1409,7 @@ static void nr_pusch_symbol_processing(void *arg)
 
     if (gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol] == 0) 
       continue;
-    int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size;
+    int soffset = (slot % RU_RX_SLOT_DEPTH) * frame_parms->symbols_per_slot * frame_parms->ofdm_symbol_size;
     inner_rx(gNB,
              ulsch_id,
              slot,
@@ -1591,7 +1591,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
 
   // extract the first dmrs for the channel level computation
   // extract the data in the OFDM frame, to the start of the array
-  int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size;
+  int soffset = (slot % RU_RX_SLOT_DEPTH) * frame_parms->symbols_per_slot * frame_parms->ofdm_symbol_size;
 
   nb_re_pusch = (nb_re_pusch + 15) & ~15;
 

openair1/PHY/NR_TRANSPORT/pucch_rx.c

@@ -501,7 +501,7 @@ void nr_decode_pucch1(c16_t **rxdataF,
    * Implement TS 38.211 Subclause 6.3.2.4.1 Sequence modulation
    *
    */
-  const int soffset = (nr_tti_tx & 3) * frame_parms->symbols_per_slot * frame_parms->ofdm_symbol_size;
+  const int soffset = (nr_tti_tx % RU_RX_SLOT_DEPTH) * frame_parms->symbols_per_slot * frame_parms->ofdm_symbol_size;
   // lprime is the index of the OFDM symbol in the slot that corresponds to the first OFDM symbol of the PUCCH transmission in the slot given by [5, TS 38.213]
   const int lprime = startingSymbolIndex;
   // mcs = 0 except for PUCCH format 0
@@ -1021,14 +1021,10 @@ void nr_decode_pucch2(PHY_VARS_gNB *gNB,
   //extract pucch and dmrs first
 
   int l2 = pucch_pdu->start_symbol_index;
-
-  int soffset = (slot & 3) * frame_parms->symbols_per_slot * frame_parms->ofdm_symbol_size;
-
+  int soffset = (slot % RU_RX_SLOT_DEPTH) * frame_parms->symbols_per_slot * frame_parms->ofdm_symbol_size;
   int re_offset[2];
   re_offset[0] =
       (12 * (pucch_pdu->prb_start + pucch_pdu->bwp_start) + frame_parms->first_carrier_offset) % frame_parms->ofdm_symbol_size;
-  if (re_offset[0]>= frame_parms->ofdm_symbol_size)
-    re_offset[0]-=frame_parms->ofdm_symbol_size;
   if (pucch_pdu->freq_hop_flag == 0)
     re_offset[1] = re_offset[0];
   else {

openair1/PHY/NR_TRANSPORT/srs_rx.c

@@ -34,6 +34,7 @@
 #include "PHY/impl_defs_nr.h"
 #include "PHY/defs_nr_common.h"
 #include "PHY/defs_gNB.h"
+#include "PHY/defs_RU.h"
 #include "PHY/CODING/nrSmallBlock/nr_small_block_defs.h"
 #include "PHY/NR_UE_TRANSPORT/srs_modulation_nr.h"
 #include "common/utils/LOG/log.h"
@@ -75,7 +76,7 @@ int nr_get_srs_signal(PHY_VARS_gNB *gNB,
   c16_t **rxdataF = gNB->common_vars.rxdataF;
   const NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms;
 
-  const uint16_t n_symbols = (slot&3)*frame_parms->symbols_per_slot;                    // number of symbols until this slot
+  const uint16_t n_symbols = (slot % RU_RX_SLOT_DEPTH) * frame_parms->symbols_per_slot; // number of symbols until this slot
   const uint8_t l0 = frame_parms->symbols_per_slot - 1 - srs_pdu->time_start_position;  // starting symbol in this slot
   const uint64_t symbol_offset = (n_symbols+l0)*frame_parms->ofdm_symbol_size;
   const uint64_t subcarrier_offset = frame_parms->first_carrier_offset + srs_pdu->bwp_start*NR_NB_SC_PER_RB;

openair1/PHY/defs_RU.h

@@ -197,7 +197,7 @@ typedef struct {
 
 #define NUMBER_OF_NR_RU_PRACH_MAX 8
 #define NUMBER_OF_NR_RU_PRACH_OCCASIONS_MAX 12
-
+#define RU_RX_SLOT_DEPTH 4
 typedef struct RU_proc_t_s {
   /// Pointer to associated RU descriptor
   struct RU_t_s *ru;

openair1/PHY/defs_gNB.h

@@ -639,8 +639,6 @@ typedef struct PHY_VARS_gNB_s {
 
   int ldpc_offload_flag;
 
-  int reorder_thread_disable;
-
   int max_ldpc_iterations;
   /// indicate the channel estimation technique in time domain
   int chest_time;
@@ -718,6 +716,7 @@ typedef struct PHY_VARS_gNB_s {
   notifiedFIFO_t L1_tx_free;
   notifiedFIFO_t L1_tx_filled;
   notifiedFIFO_t L1_tx_out;
+  notifiedFIFO_t L1_rx_out;
   notifiedFIFO_t resp_RU_tx;
   tpool_t threadPool;
   int nbSymb;
@@ -795,9 +794,7 @@ union ldpcReqUnion {
 
 typedef struct processingData_L1 {
   int frame_rx;
-  int frame_tx;
   int slot_rx;
-  int slot_tx;
   openair0_timestamp timestamp_tx;
   PHY_VARS_gNB *gNB;
 } processingData_L1_t;
@@ -811,6 +808,8 @@ typedef enum {
 typedef struct processingData_L1tx {
   int frame;
   int slot;
+  int frame_rx;
+  int slot_rx;
   openair0_timestamp timestamp_tx;
   PHY_VARS_gNB *gNB;
   nfapi_nr_dl_tti_pdcch_pdu pdcch_pdu[NFAPI_NR_MAX_NB_CORESETS];
@@ -825,4 +824,9 @@ typedef struct processingData_L1tx {
   int sched_response_id;
 } processingData_L1tx_t;
 
+typedef struct processingData_L1rx {
+  int frame_rx;
+  int slot_rx;
+  PHY_VARS_gNB *gNB;
+} processingData_L1rx_t;
 #endif

openair1/SCHED_NR/fapi_nr_l1.c

@@ -156,21 +156,9 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO)
   uint8_t number_tx_data_pdu        = (TX_req == NULL) ? 0 : TX_req->Number_of_PDUs;
 
   if (NFAPI_MODE == NFAPI_MONOLITHIC){
-
     if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) {
-      notifiedFIFO_elt_t *res=NULL;
       processingData_L1tx_t *msgTx=NULL;
-      if (!gNB->reorder_thread_disable) {
-        res = pullNotifiedFIFO(&gNB->L1_tx_free);
-        if (res == NULL)
-          return; // Tpool has been stopped, nothing to process
-        msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
-      } else {
-        msgTx = gNB->msgDataTx; //newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0, &gNB->L1_tx_out,NULL);
-      }
-      /*const time_stats_t ts = exec_time_stats_NotifiedFIFO(res);
-      merge_meas(&gNB->phy_proc_tx, &ts);
-*/
+      msgTx = gNB->msgDataTx;
       msgTx->num_pdsch_slot = 0;
       msgTx->num_dl_pdcch = 0;
       msgTx->num_ul_pdcch = number_ul_dci_pdu;
@@ -222,8 +210,6 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO)
        * released only when both threads are done with it.
        */
       inc_ref_sched_response(Sched_INFO->sched_response_id);
-      if (!gNB->reorder_thread_disable)
-	pushNotifiedFIFO(&gNB->L1_tx_filled,res);
     }
 
     for (int i = 0; i < number_ul_tti_pdu; i++) {
@@ -267,7 +253,9 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO)
   }
 
   /* this thread is done with the sched_info, decrease the reference counter */
-  deref_sched_response(Sched_INFO->sched_response_id);
-
+  if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) {
+    LOG_D(NR_PHY, "Calling dref_sched_response for id %d in %d.%d (sched_response)\n", Sched_INFO->sched_response_id, frame, slot);
+    deref_sched_response(Sched_INFO->sched_response_id);
+  }
   stop_meas(&gNB->schedule_response_stats);
 }

openair1/SCHED_NR/nr_ru_procedures.c

@@ -242,7 +242,7 @@ void nr_fep_full(RU_t *ru, int slot) {
 
   // remove_7_5_kHz(ru,proc->tti_rx<<1);
   // remove_7_5_kHz(ru,1+(proc->tti_rx<<1));
-  int offset = (proc->tti_rx&3)*(fp->symbols_per_slot * fp->ofdm_symbol_size);
+  int offset = (proc->tti_rx % RU_RX_SLOT_DEPTH) * (fp->symbols_per_slot * fp->ofdm_symbol_size);
   for (l = 0; l < fp->symbols_per_slot; l++) {
     for (aa = 0; aa < fp->nb_antennas_rx; aa++) {
       nr_slot_fep_ul(fp,
@@ -378,7 +378,7 @@ void nr_fep(void* arg) {
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPRX+aid, 1);
 
-  int offset = (tti_rx&3) * fp->symbols_per_slot * fp->ofdm_symbol_size;
+  int offset = (tti_rx % RU_RX_SLOT_DEPTH) * fp->symbols_per_slot * fp->ofdm_symbol_size;
   for (int l = startSymbol; l <= endSymbol; l++) 
       nr_slot_fep_ul(fp,
                      ru->common.rxdata[aid],

openair1/SCHED_NR/phy_procedures_nr_gNB.c

@@ -272,10 +272,11 @@ static void nr_postDecode(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req)
     }
 
     if (crc_valid && !check_abort(&ulsch_harq->abort_decode) && !gNB->pusch_vars[rdata->ulsch_id].DTX) {
-      LOG_D(PHY,
-            "[gNB %d] ULSCH: Setting ACK for SFN/SF %d.%d (rnti %x, pid %d, ndi %d, status %d, round %d, TBS %d, Max interation "
+      LOG_D(NR_PHY,
+            "[gNB %d] ULSCH %d: Setting ACK for SFN/SF %d.%d (rnti %x, pid %d, ndi %d, status %d, round %d, TBS %d, Max interation "
             "(all seg) %d)\n",
             gNB->Mod_id,
+            rdata->ulsch_id,
             ulsch->frame,
             ulsch->slot,
             ulsch->rnti,
@@ -316,7 +317,7 @@ static void nr_postDecode(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req)
         if (ulsch_harq->ulsch_pdu.mcs_index == 0 && dumpsig==1) {
           int off = ((ulsch_harq->ulsch_pdu.rb_size&1) == 1)? 4:0;
 
-          LOG_M("rxsigF0.m","rxsF0",&gNB->common_vars.rxdataF[0][(ulsch_harq->slot&3)*gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot],gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot,1,1);
+          LOG_M("rxsigF0.m","rxsF0",&gNB->common_vars.rxdataF[0][(ulsch_harq->slot%RU_RX_SLOT_DEPTH)*gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot],gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot,1,1);
           LOG_M("rxsigF0_ext.m","rxsF0_ext",
                  &gNB->pusch_vars[0].rxdataF_ext[0][ulsch_harq->ulsch_pdu.start_symbol_index*NR_NB_SC_PER_RB *
        ulsch_harq->ulsch_pdu.rb_size],ulsch_harq->ulsch_pdu.nr_of_symbols*(off+(NR_NB_SC_PER_RB *
@@ -410,9 +411,6 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int
 
 void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, uint8_t harq_pid, uint8_t crc_flag, int dtx_flag)
 {
-  if (!get_softmodem_params()->reorder_thread_disable) 
-    pthread_mutex_lock(&gNB->UL_INFO_mutex);
-
   NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ULSCH_id];
   NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
   NR_gNB_PHY_STATS_t *stats = get_phy_stats(gNB, ulsch->rnti);
@@ -467,7 +465,13 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id,
       __attribute__((unused))
       int off = ((pusch_pdu->rb_size&1) == 1)? 4:0;
 
-      LOG_M("rxsigF0.m","rxsF0",&gNB->common_vars.rxdataF[0][(slot_rx&3)*gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot],gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot,1,1);
+      LOG_M("rxsigF0.m",
+            "rxsF0",
+            &gNB->common_vars
+                 .rxdataF[0][(slot_rx % RU_RX_SLOT_DEPTH) * gNB->frame_parms.ofdm_symbol_size * gNB->frame_parms.symbols_per_slot],
+            gNB->frame_parms.ofdm_symbol_size * gNB->frame_parms.symbols_per_slot,
+            1,
+            1);
       LOG_M("chestF0.m",
             "chF0",
             &gNB->pusch_vars[0].ul_ch_estimates[0][pusch_pdu->start_symbol_index * gNB->frame_parms.ofdm_symbol_size],
@@ -494,7 +498,14 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id,
             1,
             0);
       if (gNB->frame_parms.nb_antennas_rx > 1) {
-        LOG_M("rxsigF1.m","rxsF1",&gNB->common_vars.rxdataF[1][(slot_rx&3)*gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot],gNB->frame_parms.ofdm_symbol_size*gNB->frame_parms.symbols_per_slot,1,1);
+        LOG_M(
+            "rxsigF1.m",
+            "rxsF1",
+            &gNB->common_vars
+                 .rxdataF[1][(slot_rx % RU_RX_SLOT_DEPTH) * gNB->frame_parms.ofdm_symbol_size * gNB->frame_parms.symbols_per_slot],
+            gNB->frame_parms.ofdm_symbol_size * gNB->frame_parms.symbols_per_slot,
+            1,
+            1);
         LOG_M("chestF1.m",
               "chF1",
               &gNB->pusch_vars[0].ul_ch_estimates[1][pusch_pdu->start_symbol_index * gNB->frame_parms.ofdm_symbol_size],
@@ -557,9 +568,6 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id,
   }
 
   gNB->UL_INFO.rx_ind.number_of_pdus++;
-
-  if (!get_softmodem_params()->reorder_thread_disable) 
-    pthread_mutex_unlock(&gNB->UL_INFO_mutex);
 }
 
 // Function to fill UL RB mask to be used for N0 measurements
@@ -834,9 +842,9 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx)
     NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ULSCH_id];
     NR_UL_gNB_HARQ_t *ulsch_harq = ulsch->harq_process;
     AssertFatal(ulsch_harq != NULL, "harq_pid %d is not allocated\n", ulsch->harq_pid);
-
     if ((ulsch->active == true) && (ulsch->frame == frame_rx) && (ulsch->slot == slot_rx) && (ulsch->handled == 0)) {
       LOG_D(PHY, "PUSCH ID %d with RNTI %x detection started in frame %d slot %d\n", ULSCH_id, ulsch->rnti, frame_rx, slot_rx);
+  
       int num_dmrs = 0;
       for (int s = 0; s < NR_NUMBER_OF_SYMBOLS_PER_SLOT; s++)
         num_dmrs += (ulsch_harq->ulsch_pdu.ul_dmrs_symb_pos >> s) & 1;

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -895,6 +895,7 @@ int main(int argc, char **argv)
   msgDataTx->slot = slot;
   msgDataTx->frame = frame;
   memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
+  gNB->msgDataTx = msgDataTx;
 
   // Buffers to store internal memory of slot process
   int rx_size = (((14 * frame_parms->N_RB_DL * 12 * sizeof(int32_t)) + 15) >> 4) << 4;

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -563,6 +563,7 @@ int main(int argc, char *argv[])
   notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_free, NULL);
   processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
   msgDataTx->slot = -1;
+  gNB->msgDataTx = msgDataTx;
   //gNB_config = &gNB->gNB_config;
 
   //memset((void *)&gNB->UL_INFO,0,sizeof(gNB->UL_INFO));