Merge remote-tracking branch 'origin/threading-rk' into integration_2023_w19

common/utils/system.c

@@ -232,6 +232,8 @@ void threadCreate(pthread_t* t, void * (*func)(void*), void * param, char* name,
   ret=pthread_attr_init(&attr);
   AssertFatal(ret==0,"ret: %d, errno: %d\n",ret, errno);
 
+  LOG_I(UTIL,"Creating thread %s with affinity %d and priority %d\n",name,affinity,priority);
+
   if (checkIfFedoraDistribution())
     if (checkIfGenericKernelOnFedora())
       if (checkIfInsideContainer())

executables/nr-gnb.c

@@ -110,7 +110,8 @@ time_stats_t softmodem_stats_rx_sf; // total rx time
 #define L1STATSSTRLEN 16384
 
 
-void tx_func(void *param) {
+void tx_func(void *param) 
+{
 
   processingData_L1tx_t *info = (processingData_L1tx_t *) param;
   int frame_tx = info->frame;
@@ -119,6 +120,7 @@ void tx_func(void *param) {
   int absslot_tx = info->timestamp_tx/info->gNB->frame_parms.get_samples_per_slot(slot_tx,&info->gNB->frame_parms);
   int absslot_rx = absslot_tx-info->gNB->RU_list[0]->sl_ahead;
   int rt_prof_idx = absslot_rx % RT_PROF_DEPTH;
+  start_meas(&info->gNB->phy_proc_tx);
 
   clock_gettime(CLOCK_MONOTONIC,&info->gNB->rt_L1_profiling.start_L1_TX[rt_prof_idx]);
   phy_procedures_gNB_TX(info,
@@ -127,10 +129,47 @@ void tx_func(void *param) {
                         1);
   clock_gettime(CLOCK_MONOTONIC,&info->gNB->rt_L1_profiling.return_L1_TX[rt_prof_idx]);
 
+  if (get_softmodem_params()->reorder_thread_disable) {
+    PHY_VARS_gNB *gNB = info->gNB;
+    processingData_RU_t syncMsgRU;
+    syncMsgRU.frame_tx = frame_tx;
+    syncMsgRU.slot_tx = slot_tx;
+    syncMsgRU.ru = gNB->RU_list[0];
+    syncMsgRU.timestamp_tx = info->timestamp_tx;
+    LOG_D(PHY,"gNB: %d.%d : calling RU TX function\n",syncMsgRU.frame_tx,syncMsgRU.slot_tx);
+    ru_tx_func((void*)&syncMsgRU);
+  }
   /* this thread is done with the sched_info, decrease the reference counter */
   deref_sched_response(info->sched_response_id);
+  stop_meas(&info->gNB->phy_proc_tx);
 }
 
+
+void *L1_rx_thread(void *arg) 
+{
+  PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg;
+
+  while (oai_exit == 0) {
+     notifiedFIFO_elt_t *res = pullNotifiedFIFO(&gNB->resp_L1);
+     processingData_L1_t *info = (processingData_L1_t *)NotifiedFifoData(res);
+     rx_func(info);
+     delNotifiedFIFO_elt(res);
+  }
+  return NULL;
+}
+/* to be added for URLLC, requires MAC scheduling to be split from UL indication 
+void *L1_tx_thread(void *arg) {
+  PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg;
+
+  while (oai_exit == 0) {
+     notifiedFIFO_elt_t *res = pullNotifiedFIFO(&gNB->L1_tx_out);
+     processingData_L1tx_t *info = (processingData_L1tx_t *)NotifiedFifoData(res);
+     tx_func(info);
+     delNotifiedFIFO_elt(res);
+  }
+  return NULL;
+}
+*/
 void rx_func(void *param)
 {
   processingData_L1_t *info = (processingData_L1_t *) param;
@@ -189,6 +228,8 @@ void rx_func(void *param)
     // Do PRACH RU processing
     L1_nr_prach_procedures(gNB,frame_rx,slot_rx);
 
+    //WA: comment rotation in tx/rx
+    if((gNB->num_RU == 1) && (gNB->RU_list[0]->if_south != REMOTE_IF4p5)) {
       //apply the rx signal rotation here
       for (int aa = 0; aa < gNB->frame_parms.nb_antennas_rx; aa++) {
         apply_nr_rotation_ul(&gNB->frame_parms,
@@ -197,6 +238,7 @@ void rx_func(void *param)
             0,
             gNB->frame_parms.Ncp==EXTENDED?12:14);
       }
+    }
     phy_procedures_gNB_uespec_RX(gNB, frame_rx, slot_rx);
   }
 
@@ -206,21 +248,28 @@ void rx_func(void *param)
 
   // Call the scheduler
   start_meas(&gNB->ul_indication_stats);
-  pthread_mutex_lock(&gNB->UL_INFO_mutex);
+//  pthread_mutex_lock(&gNB->UL_INFO_mutex);
   gNB->UL_INFO.frame     = frame_rx;
   gNB->UL_INFO.slot      = slot_rx;
   gNB->UL_INFO.module_id = gNB->Mod_id;
   gNB->UL_INFO.CC_id     = gNB->CC_id;
   gNB->if_inst->NR_UL_indication(&gNB->UL_INFO);
-  pthread_mutex_unlock(&gNB->UL_INFO_mutex);
+//  pthread_mutex_unlock(&gNB->UL_INFO_mutex);
   stop_meas(&gNB->ul_indication_stats);
 
-  int tx_slot_type = nr_slot_select(cfg,frame_rx,slot_tx);
+  int tx_slot_type = nr_slot_select(cfg,frame_tx,slot_tx);
   if ((tx_slot_type == NR_DOWNLINK_SLOT || tx_slot_type == NR_MIXED_SLOT) && NFAPI_MODE != NFAPI_MODE_PNF) {
     notifiedFIFO_elt_t *res;
     processingData_L1tx_t *syncMsg;
     // Its a FIFO so it maitains the order in which the MAC fills the messages
     // so no need for checking for right slot
+    if (get_softmodem_params()->reorder_thread_disable) {
+      // call the TX function directly from this thread
+      syncMsg = gNB->msgDataTx;
+      syncMsg->gNB = gNB; 
+      syncMsg->timestamp_tx = info->timestamp_tx;
+      tx_func(syncMsg);
+    } else {
       res = pullTpool(&gNB->L1_tx_filled, &gNB->threadPool);
       if (res == NULL)
         return; // Tpool has been stopped
@@ -229,6 +278,7 @@ void rx_func(void *param)
       syncMsg->timestamp_tx = info->timestamp_tx;
       res->key = slot_tx;
       pushTpool(&gNB->threadPool, res);
+    }
   } else if (get_softmodem_params()->continuous_tx) {
     notifiedFIFO_elt_t *res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool);
     if (res == NULL)
@@ -287,6 +337,11 @@ static size_t dump_L1_meas_stats(PHY_VARS_gNB *gNB, RU_t *ru, char *output, size
   const char *end = output + outputlen;
   output += print_meas_log(&gNB->phy_proc_tx, "L1 Tx processing", NULL, NULL, output, end - output);
   output += print_meas_log(&gNB->dlsch_encoding_stats, "DLSCH encoding", NULL, NULL, output, end - output);
+  output += print_meas_log(&gNB->dlsch_scrambling_stats, "DLSCH scrambling", NULL, NULL, output, end-output);
+  output += print_meas_log(&gNB->dlsch_modulation_stats, "DLSCH modulation", NULL, NULL, output, end-output);
+  output += print_meas_log(&gNB->dlsch_layer_mapping_stats, "DLSCH layer mapping", NULL, NULL, output,end-output);
+  output += print_meas_log(&gNB->dlsch_resource_mapping_stats, "DLSCH resource mapping", NULL, NULL, output,end-output);
+  output += print_meas_log(&gNB->dlsch_precoding_stats, "DLSCH precoding", NULL, NULL, output,end-output);
   output += print_meas_log(&gNB->phy_proc_rx, "L1 Rx processing", NULL, NULL, output, end - output);
   output += print_meas_log(&gNB->ul_indication_stats, "UL Indication", NULL, NULL, output, end - output);
   output += print_meas_log(&gNB->rx_pusch_stats, "PUSCH inner-receiver", NULL, NULL, output, end - output);
@@ -331,7 +386,11 @@ void *nrL1_stats_thread(void *param) {
   reset_meas(&gNB->rx_pusch_stats);
   reset_meas(&gNB->ulsch_decoding_stats);
   reset_meas(&gNB->schedule_response_stats);
-
+  reset_meas(&gNB->dlsch_scrambling_stats);
+  reset_meas(&gNB->dlsch_modulation_stats);
+  reset_meas(&gNB->dlsch_layer_mapping_stats);
+  reset_meas(&gNB->dlsch_resource_mapping_stats);
+  reset_meas(&gNB->dlsch_precoding_stats);
   while (!oai_exit) {
     sleep(1);
     dump_nr_I0_stats(fd,gNB);
@@ -414,14 +473,31 @@ void init_gNB_Tpool(int inst) {
 
   // L1 RX result FIFO 
   initNotifiedFIFO(&gNB->resp_L1);
+  if (!get_softmodem_params()->reorder_thread_disable) {
     notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->resp_L1, rx_func);
     pushNotifiedFIFO(&gNB->resp_L1, msg); // to unblock the process in the beginning
-
+  }
   // L1 TX result FIFO 
   initNotifiedFIFO(&gNB->L1_tx_free);
   initNotifiedFIFO(&gNB->L1_tx_filled);
   initNotifiedFIFO(&gNB->L1_tx_out);
  
+  if (get_softmodem_params()->reorder_thread_disable) {
+    // create the RX thread responsible for triggering RX processing and then TX processing if a single thread is used	  
+    threadCreate(&gNB->L1_rx_thread, L1_rx_thread, (void *)gNB, "L1_rx_thread",
+                 gNB->L1_rx_thread_core, OAI_PRIORITY_RT_MAX);
+    // if separate threads are used for RX and TX, create the TX thread
+ // threadCreate(&gNB->L1_tx_thread, L1_tx_thread, (void *)gNB, "L1_tx_thread",
+ //              gNB->L1_tx_thread_core, OAI_PRIORITY_RT_MAX);
+
+    notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func);
+    processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+    memset(msgDataTx, 0, sizeof(processingData_L1tx_t));
+    init_DLSCH_struct(gNB, msgDataTx);
+    memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
+    // this will be removed when the msgDataTx is not necessary anymore
+    gNB->msgDataTx = msgDataTx;
+  } else {
     // we create 2 threads for L1 tx processing
     for (int i=0; i < 2; i++) {
       notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func);
@@ -432,12 +508,13 @@ void init_gNB_Tpool(int inst) {
       pushNotifiedFIFO(&gNB->L1_tx_free, msgL1Tx); // to unblock the process in the beginning
     }
   
-  if ((!get_softmodem_params()->emulate_l1) && (!IS_SOFTMODEM_NOSTATS_BIT) && (NFAPI_MODE!=NFAPI_MODE_VNF))
-     threadCreate(&proc->L1_stats_thread,nrL1_stats_thread,(void*)gNB,"L1_stats",-1,OAI_PRIORITY_RT_LOW);
-
     LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n");
     threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder",
                   gNB->RU_list[0] ? gNB->RU_list[0]->tpcores[1] : -1, OAI_PRIORITY_RT_MAX);
+  }
+
+  if ((!get_softmodem_params()->emulate_l1) && (!IS_SOFTMODEM_NOSTATS_BIT) && (NFAPI_MODE!=NFAPI_MODE_VNF))
+     threadCreate(&proc->L1_stats_thread,nrL1_stats_thread,(void*)gNB,"L1_stats",-1,OAI_PRIORITY_RT_LOW);
 
 }
 
@@ -507,6 +584,7 @@ void init_eNB_afterRU(void) {
 
     gNB = RC.gNB[inst];
     gNB->ldpc_offload_flag = ldpc_offload_flag;
+    gNB->reorder_thread_disable = get_softmodem_params()->reorder_thread_disable;
 
     phy_init_nr_gNB(gNB);
 
@@ -585,6 +663,7 @@ void init_gNB(int single_thread_flag,int wait_for_sync) {
     gNB->prach_energy_counter = 0;
     gNB->chest_time = get_softmodem_params()->chest_time;
     gNB->chest_freq = get_softmodem_params()->chest_freq;
+
   }
   
 

executables/nr-ru.c

@@ -60,6 +60,7 @@ unsigned short config_frames[4] = {2,9,11,13};
 #endif
 
 
+
 /* these variables have to be defined before including ENB_APP/enb_paramdef.h and GNB_APP/gnb_paramdef.h */
 static int DEFBANDS[] = {7};
 static int DEFENBS[] = {0};
@@ -83,6 +84,8 @@ static int DEFRUTPCORES[] = {-1,-1,-1,-1};
 
 extern int oai_exit;
 
+uint16_t sl_ahead;
+
 extern struct timespec timespec_sub(struct timespec lhs, struct timespec rhs);
 extern struct timespec timespec_add(struct timespec lhs, struct timespec rhs);
 extern void  nr_phy_free_RU(RU_t *);
@@ -1085,7 +1088,7 @@ void *ru_thread( void *param ) {
   ru_thread_status = 0;
   // set default return value
   sprintf(threadname,"ru_thread %u",ru->idx);
-  LOG_I(PHY,"Starting RU %d (%s,%s),\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing]);
+  LOG_I(PHY,"Starting RU %d (%s,%s) on cpu %d\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing],sched_getcpu());
   memcpy((void *)&ru->config,(void *)&RC.gNB[0]->gNB_config,sizeof(ru->config));
 
   if(emulate_rf) {
@@ -1281,9 +1284,13 @@ void *ru_thread( void *param ) {
     } // end if (slot_type == NR_UPLINK_SLOT || slot_type == NR_MIXED_SLOT) {
 
     // At this point, all information for subframe has been received on FH interface
+    if (!get_softmodem_params()->reorder_thread_disable) {
       res = pullTpool(&gNB->resp_L1, &gNB->threadPool);
       if (res == NULL)
         break; // Tpool has been stopped
+    } else  {
+      res=newNotifiedFIFO_elt(sizeof(processingData_L1_t),0, &gNB->resp_L1,NULL);
+    }
     syncMsg = (processingData_L1_t *)NotifiedFifoData(res);
     syncMsg->gNB = gNB;
     syncMsg->frame_rx = proc->frame_rx;
@@ -1292,7 +1299,10 @@ void *ru_thread( void *param ) {
     syncMsg->slot_tx = proc->tti_tx;
     syncMsg->timestamp_tx = proc->timestamp_tx;
     res->key = proc->tti_rx;
+    if (!get_softmodem_params()->reorder_thread_disable) 
       pushTpool(&gNB->threadPool, res);
+    else 
+      pushNotifiedFIFO(&gNB->resp_L1, res);
   }
 
   printf( "Exiting ru_thread \n");
@@ -1307,6 +1317,7 @@ int start_streaming(RU_t *ru) {
 }
 
 int nr_start_if(struct RU_t_s *ru, struct PHY_VARS_gNB_s *gNB) {
+  if (ru->if_south <= REMOTE_IF5)
     for (int i=0;i<ru->nb_rx;i++) ru->openair0_cfg.rxbase[i] = ru->common.rxdata[i];
   ru->openair0_cfg.rxsize = ru->nr_frame_parms->samples_per_subframe*10;
   reset_meas(&ru->ifdevice.tx_fhaul);
@@ -1344,7 +1355,7 @@ void init_RU_proc(RU_t *ru) {
 
   pthread_mutex_init( &proc->mutex_emulateRF,NULL);
   pthread_cond_init( &proc->cond_emulateRF, NULL);
-  threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->tpcores[0], OAI_PRIORITY_RT_MAX );
+  threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->ru_thread_core, OAI_PRIORITY_RT_MAX );
 
   if(emulate_rf)
     threadCreate( &proc->pthread_emulateRF, emulatedRF_thread, (void *)proc, "emulateRF", -1, OAI_PRIORITY_RT );
@@ -1670,7 +1681,7 @@ void set_function_spec_param(RU_t *ru) {
     case REMOTE_IF4p5:
       ru->do_prach               = 0;
       ru->feprx                  = NULL;                // DFTs
-      ru->feptx_prec             = NULL;          // Precoding operation
+      ru->feptx_prec             = nr_feptx_prec;       // Precoding operation
       ru->feptx_ofdm             = NULL;                // no OFDM mod
       ru->fh_south_in            = fh_if4p5_south_in;   // synchronous IF4p5 reception
       ru->fh_south_out           = fh_if4p5_south_out;  // synchronous IF4p5 transmission
@@ -1782,6 +1793,7 @@ void init_NR_RU(char *rf_config_file) {
     set_function_spec_param(ru);
     LOG_I(PHY,"Starting ru_thread %d\n",ru_id);
     init_RU_proc(ru);
+    if (ru->if_south != REMOTE_IF4p5) {
       int threadCnt = ru->num_tpcores;
       if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt);
       else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt);
@@ -1799,6 +1811,7 @@ void init_NR_RU(char *rf_config_file) {
       // FEP TX result FIFO
       ru->respfeptx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
       initNotifiedFIFO(ru->respfeptx);
+    }
   } // for ru_id
 
   //  sleep(1);
@@ -1987,6 +2000,7 @@ static void NRRCconfig_RU(void) {
       RC.ru[j]->openair0_cfg.txfh_cores[0]        = *(RUParamList.paramarray[j][RU_TXFH_CORE_ID].iptr);
       RC.ru[j]->num_tpcores                       = *(RUParamList.paramarray[j][RU_NUM_TP_CORES].iptr);
       RC.ru[j]->half_slot_parallelization         = *(RUParamList.paramarray[j][RU_HALF_SLOT_PARALLELIZATION].iptr);
+      RC.ru[j]->ru_thread_core                    = *(RUParamList.paramarray[j][RU_RU_THREAD_CORE].iptr);
       printf("[RU %d] Setting half-slot parallelization to %d\n",j,RC.ru[j]->half_slot_parallelization); 
       AssertFatal(RC.ru[j]->num_tpcores <= RUParamList.paramarray[j][RU_TP_CORES].numelt, "Number of TP cores should be <=16\n");
       for (i=0; i<RC.ru[j]->num_tpcores; i++) RC.ru[j]->tpcores[i] = RUParamList.paramarray[j][RU_TP_CORES].iptr[i];

executables/softmodem-common.c

@@ -46,6 +46,7 @@ static softmodem_params_t softmodem_params;
 char *parallel_config=NULL;
 char *worker_config=NULL;
 int usrp_tx_thread = 0;
+char *nfapi_str=NULL;
 int ldpc_offload_flag=0;
 uint8_t nfapi_mode=0;
 

executables/softmodem-common.h

@@ -44,6 +44,7 @@ extern "C"
   example: -1,3 launches two working threads one floating, the second set on core 3\n\
   default 8 floating threads\n\
   use N for no pool (runs in calling thread) recommended with rfsim.\n"
+#define CONFIG_HLP_REORDER       "Disable reorder thread\n"
 #define CONFIG_HLP_ULMAXE        "set the eNodeB max ULSCH erros\n"
 #define CONFIG_HLP_CALUER        "set UE RX calibration\n"
 #define CONFIG_HLP_CALUERM       ""
@@ -110,6 +111,7 @@ extern "C"
 /*-----------------------------------------------------------------------------------------------------------------------------------------------------*/
 #define RF_CONFIG_FILE      softmodem_params.rf_config_file
 #define TP_CONFIG           softmodem_params.threadPoolConfig
+#define CONTINUOUS_TX       softmodem_params.continuous_tx
 #define PHY_TEST            softmodem_params.phy_test
 #define DO_RA               softmodem_params.do_ra
 #define SA                  softmodem_params.sa
@@ -132,7 +134,7 @@ extern "C"
 #define NON_STOP            softmodem_params.non_stop
 #define EMULATE_L1          softmodem_params.emulate_l1
 #define CONTINUOUS_TX       softmodem_params.continuous_tx
-
+#define REORDER_THREAD_DISABLE    softmodem_params.reorder_thread_disable
 #define DEFAULT_RFCONFIG_FILE    "/usr/local/etc/syriq/ue.band7.tm1.PRB100.NR40.dat";
 
 extern int usrp_tx_thread;
@@ -140,6 +142,7 @@ extern int usrp_tx_thread;
 #define CMDLINE_PARAMS_DESC {  \
   {"rf-config-file",        CONFIG_HLP_RFCFGF,        0,              .strptr=&RF_CONFIG_FILE,                .defstrval=NULL,          TYPE_STRING, 0},  \
   {"thread-pool",           CONFIG_HLP_TPOOL,         0,              .strptr=&TP_CONFIG,                     .defstrval="-1,-1,-1,-1,-1,-1,-1,-1",  TYPE_STRING, 0},     \
+  {"reorder-thread-disable",CONFIG_HLP_REORDER,     PARAMFLAG_BOOL,   .iptr=&REORDER_THREAD_DISABLE,          .defintval=0,                          TYPE_INT,    0},                     \
   {"phy-test",              CONFIG_HLP_PHYTST,        PARAMFLAG_BOOL, .iptr=&PHY_TEST,                        .defintval=0,             TYPE_INT,    0},  \
   {"do-ra",                 CONFIG_HLP_DORA,          PARAMFLAG_BOOL, .iptr=&DO_RA,                           .defintval=0,             TYPE_INT,    0},  \
   {"sa",                    CONFIG_HLP_SA,            PARAMFLAG_BOOL, .iptr=&SA,                              .defintval=0,             TYPE_INT,    0},  \
@@ -210,6 +213,7 @@ extern int usrp_tx_thread;
     { .s5 = { NULL } },                     \
     { .s5 = { NULL } },                     \
     { .s5 = { NULL } },                     \
+    { .s5 = { NULL } },                     \
     { .s3a = { config_checkstr_assign_integer, \
                {"MONOLITHIC", "PNF", "VNF","UE_STUB_PNF","UE_STUB_OFFNET","STANDALONE_PNF"}, \
                {NFAPI_MONOLITHIC, NFAPI_MODE_PNF, NFAPI_MODE_VNF,NFAPI_UE_STUB_PNF,NFAPI_UE_STUB_OFFNET,NFAPI_MODE_STANDALONE_PNF}, \
@@ -300,6 +304,7 @@ typedef struct {
   //THREAD_STRUCT  thread_struct;
   char           *rf_config_file;
   char           *threadPoolConfig;
+  int            reorder_thread_disable;
   int            phy_test;
   int            do_ra;
   int            sa;

openair1/PHY/NR_TRANSPORT/nr_dlsch.c

@@ -491,7 +491,7 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx,
         for (int rb=0; rb<rel15->rbSize; rb++) {
           //get pmi info
           uint8_t pmi;
-          if (rel15->precodingAndBeamforming.prg_size > 0)
+          if (0 /*rel15->precodingAndBeamforming.prg_size > 0*/)
             pmi = rel15->precodingAndBeamforming.prgs_list[(int)rb/rel15->precodingAndBeamforming.prg_size].pm_idx;
           else
             pmi = 0;//no precoding

openair1/PHY/defs_RU.h

@@ -655,6 +655,8 @@ typedef struct RU_t_s {
   int txfh_core_id;
   /// number of RU interfaces
   int num_fd;
+  /// Core id of ru_thread
+  int ru_thread_core;
   /// list of cores for RU ThreadPool
   int tpcores[16];
   /// number of cores for RU ThreadPool

openair1/PHY/defs_gNB.h

@@ -683,6 +683,8 @@ typedef struct PHY_VARS_gNB_s {
 
   int ldpc_offload_flag;
 
+  int reorder_thread_disable;
+
   int max_ldpc_iterations;
   /// indicate the channel estimation technique in time domain
   int chest_time;
@@ -765,6 +767,11 @@ typedef struct PHY_VARS_gNB_s {
   notifiedFIFO_t L1_tx_out;
   notifiedFIFO_t resp_RU_tx;
   tpool_t threadPool;
+  pthread_t L1_rx_thread;
+  int L1_rx_thread_core;
+  pthread_t L1_tx_thread;
+  int L1_tx_thread_core;
+  struct processingData_L1tx *msgDataTx;
   int nbDecode;
   void *scopeData;
   /// structure for analyzing high-level RT measurements

openair1/SCHED_NR/fapi_nr_l1.c

@@ -37,6 +37,7 @@
 #include "nfapi/oai_integration/vendor_ext.h"
 #include "openair2/NR_PHY_INTERFACE/nr_sched_response.h"
 
+
 extern int oai_nfapi_dl_tti_req(nfapi_nr_dl_tti_request_t *dl_config_req);
 extern int oai_nfapi_tx_data_req(nfapi_nr_tx_data_request_t *tx_data_req);
 extern int oai_nfapi_ul_dci_req(nfapi_nr_ul_dci_request_t *ul_dci_req);
@@ -157,14 +158,19 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO)
   if (NFAPI_MODE == NFAPI_MONOLITHIC){
 
     if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) {
-      notifiedFIFO_elt_t *res;
+      notifiedFIFO_elt_t *res=NULL;
+      processingData_L1tx_t *msgTx=NULL;
+      if (!gNB->reorder_thread_disable) {
 	res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool);
         if (res == NULL)
           return; // Tpool has been stopped, nothing to process
-      processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
+        msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
-      const time_stats_t ts = exec_time_stats_NotifiedFIFO(res);
+      } else {
+        msgTx = gNB->msgDataTx; //newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0, &gNB->L1_tx_out,NULL);
+      }
+      /*const time_stats_t ts = exec_time_stats_NotifiedFIFO(res);
       merge_meas(&gNB->phy_proc_tx, &ts);
-
+*/
       msgTx->num_pdsch_slot = 0;
       msgTx->num_dl_pdcch = 0;
       msgTx->num_ul_pdcch = number_ul_dci_pdu;
@@ -216,6 +222,7 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO)
        * released only when both threads are done with it.
        */
       inc_ref_sched_response(Sched_INFO->sched_response_id);
+      if (!gNB->reorder_thread_disable)
 	pushNotifiedFIFO(&gNB->L1_tx_filled,res);
     }
 

openair1/SCHED_NR/phy_procedures_nr_gNB.c

@@ -396,8 +396,10 @@ void nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int ULSCH
   stop_meas(&gNB->ulsch_decoding_stats);
 }
 
+
 void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, uint8_t harq_pid, uint8_t crc_flag, int dtx_flag)
 {
+  if (!get_softmodem_params()->reorder_thread_disable) 
     pthread_mutex_lock(&gNB->UL_INFO_mutex);
 
   NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ULSCH_id];
@@ -557,6 +559,7 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id,
 
   gNB->UL_INFO.rx_ind.number_of_pdus++;
 
+  if (!get_softmodem_params()->reorder_thread_disable) 
     pthread_mutex_unlock(&gNB->UL_INFO_mutex);
 }
 

openair2/ENB_APP/enb_paramdef.h

@@ -111,6 +111,7 @@ typedef enum {
 #define CONFIG_STRING_RU_NUM_TP_CORES             "num_tp_cores"
 #define CONFIG_STRING_RU_NUM_INTERFACES           "num_interfaces"
 #define CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION "half_slot_parallelization"
+#define CONFIG_STRING_RU_RU_THREAD_CORE            "ru_thread_core"
 
 #define HLP_RU_SF_AHEAD "LTE TX processing advance"
 #define HLP_RU_SL_AHEAD "NR TX processing advance"
@@ -122,6 +123,7 @@ typedef enum {
 #define HLP_RU_NUM_TP_CORES "Number of cores for RU ThreadPool"
 #define HLP_RU_NUM_INTERFACES "Number of network interfaces for RU"
 #define HLP_RU_HALF_SLOT_PARALLELIZATION "run half slots in parallel in RU FEP"
+#define HLP_RU_RU_THREAD_CORE "id of core to pin ru_thread, -1 is default"
 
 #define RU_LOCAL_IF_NAME_IDX          0
 #define RU_LOCAL_ADDRESS_IDX          1
@@ -164,6 +166,7 @@ typedef enum {
 #define RU_NUM_TP_CORES               38
 #define RU_NUM_INTERFACES             39
 #define RU_HALF_SLOT_PARALLELIZATION  40
+#define RU_RU_THREAD_CORE             41
 /*-----------------------------------------------------------------------------------------------------------------------------------------*/
 /*                                            RU configuration parameters                                                                  */
 /*   optname                                   helpstr   paramflags    XXXptr          defXXXval                   type      numelt        */
@@ -211,6 +214,7 @@ typedef enum {
   {CONFIG_STRING_RU_NUM_TP_CORES,              HLP_RU_NUM_TP_CORES,               0,       .uptr=NULL,       .defintval=2,                 TYPE_UINT,        0}, \
   {CONFIG_STRING_RU_NUM_INTERFACES,            HLP_RU_NUM_INTERFACES,             0,       .uptr=NULL,       .defintval=1,                 TYPE_UINT,        0}, \
   {CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION, HLP_RU_HALF_SLOT_PARALLELIZATION,  0,       .uptr=NULL,       .defintval=1,                 TYPE_UINT,        0}, \
+  {CONFIG_STRING_RU_RU_THREAD_CORE,            HLP_RU_RU_THREAD_CORE,             0,       .uptr=NULL,       .defintval=-1,                TYPE_UINT,         0}, \
 }
 // clang-format on
 

openair2/GNB_APP/L1_nr_paramdef.h

@@ -53,6 +53,8 @@
 #define CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD               "pusch_dtx_threshold"
 #define CONFIG_STRING_L1_SRS_DTX_THRESHOLD                 "srs_dtx_threshold"
 #define CONFIG_STRING_L1_MAX_LDPC_ITERATIONS               "max_ldpc_iterations"
+#define CONFIG_STRING_L1_RX_THREAD_CORE                    "L1_rx_thread_core"
+#define CONFIG_STRING_L1_TX_THREAD_CORE                    "L1_tx_thread_core"
 #define HLP_TP_SIZ "thread_pool_size paramter removed, please use --thread-pool"
 /*----------------------------------------------------------------------------------------------------------------------------------------------------*/
 /*                                            L1 configuration parameters                                                                             */
@@ -76,6 +78,8 @@
   {CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD,               NULL,       0,         .uptr=NULL,           .defintval=50,              TYPE_UINT,     0},         \
   {CONFIG_STRING_L1_SRS_DTX_THRESHOLD,                 NULL,       0,         .uptr=NULL,           .defintval=50,              TYPE_UINT,     0},         \
   {CONFIG_STRING_L1_MAX_LDPC_ITERATIONS,               NULL,       0,         .uptr=NULL,           .defintval=5,               TYPE_UINT,     0},         \
+  {CONFIG_STRING_L1_RX_THREAD_CORE,                    NULL,       0,         .uptr=NULL,           .defintval=-1,              TYPE_UINT,     0},          \
+  {CONFIG_STRING_L1_TX_THREAD_CORE,                    NULL,       0,         .uptr=NULL,           .defintval=-1,              TYPE_UINT,     0},          \
 }
 // clang-format on
 #define L1_CC_IDX                                          0
@@ -94,6 +98,8 @@
 #define L1_PUSCH_DTX_THRESHOLD                             13
 #define L1_SRS_DTX_THRESHOLD                               14
 #define L1_MAX_LDPC_ITERATIONS                             15
+#define L1_RX_THREAD_CORE                                  16
+#define L1_TX_THREAD_CORE                                  17
 
 /*----------------------------------------------------------------------------------------------------------------------------------------------------*/
 #endif

openair2/GNB_APP/gnb_config.c

@@ -770,6 +770,9 @@ void RCconfig_NR_L1(void)
       RC.gNB[j]->pusch_thres = *(L1_ParamList.paramarray[j][L1_PUSCH_DTX_THRESHOLD].uptr);
       RC.gNB[j]->srs_thres = *(L1_ParamList.paramarray[j][L1_SRS_DTX_THRESHOLD].uptr);
       RC.gNB[j]->max_ldpc_iterations = *(L1_ParamList.paramarray[j][L1_MAX_LDPC_ITERATIONS].uptr);
+      RC.gNB[j]->L1_rx_thread_core = *(L1_ParamList.paramarray[j][L1_RX_THREAD_CORE].iptr);
+      RC.gNB[j]->L1_tx_thread_core = *(L1_ParamList.paramarray[j][L1_TX_THREAD_CORE].iptr);
+      LOG_I(PHY,"L1_RX_THREAD_CORE %d (%d)\n",*(L1_ParamList.paramarray[j][L1_RX_THREAD_CORE].iptr),L1_RX_THREAD_CORE);
       if (strcmp(*(L1_ParamList.paramarray[j][L1_TRANSPORT_N_PREFERENCE_IDX].strptr), "local_mac") == 0) {
         // sf_ahead = 2; // Need 4 subframe gap between RX and TX
       } else if (strcmp(*(L1_ParamList.paramarray[j][L1_TRANSPORT_N_PREFERENCE_IDX].strptr), "nfapi") == 0) {

radio/AW2SORI/oaiori.c

@@ -22,7 +22,6 @@
 #include <unistd.h>
 #include <errno.h>
 #include <linux/sysctl.h>
-#include <sys/sysctl.h>
 #include <pthread.h>
 
 #include "common_lib.h"

radio/ETHERNET/USERSPACE/LIB/eth_udp.c

@@ -314,8 +314,14 @@ void *trx_eth_write_udp_cmd(udpTXelem_t *udpTXelem) {
   uint64_t last_rxTS = fhstate->TS[0]-fhstate->TS0;
   uint64_t TS_advance=0;
   if (timestamp > last_rxTS) TS_advance = timestamp - last_rxTS;
-
+  else {
-  if (TS_advance < (nsamps/2)) LOG_W(PHY,"Starting TX FH for TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n",(unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance);
+    LOG_W(PHY,"TS_advance is < 0  TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n", (unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance);
+    free(buff);
+    return(NULL);
+  }
+  if (TS_advance < (nsamps/2)) {
+     LOG_W(PHY,"Starting TX FH for TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n",(unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance);
+  }
      void *buff2;
 #if defined(__x86_64) || defined(__i386__)
   int nsamps2 = 256>>3;
@@ -445,7 +451,7 @@ void *udp_read_thread(void *arg) {
   char buffer[UDP_PACKET_SIZE_BYTES(256)];
   int first_read=0;
   while (oai_exit == 0) {
-    LOG_I(PHY,"UDP read thread %d, waiting for start sampling_rate_d %d, sampling_rate_n %d\n",u->thread_id,device->sampling_rate_ratio_n,device->sampling_rate_ratio_d);
+    LOG_I(PHY,"UDP read thread %d on core %d, waiting for start sampling_rate_d %d, sampling_rate_n %d\n",u->thread_id,sched_getcpu(),device->sampling_rate_ratio_n,device->sampling_rate_ratio_d);
     while (fhstate->active > 0) {
       ssize_t count = recvfrom(((eth_state_t*)device->priv)->sockfdd[0],
                                buffer,sizeof(buffer),0,
@@ -500,23 +506,23 @@ void *udp_read_thread(void *arg) {
 int trx_eth_read_udp(openair0_device *device, openair0_timestamp *timestamp, uint32_t **buff, int nsamps) {
   
   fhstate_t *fhstate = &device->fhstate;
-  openair0_timestamp prev_read_TS= fhstate->TS_read, min_TS;
+  openair0_timestamp prev_read_TS= fhstate->TS_read;
+  volatile openair0_timestamp min_TS;
   // block until FH is ready
   while (fhstate->r[0] == 0 || fhstate->r[1] == 0 || fhstate->r[2] == 0 || fhstate->r[3] == 0 ||
          fhstate->r[4] == 0 || fhstate->r[5] == 0 || fhstate->r[6] == 0 || fhstate->r[7] == 0) usleep(100);
 
   // get minimum TS over all antennas
-  min_TS = fhstate->TS[0];
+  min_TS = (volatile openair0_timestamp)fhstate->TS[0];
   for (int i=1;i<device->openair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]);
   // poll/sleep until we accumulated enough samples on each antenna port
   int count=0;
   while (fhstate->first_read == 1 && min_TS < (fhstate->TS0+prev_read_TS + nsamps)) {
-    usleep(50);
+    usleep(10);
-    min_TS = fhstate->TS[0];
+    min_TS = (volatile openair0_timestamp)fhstate->TS[0];
-    for (int i=1;i<device->openair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]);
+    for (int i=1;i<device->openair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,(volatile openair0_timestamp)fhstate->TS[i]);
     count++;
   }
-
   if (fhstate->first_read == 0) {
      *timestamp = 0;
      fhstate->TS_read = *timestamp+nsamps;

radio/USRP/USERSPACE/LIB/usrp_lib.cpp

@@ -556,6 +556,7 @@ void *trx_usrp_write_thread(void * arg){
   signed char        last_packet;
   int                flags_gpio;
 
+  printf("trx_usrp_write_thread started on cpu %d\n",sched_getcpu());
   while(1){
     pthread_mutex_lock(&write_thread->mutex_write);
     while (write_thread->count_write == 0) {
@@ -664,7 +665,6 @@ int trx_usrp_write_init(openair0_device *device){
                (char*)"trx_usrp_write_thread",
                -1,
                OAI_PRIORITY_RT_MAX);
-
   return(0);
 }