Merge branch 'trx_write_thread' into NR_RRCConfiguration_trx_thread

Conflicts:
	executables/nr-ru.c
	targets/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp

common/utils/LOG/vcd_signal_dumper.c

@@ -248,7 +248,8 @@ const char* eurecomVariablesNames[] = {
   "slot_number_TX1_gNB",
   "slot_number_RX0_gNB",
   "slot_number_RX1_gNB",
-  "ru_tx_ofdm_mask"
+  "ru_tx_ofdm_mask",
+  "usrp_send_return"
 };
 
 const char* eurecomFunctionsNames[] = {
@@ -282,6 +283,8 @@ const char* eurecomFunctionsNames[] = {
   "lock_mutex_ru",
   "lock_mutex_ru1",
   "lock_mutex_ru2",
+  /* uhd signals */
+  "trx_write_thread",
   /* simulation signals */
   "do_DL_sig",
   "do_UL_sig",

common/utils/LOG/vcd_signal_dumper.h

@@ -226,6 +226,7 @@ typedef enum {
   VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER_RX0_GNB,
   VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER_RX1_GNB,
   VCD_SIGNAL_DUMPER_VARIABLES_RU_TX_OFDM_MASK,
+  VCD_SIGNAL_DUMPER_VARIABLES_USRP_SEND_RETURN,
 
 
   VCD_SIGNAL_DUMPER_VARIABLES_END
@@ -262,6 +263,8 @@ typedef enum {
   VCD_SIGNAL_DUMPER_FUNCTIONS_LOCK_MUTEX_RU,
   VCD_SIGNAL_DUMPER_FUNCTIONS_LOCK_MUTEX_RU1,
   VCD_SIGNAL_DUMPER_FUNCTIONS_LOCK_MUTEX_RU2,
+  /* uhd signals */
+  VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE_THREAD,
   /* SIMULATION signals */ 
   VCD_SIGNAL_DUMPER_FUNCTIONS_SIM_DO_DL_SIGNAL,
   VCD_SIGNAL_DUMPER_FUNCTIONS_SIM_DO_UL_SIGNAL,

common/utils/T/T_defs.h

@@ -76,7 +76,7 @@ typedef struct {
 #define VCD_NUM_FUNCTIONS (248)
 
 /* number of VCD variables (to be kept up to date! see in T_messages.txt) */
-#define VCD_NUM_VARIABLES (186)
+#define VCD_NUM_VARIABLES (187)
 
 /* first VCD function (to be kept up to date! see in T_messages.txt) */
 #define VCD_FIRST_FUNCTION    ((uintptr_t)T_VCD_FUNCTION_RT_SLEEP)

common/utils/T/T_messages.txt

@@ -2138,6 +2138,11 @@ ID = VCD_VARIABLE_RU_TX_OFDM_MASK
     GROUP = ALL:VCD:ENB:VCD_VARIABLE
     FORMAT = ulong,value
     VCD_NAME = ru_tx_ofdm_mask
+ID = VCD_VARIABLE_USRP_SEND_RETURN
+    DESC = VCD variable USRP_SEND_RETURN
+    GROUP = ALL:VCD:ENB:VCD_VARIABLE
+    FORMAT = ulong,value
+    VCD_NAME = usrp_send_return
 
 #functions
 
@@ -2286,6 +2291,11 @@ ID = VCD_FUNCTION_LOCK_MUTEX_RU2
     GROUP = ALL:VCD:UE:VCD_FUNCTION
     FORMAT = int,value
     VCD_NAME = lock_mutex_ru2
+ID = VCD_FUNCTION_TRX_WRITE_THREAD
+    DESC = VCD function TRX_WRITE_THREAD
+    GROUP = ALL:VCD:ENB:VCD_FUNCTION
+    FORMAT = int,value
+    VCD_NAME = trx_write_thread
 ID = VCD_FUNCTION_SIM_DO_DL_SIGNAL
     DESC = VCD function SIM_DO_DL_SIGNAL
     GROUP = ALL:VCD:ENB:VCD_FUNCTION

executables/nr-ru.c

@@ -792,7 +792,7 @@ void tx_rf(RU_t *ru,int frame,int slot, uint64_t timestamp) {
       LOG_D(PHY,"[TXPATH] RU %d tx_rf, writing to TS %llu, frame %d, unwrapped_frame %d, slot %d\n",ru->idx,
 	    (long long unsigned int)timestamp,frame,proc->frame_tx_unwrap,slot);
       VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE, 0 );
-      AssertFatal(txs ==  siglen+sf_extension,"TX : Timeout (sent %u/%d)\n", txs, siglen);
+      AssertFatal(txs == 0,"trx write function error %d\n", txs);
   }
 }
 
@@ -1482,6 +1482,9 @@ void *ru_thread( void *param ) {
 
     // if this is a slave RRU, try to synchronize on the DL frequency
     if ((ru->is_slave) && (ru->if_south == LOCAL_RF)) do_ru_synch(ru);
+
+    // start trx write thread
+    ru->start_write_thread(ru);
   }
 
   pthread_mutex_lock(&proc->mutex_FH1);
@@ -1691,6 +1694,9 @@ int stop_rf(RU_t *ru) {
   return 0;
 }
 
+int start_write_thread(RU_t *ru) {
+  return(ru->rfdevice.trx_write_init(&ru->rfdevice));
+}
 
 void init_RU_proc(RU_t *ru) {
   int i=0;
@@ -2086,6 +2092,7 @@ void set_function_spec_param(RU_t *ru) {
       ru->fh_south_out           = tx_rf;                               // local synchronous RF TX
       ru->start_rf               = start_rf;                            // need to start the local RF interface
       ru->stop_rf                = stop_rf;
+      ru->start_write_thread     = start_write_thread;                  // starting RF TX in different thread
       printf("configuring ru_id %u (start_rf %p)\n", ru->idx, start_rf);
       /*
           if (ru->function == gNodeB_3GPP) { // configure RF parameters only for 3GPP eNodeB, we need to get them from RAU otherwise

openair1/PHY/defs_RU.h

@@ -540,6 +540,8 @@ typedef struct RU_t_s {
   void (*wakeup_prach_gNB)(struct PHY_VARS_gNB_s *gNB, struct RU_t_s *ru, int frame, int subframe);
   /// function pointer to wakeup routine in lte-enb.
   void (*wakeup_prach_eNB_br)(struct PHY_VARS_eNB_s *eNB, struct RU_t_s *ru, int frame, int subframe);
+  /// function pointer to start a thread of tx write for USRP.
+  int (*start_write_thread)(struct RU_t_s *ru);
 
   /// function pointer to NB entry routine
   void (*eNB_top)(struct PHY_VARS_eNB_s *eNB, int frame_rx, int subframe_rx, char *string, struct RU_t_s *ru);

targets/ARCH/COMMON/common_lib.h

@@ -48,6 +48,9 @@
 #define RAU_LOCAL_RADIO_HEAD  0
 #define RAU_REMOTE_RADIO_HEAD 1
 
+#define MAX_WRITE_THREAD_PACKAGE     10
+#define MAX_WRITE_THREAD_BUFFER_SIZE 8
+
 #ifndef MAX_CARDS
   #define MAX_CARDS 8
 #endif
@@ -276,8 +279,36 @@ typedef struct {
   void *rx;
 } if_buffer_t;
 
+typedef struct {
+  openair0_timestamp timestamp;
+  void *buff[MAX_WRITE_THREAD_BUFFER_SIZE];// buffer to be write;
+  int nsamps;
+  int cc;
+  signed char first_packet;
+  signed char last_packet;
+} openair0_write_package_t;
+
+typedef struct {
+  openair0_write_package_t write_package[MAX_WRITE_THREAD_PACKAGE];
+  int start;
+  int end;
+  /// \internal This variable is protected by \ref mutex_write
+  int count_write;
+  /// pthread struct for trx write thread
+  pthread_t pthread_write;
+  /// pthread attributes for trx write thread
+  pthread_attr_t attr_write;
+  /// condition varible for trx write thread
+  pthread_cond_t cond_write;
+  /// mutex for trx write thread
+  pthread_mutex_t mutex_write;
+} openair0_thread_t;
+
 /*!\brief structure holds the parameters to configure USRP devices */
 struct openair0_device_t {
+  /*!tx write thread*/
+  openair0_thread_t write_thread;
+
   /*!brief Module ID of this device */
   int Mod_id;
 
@@ -400,6 +431,12 @@ struct openair0_device_t {
    * \param arg pointer to capabilities or configuration
    */
   void (*configure_rru)(int idx, void *arg);
+
+  /*! \brief RRU Configuration callback
+   * \param idx RU index
+   * \param arg pointer to capabilities or configuration
+   */
+  int (*trx_write_init)(openair0_device *device);
 };
 
 /* type of device init function, implemented in shared lib */

targets/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp

@@ -49,6 +49,8 @@
 #include "common_lib.h"
 #include "assertions.h"
 
+#include "common/utils/LOG/vcd_signal_dumper.h"
+
 #include <sys/resource.h>
 
 #include "usrp_lib.h"
@@ -413,43 +415,15 @@ static int trx_usrp_write(openair0_device *device,
 			  int nsamps,
 			  int cc,
 			  int flags) {
-  int ret=0;
-  usrp_state_t *s = (usrp_state_t *)device->priv;
-  int nsamps2;  // aligned to upper 32 or 16 byte boundary
 
   int flags_lsb = flags&0xff;
   int flags_msb = (flags>>8)&0xff;
 
-#if defined(__x86_64) || defined(__i386__)
+  int end;
-  #ifdef __AVX2__
+  openair0_thread_t *write_thread = &device->write_thread;
-      nsamps2 = (nsamps+7)>>3;
+  openair0_write_package_t *write_package = write_thread->write_package;
-      __m256i buff_tx[8][nsamps2];
-  #else
-    nsamps2 = (nsamps+3)>>2;
-    __m128i buff_tx[8][nsamps2];
-  #endif
-#elif defined(__arm__)
-    nsamps2 = (nsamps+3)>>2;
-    int16x8_t buff_tx[8][nsamps2];
-#else
-#error Unsupported CPU architecture, USRP device cannot be built
-#endif
-
-  // bring RX data into 12 LSBs for softmodem RX
-  for (int i=0; i<cc; i++) {
-    for (int j=0; j<nsamps2; j++) {
-#if defined(__x86_64__) || defined(__i386__)
-#ifdef __AVX2__
-      buff_tx[i][j] = _mm256_slli_epi16(((__m256i *)buff[i])[j],4);
-#else
-      buff_tx[i][j] = _mm_slli_epi16(((__m128i *)buff[i])[j],4);
-#endif
-#elif defined(__arm__)
-      buff_tx[i][j] = vshlq_n_s16(((int16x8_t *)buff[i])[j],4);
-#endif
-    }
-  }
 
+  AssertFatal( MAX_WRITE_THREAD_BUFFER_SIZE >= cc,"Do not support more than %d cc number\n", MAX_WRITE_THREAD_BUFFER_SIZE);
 
     boolean_t first_packet_state=false,last_packet_state=false;
 
@@ -481,11 +455,111 @@ static int trx_usrp_write(openair0_device *device,
      first_packet_state = false;
      last_packet_state  = true;
     }
+  pthread_mutex_lock(&write_thread->mutex_write);
+
+  if(write_thread->count_write >= MAX_WRITE_THREAD_PACKAGE){
+    LOG_W("Buffer overflow, count_write = %d, start = %d end = %d, resetting write package\n", write_thread->count_write, write_thread->start, write_thread->end);
+    write_thread->end = write_thread->start;
+    write_thread->count_write = 0;
+  }
+
+  end = write_thread->end;
+  write_package[end].timestamp    = timestamp;
+  write_package[end].nsamps       = nsamps;
+  write_package[end].cc           = cc;
+  write_package[end].first_packet = first_packet_state;
+  write_package[end].last_packet  = last_packet_state;
+  for (int i = 0; i < cc; i++)
+    write_package[end].buff[i]    = buff[i];
+  write_thread->count_write++;
+  write_thread->end = (write_thread->end + 1)% MAX_WRITE_THREAD_PACKAGE;
+  pthread_cond_signal(&write_thread->cond_write);
+  pthread_mutex_unlock(&write_thread->mutex_write);
+
+
+  return 0;
+}
+
+//-----------------------start--------------------------
+/*! \brief Called to send samples to the USRP RF target
+      @param device pointer to the device structure specific to the RF hardware target
+      @param timestamp The timestamp at which the first sample MUST be sent
+      @param buff Buffer which holds the samples
+      @param nsamps number of samples to be sent
+      @param antenna_id index of the antenna if the device has multiple antennas
+      @param flags flags must be set to TRUE if timestamp parameter needs to be applied
+*/
+void *trx_usrp_write_thread(void * arg){ 
+  int ret=0;
+  openair0_device *device=(openair0_device *)arg;
+  openair0_thread_t *write_thread = &device->write_thread;
+  openair0_write_package_t *write_package = write_thread->write_package;
+
+    usrp_state_t *s;
+    int nsamps2;  // aligned to upper 32 or 16 byte boundary
+    int start;
+    openair0_timestamp timestamp;
+    void               **buff;
+    int                nsamps;
+    int                cc;
+    signed char        first_packet;
+    signed char        last_packet;
+
+  while(1){
+    pthread_mutex_lock(&write_thread->mutex_write);
+    while (write_thread->count_write == 0) {
+      pthread_cond_wait(&write_thread->cond_write,&write_thread->mutex_write); // this unlocks mutex_rxtx while waiting and then locks it again
+    }
+    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE_THREAD, 1 );
+    s = (usrp_state_t *)device->priv;
+    start = write_thread->start;
+    timestamp    = write_package[start].timestamp;
+    buff         = write_package[start].buff;
+    nsamps       = write_package[start].nsamps;
+    cc           = write_package[start].cc;
+    first_packet = write_package[start].first_packet;
+    last_packet  = write_package[start].last_packet;
+    write_thread->start = (write_thread->start + 1)% MAX_WRITE_THREAD_PACKAGE;
+    write_thread->count_write--;
+    pthread_mutex_unlock(&write_thread->mutex_write);
+    if(write_thread->count_write != 0){
+      LOG_W(HW,"count write = %d, start = %d, end = %d\n", write_thread->count_write, write_thread->start, write_thread->end);
+    }
+
+    #if defined(__x86_64) || defined(__i386__)
+      #ifdef __AVX2__
+        nsamps2 = (nsamps+7)>>3;
+        __m256i buff_tx[8][nsamps2];
+      #else
+        nsamps2 = (nsamps+3)>>2;
+        __m128i buff_tx[8][nsamps2];
+      #endif
+    #elif defined(__arm__)
+      nsamps2 = (nsamps+3)>>2;
+      int16x8_t buff_tx[8][nsamps2];
+    #else
+    #error Unsupported CPU architecture, USRP device cannot be built
+    #endif
+
+    // bring RX data into 12 LSBs for softmodem RX
+    for (int i=0; i<cc; i++) {
+      for (int j=0; j<nsamps2; j++) {
+        #if defined(__x86_64__) || defined(__i386__)
+          #ifdef __AVX2__
+            buff_tx[i][j] = _mm256_slli_epi16(((__m256i *)buff[i])[j],4);
+          #else
+            buff_tx[i][j] = _mm_slli_epi16(((__m128i *)buff[i])[j],4);
+          #endif
+        #elif defined(__arm__)
+          buff_tx[i][j] = vshlq_n_s16(((int16x8_t *)buff[i])[j],4);
+        #endif
+      }
+    }
 
     
     s->tx_md.has_time_spec  = true;
-    s->tx_md.start_of_burst = (s->tx_count==0) ? true : first_packet_state;
+    s->tx_md.start_of_burst = (s->tx_count==0) ? true : first_packet;
-    s->tx_md.end_of_burst   = last_packet_state;
+    s->tx_md.end_of_burst   = last_packet;
     s->tx_md.time_spec      = uhd::time_spec_t::from_ticks(timestamp, s->sample_rate);
     s->tx_count++;
 
@@ -510,11 +584,34 @@ static int trx_usrp_write(openair0_device *device,
       ret = (int)s->tx_stream->send(&(((int16_t *)buff_tx[0])[0]), nsamps, s->tx_md);
     }
 
-  if (ret != nsamps) LOG_E(HW,"[xmit] tx samples %d != %d\n",ret,nsamps);
+    if (ret != nsamps) LOG_E(HW,"[xmit] tx samples %d != %d\n",ret,nsamps);
+    VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_USRP_SEND_RETURN, ret );
+    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE_THREAD, 0 );
 
-  return ret;
+    if(0) break;
+  }
+
+  return NULL;
 }
 
+int trx_write_init(openair0_device *device){
+
+  uhd::set_thread_priority_safe(1.0);
+  openair0_thread_t *write_thread = &device->write_thread;
+  printf("initializing tx write thread\n");
+
+  write_thread->start              = 0;
+  write_thread->end                = 0;
+  write_thread->count_write        = 0;
+  printf("end of tx write thread\n");
+
+  pthread_create(&write_thread->pthread_write,NULL,trx_usrp_write_thread,(void *)device);
+
+  return(0);
+}
+
+//---------------------end-------------------------
+
 /*! \brief Receive samples from iq file.
  * Read \ref nsamps samples from each channel to buffers. buff[0] is the array for
  * the first channel. *ptimestamp is the time at which the first sample
@@ -944,6 +1041,7 @@ extern "C" {
     device->trx_stop_func  = trx_usrp_stop;
     device->trx_set_freq_func = trx_usrp_set_freq;
     device->trx_set_gains_func   = trx_usrp_set_gains;
+    device->trx_write_init = trx_write_init;
 
     if ( s->recplay_mode == RECPLAY_REPLAYMODE) {
       // Replay subframes from from file

targets/RT/USER/gNB_usrp.gtkw

@@ -1,21 +1,25 @@
 [*]
 [*] GTKWave Analyzer v3.3.61 (w)1999-2014 BSI
-[*] Tue Dec 17 15:31:51 2019
+[*] Wed Mar  4 14:05:43 2020
 [*]
-[dumpfile] "/tmp/gNB.vcd"
+[dumpfile] "/tmp/gNB_trx_thread.vcd"
-[dumpfile_mtime] "Tue Dec 17 15:25:49 2019"
+[dumpfile_mtime] "Wed Mar  4 14:05:10 2020"
-[dumpfile_size] 6343431
+[dumpfile_size] 10750468
 [savefile] "/home/wangts/openairinterface5g/targets/RT/USER/gNB_usrp.gtkw"
-[timestart] 1517712000
+[timestart] 31148700000
-[size] 1920 1018
+[size] 1916 1002
-[pos] -9 -33
+[pos] -1 -25
-*-19.276148 1518358451 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
+*-23.852516 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
 [sst_width] 386
 [signals_width] 344
 [sst_expanded] 1
-[sst_vpaned_height] 267
+[sst_vpaned_height] 262
 @28
 functions.trx_read
+functions.trx_write_thread
+@25
+variables.trx_write_flags[63:0]
+@28
 functions.trx_write
 @420
 variables.frame_number_TX0_UE[63:0]