diff --git a/common/utils/LOG/vcd_signal_dumper.c b/common/utils/LOG/vcd_signal_dumper.c index 9b41b57093..ea5c9df8a4 100644 --- a/common/utils/LOG/vcd_signal_dumper.c +++ b/common/utils/LOG/vcd_signal_dumper.c @@ -248,7 +248,8 @@ const char* eurecomVariablesNames[] = { "slot_number_TX1_gNB", "slot_number_RX0_gNB", "slot_number_RX1_gNB", - "ru_tx_ofdm_mask" + "ru_tx_ofdm_mask", + "usrp_send_return" }; const char* eurecomFunctionsNames[] = { @@ -282,6 +283,8 @@ const char* eurecomFunctionsNames[] = { "lock_mutex_ru", "lock_mutex_ru1", "lock_mutex_ru2", + /* uhd signals */ + "trx_write_thread", /* simulation signals */ "do_DL_sig", "do_UL_sig", diff --git a/common/utils/LOG/vcd_signal_dumper.h b/common/utils/LOG/vcd_signal_dumper.h index 20588bb828..88655977bb 100644 --- a/common/utils/LOG/vcd_signal_dumper.h +++ b/common/utils/LOG/vcd_signal_dumper.h @@ -226,6 +226,7 @@ typedef enum { VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER_RX0_GNB, VCD_SIGNAL_DUMPER_VARIABLES_SLOT_NUMBER_RX1_GNB, VCD_SIGNAL_DUMPER_VARIABLES_RU_TX_OFDM_MASK, + VCD_SIGNAL_DUMPER_VARIABLES_USRP_SEND_RETURN, VCD_SIGNAL_DUMPER_VARIABLES_END @@ -262,6 +263,8 @@ typedef enum { VCD_SIGNAL_DUMPER_FUNCTIONS_LOCK_MUTEX_RU, VCD_SIGNAL_DUMPER_FUNCTIONS_LOCK_MUTEX_RU1, VCD_SIGNAL_DUMPER_FUNCTIONS_LOCK_MUTEX_RU2, + /* uhd signals */ + VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE_THREAD, /* SIMULATION signals */ VCD_SIGNAL_DUMPER_FUNCTIONS_SIM_DO_DL_SIGNAL, VCD_SIGNAL_DUMPER_FUNCTIONS_SIM_DO_UL_SIGNAL, diff --git a/common/utils/T/T_defs.h b/common/utils/T/T_defs.h index 4cb4bf2bba..ad21819c7d 100644 --- a/common/utils/T/T_defs.h +++ b/common/utils/T/T_defs.h @@ -76,7 +76,7 @@ typedef struct { #define VCD_NUM_FUNCTIONS (248) /* number of VCD variables (to be kept up to date! see in T_messages.txt) */ -#define VCD_NUM_VARIABLES (186) +#define VCD_NUM_VARIABLES (187) /* first VCD function (to be kept up to date! see in T_messages.txt) */ #define VCD_FIRST_FUNCTION ((uintptr_t)T_VCD_FUNCTION_RT_SLEEP) diff --git a/common/utils/T/T_messages.txt b/common/utils/T/T_messages.txt index f8b746669c..3838518834 100644 --- a/common/utils/T/T_messages.txt +++ b/common/utils/T/T_messages.txt @@ -2138,6 +2138,11 @@ ID = VCD_VARIABLE_RU_TX_OFDM_MASK GROUP = ALL:VCD:ENB:VCD_VARIABLE FORMAT = ulong,value VCD_NAME = ru_tx_ofdm_mask +ID = VCD_VARIABLE_USRP_SEND_RETURN + DESC = VCD variable USRP_SEND_RETURN + GROUP = ALL:VCD:ENB:VCD_VARIABLE + FORMAT = ulong,value + VCD_NAME = usrp_send_return #functions @@ -2286,6 +2291,11 @@ ID = VCD_FUNCTION_LOCK_MUTEX_RU2 GROUP = ALL:VCD:UE:VCD_FUNCTION FORMAT = int,value VCD_NAME = lock_mutex_ru2 +ID = VCD_FUNCTION_TRX_WRITE_THREAD + DESC = VCD function TRX_WRITE_THREAD + GROUP = ALL:VCD:ENB:VCD_FUNCTION + FORMAT = int,value + VCD_NAME = trx_write_thread ID = VCD_FUNCTION_SIM_DO_DL_SIGNAL DESC = VCD function SIM_DO_DL_SIGNAL GROUP = ALL:VCD:ENB:VCD_FUNCTION diff --git a/executables/nr-ru.c b/executables/nr-ru.c index fe10988462..7af60c8fb8 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -792,7 +792,7 @@ void tx_rf(RU_t *ru,int frame,int slot, uint64_t timestamp) { LOG_D(PHY,"[TXPATH] RU %d tx_rf, writing to TS %llu, frame %d, unwrapped_frame %d, slot %d\n",ru->idx, (long long unsigned int)timestamp,frame,proc->frame_tx_unwrap,slot); VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE, 0 ); - AssertFatal(txs == siglen+sf_extension,"TX : Timeout (sent %u/%d)\n", txs, siglen); + AssertFatal(txs == 0,"trx write function error %d\n", txs); } } @@ -1482,6 +1482,9 @@ void *ru_thread( void *param ) { // if this is a slave RRU, try to synchronize on the DL frequency if ((ru->is_slave) && (ru->if_south == LOCAL_RF)) do_ru_synch(ru); + + // start trx write thread + ru->start_write_thread(ru); } pthread_mutex_lock(&proc->mutex_FH1); @@ -1691,6 +1694,9 @@ int stop_rf(RU_t *ru) { return 0; } +int start_write_thread(RU_t *ru) { + return(ru->rfdevice.trx_write_init(&ru->rfdevice)); +} void init_RU_proc(RU_t *ru) { int i=0; @@ -2086,6 +2092,7 @@ void set_function_spec_param(RU_t *ru) { ru->fh_south_out = tx_rf; // local synchronous RF TX ru->start_rf = start_rf; // need to start the local RF interface ru->stop_rf = stop_rf; + ru->start_write_thread = start_write_thread; // starting RF TX in different thread printf("configuring ru_id %u (start_rf %p)\n", ru->idx, start_rf); /* if (ru->function == gNodeB_3GPP) { // configure RF parameters only for 3GPP eNodeB, we need to get them from RAU otherwise diff --git a/openair1/PHY/defs_RU.h b/openair1/PHY/defs_RU.h index 10496ced20..2c48a80390 100644 --- a/openair1/PHY/defs_RU.h +++ b/openair1/PHY/defs_RU.h @@ -540,6 +540,8 @@ typedef struct RU_t_s { void (*wakeup_prach_gNB)(struct PHY_VARS_gNB_s *gNB, struct RU_t_s *ru, int frame, int subframe); /// function pointer to wakeup routine in lte-enb. void (*wakeup_prach_eNB_br)(struct PHY_VARS_eNB_s *eNB, struct RU_t_s *ru, int frame, int subframe); + /// function pointer to start a thread of tx write for USRP. + int (*start_write_thread)(struct RU_t_s *ru); /// function pointer to NB entry routine void (*eNB_top)(struct PHY_VARS_eNB_s *eNB, int frame_rx, int subframe_rx, char *string, struct RU_t_s *ru); diff --git a/targets/ARCH/COMMON/common_lib.h b/targets/ARCH/COMMON/common_lib.h index 03ab4da5f9..1125195635 100644 --- a/targets/ARCH/COMMON/common_lib.h +++ b/targets/ARCH/COMMON/common_lib.h @@ -48,6 +48,9 @@ #define RAU_LOCAL_RADIO_HEAD 0 #define RAU_REMOTE_RADIO_HEAD 1 +#define MAX_WRITE_THREAD_PACKAGE 10 +#define MAX_WRITE_THREAD_BUFFER_SIZE 8 + #ifndef MAX_CARDS #define MAX_CARDS 8 #endif @@ -276,8 +279,36 @@ typedef struct { void *rx; } if_buffer_t; +typedef struct { + openair0_timestamp timestamp; + void *buff[MAX_WRITE_THREAD_BUFFER_SIZE];// buffer to be write; + int nsamps; + int cc; + signed char first_packet; + signed char last_packet; +} openair0_write_package_t; + +typedef struct { + openair0_write_package_t write_package[MAX_WRITE_THREAD_PACKAGE]; + int start; + int end; + /// \internal This variable is protected by \ref mutex_write + int count_write; + /// pthread struct for trx write thread + pthread_t pthread_write; + /// pthread attributes for trx write thread + pthread_attr_t attr_write; + /// condition varible for trx write thread + pthread_cond_t cond_write; + /// mutex for trx write thread + pthread_mutex_t mutex_write; +} openair0_thread_t; + /*!\brief structure holds the parameters to configure USRP devices */ struct openair0_device_t { + /*!tx write thread*/ + openair0_thread_t write_thread; + /*!brief Module ID of this device */ int Mod_id; @@ -400,6 +431,12 @@ struct openair0_device_t { * \param arg pointer to capabilities or configuration */ void (*configure_rru)(int idx, void *arg); + + /*! \brief RRU Configuration callback + * \param idx RU index + * \param arg pointer to capabilities or configuration + */ + int (*trx_write_init)(openair0_device *device); }; /* type of device init function, implemented in shared lib */ diff --git a/targets/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp b/targets/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp index 0b60a73c43..0c9097fe74 100644 --- a/targets/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp +++ b/targets/ARCH/USRP/USERSPACE/LIB/usrp_lib.cpp @@ -49,6 +49,8 @@ #include "common_lib.h" #include "assertions.h" +#include "common/utils/LOG/vcd_signal_dumper.h" + #include #include "usrp_lib.h" @@ -413,43 +415,15 @@ static int trx_usrp_write(openair0_device *device, int nsamps, int cc, int flags) { - int ret=0; - usrp_state_t *s = (usrp_state_t *)device->priv; - int nsamps2; // aligned to upper 32 or 16 byte boundary int flags_lsb = flags&0xff; int flags_msb = (flags>>8)&0xff; - -#if defined(__x86_64) || defined(__i386__) - #ifdef __AVX2__ - nsamps2 = (nsamps+7)>>3; - __m256i buff_tx[8][nsamps2]; - #else - nsamps2 = (nsamps+3)>>2; - __m128i buff_tx[8][nsamps2]; - #endif -#elif defined(__arm__) - nsamps2 = (nsamps+3)>>2; - int16x8_t buff_tx[8][nsamps2]; -#else -#error Unsupported CPU architecture, USRP device cannot be built -#endif - // bring RX data into 12 LSBs for softmodem RX - for (int i=0; iwrite_thread; + openair0_write_package_t *write_package = write_thread->write_package; + AssertFatal( MAX_WRITE_THREAD_BUFFER_SIZE >= cc,"Do not support more than %d cc number\n", MAX_WRITE_THREAD_BUFFER_SIZE); boolean_t first_packet_state=false,last_packet_state=false; @@ -481,11 +455,111 @@ static int trx_usrp_write(openair0_device *device, first_packet_state = false; last_packet_state = true; } + pthread_mutex_lock(&write_thread->mutex_write); + + if(write_thread->count_write >= MAX_WRITE_THREAD_PACKAGE){ + LOG_W("Buffer overflow, count_write = %d, start = %d end = %d, resetting write package\n", write_thread->count_write, write_thread->start, write_thread->end); + write_thread->end = write_thread->start; + write_thread->count_write = 0; + } + + end = write_thread->end; + write_package[end].timestamp = timestamp; + write_package[end].nsamps = nsamps; + write_package[end].cc = cc; + write_package[end].first_packet = first_packet_state; + write_package[end].last_packet = last_packet_state; + for (int i = 0; i < cc; i++) + write_package[end].buff[i] = buff[i]; + write_thread->count_write++; + write_thread->end = (write_thread->end + 1)% MAX_WRITE_THREAD_PACKAGE; + pthread_cond_signal(&write_thread->cond_write); + pthread_mutex_unlock(&write_thread->mutex_write); + + + return 0; +} + +//-----------------------start-------------------------- +/*! \brief Called to send samples to the USRP RF target + @param device pointer to the device structure specific to the RF hardware target + @param timestamp The timestamp at which the first sample MUST be sent + @param buff Buffer which holds the samples + @param nsamps number of samples to be sent + @param antenna_id index of the antenna if the device has multiple antennas + @param flags flags must be set to TRUE if timestamp parameter needs to be applied +*/ +void *trx_usrp_write_thread(void * arg){ + int ret=0; + openair0_device *device=(openair0_device *)arg; + openair0_thread_t *write_thread = &device->write_thread; + openair0_write_package_t *write_package = write_thread->write_package; + + usrp_state_t *s; + int nsamps2; // aligned to upper 32 or 16 byte boundary + int start; + openair0_timestamp timestamp; + void **buff; + int nsamps; + int cc; + signed char first_packet; + signed char last_packet; + + while(1){ + pthread_mutex_lock(&write_thread->mutex_write); + while (write_thread->count_write == 0) { + pthread_cond_wait(&write_thread->cond_write,&write_thread->mutex_write); // this unlocks mutex_rxtx while waiting and then locks it again + } + VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE_THREAD, 1 ); + s = (usrp_state_t *)device->priv; + start = write_thread->start; + timestamp = write_package[start].timestamp; + buff = write_package[start].buff; + nsamps = write_package[start].nsamps; + cc = write_package[start].cc; + first_packet = write_package[start].first_packet; + last_packet = write_package[start].last_packet; + write_thread->start = (write_thread->start + 1)% MAX_WRITE_THREAD_PACKAGE; + write_thread->count_write--; + pthread_mutex_unlock(&write_thread->mutex_write); + if(write_thread->count_write != 0){ + LOG_W(HW,"count write = %d, start = %d, end = %d\n", write_thread->count_write, write_thread->start, write_thread->end); + } + + #if defined(__x86_64) || defined(__i386__) + #ifdef __AVX2__ + nsamps2 = (nsamps+7)>>3; + __m256i buff_tx[8][nsamps2]; + #else + nsamps2 = (nsamps+3)>>2; + __m128i buff_tx[8][nsamps2]; + #endif + #elif defined(__arm__) + nsamps2 = (nsamps+3)>>2; + int16x8_t buff_tx[8][nsamps2]; + #else + #error Unsupported CPU architecture, USRP device cannot be built + #endif + + // bring RX data into 12 LSBs for softmodem RX + for (int i=0; itx_md.has_time_spec = true; - s->tx_md.start_of_burst = (s->tx_count==0) ? true : first_packet_state; - s->tx_md.end_of_burst = last_packet_state; + s->tx_md.start_of_burst = (s->tx_count==0) ? true : first_packet; + s->tx_md.end_of_burst = last_packet; s->tx_md.time_spec = uhd::time_spec_t::from_ticks(timestamp, s->sample_rate); s->tx_count++; @@ -510,11 +584,34 @@ static int trx_usrp_write(openair0_device *device, ret = (int)s->tx_stream->send(&(((int16_t *)buff_tx[0])[0]), nsamps, s->tx_md); } - if (ret != nsamps) LOG_E(HW,"[xmit] tx samples %d != %d\n",ret,nsamps); + if (ret != nsamps) LOG_E(HW,"[xmit] tx samples %d != %d\n",ret,nsamps); + VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_USRP_SEND_RETURN, ret ); + VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_TRX_WRITE_THREAD, 0 ); - return ret; + if(0) break; + } + + return NULL; } +int trx_write_init(openair0_device *device){ + + uhd::set_thread_priority_safe(1.0); + openair0_thread_t *write_thread = &device->write_thread; + printf("initializing tx write thread\n"); + + write_thread->start = 0; + write_thread->end = 0; + write_thread->count_write = 0; + printf("end of tx write thread\n"); + + pthread_create(&write_thread->pthread_write,NULL,trx_usrp_write_thread,(void *)device); + + return(0); +} + +//---------------------end------------------------- + /*! \brief Receive samples from iq file. * Read \ref nsamps samples from each channel to buffers. buff[0] is the array for * the first channel. *ptimestamp is the time at which the first sample @@ -944,6 +1041,7 @@ extern "C" { device->trx_stop_func = trx_usrp_stop; device->trx_set_freq_func = trx_usrp_set_freq; device->trx_set_gains_func = trx_usrp_set_gains; + device->trx_write_init = trx_write_init; if ( s->recplay_mode == RECPLAY_REPLAYMODE) { // Replay subframes from from file diff --git a/targets/RT/USER/gNB_usrp.gtkw b/targets/RT/USER/gNB_usrp.gtkw index e6726e7d9f..6ece894bb6 100644 --- a/targets/RT/USER/gNB_usrp.gtkw +++ b/targets/RT/USER/gNB_usrp.gtkw @@ -1,21 +1,25 @@ [*] [*] GTKWave Analyzer v3.3.61 (w)1999-2014 BSI -[*] Tue Dec 17 15:31:51 2019 +[*] Wed Mar 4 14:05:43 2020 [*] -[dumpfile] "/tmp/gNB.vcd" -[dumpfile_mtime] "Tue Dec 17 15:25:49 2019" -[dumpfile_size] 6343431 +[dumpfile] "/tmp/gNB_trx_thread.vcd" +[dumpfile_mtime] "Wed Mar 4 14:05:10 2020" +[dumpfile_size] 10750468 [savefile] "/home/wangts/openairinterface5g/targets/RT/USER/gNB_usrp.gtkw" -[timestart] 1517712000 -[size] 1920 1018 -[pos] -9 -33 -*-19.276148 1518358451 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 +[timestart] 31148700000 +[size] 1916 1002 +[pos] -1 -25 +*-23.852516 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 [sst_width] 386 [signals_width] 344 [sst_expanded] 1 -[sst_vpaned_height] 267 +[sst_vpaned_height] 262 @28 functions.trx_read +functions.trx_write_thread +@25 +variables.trx_write_flags[63:0] +@28 functions.trx_write @420 variables.frame_number_TX0_UE[63:0]