From d1e27f4e8c8c945f54f3ed98ddc682a4a00c0b49 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 19 Apr 2019 03:18:06 +0200 Subject: [PATCH] changes in PARALLEL_RU_L!_TRX_SPLIT configuration. Correction of a few bugs related to passing of variables between threads --- openair1/PHY/defs_common.h | 29 ++++++++++++++++- openair1/PHY/defs_eNB.h | 5 ++- targets/RT/USER/lte-enb.c | 67 ++++++++++++++++++++++++++++++-------- targets/RT/USER/lte-ru.c | 62 +++++++++++++++++------------------ targets/RT/USER/lte-ue.c | 4 +-- 5 files changed, 117 insertions(+), 50 deletions(-) diff --git a/openair1/PHY/defs_common.h b/openair1/PHY/defs_common.h index 64ca5afbb6..c7e5897fff 100644 --- a/openair1/PHY/defs_common.h +++ b/openair1/PHY/defs_common.h @@ -1052,15 +1052,42 @@ static inline void wait_sync(char *thread_name) { fflush(stderr); } -static inline int wakeup_thread(pthread_mutex_t *mutex,pthread_cond_t *cond,int *instance_cnt,char *name) { +static inline int wakeup_thread(pthread_mutex_t *mutex,pthread_cond_t *cond,int *instance_cnt,char *name, int sleeptime,int sleep_cnt_max) { int rc; + int sleep_cnt=0; + if ((rc = pthread_mutex_lock(mutex)) != 0) { LOG_E(PHY, "wakeup_thread(): error locking mutex for %s (%d %s, %p)\n", name, rc, strerror(rc), (void *)mutex); exit_fun("nothing to add"); return(-1); } + while (*instance_cnt == 0) { + if ((rc = pthread_mutex_unlock(mutex)) != 0) { + LOG_E(PHY, "wakeup_thread(): error unlocking mutex for %s (%d %s, %p)\n", + name, rc, strerror(rc), (void *)mutex); + exit_fun("nothing to add"); + return(-1); + } + sleep_cnt++; + if (sleep_cnt>sleep_cnt_max) return(-1); + usleep(sleeptime); + if ((rc = pthread_mutex_lock(mutex)) != 0) { + LOG_E(PHY, "wakeup_thread(): error locking mutex for %s (%d %s, %p)\n", + name, rc, strerror(rc), (void *)mutex); + exit_fun("nothing to add"); + return(-1); + } + } *instance_cnt = *instance_cnt + 1; + + if ((rc = pthread_mutex_unlock(mutex)) != 0) { + LOG_E(PHY, "wakeup_thread(): error locking mutex for %s (%d %s, %p)\n", + name, rc, strerror(rc), (void *)mutex); + exit_fun("nothing to add"); + return(-1); + } + // the thread can now be woken up if (pthread_cond_signal(cond) != 0) { LOG_E( PHY, "ERROR pthread_cond_signal\n"); diff --git a/openair1/PHY/defs_eNB.h b/openair1/PHY/defs_eNB.h index d3157d5e82..4f2f10e1af 100644 --- a/openair1/PHY/defs_eNB.h +++ b/openair1/PHY/defs_eNB.h @@ -403,7 +403,10 @@ typedef struct RU_t_s{ RU_proc_t proc; /// stats thread pthread descriptor pthread_t ru_stats_thread; - + /// sleep time in us for delaying L1 wakeup + int wakeup_L1_sleeptime; + /// maximum number of sleeps + int wakeup_L1_sleep_cnt_max; } RU_t; diff --git a/targets/RT/USER/lte-enb.c b/targets/RT/USER/lte-enb.c index 8ae3d8956c..d523bb2473 100644 --- a/targets/RT/USER/lte-enb.c +++ b/targets/RT/USER/lte-enb.c @@ -133,7 +133,7 @@ void init_eNB(int,int); void stop_eNB(int nb_inst); int wakeup_tx(PHY_VARS_eNB *eNB); -int wakeup_txfh(L1_rxtx_proc_t *proc,PHY_VARS_eNB *eNB); +int wakeup_txfh(L1_rxtx_proc_t *proc,int frame_tx,int subframe_tx,uint64_t timestamp_tx,PHY_VARS_eNB *eNB); void wakeup_prach_eNB(PHY_VARS_eNB *eNB,RU_t *ru,int frame,int subframe); #if (LTE_RRC_VERSION >= MAKE_VERSION(14, 0, 0)) void wakeup_prach_eNB_br(PHY_VARS_eNB *eNB,RU_t *ru,int frame,int subframe); @@ -323,6 +323,7 @@ static void *L1_thread_tx(void *param) { //wait_sync("tx_thread"); while (!oai_exit) { + LOG_D(PHY,"Waiting for TX (IC %d)\n",proc->instance_cnt); if (wait_on_condition(&proc->mutex,&proc->cond,&proc->instance_cnt,thread_name)<0) break; if (oai_exit) break; @@ -336,10 +337,17 @@ static void *L1_thread_tx(void *param) { VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_SUBFRAME_NUMBER_RX1_ENB,proc->subframe_rx); VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_TX1_ENB,proc->frame_tx); VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_FRAME_NUMBER_RX1_ENB,proc->frame_rx); + LOG_D(PHY,"L1 TX processing %d.%d\n",proc->frame_tx,proc->subframe_tx); phy_procedures_eNB_TX(eNB, proc, 1); pthread_mutex_lock( &proc->mutex ); + + int subframe_tx = proc->subframe_tx; + int frame_tx = proc->frame_tx; + uint64_t timestamp_tx = proc->timestamp_tx; + proc->instance_cnt = -1; + LOG_D(PHY,"L1 TX signaling done for %d.%d\n",proc->frame_tx,proc->subframe_tx); // the thread can now be woken up if (pthread_cond_signal(&proc->cond) != 0) { LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n"); @@ -347,7 +355,7 @@ static void *L1_thread_tx(void *param) { } pthread_mutex_unlock( &proc->mutex ); - wakeup_txfh(proc,eNB); + wakeup_txfh(proc,frame_tx,subframe_tx,timestamp_tx,eNB); } return 0; @@ -387,8 +395,10 @@ static void *L1_thread( void *param ) { VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 ); T(T_ENB_MASTER_TICK, T_INT(0), T_INT(proc->frame_rx), T_INT(proc->subframe_rx)); + LOG_D(PHY,"L1RX waiting for RU RX\n"); if (wait_on_condition(&proc->mutex,&proc->cond,&proc->instance_cnt,thread_name)<0) break; + LOG_D(PHY,"L1RX starting in %d.%d\n",proc->frame_rx,proc->subframe_rx); VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_CPUID_ENB_THREAD_RXTX,sched_getcpu()); VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 1 ); VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_SUBFRAME_NUMBER_TX0_ENB,proc->subframe_tx); @@ -402,14 +412,19 @@ static void *L1_thread( void *param ) { if (rxtx(eNB,proc,thread_name) < 0) break; } + LOG_D(PHY,"L1 RX %d.%d done\n",proc->frame_rx,proc->subframe_rx); + if(get_thread_parallel_conf() == PARALLEL_RU_L1_SPLIT) phy_procedures_eNB_TX(eNB, proc, 1); - if (release_thread(&proc->mutex,&proc->instance_cnt,thread_name)<0) break; if (NFAPI_MODE!=NFAPI_MODE_VNF) { if(get_thread_parallel_conf() == PARALLEL_RU_L1_TRX_SPLIT) wakeup_tx(eNB); - else if(get_thread_parallel_conf() == PARALLEL_RU_L1_SPLIT) wakeup_txfh(proc,eNB); + else if(get_thread_parallel_conf() == PARALLEL_RU_L1_SPLIT) wakeup_txfh(proc,proc->frame_tx,proc->subframe_tx,proc->timestamp_tx,eNB); } + + if (release_thread(&proc->mutex,&proc->instance_cnt,thread_name)<0) break; + + } // while !oai_exit VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_eNB_PROC_RXTX0+(proc->subframe_rx&1), 0 ); @@ -442,15 +457,19 @@ void eNB_top(PHY_VARS_eNB *eNB, int frame_rx, int subframe_rx, char *string,RU_t } } -int wakeup_txfh(L1_rxtx_proc_t *proc,PHY_VARS_eNB *eNB) { +int wakeup_txfh(L1_rxtx_proc_t *proc,int frame_tx,int subframe_tx,uint64_t timestamp_tx,PHY_VARS_eNB *eNB) { RU_t *ru; RU_proc_t *ru_proc; struct timespec wait; wait.tv_sec=0; wait.tv_nsec=5000000L; + LOG_D(PHY,"L1 TX Waking up TX FH %d.%d (IC RU TX %d)\n",frame_tx,subframe_tx,proc->instance_cnt_RUs); + + // grab the information for the RU due to the following wait + if(wait_on_condition(&proc->mutex_RUs,&proc->cond_RUs,&proc->instance_cnt_RUs,"wakeup_txfh")<0) { - LOG_E(PHY,"Frame %d, subframe %d: TX FH not ready\n", proc->frame_tx, proc->subframe_tx); + LOG_E(PHY,"Frame %d, subframe %d: TX FH not ready\n", frame_tx, subframe_tx); return(-1); } @@ -476,10 +495,11 @@ int wakeup_txfh(L1_rxtx_proc_t *proc,PHY_VARS_eNB *eNB) { } ru_proc->instance_cnt_eNBs = 0; - ru_proc->timestamp_tx = proc->timestamp_tx; - ru_proc->subframe_tx = proc->subframe_tx; - ru_proc->frame_tx = proc->frame_tx; + ru_proc->timestamp_tx = timestamp_tx; + ru_proc->subframe_tx = subframe_tx; + ru_proc->frame_tx = frame_tx; + LOG_D(PHY,"L1 TX Waking up TX FH (2) %d.%d\n",frame_tx,subframe_tx); // the thread can now be woken up if (pthread_cond_signal(&ru_proc->cond_eNBs) != 0) { LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n"); @@ -507,10 +527,14 @@ int wakeup_tx(PHY_VARS_eNB *eNB) { return(-1); } + LOG_D(PHY,"L1 RX %d.%d Waiting to wake up L1 TX %d.%d (IC L1TX %d)\n",L1_proc->frame_rx,L1_proc->subframe_rx,L1_proc->frame_tx,L1_proc->subframe_tx,L1_proc_tx->instance_cnt); + while(L1_proc_tx->instance_cnt == 0) { pthread_cond_wait(&L1_proc_tx->cond,&L1_proc_tx->mutex); } + LOG_D(PHY,"L1 RX Got signal that TX %d.%d is done\n",L1_proc_tx->frame_tx,L1_proc_tx->subframe_tx); + L1_proc_tx->instance_cnt = 0; L1_proc_tx->subframe_rx = L1_proc->subframe_rx; L1_proc_tx->frame_rx = L1_proc->frame_rx; @@ -519,6 +543,8 @@ int wakeup_tx(PHY_VARS_eNB *eNB) { L1_proc_tx->timestamp_tx = L1_proc->timestamp_tx; // the thread can now be woken up + LOG_D(PHY,"L1 RX Waking up L1 TX %d.%d\n",L1_proc->frame_tx,L1_proc->subframe_tx); + if (pthread_cond_signal(&L1_proc_tx->cond) != 0) { LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n"); exit_fun( "ERROR pthread_cond_signal" ); @@ -536,6 +562,8 @@ int wakeup_rxtx(PHY_VARS_eNB *eNB,RU_t *ru) { LTE_DL_FRAME_PARMS *fp = &eNB->frame_parms; int i; struct timespec wait; + int sleep_cnt=0; + pthread_mutex_lock(&proc->mutex_RU); for (i=0; inum_RU; i++) { @@ -560,10 +588,6 @@ int wakeup_rxtx(PHY_VARS_eNB *eNB,RU_t *ru) { wait.tv_sec=0; wait.tv_nsec=5000000L; - if (L1_proc->instance_cnt == 0) { - LOG_E(PHY,"Frame %d, subframe %d: RXTX0 thread busy, dropping\n",L1_proc->frame_rx,L1_proc->subframe_rx); - return(-1); - } // wake up TX for subframe n+sf_ahead // lock the TX mutex and make sure the thread is ready @@ -573,6 +597,23 @@ int wakeup_rxtx(PHY_VARS_eNB *eNB,RU_t *ru) { return(-1); } + LOG_D(PHY,"RU RX Waking up rxtx %d.%d (IC L1RX %d)\n",ru_proc->frame_rx,ru_proc->subframe_rx,L1_proc->instance_cnt); + + while (L1_proc->instance_cnt == 0) { + pthread_mutex_unlock( &L1_proc->mutex); + if (++sleep_cnt == ru->wakeup_L1_sleep_cnt_max) { + LOG_E(PHY,"Frame %d, subframe %d: RXTX0 thread busy, dropping\n",L1_proc->frame_rx,L1_proc->subframe_rx); + return(-1); + } + usleep(ru->wakeup_L1_sleeptime); + + if (pthread_mutex_timedlock(&L1_proc->mutex,&wait) != 0) { + LOG_E( PHY, "[eNB] ERROR pthread_mutex_lock for eNB RXTX thread %d (IC %d)\n", L1_proc->subframe_rx&1,L1_proc->instance_cnt ); + exit_fun( "error locking mutex_rxtx" ); + return(-1); + } + } + ++L1_proc->instance_cnt; // We have just received and processed the common part of a subframe, say n. // TS_rx is the last received timestamp (start of 1st slot), TS_tx is the desired diff --git a/targets/RT/USER/lte-ru.c b/targets/RT/USER/lte-ru.c index 93a2440b94..10fc606bbb 100644 --- a/targets/RT/USER/lte-ru.c +++ b/targets/RT/USER/lte-ru.c @@ -759,14 +759,7 @@ void rx_rf(RU_t *ru,int *frame,int *subframe) { proc->timestamp_phy_tx = proc->timestamp_rx+((sf_ahead-1)*fp->samples_per_tti); proc->subframe_phy_tx = (proc->subframe_rx+(sf_ahead-1))%10; proc->frame_phy_tx = (proc->subframe_rx>(9-(sf_ahead-1))) ? (proc->frame_rx+1)&1023 : proc->frame_rx; -#else - proc->timestamp_tx = proc->timestamp_rx+(sf_ahead*fp->samples_per_tti); - proc->subframe_tx = (proc->subframe_rx+sf_ahead)%10; - proc->frame_tx = (proc->subframe_rx>(9-sf_ahead)) ? (proc->frame_rx+1)&1023 : proc->frame_rx; #endif - //proc->timestamp_tx = proc->timestamp_rx+(sf_ahead*fp->samples_per_tti); - //proc->subframe_tx = (proc->subframe_rx+sf_ahead)%10; - //proc->frame_tx = (proc->subframe_rx>(9-sf_ahead)) ? (proc->frame_rx+1)&1023 : proc->frame_rx; LOG_D(PHY,"RU %d/%d TS %llu (off %d), frame %d, subframe %d\n", ru->idx, 0, @@ -1197,7 +1190,7 @@ void do_ru_synch(RU_t *ru) { void wakeup_L1s(RU_t *ru) { int i; PHY_VARS_eNB **eNB_list = ru->eNB_list; - LOG_D(PHY,"wakeup_L1s (num %d) for RU %d ru->eNB_top:%p\n",ru->num_eNB,ru->idx, ru->eNB_top); + LOG_D(PHY,"wakeup_L1s (num %d) for RU %d (%d.%d)\n",ru->num_eNB,ru->idx, ru->proc.frame_rx,ru->proc.subframe_rx); if (ru->num_eNB==1 && ru->eNB_top!=0 && get_thread_parallel_conf() == PARALLEL_SINGLE_THREAD) { // call eNB function directly @@ -1484,7 +1477,6 @@ static void *ru_thread_tx( void *param ) { //pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); //wait_sync("ru_thread_tx"); wait_on_condition(&proc->mutex_FH1,&proc->cond_FH1,&proc->instance_cnt_FH1,"ru_thread_tx"); - printf( "ru_thread_tx ready\n"); while (!oai_exit) { VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME(VCD_SIGNAL_DUMPER_VARIABLES_CPUID_RU_THREAD_TX,sched_getcpu()); @@ -1495,6 +1487,8 @@ static void *ru_thread_tx( void *param ) { // wait until eNBs are finished subframe RX n and TX n+4 wait_on_condition(&proc->mutex_eNBs,&proc->cond_eNBs,&proc->instance_cnt_eNBs,"ru_thread_tx"); + LOG_D(PHY,"ru_thread_tx: TX in %d.%d\n",ru->proc.frame_tx,ru->proc.subframe_tx); + if (oai_exit) break; // do TX front-end processing if needed (precoding and/or IDFTs) @@ -1509,7 +1503,7 @@ static void *ru_thread_tx( void *param ) { if (ru->fh_north_out) ru->fh_north_out(ru); } - + LOG_D(PHY,"ru_thread_tx: releasing RU TX in %d.%d\n",proc->frame_tx,proc->subframe_tx); release_thread(&proc->mutex_eNBs,&proc->instance_cnt_eNBs,"ru_thread_tx"); for(int i = 0; inum_eNB; i++) { @@ -1536,6 +1530,7 @@ static void *ru_thread_tx( void *param ) { pthread_mutex_lock( &L1_proc->mutex_RUs); L1_proc->instance_cnt_RUs = 0; + LOG_D(PHY,"ru_thread_tx: Signaling RU TX done in %d.%d\n",proc->frame_tx,proc->subframe_tx); // the thread can now be woken up if (pthread_cond_signal(&L1_proc->cond_RUs) != 0) { LOG_E( PHY, "[eNB] ERROR pthread_cond_signal for eNB TXnp4 thread\n"); @@ -1943,32 +1938,31 @@ static void *eNB_thread_phy_tx( void *param ) { LOG_D(PHY,"Running eNB phy tx procedures\n"); - if(ru->num_eNB == 1) { - L1_proc.subframe_tx = proc->subframe_phy_tx; - L1_proc.frame_tx = proc->frame_phy_tx; - phy_procedures_eNB_TX(eNB_list[0], &L1_proc, 1); - phy_tx_txdataF_end = 1; + AssertFatal(ru->num_eNB == 1, "handle multiple L1 case\n"); + L1_proc.subframe_tx = proc->subframe_phy_tx; + L1_proc.frame_tx = proc->frame_phy_tx; + phy_procedures_eNB_TX(eNB_list[0], &L1_proc, 1); + phy_tx_txdataF_end = 1; - if(pthread_mutex_lock(&ru->proc.mutex_rf_tx) != 0) { - LOG_E( PHY, "[RU] ERROR pthread_mutex_lock for rf tx thread (IC %d)\n", ru->proc.instance_cnt_rf_tx); - exit_fun( "error locking mutex_rf_tx" ); - } - - if (ru->proc.instance_cnt_rf_tx==-1) { - ++ru->proc.instance_cnt_rf_tx; - ru->proc.frame_tx = proc->frame_phy_tx; - ru->proc.subframe_tx = proc->subframe_phy_tx; - ru->proc.timestamp_tx = proc->timestamp_phy_tx; - // the thread can now be woken up - AssertFatal(pthread_cond_signal(&ru->proc.cond_rf_tx) == 0, "ERROR pthread_cond_signal for rf_tx thread\n"); - } else { - LOG_E(PHY,"rf tx thread busy, skipping\n"); - late_control=STATE_BURST_TERMINATE; - } - - pthread_mutex_unlock( &ru->proc.mutex_rf_tx ); + if(pthread_mutex_lock(&ru->proc.mutex_rf_tx) != 0) { + LOG_E( PHY, "[RU] ERROR pthread_mutex_lock for rf tx thread (IC %d)\n", ru->proc.instance_cnt_rf_tx); + exit_fun( "error locking mutex_rf_tx" ); } + if (ru->proc.instance_cnt_rf_tx==-1) { + ++ru->proc.instance_cnt_rf_tx; + ru->proc.frame_tx = proc->frame_phy_tx; + ru->proc.subframe_tx = proc->subframe_phy_tx; + ru->proc.timestamp_tx = proc->timestamp_phy_tx; + // the thread can now be woken up + AssertFatal(pthread_cond_signal(&ru->proc.cond_rf_tx) == 0, "ERROR pthread_cond_signal for rf_tx thread\n"); + } else { + LOG_E(PHY,"rf tx thread busy, skipping\n"); + late_control=STATE_BURST_TERMINATE; + } + + pthread_mutex_unlock( &ru->proc.mutex_rf_tx ); + if (release_thread(&proc->mutex_phy_tx,&proc->instance_cnt_phy_tx,"eNB_thread_phy_tx") < 0) break; phy_tx_end = 1; @@ -2668,6 +2662,8 @@ void init_RU(char *rf_config_file) { // use eNB_list[0] as a reference for RU frame parameters // NOTE: multiple CC_id are not handled here yet! + ru->wakeup_L1_sleeptime = 2000; + ru->wakeup_L1_sleep_cnt_max = 3; if (ru->num_eNB > 0) { LOG_D(PHY, "%s() RC.ru[%d].num_eNB:%d ru->eNB_list[0]:%p RC.eNB[0][0]:%p rf_config_file:%s\n", __FUNCTION__, ru_id, ru->num_eNB, ru->eNB_list[0], RC.eNB[0][0], ru->rf_config_file); diff --git a/targets/RT/USER/lte-ue.c b/targets/RT/USER/lte-ue.c index 0a6df78682..42979399b1 100644 --- a/targets/RT/USER/lte-ue.c +++ b/targets/RT/USER/lte-ue.c @@ -890,7 +890,7 @@ void ue_stub_rx_handler(unsigned int num_bytes, char *rx_buffer) { switch (((UE_tport_header_t *)rx_buffer)->packet_type) { case TTI_SYNC: emulator_absSF = ((UE_tport_header_t *)rx_buffer)->absSF; - wakeup_thread(&UE->timer_mutex,&UE->timer_cond,&UE->instance_cnt_timer,"timer_thread"); + wakeup_thread(&UE->timer_mutex,&UE->timer_cond,&UE->instance_cnt_timer,"timer_thread",100,1); break; case SLSCH: @@ -1736,7 +1736,7 @@ void *UE_thread(void *arg) { LOG_E(PHY,"Missed real time\n"); continue; } else { - LOG_E(PHY,"System error\n"); + LOG_E(PHY,"System error %s\n",strerror(errno)); abort(); } }