maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes for thread-core pinning

Browse Source
This commit is contained in:
Raymond Knopp
2022-06-11 08:08:03 +02:00
parent 7b8f776246
commit 6236fe8e77
8 changed files with 57 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
executables/nr-gnb.c
+5 -1
View File
@@ -375,6 +375,8 @@ void *tx_reorder_thread(void* param) {
resL1Reserve=pullTpool(gNB->L1_tx_out, gNB->threadPool);
int next_tx_slot=((processingData_L1tx_t *)NotifiedFifoData(resL1Reserve))->slot;
LOG_I(PHY,"tx_reorder_thread started\n");
while (!oai_exit) {
notifiedFIFO_elt_t *resL1;
if (resL1Reserve) {
@@ -433,6 +435,7 @@ void init_gNB_Tpool(int inst) {
s_offset += 3;
}
if (getenv("noThreads")) strcpy(pool, "n");
LOG_I(PHY,"Initializing gNB ThreadPool with %s\n",pool);
initTpool(pool, gNB->threadPool, cpumeas(CPUMEAS_GETSTATE));
// ULSCH decoder result FIFO
gNB->respDecode = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
@@ -465,7 +468,8 @@ void init_gNB_Tpool(int inst) {
if (!get_softmodem_params()->emulate_l1)
threadCreate(&proc->L1_stats_thread,nrL1_stats_thread,(void*)gNB,"L1_stats",-1,OAI_PRIORITY_RT_LOW);
threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", -1, OAI_PRIORITY_RT_MAX);
LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n");
threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", gNB->RU_list[0]->tpcores[1], OAI_PRIORITY_RT_MAX);
}
executables/nr-ru.c
+16 -9
View File
@@ -64,7 +64,7 @@ unsigned short config_frames[4] = {2,9,11,13};
static int DEFBANDS[] = {7};
static int DEFENBS[] = {0};
static int DEFBFW[] = {0x00007fff};
static int DEFRUTPCORES[] = {2,4,6,8};
//static int DEFNRBANDS[] = {7};
//static int DEFGNBS[] = {0};
@@ -279,6 +279,7 @@ void fh_if5_south_out(RU_t *ru, int frame, int slot, uint64_t timestamp) {
if (ru == RC.ru[0]) VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_TRX_TST, ru->proc.timestamp_tx&0xffffffff );
int offset = ru->nr_frame_parms->get_samples_slot_timestamp(slot,ru->nr_frame_parms,0);
start_meas(&ru->tx_fhaul);
/*
for (int aid=0;aid<ru->nb_tx;aid++)
ru->ifdevice.trx_write_func2(&ru->ifdevice,
timestamp,
@@ -286,6 +287,7 @@ void fh_if5_south_out(RU_t *ru, int frame, int slot, uint64_t timestamp) {
aid,
ru->nr_frame_parms->get_samples_per_slot(slot,ru->nr_frame_parms),
0);
*/
stop_meas(&ru->tx_fhaul);
LOG_D(PHY,"IF5 TX %d.%d\n",frame,slot);
}
@@ -338,7 +340,8 @@ void fh_if5_south_in(RU_t *ru,
}
stop_meas(&ru->rx_fhaul);
LOG_D(PHY,"IF5 %d.%d => RX %d.%d first_rx %d: time %f\n",*frame,*tti,proc->frame_rx,proc->tti_rx,proc->first_rx,ru->rx_fhaul.p_time/(cpu_freq_GHz*1000.0));
double fhtime = ru->rx_fhaul.p_time/(cpu_freq_GHz*1000.0);
if (fhtime > 800) LOG_W(PHY,"IF5 %d.%d => RX %d.%d first_rx %d: time %f\n",*frame,*tti,proc->frame_rx,proc->tti_rx,proc->first_rx,ru->rx_fhaul.p_time/(cpu_freq_GHz*1000.0));
VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_TRX_TS, proc->timestamp_rx&0xffffffff );
VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_RECV_IF5, 0 );
@@ -1555,7 +1558,7 @@ void init_RU_proc(RU_t *ru) {
pthread_mutex_init( &proc->mutex_emulateRF,NULL);
pthread_cond_init( &proc->cond_emulateRF, NULL);
threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "thread_FH", -1, OAI_PRIORITY_RT_MAX );
threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->tpcores[0], OAI_PRIORITY_RT_MAX );
if(emulate_rf)
threadCreate( &proc->pthread_emulateRF, emulatedRF_thread, (void *)proc, "emulateRF", -1, OAI_PRIORITY_RT );
@@ -1872,7 +1875,7 @@ void set_function_spec_param(RU_t *ru) {
case REMOTE_IF5: // the remote unit is IF5 RRU
ru->do_prach = 0;
ru->txfh_in_fep = 1;
ru->txfh_in_fep = 0;
#ifdef USE_TPOOL
ru->feprx = (get_thread_worker_conf() == WORKER_ENABLE) ? nr_fep_tp : nr_fep_full; // this is frequency-shift + DFTs
ru->feptx_prec = (get_thread_worker_conf() == WORKER_ENABLE) ? NULL : nr_feptx_prec; // need to do transmit Precoding + IDFTs
@@ -2020,16 +2023,16 @@ void init_NR_RU(char *rf_config_file) {
set_function_spec_param(ru);
LOG_I(PHY,"Starting ru_thread %d\n",ru_id);
init_RU_proc(ru);
int threadCnt = ru->nb_rx+ru->nb_tx;
int threadCnt = ru->num_tpcores;
if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt);
else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt);
char pool[80];
sprintf(pool,"-1");
int s_offset = 0;
int s_offset = sprintf(pool,"%d",ru->tpcores[0]);
for (int icpu=1; icpu<threadCnt; icpu++) {
sprintf(pool+2+s_offset,",-1");
s_offset += 3;
s_offset+=sprintf(pool+s_offset,",%d",ru->tpcores[icpu]);
}
#ifdef USE_TPOOL
LOG_I(PHY,"RU thread-pool core string %s\n",pool);
ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t));
initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE));
// FEP RX result FIFO
@@ -2211,6 +2214,10 @@ static void NRRCconfig_RU(void) {
RC.ru[j]->openair0_cfg.nr_band = RC.ru[j]->band[0];
RC.ru[j]->openair0_cfg.nr_scs_for_raster = *(RUParamList.paramarray[j][RU_NR_SCS_FOR_RASTER].iptr);
printf("[RU %d] Setting nr_flag %d, nr_band %d, nr_scs_for_raster %d\n",j,RC.ru[j]->openair0_cfg.nr_flag,RC.ru[j]->openair0_cfg.nr_band,RC.ru[j]->openair0_cfg.nr_scs_for_raster);
RC.ru[j]->openair0_cfg.fh_cores[0] = *(RUParamList.paramarray[j][RU_RXFH_CORE_ID].iptr);
RC.ru[j]->num_tpcores = *(RUParamList.paramarray[j][RU_NUM_TP_CORES].iptr);
AssertFatal(RC.ru[j]->num_tpcores <= RUParamList.paramarray[j][RU_TP_CORES].numelt, "Number of TP cores should be <=16\n");
for (i=0; i<RC.ru[j]->num_tpcores; i++) RC.ru[j]->tpcores[i] = RUParamList.paramarray[j][RU_TP_CORES].iptr[i];
if (config_isparamset(RUParamList.paramarray[j], RU_BF_WEIGHTS_LIST_IDX)) {
RC.ru[j]->nb_bfw = RUParamList.paramarray[j][RU_BF_WEIGHTS_LIST_IDX].numelt;
openair1/PHY/defs_RU.h
+6 -1
View File
@@ -668,7 +668,12 @@ typedef struct RU_t_s {
notifiedFIFO_t *respfeprx;
/// to signal end of feptx
notifiedFIFO_t *respfeptx;
/// core id for RX fhaul (IF5 ECPRI)
int rxfh_core_id;
/// list of cores for RU ThreadPool
int tpcores[16];
/// number of cores for RU ThreadPool
int num_tpcores;
} RU_t;
openair2/ENB_APP/enb_paramdef.h
+12
View File
@@ -103,11 +103,17 @@ typedef enum {
#define CONFIG_STRING_RU_SL_AHEAD "sl_ahead"
#define CONFIG_STRING_RU_NR_FLAG "nr_flag"
#define CONFIG_STRING_RU_NR_SCS_FOR_RASTER "nr_scs_for_raster"
#define CONFIG_STRING_RU_RXFH_CORE_ID "rxfh_core_id"
#define CONFIG_STRING_RU_TP_CORES "tp_cores"
#define CONFIG_STRING_RU_NUM_TP_CORES "num_tp_cores"
#define HLP_RU_SF_AHEAD "LTE TX processing advance"
#define HLP_RU_SL_AHEAD "NR TX processing advance"
#define HLP_RU_NR_FLAG "Use NR numerology (for AW2SORI)"
#define HLP_RU_NR_SCS_FOR_RASTER "NR SCS for raster (for AW2SORI)"
#define HLP_RU_RXFH_CORE_ID "Core ID for RX Fronthaul thread (ECPRI IF5)"
#define HLP_RU_TP_CORES "List of cores for RU ThreadPool"
#define HLP_RU_NUM_TP_CORES "Number of cores for RU ThreadPool"
#define RU_LOCAL_IF_NAME_IDX 0
#define RU_LOCAL_ADDRESS_IDX 1
@@ -142,6 +148,9 @@ typedef enum {
#define RU_SL_AHEAD 30
#define RU_NR_FLAG 31
#define RU_NR_SCS_FOR_RASTER 32
#define RU_RXFH_CORE_ID 33
#define RU_TP_CORES 34
#define RU_NUM_TP_CORES 35
/*-----------------------------------------------------------------------------------------------------------------------------------------*/
/* RU configuration parameters */
/* optname helpstr paramflags XXXptr defXXXval type numelt */
@@ -180,6 +189,9 @@ typedef enum {
{CONFIG_STRING_RU_SL_AHEAD, HLP_RU_SL_AHEAD, 0, iptr:NULL, defintval:6, TYPE_INT, 0}, \
{CONFIG_STRING_RU_NR_FLAG, HLP_RU_NR_FLAG, 0, iptr:NULL, defintval:0, TYPE_INT, 0}, \
{CONFIG_STRING_RU_NR_SCS_FOR_RASTER, HLP_RU_NR_SCS_FOR_RASTER, 0, iptr:NULL, defintval:1, TYPE_INT, 0}, \
{CONFIG_STRING_RU_RXFH_CORE_ID, HLP_RU_RXFH_CORE_ID, 0, uptr:NULL, defintval:0, TYPE_UINT, 0}, \
{CONFIG_STRING_RU_TP_CORES, HLP_RU_TP_CORES, 0, uptr:NULL, defintarrayval:DEFRUTPCORES, TYPE_INTARRAY, 8}, \
{CONFIG_STRING_RU_NUM_TP_CORES, HLP_RU_NUM_TP_CORES, 0, uptr:NULL, defintval:4, TYPE_UINT, 0}, \
}
/*---------------------------------------------------------------------------------------------------------------------------------------*/
targets/ARCH/COMMON/common_lib.h
+2
View File
@@ -254,6 +254,8 @@ typedef struct {
int nr_band;
//! NR scs for raster
int nr_scs_for_raster;
//! Core IDs for RX FH
int fh_cores[4];
} openair0_config_t;
/*! \brief RF mapping */
targets/ARCH/ETHERNET/USERSPACE/LIB/eth_udp.c
+5 -4
View File
@@ -43,6 +43,7 @@
#include <unistd.h>
#include <errno.h>
#include "common/utils/LOG/vcd_signal_dumper.h"
#include <sched.h>
#include "common_lib.h"
#include "ethernet_lib.h"
@@ -148,11 +149,11 @@ int eth_socket_init_udp(openair0_device *device) {
printf("ETHERNET: Cannot set SO_NO_CHECK option on socket (user %d)",i);
exit(0);
}
#if 0 /*def SO_ATTACH_REUSEPORT_EBPF*/
if (setsockopt(eth->sockfdd[i], SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(int))) {
printf("ETHERNET: Cannot set SO_REUSEPORT option on socket (user %d)",i);
exit(0);
}
#ifdef SO_ATTACH_REUSEPORT_EBPF*/
struct sock_filter code[]={
{ BPF_LD | BPF_W | BPF_ABS, 0, 0, SKF_AD_OFF + SKF_AD_CPU }, // A = #cpu
{ BPF_RET | BPF_A, 0, 0, 0 }, // return A
@@ -451,8 +452,8 @@ void *udp_read_thread(void *arg) {
// zero-copy and corrected the header component.
memcpy((void*)(device->openair0_cfg->rxbase[aid]+offset),
(void*)&buffer[APP_HEADER_SIZE_BYTES],
count-APP_HEADER_SIZE_BYTES);
LOG_D(PHY,"thread_id %d, aid %d, TS %llu, TS0 %llu, offset %d\n",u->thread_id,aid,(unsigned long long)TS,fhstate->TS0,offset);
count-APP_HEADER_SIZE_BYTES);
LOG_D(PHY,"thread_id %d (%d), aid %d, TS %llu, TS0 %llu, offset %d\n",u->thread_id,sched_getcpu(),aid,(unsigned long long)TS,fhstate->TS0,offset);
}
sleep(1);
}
@@ -473,7 +474,7 @@ int trx_eth_read_udp(openair0_device *device, openair0_timestamp *timestamp, uin
// poll/sleep until we accumulated enough samples on each antenna port
int count=0;
while (fhstate->first_read == 1 && min_TS < (fhstate->TS0+prev_read_TS + nsamps)) {
usleep(100);
usleep(50);
min_TS = fhstate->TS[0];
for (int i=1;i<device->openair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]);
count++;
targets/ARCH/ETHERNET/USERSPACE/LIB/ethernet_lib.c
+9 -6
View File
@@ -65,13 +65,13 @@ int trx_eth_start(openair0_device *device)
AssertFatal(device->thirdparty_init(device) == 0, "third-party init failed\n");
device->openair0_cfg->samples_per_packet = 256;
eth->num_fd = 1; //max(device->openair0_cfg->rx_num_channels,device->openair0_cfg->tx_num_channels);
printf("Creating %d UDP threads ...\n",device->openair0_cfg->rx_num_channels);
udp_read_t *u[device->openair0_cfg->rx_num_channels];
for (int i=0;i<(device->openair0_cfg->rx_num_channels);i++) {
udp_read_t *u[2];
for (int i=0;i<eth->num_fd;i++) {
u[i] = malloc(sizeof(udp_read_t));
u[i]->thread_id=i;
u[i]->device = device;
threadCreate(&u[i]->pthread,udp_read_thread,u[i],"udp read thread",-1,OAI_PRIORITY_RT_MAX);
printf("UDP Read Thread %d on core %d\n",i,device->openair0_cfg->fh_cores[i]);
threadCreate(&u[i]->pthread,udp_read_thread,u[i],"udp read thread",device->openair0_cfg->fh_cores[i],OAI_PRIORITY_RT_MAX);
}
device->sampling_rate_ratio_n=1;
device->sampling_rate_ratio_d=1;
@@ -112,6 +112,7 @@ int trx_eth_start(openair0_device *device)
}
else AssertFatal(1==0,"num_rb_dl %d not ok with ECPRI\n",device->openair0_cfg->num_rb_dl);
}
#ifdef USE_TX_TPOOL
int threadCnt = device->openair0_cfg->tx_num_channels;
if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt);
char pool[80];
@@ -126,6 +127,7 @@ int trx_eth_start(openair0_device *device)
// ULSCH decoder result FIFO
device->respudpTX = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
initNotifiedFIFO(device->respudpTX);
#endif
}
/* initialize socket */
if (eth->flags == ETH_RAW_MODE) {
@@ -211,6 +213,7 @@ int trx_eth_start(openair0_device *device)
if(ethernet_tune (device, RCV_BUF_SIZE,67108864)!=0) return -1;
if(ethernet_tune (device, KERNEL_SND_BUF_MAX_SIZE, 67108864)!=0) return -1;
if(ethernet_tune (device, KERNEL_RCV_BUF_MAX_SIZE, 67108864)!=0) return -1;
if(ethernet_tune (device, TX_Q_LEN, 10000)!=0) return -1;
return 0;
@@ -445,7 +448,7 @@ int ethernet_tune(openair0_device *device,
}
break;
case KERNEL_SND_BUF_MAX_SIZE:
ret=snprintf(system_cmd,sizeof(system_cmd),"sysctl -w net.core.wmem_max=%d;sysctl -w net.core.wmem_default=%d;sysctl -w net.core.netdev_max_backlog=416384;sysctl -w net.core.optmem_max=524288;",value,value);
ret=snprintf(system_cmd,sizeof(system_cmd),"sysctl -w net.core.wmem_max=%d;sysctl -w net.core.wmem_default=%d;sysctl -w net.core.netdev_max_backlog=5000;sysctl -w net.core.optmem_max=524288;",value,value);
if (ret > 0) {
ret=system(system_cmd);
if (ret == -1) {
@@ -487,7 +490,7 @@ int transport_init(openair0_device *device,
eth->compression = ALAW_COMPRESS;
}
eth->num_fd = max(openair0_cfg->rx_num_channels,openair0_cfg->tx_num_channels);
eth->num_fd = 1;
printf("[ETHERNET]: Initializing openair0_device for %s ...\n", ((device->host_type == RAU_HOST) ? "RAU": "RRU"));
printf("[ETHERNET]: num_fd %d\n",eth->num_fd);
targets/RT/USER/lte-ru.c
+2
View File
@@ -66,6 +66,8 @@
static int DEFBANDS[] = {7};
static int DEFENBS[] = {0};
static int DEFBFW[] = {0x00007fff};
static int DEFRUTPCORES[] = {2,4,6,8};
#include "ENB_APP/enb_paramdef.h"
#include "common/config/config_userapi.h"