diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/CMakeLists.txt b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/CMakeLists.txt index 5b89d36f38..f64384568a 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/CMakeLists.txt +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/CMakeLists.txt @@ -7,7 +7,7 @@ add_boolean_option(ENABLE_LDPC_T2 OFF "Build support for LDPC Offload to T2 libr if (ENABLE_LDPC_T2) - pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.9) + pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk>=20.11.9) find_library(PMD_T2 NAMES rte_baseband_accl_ldpc HINTS ${LIBDPDK_T2_LIBRARY_DIRS}) if (NOT PMD_T2) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c index 6eb8f0dacc..04ad571de0 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c @@ -45,8 +45,6 @@ #define SYNC_WAIT 0 #define SYNC_START 1 #define TIME_OUT_POLL 1e8 -/* Increment for next code block in external HARQ memory */ -#define HARQ_INCR 32768 /* Headroom for filler LLRs insertion in HARQ buffer */ #define FILLER_HEADROOM 1024 /* Number of segments that could be stored in HARQ combined buffers */ @@ -55,15 +53,6 @@ pthread_mutex_t encode_mutex; pthread_mutex_t decode_mutex; -const char *typeStr[] = { - "RTE_BBDEV_OP_NONE", /**< Dummy operation that does nothing */ - "RTE_BBDEV_OP_TURBO_DEC", /**< Turbo decode */ - "RTE_BBDEV_OP_TURBO_ENC", /**< Turbo encode */ - "RTE_BBDEV_OP_LDPC_DEC", /**< LDPC decode */ - "RTE_BBDEV_OP_LDPC_ENC", /**< LDPC encode */ - "RTE_BBDEV_OP_TYPE_COUNT", /**< Count of different op types */ -}; - /* Represents tested active devices */ struct active_device { const char *driver_name; @@ -76,9 +65,6 @@ struct active_device { struct rte_mempool *bbdev_enc_op_pool; struct rte_mempool *in_mbuf_pool; struct rte_mempool *hard_out_mbuf_pool; - struct rte_mempool *soft_out_mbuf_pool; - struct rte_mempool *harq_in_mbuf_pool; - struct rte_mempool *harq_out_mbuf_pool; } active_devs[RTE_BBDEV_MAX_DEVS]; static int nb_active_devs; @@ -86,23 +72,12 @@ static int nb_active_devs; struct data_buffers { struct rte_bbdev_op_data *inputs; struct rte_bbdev_op_data *hard_outputs; - struct rte_bbdev_op_data *soft_outputs; - struct rte_bbdev_op_data *harq_inputs; - struct rte_bbdev_op_data *harq_outputs; }; /* Operation parameters specific for given test case */ struct test_op_params { - struct rte_mempool *mp_dec; - struct rte_mempool *mp_enc; - struct rte_bbdev_dec_op *ref_dec_op; - struct rte_bbdev_enc_op *ref_enc_op; - uint16_t burst_sz; - uint16_t num_to_process; uint16_t num_lcores; - int vector_mask; rte_atomic16_t sync; - struct data_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; }; /* Contains per lcore params */ @@ -113,18 +88,14 @@ struct thread_params { nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters; nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters; uint8_t iter_count; - rte_atomic16_t nb_dequeued; - rte_atomic16_t processing_status; - rte_atomic16_t burst_sz; struct test_op_params *op_params; - struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; - struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; + struct data_buffers *data_buffers; + struct rte_mempool *bbdev_op_pool; }; static uint16_t nb_segments_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters) { uint16_t nb_segments = 0; - for (uint16_t h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) - { + for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { nb_segments+=nrLDPC_slot_decoding_parameters->TBs[h].C; } return nb_segments; @@ -132,33 +103,12 @@ static uint16_t nb_segments_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_s static uint16_t nb_segments_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { uint16_t nb_segments = 0; - for (uint16_t h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h) - { + for (int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h) { nb_segments+=nrLDPC_slot_encoding_parameters->TBs[h].C; } return nb_segments; } -// DPDK BBDEV copy -static inline void -mbuf_reset(struct rte_mbuf *m) -{ - m->pkt_len = 0; - - do { - m->data_len = 0; - m = m->next; - } while (m != NULL); -} - -/* Read flag value 0/1 from bitmap */ -// DPDK BBDEV copy -static inline bool -check_bit(uint32_t bitmap, uint32_t bitmask) -{ - return bitmap & bitmask; -} - /* calculates optimal mempool size not smaller than the val */ // DPDK BBDEV copy static unsigned int @@ -298,7 +248,7 @@ static int add_dev(uint8_t dev_id, struct rte_bbdev_info *info) /* Display for debug the capabilities of the card */ for (int i = 0; info->drv.capabilities[i].type != RTE_BBDEV_OP_NONE; i++) { - printf("device: %d, capability[%d]=%s\n", dev_id, i, typeStr[info->drv.capabilities[i].type]); + printf("device: %d, capability[%d]=%s\n", dev_id, i, rte_bbdev_op_type_str(info->drv.capabilities[i].type)); if (info->drv.capabilities[i].type == RTE_BBDEV_OP_LDPC_ENC) { const struct rte_bbdev_op_cap_ldpc_enc cap = info->drv.capabilities[i].cap.ldpc_enc; printf(" buffers: src = %d, dst = %d\n capabilites: ", cap.num_buffers_src, cap.num_buffers_dst); @@ -376,9 +326,9 @@ static int init_op_data_objs_dec(struct rte_bbdev_op_data *bufs, uint16_t min_alignment) { bool large_input = false; - uint16_t j = 0; - for (uint16_t h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { - for (uint16_t i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { + int j = 0; + for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { + for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { uint32_t data_len = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].E; char *data; struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); @@ -395,9 +345,9 @@ static int init_op_data_objs_dec(struct rte_bbdev_op_data *bufs, bufs[j].data = m_head; bufs[j].offset = 0; bufs[j].length = 0; - - if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { - if ((op_type == DATA_INPUT) && large_input) { + + if (op_type == DATA_INPUT) { + if (large_input) { /* Allocate a fake overused mbuf */ data = rte_malloc(NULL, data_len, 0); AssertFatal(data != NULL, "rte malloc failed with %u bytes", data_len); @@ -427,14 +377,13 @@ static int init_op_data_objs_dec(struct rte_bbdev_op_data *bufs, // based on DPDK BBDEV init_op_data_objs static int init_op_data_objs_enc(struct rte_bbdev_op_data *bufs, nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, - struct rte_mbuf *m_head, struct rte_mempool *mbuf_pool, enum op_data_type op_type, uint16_t min_alignment) { bool large_input = false; - uint16_t j = 0; - for (uint16_t h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h) { + int j = 0; + for (int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h) { for (int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) { uint32_t data_len = (nrLDPC_slot_encoding_parameters->TBs[h].K - nrLDPC_slot_encoding_parameters->TBs[h].F + 7) / 8; char *data; @@ -452,9 +401,9 @@ static int init_op_data_objs_enc(struct rte_bbdev_op_data *bufs, bufs[j].data = m_head; bufs[j].offset = 0; bufs[j].length = 0; - - if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { - if ((op_type == DATA_INPUT) && large_input) { + + if (op_type == DATA_INPUT) { + if (large_input) { /* Allocate a fake overused mbuf */ data = rte_malloc(NULL, data_len, 0); AssertFatal(data != NULL, "rte malloc failed with %u bytes", data_len); @@ -502,40 +451,24 @@ static int allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const } // DPDK BBDEV copy -static void -free_buffers(struct active_device *ad, struct test_op_params *op_params) +static void free_mempools(struct active_device *ad) { rte_mempool_free(ad->bbdev_dec_op_pool); rte_mempool_free(ad->bbdev_enc_op_pool); rte_mempool_free(ad->in_mbuf_pool); rte_mempool_free(ad->hard_out_mbuf_pool); - rte_mempool_free(ad->soft_out_mbuf_pool); - rte_mempool_free(ad->harq_in_mbuf_pool); - rte_mempool_free(ad->harq_out_mbuf_pool); - - for (int i = 2; i < rte_lcore_count(); ++i) { - for (int j = 0; j < RTE_MAX_NUMA_NODES; ++j) { - rte_free(op_params->q_bufs[j][i].inputs); - rte_free(op_params->q_bufs[j][i].hard_outputs); - rte_free(op_params->q_bufs[j][i].soft_outputs); - rte_free(op_params->q_bufs[j][i].harq_inputs); - rte_free(op_params->q_bufs[j][i].harq_outputs); - } - } } // based on DPDK BBDEV copy_reference_ldpc_dec_op static void set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, - unsigned int start_idx, - struct data_buffers *bufs, + struct rte_bbdev_op_data *inputs, + struct rte_bbdev_op_data *outputs, nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters) { - unsigned int h; - unsigned int i; - unsigned int j = 0; - for (h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){ - for (i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { + int j = 0; + for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { + for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { ops[j]->ldpc_dec.cb_params.e = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].E; ops[j]->ldpc_dec.basegraph = nrLDPC_slot_decoding_parameters->TBs[h].BG; ops[j]->ldpc_dec.z_c = nrLDPC_slot_decoding_parameters->TBs[h].Z; @@ -571,16 +504,8 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, ops[j]->ldpc_dec.harq_combined_input.offset = harq_combined_offset; ops[j]->ldpc_dec.harq_combined_output.offset = harq_combined_offset; - if (bufs->hard_outputs != NULL) - ops[j]->ldpc_dec.hard_output = bufs->hard_outputs[start_idx + j]; - if (bufs->inputs != NULL) - ops[j]->ldpc_dec.input = bufs->inputs[start_idx + j]; - if (bufs->soft_outputs != NULL) - ops[j]->ldpc_dec.soft_output = bufs->soft_outputs[start_idx + j]; - if (bufs->harq_inputs != NULL) - ops[j]->ldpc_dec.harq_combined_input = bufs->harq_inputs[start_idx + j]; - if (bufs->harq_outputs != NULL) - ops[j]->ldpc_dec.harq_combined_output = bufs->harq_outputs[start_idx + j]; + ops[j]->ldpc_dec.hard_output = outputs[j]; + ops[j]->ldpc_dec.input = inputs[j]; ++j; } } @@ -589,16 +514,13 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, // based on DPDK BBDEV copy_reference_ldpc_enc_op static void set_ldpc_enc_op(struct rte_bbdev_enc_op **ops, - unsigned int start_idx, struct rte_bbdev_op_data *inputs, struct rte_bbdev_op_data *outputs, nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { - unsigned int h; - unsigned int i; - unsigned int j = 0; - for (h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){ - for (i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) { + int j = 0; + for (int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h) { + for (int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) { ops[j]->ldpc_enc.cb_params.e = nrLDPC_slot_encoding_parameters->TBs[h].segments[i].E; ops[j]->ldpc_enc.basegraph = nrLDPC_slot_encoding_parameters->TBs[h].BG; ops[j]->ldpc_enc.z_c = nrLDPC_slot_encoding_parameters->TBs[h].Z; @@ -612,34 +534,23 @@ set_ldpc_enc_op(struct rte_bbdev_enc_op **ops, ops[j]->ldpc_enc.rv_index = nrLDPC_slot_encoding_parameters->TBs[h].rv_index; ops[j]->ldpc_enc.op_flags = RTE_BBDEV_LDPC_RATE_MATCH; ops[j]->ldpc_enc.code_block_mode = 1; - ops[j]->ldpc_enc.output = outputs[start_idx + j]; - ops[j]->ldpc_enc.input = inputs[start_idx + j]; + ops[j]->ldpc_enc.output = outputs[j]; + ops[j]->ldpc_enc.input = inputs[j]; ++j; } } } -static int -retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, - const int vector_mask, - nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters) +static int retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters) { - struct rte_bbdev_op_data *hard_output; - uint16_t data_len = 0; - struct rte_mbuf *m; - char *data; - unsigned int h; - unsigned int i; - unsigned int j = 0; - for (h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){ - for (i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { - hard_output = &ops[j]->ldpc_dec.hard_output; - m = hard_output->data; - data_len = rte_pktmbuf_data_len(m) - hard_output->offset; - data = m->buf_addr; - memcpy(nrLDPC_slot_decoding_parameters->TBs[h].segments[i].c, data + m->data_off, data_len); - rte_pktmbuf_free(ops[j]->ldpc_dec.hard_output.data); - rte_pktmbuf_free(ops[j]->ldpc_dec.input.data); + int j = 0; + for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { + for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { + struct rte_bbdev_op_data *hard_output = &ops[j]->ldpc_dec.hard_output; + struct rte_mbuf *m = hard_output->data; + uint16_t data_len = rte_pktmbuf_data_len(m) - hard_output->offset; + uint8_t *data = rte_pktmbuf_mtod_offset(m, uint8_t *, hard_output->offset); + memcpy(nrLDPC_slot_decoding_parameters->TBs[h].segments[i].c, data, data_len); ++j; } } @@ -651,17 +562,17 @@ retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { uint8_t *p_out = NULL; - unsigned int j = 0; - for (unsigned int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){ + int j = 0; + for (int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){ int E_sum = 0; int bit_offset = 0; int byte_offset = 0; p_out = nrLDPC_slot_encoding_parameters->TBs[h].output; - for (unsigned int r = 0; r < nrLDPC_slot_encoding_parameters->TBs[h].C; ++r) { + for (int r = 0; r < nrLDPC_slot_encoding_parameters->TBs[h].C; ++r) { struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output; struct rte_mbuf *m = output->data; uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset; - uint8_t *data = m->buf_addr + m->data_off; + uint8_t *data = rte_pktmbuf_mtod_offset(m, uint8_t *, output->offset); reverse_bits_u8(data, data_len, data); if (bit_offset == 0) { memcpy(&p_out[byte_offset], data, data_len); @@ -688,93 +599,58 @@ retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[r].E; byte_offset = (E_sum + 7) / 8; bit_offset = E_sum % 8; - rte_pktmbuf_free(m); - rte_pktmbuf_free(ops[j]->ldpc_enc.input.data); ++j; } } return 0; } -// based on DPDK BBDEV init_test_op_params -static int init_test_op_params(struct test_op_params *op_params, - enum rte_bbdev_op_type op_type, - struct rte_mempool *ops_mp, - uint16_t burst_sz, - uint16_t num_to_process, - uint16_t num_lcores) -{ - int ret = 0; - if (op_type == RTE_BBDEV_OP_LDPC_DEC) { - ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, &op_params->ref_dec_op, num_to_process); - op_params->mp_dec = ops_mp; - } else { - ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, &op_params->ref_enc_op, 1); - op_params->mp_enc = ops_mp; - } - - AssertFatal(ret == 0, "rte_bbdev_op_alloc_bulk() failed"); - - op_params->burst_sz = burst_sz; - op_params->num_to_process = num_to_process; - op_params->num_lcores = num_lcores; - return 0; -} - // based on DPDK BBDEV throughput_pmd_lcore_ldpc_dec static int pmd_lcore_ldpc_dec(void *arg) { struct thread_params *tp = arg; nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters = tp->nrLDPC_slot_decoding_parameters; - uint16_t enq, deq; int time_out = 0; const uint16_t queue_id = tp->queue_id; const uint16_t num_segments = nb_segments_decoding(nrLDPC_slot_decoding_parameters); - struct rte_bbdev_dec_op **ops_enq; - struct rte_bbdev_dec_op **ops_deq; - ops_enq = (struct rte_bbdev_dec_op **)rte_calloc("struct rte_bbdev_dec_op **ops_enq", num_segments, sizeof(struct rte_bbdev_dec_op *), RTE_CACHE_LINE_SIZE); - ops_deq = (struct rte_bbdev_dec_op **)rte_calloc("struct rte_bbdev_dec_op **ops_dec", num_segments, sizeof(struct rte_bbdev_dec_op *), RTE_CACHE_LINE_SIZE); - struct data_buffers *bufs = NULL; - uint16_t h, i, j; - int ret; - struct rte_bbdev_info info; - uint16_t num_to_enq; + struct rte_bbdev_dec_op *ops_enq[num_segments]; + struct rte_bbdev_dec_op *ops_deq[num_segments]; + struct data_buffers *bufs = tp->data_buffers; AssertFatal((num_segments < MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); + + struct rte_bbdev_info info; rte_bbdev_info_get(tp->dev_id, &info); - bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) rte_pause(); - ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp_dec, ops_enq, num_segments); + int ret = rte_bbdev_dec_op_alloc_bulk(tp->bbdev_op_pool, ops_enq, num_segments); AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments); - set_ldpc_dec_op(ops_enq, 0, bufs, nrLDPC_slot_decoding_parameters); - - for (enq = 0, deq = 0; enq < num_segments;) { - num_to_enq = num_segments; - if (unlikely(num_segments - enq < num_to_enq)) - num_to_enq = num_segments - enq; + set_ldpc_dec_op(ops_enq, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_decoding_parameters); + uint16_t enq = 0, deq = 0; + while (enq < num_segments) { + uint16_t num_to_enq = num_segments - enq; enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, queue_id, &ops_enq[enq], num_to_enq); deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, queue_id, &ops_deq[deq], enq - deq); } - /* dequeue the remaining */ while (deq < enq) { deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, queue_id, &ops_deq[deq], enq - deq); time_out++; DevAssert(time_out <= TIME_OUT_POLL); } + if (deq == enq) { - ret = retrieve_ldpc_dec_op(ops_deq, tp->op_params->vector_mask, nrLDPC_slot_decoding_parameters); + ret = retrieve_ldpc_dec_op(ops_deq, nrLDPC_slot_decoding_parameters); AssertFatal(ret == 0, "LDPC offload decoder failed!"); tp->iter_count = 0; /* get the max of iter_count for all dequeued ops */ - j = 0; - for (h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){ - for (i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { + int j = 0; + for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { + for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) { bool *status = &nrLDPC_slot_decoding_parameters->TBs[h].segments[i].decodeSuccess; tp->iter_count = RTE_MAX(ops_enq[j]->ldpc_dec.iter_count, tp->iter_count); @@ -804,8 +680,6 @@ pmd_lcore_ldpc_dec(void *arg) } rte_bbdev_dec_op_free_bulk(ops_enq, num_segments); - rte_free(ops_enq); - rte_free(ops_deq); // Return the worst decoding number of iterations for all segments return tp->iter_count; } @@ -815,37 +689,33 @@ static int pmd_lcore_ldpc_enc(void *arg) { struct thread_params *tp = arg; nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters = tp->nrLDPC_slot_encoding_parameters; - uint16_t enq, deq; int time_out = 0; const uint16_t queue_id = tp->queue_id; const uint16_t num_segments = nb_segments_encoding(nrLDPC_slot_encoding_parameters); - struct rte_bbdev_enc_op **ops_enq; - struct rte_bbdev_enc_op **ops_deq; - ops_enq = (struct rte_bbdev_enc_op **)rte_calloc("struct rte_bbdev_dec_op **ops_enq", num_segments, sizeof(struct rte_bbdev_enc_op *), RTE_CACHE_LINE_SIZE); - ops_deq = (struct rte_bbdev_enc_op **)rte_calloc("struct rte_bbdev_dec_op **ops_dec", num_segments, sizeof(struct rte_bbdev_enc_op *), RTE_CACHE_LINE_SIZE); - struct rte_bbdev_info info; - int ret; - struct data_buffers *bufs = NULL; - uint16_t num_to_enq; + struct rte_bbdev_enc_op *ops_enq[num_segments]; + struct rte_bbdev_enc_op *ops_deq[num_segments]; + struct data_buffers *bufs = tp->data_buffers; AssertFatal((num_segments < MAX_BURST), "BURST_SIZE should be <= %u", MAX_BURST); + struct rte_bbdev_info info; rte_bbdev_info_get(tp->dev_id, &info); - bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) rte_pause(); - ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp_enc, ops_enq, num_segments); - AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments); - set_ldpc_enc_op(ops_enq, 0, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters); + int ret = rte_bbdev_enc_op_alloc_bulk(tp->bbdev_op_pool, ops_enq, num_segments); + AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments); + set_ldpc_enc_op(ops_enq, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters); + if (nrLDPC_slot_encoding_parameters->tprep != NULL) stop_meas(nrLDPC_slot_encoding_parameters->tprep); if (nrLDPC_slot_encoding_parameters->tparity != NULL) start_meas(nrLDPC_slot_encoding_parameters->tparity); - for (enq = 0, deq = 0; enq < num_segments;) { - num_to_enq = num_segments; - if (unlikely(num_segments - enq < num_to_enq)) - num_to_enq = num_segments - enq; + + uint16_t enq = 0, deq = 0; + while (enq < num_segments) { + uint16_t num_to_enq = num_segments - enq; enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, queue_id, &ops_enq[enq], num_to_enq); deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, queue_id, &ops_deq[deq], enq - deq); } @@ -864,21 +734,18 @@ static int pmd_lcore_ldpc_enc(void *arg) if (nrLDPC_slot_encoding_parameters->toutput != NULL) stop_meas(nrLDPC_slot_encoding_parameters->toutput); rte_bbdev_enc_op_free_bulk(ops_enq, num_segments); - rte_free(ops_enq); - rte_free(ops_deq); return ret; } // based on DPDK BBDEV throughput_pmd_lcore_dec int start_pmd_dec(struct active_device *ad, struct test_op_params *op_params, + struct data_buffers *data_buffers, nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters) { - int ret; unsigned int lcore_id, used_cores = 0; - uint16_t num_lcores; /* Set number of lcores */ - num_lcores = (ad->nb_queues < (op_params->num_lcores)) ? ad->nb_queues : op_params->num_lcores; + int num_lcores = (ad->nb_queues < (op_params->num_lcores)) ? ad->nb_queues : op_params->num_lcores; /* Allocate memory for thread parameters structure */ struct thread_params *t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), RTE_CACHE_LINE_SIZE); AssertFatal(t_params != 0, @@ -889,6 +756,8 @@ int start_pmd_dec(struct active_device *ad, t_params[0].dev_id = ad->dev_id; t_params[0].lcore_id = rte_lcore_id(); t_params[0].op_params = op_params; + t_params[0].data_buffers = data_buffers; + t_params[0].bbdev_op_pool = ad->bbdev_dec_op_pool; t_params[0].queue_id = ad->dec_queue; t_params[0].iter_count = 0; t_params[0].nrLDPC_slot_decoding_parameters = nrLDPC_slot_decoding_parameters; @@ -900,13 +769,15 @@ int start_pmd_dec(struct active_device *ad, t_params[used_cores].dev_id = ad->dev_id; t_params[used_cores].lcore_id = lcore_id; t_params[used_cores].op_params = op_params; + t_params[used_cores].data_buffers = data_buffers; + t_params[used_cores].bbdev_op_pool = ad->bbdev_dec_op_pool; t_params[used_cores].queue_id = ad->queue_ids[used_cores]; t_params[used_cores].iter_count = 0; t_params[used_cores].nrLDPC_slot_decoding_parameters = nrLDPC_slot_decoding_parameters; rte_eal_remote_launch(pmd_lcore_ldpc_dec, &t_params[used_cores++], lcore_id); } rte_atomic16_set(&op_params->sync, SYNC_START); - ret = pmd_lcore_ldpc_dec(&t_params[0]); + int ret = pmd_lcore_ldpc_dec(&t_params[0]); /* Master core is always used */ // for (used_cores = 1; used_cores < num_lcores; used_cores++) // ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); @@ -917,6 +788,7 @@ int start_pmd_dec(struct active_device *ad, // based on DPDK BBDEV throughput_pmd_lcore_enc int32_t start_pmd_enc(struct active_device *ad, struct test_op_params *op_params, + struct data_buffers *data_buffers, nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { unsigned int lcore_id, used_cores = 0; @@ -928,6 +800,8 @@ int32_t start_pmd_enc(struct active_device *ad, t_params[0].dev_id = ad->dev_id; t_params[0].lcore_id = rte_lcore_id() + 1; t_params[0].op_params = op_params; + t_params[0].data_buffers = data_buffers; + t_params[0].bbdev_op_pool = ad->bbdev_enc_op_pool; t_params[0].queue_id = ad->enc_queue; t_params[0].iter_count = 0; t_params[0].nrLDPC_slot_encoding_parameters = nrLDPC_slot_encoding_parameters; @@ -939,6 +813,8 @@ int32_t start_pmd_enc(struct active_device *ad, t_params[used_cores].dev_id = ad->dev_id; t_params[used_cores].lcore_id = lcore_id; t_params[used_cores].op_params = op_params; + t_params[used_cores].data_buffers = data_buffers; + t_params[used_cores].bbdev_op_pool = ad->bbdev_enc_op_pool; t_params[used_cores].queue_id = ad->queue_ids[1]; t_params[used_cores].iter_count = 0; t_params[used_cores].nrLDPC_slot_encoding_parameters = nrLDPC_slot_encoding_parameters; @@ -952,8 +828,6 @@ int32_t start_pmd_enc(struct active_device *ad, struct test_op_params *op_params = NULL; -struct rte_mbuf *m_head[DATA_NUM_TYPES]; - // OAI CODE int32_t nrLDPC_coding_init() { @@ -1008,12 +882,7 @@ int32_t nrLDPC_coding_init() printf("Couldn't create mempools"); return -1; } - f_ret = init_test_op_params(op_params, RTE_BBDEV_OP_LDPC_DEC, ad->bbdev_dec_op_pool, num_queues, num_queues, 1); - f_ret = init_test_op_params(op_params, RTE_BBDEV_OP_LDPC_ENC, ad->bbdev_enc_op_pool, num_queues, num_queues, 1); - if (f_ret != 0) { - printf("Couldn't init test op params"); - return -1; - } + op_params->num_lcores = 1; return 0; } @@ -1022,7 +891,7 @@ int32_t nrLDPC_coding_shutdown() struct active_device *ad = active_devs; int dev_id = 0; struct rte_bbdev_stats stats; - free_buffers(ad, op_params); + free_mempools(ad); rte_free(op_params); rte_bbdev_stats_get(dev_id, &stats); rte_bbdev_stop(dev_id); @@ -1032,20 +901,17 @@ int32_t nrLDPC_coding_shutdown() return 0; } -int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters){ - +int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters) +{ pthread_mutex_lock(&decode_mutex); - uint16_t num_blocks = 0; - for(uint16_t h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){ - num_blocks += nrLDPC_slot_decoding_parameters->TBs[h].C; - } - uint16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); + const uint16_t num_segments = nb_segments_decoding(nrLDPC_slot_decoding_parameters); + /* It is not unlikely that l_ol becomes big enough to overflow the stack * If you observe this behavior then move it to the heap * Then you would better do a persistent allocation to limit the overhead */ - uint8_t l_ol[num_blocks * LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); + uint8_t l_ol[num_segments * LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); // hardcoded we use first device struct active_device *ad = active_devs; @@ -1054,35 +920,26 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec rte_bbdev_info_get(ad->dev_id, &info); int socket_id = GET_SOCKET(info.socket_id); // fill_queue_buffers -> init_op_data_objs - struct rte_mempool *in_mp = ad->in_mbuf_pool; - struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool; - struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool; - struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool; - struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool; - struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {in_mp, soft_out_mp, hard_out_mp, harq_in_mp, harq_out_mp}; - uint8_t queue_id = ad->dec_queue; - struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&op_params->q_bufs[socket_id][queue_id].inputs, - &op_params->q_bufs[socket_id][queue_id].soft_outputs, - &op_params->q_bufs[socket_id][queue_id].hard_outputs, - &op_params->q_bufs[socket_id][queue_id].harq_inputs, - &op_params->q_bufs[socket_id][queue_id].harq_outputs}; + struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {ad->in_mbuf_pool, ad->hard_out_mbuf_pool}; + struct data_buffers data_buffers; + struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&data_buffers.inputs, &data_buffers.hard_outputs}; int offset = 0; - for(uint16_t h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){ + for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) { for (int r = 0; r < nrLDPC_slot_decoding_parameters->TBs[h].C; r++) { + uint16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); memcpy(z_ol, nrLDPC_slot_decoding_parameters->TBs[h].segments[r].llr, nrLDPC_slot_decoding_parameters->TBs[h].segments[r].E * sizeof(uint16_t)); simde__m128i *pv_ol128 = (simde__m128i *)z_ol; simde__m128i *pl_ol128 = (simde__m128i *)&l_ol[offset]; - int kc = nrLDPC_slot_decoding_parameters->TBs[h].BG == 2 ? 52 : 68; - for (int i = 0, j = 0; j < ((kc * nrLDPC_slot_decoding_parameters->TBs[h].Z) >> 4) + 1; i += 2, j++) { + for (int i = 0, j = 0; j < ((nrLDPC_slot_decoding_parameters->TBs[h].segments[r].E + 15) >> 4); i += 2, j++) { pl_ol128[j] = simde_mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]); } offset += LDPC_MAX_CB_SIZE; } } - for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) { - ret = allocate_buffers_on_socket(queue_ops[type], num_blocks * sizeof(struct rte_bbdev_op_data), socket_id); + for (enum op_data_type type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { + ret = allocate_buffers_on_socket(queue_ops[type], num_segments * sizeof(struct rte_bbdev_op_data), socket_id); AssertFatal(ret == 0, "Couldn't allocate memory for rte_bbdev_op_data structs"); ret = init_op_data_objs_dec(*queue_ops[type], l_ol, @@ -1093,11 +950,17 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec AssertFatal(ret == 0, "Couldn't init rte_bbdev_op_data structs"); } - ret = start_pmd_dec(ad, op_params, nrLDPC_slot_decoding_parameters); + ret = start_pmd_dec(ad, op_params, &data_buffers, nrLDPC_slot_decoding_parameters); if (ret < 0) { LOG_E(PHY, "Couldn't start pmd dec\n"); } + for (enum op_data_type type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { + for (int segment = 0; segment < num_segments; ++segment) + rte_pktmbuf_free((*queue_ops[type])[segment].data); + rte_free(*queue_ops[type]); + } + pthread_mutex_unlock(&decode_mutex); return 0; } @@ -1107,6 +970,9 @@ int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_enc pthread_mutex_lock(&encode_mutex); if (nrLDPC_slot_encoding_parameters->tprep != NULL) start_meas(nrLDPC_slot_encoding_parameters->tprep); + + const uint16_t num_segments = nb_segments_encoding(nrLDPC_slot_encoding_parameters); + // hardcoded to use the first found board struct active_device *ad = active_devs; int ret; @@ -1114,34 +980,25 @@ int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_enc rte_bbdev_info_get(ad->dev_id, &info); int socket_id = GET_SOCKET(info.socket_id); // fill_queue_buffers -> init_op_data_objs - struct rte_mempool *in_mp = ad->in_mbuf_pool; - struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool; - struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool; - struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool; - struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool; - struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {in_mp, soft_out_mp, hard_out_mp, harq_in_mp, harq_out_mp}; - uint8_t queue_id = ad->enc_queue; - struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&op_params->q_bufs[socket_id][queue_id].inputs, - &op_params->q_bufs[socket_id][queue_id].soft_outputs, - &op_params->q_bufs[socket_id][queue_id].hard_outputs, - &op_params->q_bufs[socket_id][queue_id].harq_inputs, - &op_params->q_bufs[socket_id][queue_id].harq_outputs}; - uint16_t num_blocks = 0; - for(uint16_t h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){ - num_blocks += nrLDPC_slot_encoding_parameters->TBs[h].C; - } - for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) { - ret = allocate_buffers_on_socket(queue_ops[type], num_blocks * sizeof(struct rte_bbdev_op_data), socket_id); + struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {ad->in_mbuf_pool, ad->hard_out_mbuf_pool}; + struct data_buffers data_buffers; + struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&data_buffers.inputs, &data_buffers.hard_outputs}; + + for (enum op_data_type type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { + ret = allocate_buffers_on_socket(queue_ops[type], num_segments * sizeof(struct rte_bbdev_op_data), socket_id); AssertFatal(ret == 0, "Couldn't allocate memory for rte_bbdev_op_data structs"); - ret = init_op_data_objs_enc(*queue_ops[type], - nrLDPC_slot_encoding_parameters, - m_head[type], - mbuf_pools[type], - type, - info.drv.min_alignment); + ret = init_op_data_objs_enc(*queue_ops[type], nrLDPC_slot_encoding_parameters, mbuf_pools[type], type, info.drv.min_alignment); AssertFatal(ret == 0, "Couldn't init rte_bbdev_op_data structs"); } - ret = start_pmd_enc(ad, op_params, nrLDPC_slot_encoding_parameters); + + ret = start_pmd_enc(ad, op_params, &data_buffers, nrLDPC_slot_encoding_parameters); + + for (enum op_data_type type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { + for (int segment = 0; segment < num_segments; ++segment) + rte_pktmbuf_free((*queue_ops[type])[segment].data); + rte_free(*queue_ops[type]); + } + pthread_mutex_unlock(&encode_mutex); return ret; } diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.h b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.h index 9da33f4671..642ac8f55a 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.h +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.h @@ -16,10 +16,7 @@ enum op_data_type { DATA_INPUT = 0, - DATA_SOFT_OUTPUT, DATA_HARD_OUTPUT, - DATA_HARQ_INPUT, - DATA_HARQ_OUTPUT, DATA_NUM_TYPES, };