From bbbbda70f5352d99224d0b144b0ab0c4ab75d753 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Tue, 13 May 2025 13:20:34 +0000 Subject: [PATCH] feat(nrLDPC_coding_segment_encoder): Dense Rate Matching and Dense and Concatenated Output 1. 3GPP Rate-matching/interleaving on 8-segment blocks, deinterleaving at output of multi-segment processing. 2. Bit-packing done in nrLDPC_coding_segment_encoder.c now instead of nr_scrambling. Interestingly, it improves performance significantly too. Tested for AVX512 only for now. --- .../nrLDPC_coding_segment_encoder.c | 485 +++++++++++++++--- .../ldpc_encoder_optim8segmulti.c | 98 +--- openair1/PHY/NR_TRANSPORT/nr_scrambling.c | 25 +- openair1/SIMULATION/NR_PHY/dlschsim.c | 6 +- 4 files changed, 443 insertions(+), 171 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index 30ace31054..c962074560 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -41,8 +41,8 @@ #include -// #define DEBUG_LDPC_ENCODING -// #define DEBUG_LDPC_ENCODING_FREE 1 +#define DEBUG_LDPC_ENCODING +//#define DEBUG_LDPC_ENCODING_FREE 1 typedef struct ldpc8blocks_args_s { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters; @@ -64,98 +64,442 @@ static void ldpc8blocks(void *p) // nrLDPC_encoder output is in "d" // let's make this interface happy! - uint8_t tmp[8][68 * 384] __attribute__((aligned(64))); - uint8_t *d[impp->n_segments]; - for (int rr = impp->macro_num * 8, i = 0; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++, i++) - d[rr] = tmp[i]; - + uint8_t d[68 * 384] __attribute__((aligned(64))); + uint8_t *dp[2]; + dp[0] = &d[0]; uint8_t *c[nrLDPC_TB_encoding_parameters->C]; + unsigned int macro_segment, macro_segment_end; + + + macro_segment = 8*impp->macro_num; + macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) c[r] = nrLDPC_TB_encoding_parameters->segments[r].c; start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); - LDPCencoder(c, d, impp); + LDPCencoder(c, dp, impp); stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); // Compute where to place in output buffer that is concatenation of all segments - uint32_t r_offset = 0; - for (int i = 0; i < impp->macro_num * 8; i++) - r_offset += nrLDPC_TB_encoding_parameters->segments[i].E; - for (int rr = impp->macro_num * 8; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++) { - if (impp->F > 0) { - // writing into positions d[r][k-2Zc] as in clause 5.3.2 step 2) in 38.212 - memset(&d[rr][impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F); - } + + if (impp->F>0) { + // writing into positions d[k-2Zc] as in clause 5.3.2 step 2) in 38.212 + memset(&d[impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F); + } #ifdef DEBUG_LDPC_ENCODING - LOG_D(PHY, "rvidx in encoding = %d\n", rel15->rvIndex[0]); + LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index); #endif - uint32_t E = nrLDPC_TB_encoding_parameters->segments[rr].E; - LOG_D(NR_PHY, - "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, Filler bits %d, Filler offset %d mod_order %d, nb_rb " + const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; + uint32_t E2=E,E2_first_segment=macro_segment_end-macro_segment; + bool Eshift=false; + for (int s=macro_segment;ssegments[s].E != E) { + E2=nrLDPC_TB_encoding_parameters->segments[s].E; + Eshift=true; + E2_first_segment = s-macro_segment; + break; + } + + + LOG_D(NR_PHY, + "Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb " "%d,nrOfLayer %d)...\n", - rr, - impp->n_segments, - G, - E, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - mod_order, - nb_rb, - nrLDPC_TB_encoding_parameters->nb_layers); + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E,E2, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb, + nrLDPC_TB_encoding_parameters->nb_layers); +/* + printf("Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb " + "%d,nrOfLayer %d)...\n", + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E,E2, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb, + nrLDPC_TB_encoding_parameters->nb_layers); +*/ + uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - - uint8_t e[E]; - bzero(e, E); - start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match); + uint8_t e[E] __attribute__((aligned(64))); + uint8_t f[E+64] __attribute__((aligned(64))); + uint8_t e2[E2] __attribute__((aligned(64))); + uint8_t f2[E2+64] __attribute__((aligned(64))); + bzero(e, E); + if (Eshift) bzero(e2,E2); + start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); + nr_rate_matching_ldpc(Tbslbrm, + impp->BG, + impp->Zc, + d, + e, + impp->n_segments, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + nrLDPC_TB_encoding_parameters->rv_index, + E); + if (Eshift) nr_rate_matching_ldpc(Tbslbrm, impp->BG, impp->Zc, - d[rr], - e, + d, + e2, impp->n_segments, impp->F, impp->K - impp->F - 2 * impp->Zc, nrLDPC_TB_encoding_parameters->rv_index, - E); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match); - if (impp->K - impp->F - 2 * impp->Zc > E) { - LOG_E(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp->K, G, nb_rb, (int)mod_order); + E2); - LOG_E(NR_PHY, - "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, K %d, Filler bits %d, Filler offset %d mod_order %d, " - "nb_rb %d)...\n", - rr, - impp->n_segments, - G, - E, - impp->K, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - mod_order, - nb_rb); - } -#ifdef DEBUG_LDPC_ENCODING - - for (int i = 0; i < 16; i++) - printf("output ratematching e[%d]= %d r_offset %u\n", i, e[i], r_offset); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); + if (impp->K - impp->F - 2 * impp->Zc > E) { + LOG_E(PHY, + "dlsch coding A %d Kr %d G %d (nb_rb %d, mod_order %d)\n", + A, + impp->K, + G, + nb_rb, + (int)mod_order); + LOG_E(NR_PHY, + "Rate Matching, Code segments %d...%d/%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, " + "nb_rb %d)...\n", + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E, + impp->K, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb); + } + start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); + nr_interleaving_ldpc(E, + mod_order, + e, + f); + if (Eshift) + nr_interleaving_ldpc(E2, + mod_order, + e2, + f2); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); + if(impp->toutput != NULL) start_meas(impp->toutput); +/* + for (int i=0;i<16;i++) + for (int s=0;s>s)&1); +*/ + // information part and puncture columns + + uint32_t Eoffset=0; + for (int s=0; ssegments[s].E); +#ifdef __AVX512VBMI__ + uint64_t *output_p = (uint64_t*)impp->output; + int i=0; + __m512i bitperm[8]; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + bitperm[0] = _mm512_set1_epi64(0x3830282018100800); + __m512i inc = _mm512_set1_epi8(0x1); + Eoffset2[0] = Eoffset>>6; + Eoffset2_bit[0] = Eoffset&63; + //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); + for (int n=1;n>6; + Eoffset2_bit[n] = Eoffset&63; + //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + if (Eshift) { + bitperm[E2_first_segment] = _mm512_add_epi8(bitperm[E2_first_segment-1],inc); + Eoffset += E; + Eoffset2[E2_first_segment] = Eoffset>>6; + Eoffset2_bit[E2_first_segment] = Eoffset &63; + //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); + } + for (int n=E2_first_segment+1;n>6; + Eoffset2_bit[n] = Eoffset&63; + //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + int i2=0; + __m64 tmp; + for (i=0;i>6), + (Eoffset2_bit[j] + i)&63, + (i<=(E-64)) ? 63 : (Eoffset2_bit[j] + E-1)&63); + if (Eoffset2_bit[j] > 0) + printf("segment %d : qword %d, first bit %d last bit %d\n",j, + 1+Eoffset2[j]+(i>>6), + 0, + ((Eoffset2_bit[j] + i)&63) - 1); +#endif + // Note: Here and below for AVX2, we are using the 64-bit SIMD instruction + // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i2],bitperm[j]); + *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); + *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); +#ifdef DEBUG_BIT_INTERLEAVE + if (j<=1) { + printf("pos : %d, i2 %d, j %d\n",(int)(output_p + Eoffset2[j]-(uint64_t*)impp->output),i2,j); + printf("i : %x.%x.%x.%x.%x.%x.%x.%x\n", + ((uint8_t*)(output_p + Eoffset2[j]))[0], + ((uint8_t*)(output_p + Eoffset2[j]))[1], + ((uint8_t*)(output_p + Eoffset2[j]))[2], + ((uint8_t*)(output_p + Eoffset2[j]))[3], + ((uint8_t*)(output_p + Eoffset2[j]))[4], + ((uint8_t*)(output_p + Eoffset2[j]))[5], + ((uint8_t*)(output_p + Eoffset2[j]))[6], + ((uint8_t*)(output_p + Eoffset2[j]))[7]); + printf("i+1 : %x.%x.%x.%x.%x.%x.%x.%x\n", + ((uint8_t*)(output_p + Eoffset2[j]+1))[0], + ((uint8_t*)(output_p + Eoffset2[j]+1))[1], + ((uint8_t*)(output_p + Eoffset2[j]+1))[2], + ((uint8_t*)(output_p + Eoffset2[j]+1))[3], + ((uint8_t*)(output_p + Eoffset2[j]+1))[4], + ((uint8_t*)(output_p + Eoffset2[j]+1))[5], + ((uint8_t*)(output_p + Eoffset2[j]+1))[6], + ((uint8_t*)(output_p + Eoffset2[j]+1))[7]); + printf("%x.%x.%x.%x.%x.%x.%x.%x\n", + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),0)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),1)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),2)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),3)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),4)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),5)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),6)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),7)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),8)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),9)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),10)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),11)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),12)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),13)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),14)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),15)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),16)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),17)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),18)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),19)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),20)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),21)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),22)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),23)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),24)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),25)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),26)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),27)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),28)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),29)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),30)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),31)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),0)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),1)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),2)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),3)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),4)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),5)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),6)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),7)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),8)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),9)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),10)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),11)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),12)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),13)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),14)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),15)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),16)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),17)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),18)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),19)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),20)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),21)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),22)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),23)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),24)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),25)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),26)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),27)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),28)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),29)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),30)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),31)<<(7-j))&128)); + } #endif - start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave); - nr_interleaving_ldpc(E, mod_order, e, impp->output + r_offset); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave); -#ifdef DEBUG_LDPC_ENCODING - - for (int i = 0; i < 16; i++) - printf("output interleaving f[%d]= %d r_offset %u\n", i, impp->output[i + r_offset], r_offset); - - if (r == impp->n_segments - 1) - write_output("enc_output.m", "enc", impp->output, G, 1, 4); - + } //for (int j=0; + } // if (i < E) + for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { +#ifdef DEBUG_BIT_INTERLEAVE + printf("segment %d : qword %d, first bit %d last bit %d\n",j, + Eoffset2[j]+(i>>6), + (Eoffset2_bit[j] + i)&63, + (i<=(E-64)) ? 63 : (Eoffset2_bit[j] + E-1)&63); + if (Eoffset2_bit[j] > 0) + printf("segment %d : qword %d, first bit %d last bit %d\n",j, + 1+Eoffset2[j]+(i>>6), + 0, + ((Eoffset2_bit[j] + i)&63) - 1); #endif - r_offset += E; + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i2],bitperm[j]); + *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]), + _mm_slli_si64(tmp,Eoffset2_bit[j])); + *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); + } + output_p++; } +#elif defined(__aarch64__) + uint16_t *output_p = (uint16_t*)impp->output; + int i=0; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + Eoffset2[0] = Eoffset>>4; + Eoffset2_bit[0] = Eoffset&15; + //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); + for (int n=1;n>4; + Eoffset2_bit[n] = Eoffset&15; + //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + if (Eshift) { + Eoffset += E; + Eoffset2[E2_first_segment] = Eoffset>>4; + Eoffset2_bit[E2_first_segment] = Eoffset &15; + //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); + } + for (int n=E2_first_segment+1;n>4; + Eoffset2_bit[n] = Eoffset&15; + //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + int i2=0; + const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = { + {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0 + {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1 + {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2 + {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3 + {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4 + {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5 + {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6 + {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7 + const uint8_t __attribute__ ((aligned (16))) masks[16] = + {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80}; + int8x16_t vshift[8]; + for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]); + uint8x16_t vmask = vld1q_u8(masks); + + int32_t tmp; + uint8x16_t cshift; + for (i=0;i>(16-Eoffset2_bit[j])); + } + } + for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { + cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i2],vshift[j]),vmask); + tmp = (int)vaddv_u8(vget_low_u8(cshift)); + tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); + *(output_p + Eoffset2[j]) |= (uint16_t)(tmp<>(16-Eoffset2_bit[j])); + } + output_p++; + } + +#else + uint32_t *output_p = (uint32_t*)impp->output; + int i=0; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + Eoffset2[0] = Eoffset>>5; + Eoffset2_bit[0] = Eoffset&31; + //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); + for (int n=1;n>5; + Eoffset2_bit[n] = Eoffset&31; + //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + if (Eshift) { + Eoffset += E; + Eoffset2[E2_first_segment] = Eoffset>>5; + Eoffset2_bit[E2_first_segment] = Eoffset &31; + //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); + } + for (int n=E2_first_segment+1;n>5; + Eoffset2_bit[n] = Eoffset&31; + //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + int i2=0; + int tmp; + __m64 tmp64,tmp64b,tmp64c; + + for (i=0;i>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2],7-j)); + tmp64=_mm_set1_pi32(tmp); + tmp64b = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_pi32(tmp64,Eoffset2_bit[j])); + tmp64c = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_srli_pi32(tmp64,(32-Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j]+1) = _m_to_int(_mm_srli_si64(tmp64c,32)); + } + } + for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2],7-j)); + tmp64=_mm_set1_pi32(tmp); + tmp64b = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_pi32(tmp64,Eoffset2_bit[j])); + tmp64c = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_srli_pi32(tmp64,(32-Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j]+1) = _m_to_int(_mm_srli_si64(tmp64c,32)); + } + output_p++; + } + + +#endif + + if(impp->toutput != NULL) stop_meas(impp->toutput); +/* + printf("E %d (mod16 %d) F %d\n",E,E&15,impp->F); + for (int i=0;i<16;i++) + for (int s=0;sans); } @@ -176,7 +520,7 @@ static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_ impp.Kb = nrLDPC_TB_encoding_parameters->Kb; impp.Zc = nrLDPC_TB_encoding_parameters->Z; impp.BG = nrLDPC_TB_encoding_parameters->BG; - impp.output = nrLDPC_TB_encoding_parameters->segments->output; + impp.output = nrLDPC_TB_encoding_parameters->segments->output; impp.K = nrLDPC_TB_encoding_parameters->K; impp.F = nrLDPC_TB_encoding_parameters->F; @@ -201,6 +545,7 @@ static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { int nbTasks = 0; + for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index bd25ecf646..268d2f9dc6 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -203,102 +203,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) return(-1); } } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - /* - memcpy(&output[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&output[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - */ -#ifdef __AVX512F__ - - uint32_t l1 = (block_length-(2*Zc)); - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit); - if ((((2*Zc)&63) == 0) && (((block_length-(2*Zc))&63) == 0)) { - __m512i *c512p = (__m512i *)&cc[2*Zc]; - __m512i *d512p = (__m512i *)&dd[0]; - __m512i mask0 = _mm512_set1_epi8(0x1); - - for (int i=0;i>6;i++) - for (int j=macro_segment; j < macro_segment_end; j++) - ((__m512i *)output[j])[i] = _mm512_and_si512(_mm512_srai_epi16(c512p[i],j-macro_segment),mask0); - for (int i1=0, i=l1>>6;i1>6;i1++,i++) - for (int j=macro_segment; j < macro_segment_end; j++) - ((__m512i *)output[j])[i] = _mm512_and_si512(_mm512_srai_epi16(d512p[i1],j-macro_segment),mask0); - } - else if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) { - uint32_t l1 = (block_length-(2*Zc))>>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&cc[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i0) l2++; - - for (int i1=0, i=l1;i1>4; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>4; - simde__m128i *c128p = (simde__m128i *)&cc[2*Zc]; - simde__m128i *d128p = (simde__m128i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&cc[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i0) l2++; - - for (int i1=0, i=l1;i1>(j-macro_segment))&1; - for (int i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++) - //for (j=0; j>(j-macro_segment))&1; - } - - if(impp->toutput != NULL) stop_meas(impp->toutput); + memcpy(output[0],&cc[2*Zc],(block_length-(2*Zc))); + memcpy(output[0]+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); return 0; } diff --git a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c index b71ee0cce0..821d92a44d 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c +++ b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c @@ -33,12 +33,33 @@ void nr_codeword_scrambling(uint8_t *in, { const int roundedSz = (size + 31) / 32; uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz); + for (int i = 0; i < roundedSz; i++) + out[i] = ((uint32_t*)in)[i] ^ seq[i]; +#if 0 +//#elif __AVX2__ for (int i = 0; i < roundedSz; i++) { - simde__m256i c = ((simde__m256i*)in)[i]; - uint32_t in32 = simde_mm256_movemask_epi8(simde_mm256_slli_epi16(c, 7)); + __m256i c = ((__m256i*)in)[i]; + uint32_t in32 = _mm256_movemask_epi8(_mm256_slli_epi16(c, 7)); out[i] = in32 ^ seq[i]; DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); } +//#elif defined (__aarch64__) + const int8_t __attribute__ ((aligned (16))) ucShift[] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}; + int8x16_t vshift = vld1q_s8(ucShift); + uint8x16_t *c = (uint8x16_t*)in; + uint8x16_t cshift; + uint32_t in32; + for (int i = 0; i < roundedSz; i++) { + cshift = vshlq_u8(*c++, vshift); + in32 = vaddv_u8(vget_low_u8(cshift)); + in32 += (vaddv_u8(vget_high_u8(cshift)) << 8); + cshift = vshlq_u8(*c++, vshift); + in32 += (vaddv_u8(vget_low_u8(cshift)) << 16); + in32 += (vaddv_u8(vget_high_u8(cshift)) << 24); + out[i] = in32 ^ seq[i]; + DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); + } +#endif } void nr_codeword_unscrambling(int16_t* llr, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI) diff --git a/openair1/SIMULATION/NR_PHY/dlschsim.c b/openair1/SIMULATION/NR_PHY/dlschsim.c index 36fe9daa50..f7b10e5fb1 100644 --- a/openair1/SIMULATION/NR_PHY/dlschsim.c +++ b/openair1/SIMULATION/NR_PHY/dlschsim.c @@ -512,7 +512,7 @@ int main(int argc, char **argv) //printf("crc32: [0]->0x%08x\n",crc24c(test_input, 32)); // generate signal - unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(32))); + unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(64))); bzero(output,rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS); if (input_fd == NULL) { msgDataTx.num_pdsch_slot = 1; @@ -533,12 +533,12 @@ int main(int argc, char **argv) //if (i<16) // printf("encoder output f[%d] = %d\n",i,dlsch->harq_processes[0]->f[i]); - if (output[i] == 0) + if ((output[i]&1) == 0) modulated_input[i] = 1.0; ///sqrt(2); //QPSK else modulated_input[i] = -1.0; ///sqrt(2); - //if (i<16) printf("modulated_input[%d] = %d\n",i,modulated_input[i]); + //if (i<16) printf("modulated_input[%d] = %f (%d)\n",i,modulated_input[i],output[i]); //SNR =10; SNR_lin = pow(10, SNR / 10.0); sigma = 1.0 / sqrt(2 * SNR_lin);