maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(nrLDPC_coding_segment_encoder): address review

Browse Source 
1. nrLDPC_coding_segment_encoding: Clarifying encoder parameters
   The way the encoder implementation params was used could be made clearer:
   * `macro_num` was always used in order to calculate segment indexes.
     It is clearer to pass directly the first segment index.
   * The initialisation of a common implementation parameters structure can be made clearer.

2. nrLDPC_coding_segment_encoder: Remove multi segment dimension in encoder output
   The output of `LDPCencoder` was a 2D unsigned char array.
   But all the segments were actually packed in the first segment, 8 bits of different segments per byte of the first segment.
   So the first dimension was useless. This commit removes this dimension.

3. nrLDPC_coding_segment_encoder: Rename `nrLDPC_prepare_TB_encoding` into `nrLDPC_launch_TB_encoding`

4. nrLDPC_coding_segment_encoder: Remove the useless second rate matching for E2, one rate matching with max(E, E2) does the job.

5. nr_interleaving_ldpc: cleanup
   Rewrite code to use different SIMD sizes in a linear fashion.
   This means cascading preprocessor directives are used to start from a high SIMD size an decrease it progressively.

6. ldpc_encoder_optim8segmulti: linear usage of SIMD
   Use cascading preprocessor directives to include different SIMD sizes and types depending on which are available on the system.
   The different loops use a common index that is progressively incremented.
   Also add AVX512 version of bits distribution from 8 segments into one byte.

Co-authored-by: Laurent Thomas <laurent.thomas@open-cells.com>
This commit is contained in:
Romain Beurdouche
2025-05-13 14:08:33 +00:00
co-authored by Laurent Thomas
parent 1b31ac417b
commit 06c81030dc
6 changed files with 515 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openair1/PHY/CODING/TESTBENCH/ldpctest.c
+14 -16
View File
@@ -123,7 +123,7 @@ one_measurement_t test_ldpc(short max_iterations,
uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
memset(estimated_output, 0, sizeof(estimated_output));
uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
uint8_t *channel_input_optim[MAX_NUM_DLSCH_SEGMENTS];
uint8_t *channel_input_optim;
// double channel_output[68 * 384];
double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
@@ -265,9 +265,9 @@ one_measurement_t test_ldpc(short max_iterations,
memset(test_input[j], 0, ((K + 7) & ~7) / 8);
channel_input[j] = malloc16(68 * 384);
memset(channel_input[j], 0, 68 * 384);
channel_input_optim[j] = malloc16(68 * 384);
memset(channel_input_optim[j], 0, 68 * 384);
}
channel_input_optim = malloc16(68 * 384);
memset(channel_input_optim, 0, 68 * 384);
// Fill input segments with random values
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
@@ -285,7 +285,7 @@ one_measurement_t test_ldpc(short max_iterations,
impp.gen_code = 1;
if (ntrials == 0)
ldpc_orig.LDPCencoder(test_input, channel_input, &impp);
ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp);
impp.gen_code = 0;
decode_abort_t dec_abort;
init_abort(&dec_abort);
@@ -294,23 +294,21 @@ one_measurement_t test_ldpc(short max_iterations,
//// encoder
start_meas(&time);
for (int j = 0; j < n_segments; j++) {
ldpc_orig.LDPCencoder(&test_input[j], &channel_input[j], &impp);
ldpc_orig.LDPCencoder(&test_input[j], channel_input[j], &impp);
}
stop_meas(&time);
impp.n_segments = n_segments;
for (int j = 0; j < (n_segments / 8 + 1); j++) {
start_meas(&ret.time_optim);
impp.macro_num = j;
ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
stop_meas(&ret.time_optim);
}
start_meas(&ret.time_optim);
impp.first_seg = 0;
ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
stop_meas(&ret.time_optim);
if (ntrials == 1)
for (int j = 0; j < n_segments; j++)
for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
if (channel_input[j][i] != channel_input_optim[j][i]) {
printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], channel_input_optim[j][i]);
if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) {
printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1);
return ret;
}
}
@@ -322,7 +320,7 @@ one_measurement_t test_ldpc(short max_iterations,
printf("\ne %d..%d: ", i, i + 15);
#endif
if (channel_input_optim[j][i - 2 * Zc] == 0)
if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0)
modulated_input[j][i] = 1.0; /// sqrt(2); //QPSK
else
modulated_input[j][i] = -1.0; /// sqrt(2);
@@ -332,7 +330,7 @@ one_measurement_t test_ldpc(short max_iterations,
// Uncoded BER
uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0;
if (channel_output_uncoded != channel_input_optim[j][i - 2 * Zc])
if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1))
ret.errors_bit_uncoded++;
}
@@ -396,8 +394,8 @@ one_measurement_t test_ldpc(short max_iterations,
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
free(test_input[j]);
free(channel_input[j]);
free(channel_input_optim[j]);
}
free(channel_input_optim);
print_meas(&time, "ldpc_encoder", NULL, NULL);
print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL);
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
+33 -45
View File
@@ -65,19 +65,17 @@ static void ldpc8blocks(void *p)
// nrLDPC_encoder output is in "d"
// let's make this interface happy!
uint8_t d[68 * 384] __attribute__((aligned(64)));
uint8_t *dp[2];
dp[0] = &d[0];
uint8_t *c[nrLDPC_TB_encoding_parameters->C];
unsigned int macro_segment, macro_segment_end;
macro_segment = 8*impp->macro_num;
macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments;
macro_segment = impp->first_seg;
macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments;
for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++)
c[r] = nrLDPC_TB_encoding_parameters->segments[r].c;
start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode);
LDPCencoder(c, dp, impp);
stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode);
start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode);
LDPCencoder(c, d, impp);
stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode);
// Compute where to place in output buffer that is concatenation of all segments
#ifdef DEBUG_LDPC_ENCODING
@@ -86,11 +84,14 @@ static void ldpc8blocks(void *p)
const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E;
uint32_t E2=E,E2_first_segment=macro_segment_end-macro_segment;
bool Eshift=false;
uint32_t Emax = E;
for (int s=macro_segment;s<macro_segment_end;s++)
if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
E2=nrLDPC_TB_encoding_parameters->segments[s].E;
Eshift=true;
E2_first_segment = s-macro_segment;
if(E2 > Emax)
Emax = E2;
break;
}
@@ -124,14 +125,12 @@ static void ldpc8blocks(void *p)
*/
uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
uint8_t e[E] __attribute__((aligned(64)));
uint8_t e[Emax] __attribute__((aligned(64)));
uint8_t f[E+64] __attribute__((aligned(64)));
uint8_t e2[E2] __attribute__((aligned(64)));
uint8_t f2[E2+64] __attribute__((aligned(64)));
bzero(e, E);
bzero(e, Emax);
bzero(f, E + 64);
if (Eshift) {
bzero(e2, E2);
bzero(f2, E2 + 64);
}
start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match);
@@ -144,18 +143,7 @@ static void ldpc8blocks(void *p)
impp->F,
impp->K - impp->F - 2 * impp->Zc,
nrLDPC_TB_encoding_parameters->rv_index,
E);
if (Eshift)
nr_rate_matching_ldpc(Tbslbrm,
impp->BG,
impp->Zc,
d,
e2,
impp->n_segments,
impp->F,
impp->K - impp->F - 2 * impp->Zc,
nrLDPC_TB_encoding_parameters->rv_index,
E2);
Emax);
stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match);
if (impp->K - impp->F - 2 * impp->Zc > E) {
@@ -189,7 +177,7 @@ static void ldpc8blocks(void *p)
if (Eshift)
nr_interleaving_ldpc(E2,
mod_order,
e2,
e,
f2);
stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave);
if(impp->toutput != NULL) start_meas(impp->toutput);
@@ -504,42 +492,42 @@ static void ldpc8blocks(void *p)
completed_task_ans(impp->ans);
}
static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters,
static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters,
int dlsch_id,
thread_info_tm_t *t_info)
{
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
encoder_implemparams_t impp = {0};
encoder_implemparams_t common_segment_params = {
.n_segments = nrLDPC_TB_encoding_parameters->C,
.tinput = nrLDPC_slot_encoding_parameters->tinput,
.tprep = nrLDPC_slot_encoding_parameters->tprep,
.tparity = nrLDPC_slot_encoding_parameters->tparity,
.toutput = nrLDPC_slot_encoding_parameters->toutput,
.Kb = nrLDPC_TB_encoding_parameters->Kb,
.Zc = nrLDPC_TB_encoding_parameters->Z,
.BG = nrLDPC_TB_encoding_parameters->BG,
.output = nrLDPC_TB_encoding_parameters->output,
.K = nrLDPC_TB_encoding_parameters->K,
.F = nrLDPC_TB_encoding_parameters->F,
};
impp.n_segments = nrLDPC_TB_encoding_parameters->C;
impp.tinput = nrLDPC_slot_encoding_parameters->tinput;
impp.tprep = nrLDPC_slot_encoding_parameters->tprep;
impp.tparity = nrLDPC_slot_encoding_parameters->tparity;
impp.toutput = nrLDPC_slot_encoding_parameters->toutput;
impp.Kb = nrLDPC_TB_encoding_parameters->Kb;
impp.Zc = nrLDPC_TB_encoding_parameters->Z;
impp.BG = nrLDPC_TB_encoding_parameters->BG;
impp.output = nrLDPC_TB_encoding_parameters->output;
impp.K = nrLDPC_TB_encoding_parameters->K;
impp.F = nrLDPC_TB_encoding_parameters->F;
const size_t n_tasks = (common_segment_params.n_segments + 7) >> 3;
size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1));
for (int j = 0; j < n_seg; j++) {
for (int j = 0; j < n_tasks; j++) {
ldpc8blocks_args_t *perJobImpp = &((ldpc8blocks_args_t *)t_info->buf)[t_info->len];
DevAssert(t_info->len < t_info->cap);
impp.ans = t_info->ans;
t_info->len += 1;
impp.macro_num = j;
perJobImpp->impp = impp;
perJobImpp->impp = common_segment_params;
perJobImpp->impp.first_seg = j * 8;
perJobImpp->impp.ans = t_info->ans;
perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters;
task_t t = {.func = ldpc8blocks, .args = perJobImpp};
pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t);
}
return n_seg;
return n_tasks;
}
int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
@@ -558,7 +546,7 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
int nbEncode = 0;
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
nbEncode += nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info);
nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info);
}
if (nbEncode < nbTasks) {
completed_many_task_ans(&ans, nbTasks - nbEncode);
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c
+390 -417
View File
@@ -29,446 +29,419 @@ static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
{
uint32_t EQm;
EQm = E / Qm;
const uint32_t EQm = E / Qm;
memset(f, 0, E * sizeof(uint8_t));
uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7;
uint8_t *fp;
int j,k;
#if 1 //def __WASAVX2__
#ifdef __AVX512VBMI__
__m512i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5;
__m512i *e0_512,*e1_512,*e2_512,*e3_512,*e4_512,*e5_512,*e6_512,*e7_512;
__m512i *f_512=(__m512i *)f;
#else
simde__m128i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5,tmp6,tmp7;
simde__m128i *e0_128,*e1_128,*e2_128,*e3_128,*e4_128,*e5_128,*e6_128,*e7_128;
simde__m128i *f_128=(simde__m128i *)f;
#endif
switch(Qm) {
case 2:
e0=e;
e1=e0+EQm;
case 2: {
uint8_t *e0 = e;
uint8_t *e1 = e0 + EQm;
int i = 0;
#ifdef __AVX512VBMI__
e0_512=(__m512i *)e0;
e1_512=(__m512i *)e1;
__m512i p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0);
__m512i p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32);
__m512i e0j,e1j;
for (k=0,j=0;j<EQm>>6;j++,k+=2) {
e0j = _mm512_loadu_si512(e0_512+j);
e1j = _mm512_loadu_si512(e1_512+j);
_mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi8(e0j,p8a,e1j)); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
_mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi8(e0j,p8b,e1j));
}
if ((j<<6) != EQm) {
int k2=k<<6;
fp = &f[k2];
for (int j2=(j<<6) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
// clang-format off
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24,
87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16,
79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8,
71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120,
56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112,
48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104,
40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
// clang-format on
__m512i *f_512 = (__m512i *)f;
__m512i *e0_512 = (__m512i *)e0;
__m512i *e1_512 = (__m512i *)e1;
for (; i < (EQm & ~63); i += 64) {
__m512i e0j = _mm512_loadu_si512(e0_512++);
__m512i e1j = _mm512_loadu_si512(e1_512++);
// e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j));
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j));
}
}
#else
e0_128=(simde__m128i *)e0;
e1_128=(simde__m128i *)e1;
simde__m128i e0j,e1j;
for (k=0,j=0;j<EQm>>4;j++,k+=2) {
e0j=simde_mm_loadu_si128(e0_128+j);
e1j=simde_mm_loadu_si128(e1_128+j);
simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi8(e0j,e1j));
simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi8(e0j,e1j));
}
if ((j<<4) != EQm) {
int k2=k<<4;
fp = &f[k2];
for (int j2=(j<<4) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
}
}
e0 = (uint8_t *)e0_512;
e1 = (uint8_t *)e1_512;
f = (uint8_t *)f_512;
#endif
break;
case 4:
e0=e;
e1=e0+EQm;
e2=e1+EQm;
e3=e2+EQm;
#ifdef USE128BIT
simde__m128i *f_128 = (simde__m128i *)f;
simde__m128i *e0_128 = (simde__m128i *)e0;
simde__m128i *e1_128 = (simde__m128i *)e1;
for (; i < (EQm & ~15); i += 64) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j));
}
e0 = (uint8_t *)e0_128;
e1 = (uint8_t *)e1_128;
f = (uint8_t *)f_128;
#endif
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
}
} break;
case 4: {
uint8_t *e0 = e;
uint8_t *e1 = e0 + EQm;
uint8_t *e2 = e1 + EQm;
uint8_t *e3 = e2 + EQm;
int i = 0;
#ifdef __AVX512VBMI__
e0_512=(__m512i *)e0;
e1_512=(__m512i *)e1;
e2_512=(__m512i *)e2;
e3_512=(__m512i *)e3;
p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0);
p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32);
__m512i p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0);
__m512i p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16);
__m512i p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0);
__m512i p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8);
__m512i e2j,e3j;
for (k=0,j=0;j<EQm>>6;j++,k+=4) {
e0j = _mm512_loadu_si512(e0_512+j);
e1j = _mm512_loadu_si512(e1_512+j);
e2j = _mm512_loadu_si512(e2_512+j);
e3j = _mm512_loadu_si512(e3_512+j);
tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
_mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
_mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp0,p16b,tmp1));
tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
_mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
_mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp0,p16b,tmp1));
}
if ((j<<6) != EQm) {
int k2=k<<6;
fp = &f[k2];
for (int j2=(j<<6) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
*fp++ = e2[j2];
*fp++ = e3[j2];
// clang-format off
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47,
110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102,
38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
// clang-format on
__m512i *e0_512 = (__m512i *)e0;
__m512i *e1_512 = (__m512i *)e1;
__m512i *e2_512 = (__m512i *)e2;
__m512i *e3_512 = (__m512i *)e3;
__m512i *f_512 = (__m512i *)f;
for (; i < (EQm & ~63); i += 64) {
__m512i e0j = _mm512_loadu_si512(e0_512++);
__m512i e1j = _mm512_loadu_si512(e1_512++);
__m512i e2j = _mm512_loadu_si512(e2_512++);
__m512i e3j = _mm512_loadu_si512(e3_512++);
__m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
__m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
// e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
// e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
}
}
#else
e0_128=(simde__m128i *)e0;
e1_128=(simde__m128i *)e1;
e2_128=(simde__m128i *)e2;
e3_128=(simde__m128i *)e3;
simde__m128i e2j,e3j;
for (k=0,j=0;j<EQm>>4;j++,k+=4) {
e0j = simde_mm_loadu_si128(e0_128+j);
e1j = simde_mm_loadu_si128(e1_128+j);
e2j = simde_mm_loadu_si128(e2_128+j);
e3j = simde_mm_loadu_si128(e3_128+j);
tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi16(tmp0,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi16(tmp0,tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31)
simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi16(tmp0,tmp1));
simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi16(tmp0,tmp1));
}
if ((j<<4) != EQm) {
int k2=k<<4;
fp = &f[k2];
for (int j2=(j<<4) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
*fp++ = e2[j2];
*fp++ = e3[j2];
}
}
e0 = (uint8_t *)e0_512;
e1 = (uint8_t *)e1_512;
e2 = (uint8_t *)e2_512;
e3 = (uint8_t *)e3_512;
f = (uint8_t *)f_512;
#endif
break;
case 6:
e0=e;
e1=e0+EQm;
e2=e1+EQm;
e3=e2+EQm;
e4=e3+EQm;
e5=e4+EQm;
#ifdef USE128BIT
simde__m128i *e0_128 = (simde__m128i *)e0;
simde__m128i *e1_128 = (simde__m128i *)e1;
simde__m128i *e2_128 = (simde__m128i *)e2;
simde__m128i *e3_128 = (simde__m128i *)e3;
simde__m128i *f_128 = (simde__m128i *)f;
for (; i < (EQm & ~15); i += 16) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
simde_mm_storeu_si128(f_128++,
simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
simde_mm_storeu_si128(
f_128++,
simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31)
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1));
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1));
}
e0 = (uint8_t *)e0_128;
e1 = (uint8_t *)e1_128;
e2 = (uint8_t *)e2_128;
e3 = (uint8_t *)e3_128;
f = (uint8_t *)f_128;
#endif
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
}
} break;
case 6: {
uint8_t *e0 = e;
uint8_t *e1 = e0 + EQm;
uint8_t *e2 = e1 + EQm;
uint8_t *e3 = e2 + EQm;
uint8_t *e4 = e3 + EQm;
uint8_t *e5 = e4 + EQm;
int i = 0;
#ifdef __AVX512VBMI__
e0_512=(__m512i *)e0;
e1_512=(__m512i *)e1;
e2_512=(__m512i *)e2;
e3_512=(__m512i *)e3;
e4_512=(__m512i *)e4;
e5_512=(__m512i *)e5;
__m512i e4j,e5j;
p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0);
p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32);
// clang-format off
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110,
46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0);
const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42);
const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0,
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11);
// clang-format on
__m512i *e0_512 = (__m512i *)e0;
__m512i *e1_512 = (__m512i *)e1;
__m512i *e2_512 = (__m512i *)e2;
__m512i *e3_512 = (__m512i *)e3;
__m512i *e4_512 = (__m512i *)e4;
__m512i *e5_512 = (__m512i *)e5;
__m512i *f_512 = (__m512i *)f;
for (; i < (EQm & ~63); i += 64) {
__m512i e0j = _mm512_loadu_si512(e0_512++);
__m512i e1j = _mm512_loadu_si512(e1_512++);
__m512i e2j = _mm512_loadu_si512(e2_512++);
__m512i e3j = _mm512_loadu_si512(e3_512++);
__m512i e4j = _mm512_loadu_si512(e4_512++);
__m512i e5j = _mm512_loadu_si512(e5_512++);
__m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31)
__m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31)
__m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31)
// e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
__m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
// e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
__m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15)
// e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
__m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
// e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20)
// e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
// e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0);
p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16);
__m512i p16c = _mm512_set_epi16(21,20,41,19,18,40,17,16,39,15,14,38,13,12,37,11,10,36,9,8,35,7,6,34,5,4,33,3,2,32,1,0);
__m512i p16d = _mm512_set_epi16(10,52,9,8,51,7,6,50,5,4,49,3,2,48,1,0,47,31,30,46,29,28,45,27,26,44,25,24,43,23,22,42);
__m512i p16d2 = _mm512_set_epi16(32+10,30,32+9,32+8,27,32+7,32+6,24,32+5,32+4,21,32+3,32+2,18,32+1,32+0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
__m512i p16e = _mm512_set_epi16(63,31,30,62,29,28,61,27,26,60,25,24,59,23,22,58,21,20,57,19,18,56,17,16,55,15,14,54,13,12,53,11);
for (j=0,k=0;j<EQm>>6;j++,k+=6) {
e0j = _mm512_loadu_si512(e0_512+j);
e1j = _mm512_loadu_si512(e1_512+j);
e2j = _mm512_loadu_si512(e2_512+j);
e3j = _mm512_loadu_si512(e3_512+j);
e4j = _mm512_loadu_si512(e4_512+j);
e5j = _mm512_loadu_si512(e5_512+j);
tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31)
tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31)
tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31)
tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
_mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
_mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
_mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63)
tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63)
tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63)
tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
_mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
_mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
_mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
}
if ((j<<6) != EQm) {
int k2=k<<6;
fp = &f[k2];
for (int j2=(j<<6) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
*fp++ = e2[j2];
*fp++ = e3[j2];
*fp++ = e4[j2];
*fp++ = e5[j2];
tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63)
tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63)
tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63)
tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
// e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
// e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
// e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20)
// e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
// e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
}
}
#else
fp = f;
for (int j = 0; j< EQm; j++){
*fp++ = e0[j];
*fp++ = e1[j];
*fp++ = e2[j];
*fp++ = e3[j];
*fp++ = e4[j];
*fp++ = e5[j];
}
e0 = (uint8_t *)e0_512;
e1 = (uint8_t *)e1_512;
e2 = (uint8_t *)e2_512;
e3 = (uint8_t *)e3_512;
e4 = (uint8_t *)e4_512;
e5 = (uint8_t *)e5_512;
f = (uint8_t *)f_512;
#endif
break;
case 8:
e0=e;
e1=e0+EQm;
e2=e1+EQm;
e3=e2+EQm;
e4=e3+EQm;
e5=e4+EQm;
e6=e5+EQm;
e7=e6+EQm;
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
*f++ = *e4++;
*f++ = *e5++;
}
} break;
case 8: {
uint8_t *e0 = e;
uint8_t *e1 = e0 + EQm;
uint8_t *e2 = e1 + EQm;
uint8_t *e3 = e2 + EQm;
uint8_t *e4 = e3 + EQm;
uint8_t *e5 = e4 + EQm;
uint8_t *e6 = e5 + EQm;
uint8_t *e7 = e6 + EQm;
int i = 0;
#ifdef __AVX512VBMI__
e0_512=(__m512i *)e0;
e1_512=(__m512i *)e1;
e2_512=(__m512i *)e2;
e3_512=(__m512i *)e3;
e4_512=(__m512i *)e4;
e5_512=(__m512i *)e5;
e6_512=(__m512i *)e6;
e7_512=(__m512i *)e7;
__m512i e6j,e7j;
p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0);
p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32);
// clang-format off
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55,
118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46,
109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0);
p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16);
const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0);
p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8);
for (k=0,j=0;j<EQm>>6;j++,k+=8) {
e0j = _mm512_loadu_si512(e0_512+j);
e1j = _mm512_loadu_si512(e1_512+j);
e2j = _mm512_loadu_si512(e2_512+j);
e3j = _mm512_loadu_si512(e3_512+j);
e4j = _mm512_loadu_si512(e4_512+j);
e5j = _mm512_loadu_si512(e5_512+j);
e6j = _mm512_loadu_si512(e6_512+j);
e7j = _mm512_loadu_si512(e7_512+j);
tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
tmp3 = _mm512_permutex2var_epi8(e6j,p8a,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
_mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
_mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
// clang-format on
__m512i *e0_512 = (__m512i *)e0;
__m512i *e1_512 = (__m512i *)e1;
__m512i *e2_512 = (__m512i *)e2;
__m512i *e3_512 = (__m512i *)e3;
__m512i *e4_512 = (__m512i *)e4;
__m512i *e5_512 = (__m512i *)e5;
__m512i *e6_512 = (__m512i *)e6;
__m512i *e7_512 = (__m512i *)e7;
__m512i *f_512 = (__m512i *)f;
for (; i < (EQm & ~63); i += 64) {
__m512i e0j = _mm512_loadu_si512(e0_512++);
__m512i e1j = _mm512_loadu_si512(e1_512++);
__m512i e2j = _mm512_loadu_si512(e2_512++);
__m512i e3j = _mm512_loadu_si512(e3_512++);
__m512i e4j = _mm512_loadu_si512(e4_512++);
__m512i e5j = _mm512_loadu_si512(e5_512++);
__m512i e6j = _mm512_loadu_si512(e6_512++);
__m512i e7j = _mm512_loadu_si512(e7_512++);
__m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
__m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
__m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
__m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
__m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
// e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3)
__m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
// e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)...
// e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
// e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
// e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
// e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11)
// e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
// e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15)
// e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
_mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
_mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
tmp3 = _mm512_permutex2var_epi8(e6j,p8b,e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
_mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
_mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
_mm512_storeu_si512(f_512+k+6,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
_mm512_storeu_si512(f_512+k+7,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
}
if ((j<<6) != EQm) {
int k2=k<<6;
fp=&f[k2];
for (int j2=(j<<6) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
*fp++ = e2[j2];
*fp++ = e3[j2];
*fp++ = e4[j2];
*fp++ = e5[j2];
*fp++ = e6[j2];
*fp++ = e7[j2];
tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
// e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
// e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
// e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19)
// e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
// e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23)
// e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
// e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
// e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
// e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27)
// e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
// e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31)
// e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
}
}
#else
e0_128=(simde__m128i *)e0;
e1_128=(simde__m128i *)e1;
e2_128=(simde__m128i *)e2;
e3_128=(simde__m128i *)e3;
e4_128=(simde__m128i *)e4;
e5_128=(simde__m128i *)e5;
e6_128=(simde__m128i *)e6;
e7_128=(simde__m128i *)e7;
simde__m128i e4j,e5j,e6j,e7j;
for (k=0,j=0;j<EQm>>4;j++,k+=8) {
e0j = simde_mm_loadu_si128(e0_128+j);
e1j = simde_mm_loadu_si128(e1_128+j);
e2j = simde_mm_loadu_si128(e2_128+j);
e3j = simde_mm_loadu_si128(e3_128+j);
e4j = simde_mm_loadu_si128(e4_128+j);
e5j = simde_mm_loadu_si128(e5_128+j);
e6j = simde_mm_loadu_si128(e6_128+j);
e7j = simde_mm_loadu_si128(e7_128+j);
tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
tmp4 = simde_mm_unpacklo_epi8(e4j,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7)
tmp5 = simde_mm_unpacklo_epi8(e6j,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7)
tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15)
tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
tmp4 = simde_mm_unpackhi_epi8(e4j,e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7)
tmp5 = simde_mm_unpackhi_epi8(e6j,e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15)
tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
simde_mm_storeu_si128(f_128+k+4,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
simde_mm_storeu_si128(f_128+k+5,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
simde_mm_storeu_si128(f_128+k+6,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
simde_mm_storeu_si128(f_128+k+7,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
}
if ((j<<4) != EQm) {
int k2=k<<4;
fp=&f[k2];
for (int j2=(j<<4) ; j2< EQm ; j2++) {
*fp++ = e0[j2];
*fp++ = e1[j2];
*fp++ = e2[j2];
*fp++ = e3[j2];
*fp++ = e4[j2];
*fp++ = e5[j2];
*fp++ = e6[j2];
*fp++ = e7[j2];
}
}
e0 = (uint8_t *)e0_512;
e1 = (uint8_t *)e1_512;
e2 = (uint8_t *)e2_512;
e3 = (uint8_t *)e3_512;
e4 = (uint8_t *)e4_512;
e5 = (uint8_t *)e5_512;
e6 = (uint8_t *)e6_512;
e7 = (uint8_t *)e7_512;
f = (uint8_t *)f_512;
#endif
break;
default: AssertFatal(1==0,"Should be here!\n");
}
#ifdef USE128BIT
e0_128 = (simde__m128i *)e0;
e1_128 = (simde__m128i *)e1;
e2_128 = (simde__m128i *)e2;
e3_128 = (simde__m128i *)e3;
e4_128 = (simde__m128i *)e4;
e5_128 = (simde__m128i *)e5;
e6_128 = (simde__m128i *)e6;
e7_128 = (simde__m128i *)e7;
for (; i < (EQm & ~15); i += 16) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
simde__m128i tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1);
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
simde__m128i tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1);
simde__m128i tmp4 = simde_mm_unpacklo_epi8(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7)
simde__m128i tmp5 = simde_mm_unpacklo_epi8(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7)
// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
simde__m128i tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5);
// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
simde__m128i tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5);
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6));
// e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6));
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7));
// e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7));
#else
// original unoptimized loops
/*
for (int j = 0; j< EQm; j++,j2+=2){
for (int i = 0; i< Qm; i++){
f[(i+j*Qm)] = e[(i*EQm + j)];
}
}
*/
fp = f;
switch (Qm) {
case 2:
e0 = e;
e1 = e0 + EQm;
for (int j = 0, j2 = 0; j < EQm; j++, j2 += 2) {
fp = &f[j2];
fp[0] = e0[j];
fp[1] = e1[j];
tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15)
// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1);
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1);
tmp4 = simde_mm_unpackhi_epi8(e4j, e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7)
tmp5 = simde_mm_unpackhi_epi8(e6j, e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15)
// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5);
// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5);
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6));
// e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6));
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7));
// e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7));
}
break;
case 4:
e0 = e;
e1 = e0 + EQm;
e2 = e1 + EQm;
e3 = e2 + EQm;
for (int j = 0, j2 = 0; j < EQm; j++, j2 += 4) {
fp = &f[j2];
fp[0] = e0[j];
fp[1] = e1[j];
fp[2] = e2[j];
fp[3] = e3[j];
e0 = (uint8_t *)e0_128;
e1 = (uint8_t *)e1_128;
e2 = (uint8_t *)e2_128;
e3 = (uint8_t *)e3_128;
e4 = (uint8_t *)e4_128;
e5 = (uint8_t *)e5_128;
e6 = (uint8_t *)e6_128;
e7 = (uint8_t *)e7_128;
f = (uint8_t *)f_128;
#endif
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
*f++ = *e4++;
*f++ = *e5++;
*f++ = *e6++;
*f++ = *e7++;
}
break;
case 6:
e0 = e;
e1 = e0 + EQm;
e2 = e1 + EQm;
e3 = e2 + EQm;
e4 = e3 + EQm;
e5 = e4 + EQm;
fp = f;
for (int j = 0; j < EQm; j++) {
*fp++ = e0[j];
*fp++ = e1[j];
*fp++ = e2[j];
*fp++ = e3[j];
*fp++ = e4[j];
*fp++ = e5[j];
}
break;
case 8:
e0 = e;
e1 = e0 + EQm;
e2 = e1 + EQm;
e3 = e2 + EQm;
e4 = e3 + EQm;
e5 = e4 + EQm;
e6 = e5 + EQm;
e7 = e6 + EQm;
for (int j = 0, j2 = 0; j < EQm; j++, j2 += 8) {
fp = &f[j2];
fp[0] = e0[j];
fp[1] = e1[j];
fp[2] = e2[j];
fp[3] = e3[j];
fp[4] = e4[j];
fp[5] = e5[j];
fp[6] = e6[j];
fp[7] = e7[j];
}
break;
} break;
default:
AssertFatal(1 == 0, "Should never be here!\n");
AssertFatal(false, "Should be here!\n");
}
#endif
}
void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
openair1/PHY/CODING/nrLDPC_defs.h
+3 -3
View File
@@ -28,7 +28,7 @@
/**
\brief LDPC encoder parameter structure
\var n_segments number of segments in the transport block
\var macro_num index of the subset of segments to encode
\var first_seg index of the first segment of the subset to encode
within the transport block
\var gen_code flag to generate parity check code
0 -> encoding
@@ -52,7 +52,7 @@
*/
typedef struct {
unsigned int n_segments; // optim8seg
unsigned int macro_num; // optim8segmulti
unsigned int first_seg; // optim8segmulti
unsigned char gen_code; //orig
time_stats_t *tinput;
time_stats_t *tprep;
@@ -92,6 +92,6 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams,
int8_t *p_out,
t_nrLDPC_time_stats *,
decode_abort_t *ab);
typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t **, encoder_implemparams_t *);
typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *);
#endif
openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c
+2 -2
View File
@@ -41,11 +41,11 @@
#include "openair1/PHY/CODING/nrLDPC_extern.h"
#include "ldpc_generate_coefficient.c"
int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder_implemparams_t *impp)
int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_implemparams_t *impp)
{
const unsigned char *input = inputArray[0];
// channel input is the output of this function!
unsigned char *output = outputArray[0];
unsigned char *output = outputArray;
const int Zc = impp->Zc;
const int Kb = impp->Kb;
const short block_length = impp->K;
openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
+73 -69
View File
@@ -43,7 +43,7 @@
#include "ldpc_encode_parity_check.c"
#include "ldpc_generate_coefficient.c"
int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
{
//set_log(PHY, 4);
@@ -61,41 +61,10 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
unsigned int macro_segment, macro_segment_end;
macro_segment = 8*impp->macro_num;
macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments;
macro_segment = impp->first_seg;
macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments;
///printf("macro_segment: %d\n", macro_segment);
///printf("macro_segment_end: %d\n", macro_segment_end );
#ifdef __aarch64__
simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
simde__m128i andmask = simde_mm_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
simde__m128i zero128 = simde_mm_setzero_si128();
simde__m128i masks[8];
register simde__m128i c128;
masks[0] = simde_mm_set1_epi8(0x1);
masks[1] = simde_mm_set1_epi8(0x2);
masks[2] = simde_mm_set1_epi8(0x4);
masks[3] = simde_mm_set1_epi8(0x8);
masks[4] = simde_mm_set1_epi8(0x10);
masks[5] = simde_mm_set1_epi8(0x20);
masks[6] = simde_mm_set1_epi8(0x40);
masks[7] = simde_mm_set1_epi8(0x80);
#else
simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
simde__m256i zero256 = simde_mm256_setzero_si256();
simde__m256i masks[8];
register simde__m256i c256;
masks[0] = simde_mm256_set1_epi8(0x1);
masks[1] = simde_mm256_set1_epi8(0x2);
masks[2] = simde_mm256_set1_epi8(0x4);
masks[3] = simde_mm256_set1_epi8(0x8);
masks[4] = simde_mm256_set1_epi8(0x10);
masks[5] = simde_mm256_set1_epi8(0x20);
masks[6] = simde_mm256_set1_epi8(0x40);
masks[7] = simde_mm256_set1_epi8(0x80);
#endif
//determine number of bits in codeword
if (BG==1)
@@ -136,55 +105,90 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
if(impp->tinput != NULL) start_meas(impp->tinput);
//interleave up to 8 transport-block segements at a time
#if 0
for (i=0; i<block_length; i++) {
//for (j=0; j<n_segments; j++) {
for (j=macro_segment; j < macro_segment_end; j++) {
temp = (input[j][i/8]&(1<<(i&7)))>>(i&7);
//printf("c(%d,%d)=%d\n",j,i,temp);
c[i] |= (temp << (j-macro_segment));
}
}
#elif defined(__aarch64__)
for (int i=0; i<block_length>>4; i++) {
c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]);
//for (j=1; j<n_segments; j++) {
for (int j=macro_segment+1; j < macro_segment_end; j++) {
c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128);
}
((simde__m128i *)cc)[i] = c128;
}
unsigned int i_byte = 0;
for (int i=(block_length>>4)<<4;i<block_length;i++) {
//for (j=0; j<n_segments; j++) {
for (int j=macro_segment; j < macro_segment_end; j++) {
#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
const __m512i masks5[8] = { _mm512_set1_epi8(0x1), _mm512_set1_epi8(0x2),
_mm512_set1_epi8(0x4), _mm512_set1_epi8(0x8),
_mm512_set1_epi8(0x10), _mm512_set1_epi8(0x20),
_mm512_set1_epi8(0x40), _mm512_set1_epi8(0x80)};
const __m512i zero512 = _mm512_setzero_si512();
const uint8_t perm[64]__attribute__((aligned(64))) = {7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8,
23,22,21,20,19,18,17,16, 31,30,29,28,27,26,25,24,
39,38,37,36,35,34,33,32, 47,46,45,44,43,42,41,40,
55,54,53,52,51,50,49,48, 63,62,61,60,59,58,57,56};
register __m512i c512;
temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
//printf("c(%d,%d)=%d\n",j,i,temp);
cc[i] |= (temp << (j-macro_segment));
for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) {
unsigned int i = i_byte >> 6;
c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]);
for (int j = macro_segment + 1; j < macro_segment_end; j++) {
c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment]));
}
c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512);
((__m512i *)cc)[i] = c512;
}
#else
for (int i=0; i<block_length>>5; i++) {
#endif
#ifndef __aarch64__
simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
simde__m256i zero256 = simde_mm256_setzero_si256();
simde__m256i masks[8];
register simde__m256i c256;
masks[0] = simde_mm256_set1_epi8(0x1);
masks[1] = simde_mm256_set1_epi8(0x2);
masks[2] = simde_mm256_set1_epi8(0x4);
masks[3] = simde_mm256_set1_epi8(0x8);
masks[4] = simde_mm256_set1_epi8(0x10);
masks[5] = simde_mm256_set1_epi8(0x20);
masks[6] = simde_mm256_set1_epi8(0x40);
masks[7] = simde_mm256_set1_epi8(0x80);
for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) {
unsigned int i = i_byte >> 5;
c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]);
//for (j=1; j<n_segments; j++) {
for (int j=macro_segment+1; j < macro_segment_end; j++) {
c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[j])[i]), shufmask),andmask),zero256),masks[j-macro_segment]),c256);
}
((simde__m256i *)cc)[i] = c256;
}
#endif
for (int i=(block_length>>5)<<5;i<block_length;i++) {
//for (j=0; j<n_segments; j++) {
for (int j=macro_segment; j < macro_segment_end; j++) {
#ifdef __aarch64__
simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
simde__m128i andmask = simde_mm_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
simde__m128i zero128 = simde_mm_setzero_si128();
simde__m128i masks[8];
register simde__m128i c128;
masks[0] = simde_mm_set1_epi8(0x1);
masks[1] = simde_mm_set1_epi8(0x2);
masks[2] = simde_mm_set1_epi8(0x4);
masks[3] = simde_mm_set1_epi8(0x8);
masks[4] = simde_mm_set1_epi8(0x10);
masks[5] = simde_mm_set1_epi8(0x20);
masks[6] = simde_mm_set1_epi8(0x40);
masks[7] = simde_mm_set1_epi8(0x80);
temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
//printf("c(%d,%d)=%d\n",j,i,temp);
for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) {
unsigned int i = i_byte >> 4;
c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]);
for (int j=macro_segment+1; j < macro_segment_end; j++) {
c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128);
}
((simde__m128i *)cc)[i] = c128;
}
#endif
for (; i_byte < block_length; i_byte++) {
unsigned int i = i_byte;
for (int j = macro_segment; j < macro_segment_end; j++) {
temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
cc[i] |= (temp << (j-macro_segment));
}
}
#endif
if(impp->tinput != NULL) stop_meas(impp->tinput);
@@ -203,8 +207,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
return(-1);
}
}
memcpy(output[0],&cc[2*Zc],(block_length-(2*Zc)));
memcpy(output[0]+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
memcpy(output,&cc[2*Zc],(block_length-(2*Zc)));
memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
return 0;
}