Merge remote-tracking branch 'origin/ldpc_enc_avx512' into integration_2025_w20 (3168)
Improvements for LDPC encoding AVX512 modifications for LDPC encoding: interleaving, ZC384 BG1, output formatting for encoder, segmentation (memcpy instead of loop). some improvement in TX for aarch64 in same places where AVX512 support was added. Also, rate matching and interleaving are done on bytes containing 8 segments and reformatting of the output is done at the end of segment processing instead of after ldpc encoding. This improves the overall performance of the NR DL transmitter in gNB See the merge request description for numbers.
This commit is contained in:
Robert Schmidt
+14
-2
@@ -108,6 +108,7 @@ add_list_string_option(CMAKE_BUILD_TYPE "RelWithDebInfo" "Choose the type of bui
|
||||
# -so to avoid AVX512 instructions generation by gcc
|
||||
if(EXISTS "/proc/cpuinfo" AND NOT CROSS_COMPILE)
|
||||
file(STRINGS "/proc/cpuinfo" CPUFLAGS REGEX flags LIMIT_COUNT 1)
|
||||
file(STRINGS "/proc/cpuinfo" CPUPART REGEX "CPU part" LIMIT_COUNT 1)
|
||||
else()
|
||||
message(WARNING "did not find /proc/cpuinfo -- not setting any x86-specific compilation variables")
|
||||
endif()
|
||||
@@ -146,9 +147,20 @@ elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSSE3_NATIVE")
|
||||
endif()
|
||||
elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7l")
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt")
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt")
|
||||
elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -lgcc -lrt")
|
||||
# Neoverse CPU part codes available at https://github.com/util-linux/util-linux/blob/master/sys-utils/lscpu-arm.c
|
||||
if (CPUPART MATCHES "0xd40") # Neoverse-V1
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v1 -lgcc -lrt")
|
||||
elseif (CPUPART MATCHES "0xd4f") # Neoverse-V2
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v2 -ftree-vectorize -lgcc -lrt")
|
||||
elseif (CPUPART MATCHES "0xd0c") # Neoverse-N1
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n1 -lgcc -lrt")
|
||||
elseif (CPUPART MATCHES "0xd49") # Neoverse-N2
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n2 -ftree-vectorize -lgcc -lrt")
|
||||
else ()
|
||||
set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=native -lgcc -lrt")
|
||||
endif()
|
||||
else()
|
||||
message(FATAL_ERROR "compile for CPU architecture ${CPUARCH}, CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
|
||||
@@ -11,8 +11,8 @@ Ref :
|
||||
feptx_prec : 0.0
|
||||
feptx_ofdm : 60.0
|
||||
feptx_total : 150.0
|
||||
L1 Tx processing : 530.0
|
||||
DLSCH encoding : 220.0
|
||||
L1 Tx processing : 475.0
|
||||
DLSCH encoding : 200.0
|
||||
L1 Rx processing : 530.0
|
||||
PUSCH inner-receiver : 360.0
|
||||
Schedule Response : 1.0
|
||||
|
||||
@@ -11,8 +11,8 @@ Ref :
|
||||
feptx_prec : 15.0
|
||||
feptx_ofdm : 30.0
|
||||
feptx_total : 45.0
|
||||
L1 Tx processing : 205.0
|
||||
DLSCH encoding : 140.0
|
||||
L1 Tx processing : 175.0
|
||||
DLSCH encoding : 110.0
|
||||
L1 Rx processing : 345.0
|
||||
PUSCH inner-receiver : 150.0
|
||||
Schedule Response : 1.0
|
||||
|
||||
@@ -11,8 +11,8 @@ Ref :
|
||||
feptx_prec : 14.0
|
||||
feptx_ofdm : 30.0
|
||||
feptx_total : 80.0
|
||||
L1 Tx processing : 315.0
|
||||
DLSCH encoding : 155.0
|
||||
L1 Tx processing : 280.0
|
||||
DLSCH encoding : 140.0
|
||||
L1 Rx processing : 345.0
|
||||
PUSCH inner-receiver : 155.0
|
||||
Schedule Response : 1.0
|
||||
|
||||
@@ -11,8 +11,8 @@ Ref :
|
||||
feptx_prec : 13.0
|
||||
feptx_ofdm : 30.0
|
||||
feptx_total : 45.0
|
||||
L1 Tx processing : 160.0
|
||||
DLSCH encoding : 100.0
|
||||
L1 Tx processing : 145.0
|
||||
DLSCH encoding : 90.0
|
||||
L1 Rx processing : 290.0
|
||||
PUSCH inner-receiver : 115.0
|
||||
Schedule Response : 1.0
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
#include "nr_common.h"
|
||||
#include <limits.h>
|
||||
#include <math.h>
|
||||
#include <simde/x86/gfni.h>
|
||||
|
||||
#define C_SRS_NUMBER (64)
|
||||
#define B_SRS_NUMBER (4)
|
||||
@@ -131,8 +132,24 @@ void reverse_bits_u8(uint8_t const* in, size_t sz, uint8_t* out)
|
||||
DevAssert(in != NULL);
|
||||
DevAssert(out != NULL);
|
||||
|
||||
// Bit reversal implementation based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/
|
||||
#if defined(__GFNI__) && defined(__AVX512F__)
|
||||
int simde_sz = 64;
|
||||
int i = 0;
|
||||
int simde_bound = sz - simde_sz;
|
||||
for (; i <= simde_bound; i += simde_sz) {
|
||||
__m512i input = _mm512_loadu_epi8(&in[i]);
|
||||
__m512i reversed = _mm512_gf2p8affine_epi64_epi8(input, _mm512_set1_epi64(0x8040201008040201), 0);
|
||||
_mm512_storeu_epi8(&out[i], reversed);
|
||||
}
|
||||
|
||||
for (; i < sz; ++i) {
|
||||
out[i] = bit_reverse_table_256[in[i]];
|
||||
}
|
||||
#else
|
||||
for(size_t i = 0; i < sz; ++i)
|
||||
out[i] = bit_reverse_table_256[in[i]];
|
||||
#endif
|
||||
}
|
||||
|
||||
// Reverse bits implementation based on http://graphics.stanford.edu/~seander/bithacks.html
|
||||
|
||||
@@ -2133,10 +2133,8 @@ INPUT = \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc176_byte.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc352_byte.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc128_byte_128.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc224_byte.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_generate_coefficient.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc104_byte.c \
|
||||
@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc320_byte_128.c \
|
||||
|
||||
@@ -17,20 +17,6 @@ add_library(ldpc_orig MODULE
|
||||
set_target_properties(ldpc_orig PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS)
|
||||
|
||||
add_library(ldpc_optim MODULE
|
||||
nrLDPC_decoder/nrLDPC_decoder.c
|
||||
nrLDPC_encoder/ldpc_encoder_optim.c
|
||||
)
|
||||
set_target_properties(ldpc_optim PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
target_link_libraries(ldpc_optim PRIVATE ldpc_segment ldpc_gen_HEADERS)
|
||||
|
||||
add_library(ldpc_optim8seg MODULE
|
||||
nrLDPC_decoder/nrLDPC_decoder.c
|
||||
nrLDPC_encoder/ldpc_encoder_optim8seg.c
|
||||
)
|
||||
set_target_properties(ldpc_optim8seg PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
target_link_libraries(ldpc_optim8seg PRIVATE ldpc_segment ldpc_gen_HEADERS)
|
||||
|
||||
add_library(ldpc MODULE
|
||||
nrLDPC_decoder/nrLDPC_decoder.c
|
||||
nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
|
||||
@@ -38,13 +24,13 @@ add_library(ldpc MODULE
|
||||
set_target_properties(ldpc PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
target_link_libraries(ldpc PRIVATE ldpc_segment ldpc_gen_HEADERS)
|
||||
|
||||
add_dependencies(ldpctest ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(nr-softmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(nr-uesoftmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(nr_ulsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(nr_ulschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(nr_dlsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(nr_dlschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
|
||||
add_dependencies(ldpctest ldpc ldpc_orig)
|
||||
add_dependencies(nr-softmodem ldpc ldpc_orig)
|
||||
add_dependencies(nr-uesoftmodem ldpc ldpc_orig)
|
||||
add_dependencies(nr_ulsim ldpc ldpc_orig)
|
||||
add_dependencies(nr_ulschsim ldpc ldpc_orig)
|
||||
add_dependencies(nr_dlsim ldpc ldpc_orig)
|
||||
add_dependencies(nr_dlschsim ldpc ldpc_orig)
|
||||
|
||||
add_custom_target(nrLDPC_decoder_kernels_CL
|
||||
COMMAND gcc nrLDPC_decoder/nrLDPC_decoder_CL.c -dD -DNRLDPC_KERNEL_SOURCE -E -o ${CMAKE_CURRENT_BINARY_DIR}/nrLDPC_decoder_kernels_CL.clc
|
||||
|
||||
@@ -120,16 +120,16 @@ By default the function `int load_nrLDPClib(void)` looks for `libldpc.so`, this
|
||||
|
||||
#### Examples of ldpc shared lib selection when running nr softmodem's:
|
||||
|
||||
loading `libldpc_optim8seg.so` instead of `libldpc.so`:
|
||||
loading `libldpc_orig.so` instead of `libldpc.so`:
|
||||
|
||||
```
|
||||
./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5 --loader.ldpc.shlibversion _optim8seg
|
||||
./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5 --loader.ldpc.shlibversion _orig
|
||||
.......................
|
||||
[CONFIG] loader.ldpc.shlibversion set to default value ""
|
||||
[LIBCONFIG] loader.ldpc: 2/2 parameters successfully set, (1 to default value)
|
||||
[CONFIG] shlibversion set to _optim8seg from command line
|
||||
[CONFIG] shlibversion set to _orig from command line
|
||||
[CONFIG] loader.ldpc 1 options set from command line
|
||||
[LOADER] library libldpc_optim8seg.so successfully loaded
|
||||
[LOADER] library libldpc_orig.so successfully loaded
|
||||
........................
|
||||
```
|
||||
|
||||
|
||||
@@ -123,7 +123,7 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
|
||||
memset(estimated_output, 0, sizeof(estimated_output));
|
||||
uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
|
||||
uint8_t *channel_input_optim[MAX_NUM_DLSCH_SEGMENTS];
|
||||
uint8_t *channel_input_optim;
|
||||
// double channel_output[68 * 384];
|
||||
double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
|
||||
int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
|
||||
@@ -265,9 +265,9 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
memset(test_input[j], 0, ((K + 7) & ~7) / 8);
|
||||
channel_input[j] = malloc16(68 * 384);
|
||||
memset(channel_input[j], 0, 68 * 384);
|
||||
channel_input_optim[j] = malloc16(68 * 384);
|
||||
memset(channel_input_optim[j], 0, 68 * 384);
|
||||
}
|
||||
channel_input_optim = malloc16(68 * 384);
|
||||
memset(channel_input_optim, 0, 68 * 384);
|
||||
|
||||
// Fill input segments with random values
|
||||
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
|
||||
@@ -282,10 +282,10 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
}
|
||||
|
||||
encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K};
|
||||
impp.gen_code = 2;
|
||||
impp.gen_code = 1;
|
||||
|
||||
if (ntrials == 0)
|
||||
ldpc_orig.LDPCencoder(test_input, channel_input, &impp);
|
||||
ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp);
|
||||
impp.gen_code = 0;
|
||||
decode_abort_t dec_abort;
|
||||
init_abort(&dec_abort);
|
||||
@@ -294,23 +294,21 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
//// encoder
|
||||
start_meas(&time);
|
||||
for (int j = 0; j < n_segments; j++) {
|
||||
ldpc_orig.LDPCencoder(&test_input[j], &channel_input[j], &impp);
|
||||
ldpc_orig.LDPCencoder(&test_input[j], channel_input[j], &impp);
|
||||
}
|
||||
stop_meas(&time);
|
||||
|
||||
impp.n_segments = n_segments;
|
||||
for (int j = 0; j < (n_segments / 8 + 1); j++) {
|
||||
start_meas(&ret.time_optim);
|
||||
impp.macro_num = j;
|
||||
ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
|
||||
stop_meas(&ret.time_optim);
|
||||
}
|
||||
start_meas(&ret.time_optim);
|
||||
impp.first_seg = 0;
|
||||
ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
|
||||
stop_meas(&ret.time_optim);
|
||||
|
||||
if (ntrials == 1)
|
||||
for (int j = 0; j < n_segments; j++)
|
||||
for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
|
||||
if (channel_input[j][i] != channel_input_optim[j][i]) {
|
||||
printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], channel_input_optim[j][i]);
|
||||
if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) {
|
||||
printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
@@ -322,7 +320,7 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
printf("\ne %d..%d: ", i, i + 15);
|
||||
#endif
|
||||
|
||||
if (channel_input_optim[j][i - 2 * Zc] == 0)
|
||||
if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0)
|
||||
modulated_input[j][i] = 1.0; /// sqrt(2); //QPSK
|
||||
else
|
||||
modulated_input[j][i] = -1.0; /// sqrt(2);
|
||||
@@ -332,7 +330,7 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
|
||||
// Uncoded BER
|
||||
uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0;
|
||||
if (channel_output_uncoded != channel_input_optim[j][i - 2 * Zc])
|
||||
if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1))
|
||||
ret.errors_bit_uncoded++;
|
||||
}
|
||||
|
||||
@@ -396,8 +394,8 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
|
||||
free(test_input[j]);
|
||||
free(channel_input[j]);
|
||||
free(channel_input_optim[j]);
|
||||
}
|
||||
free(channel_input_optim);
|
||||
|
||||
print_meas(&time, "ldpc_encoder", NULL, NULL);
|
||||
print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL);
|
||||
|
||||
@@ -140,9 +140,7 @@ typedef struct nrLDPC_slot_decoding_parameters_s{
|
||||
* \struct nrLDPC_segment_encoding_parameters_s
|
||||
* \brief encoding parameter of segments
|
||||
* \var E input llr segment size
|
||||
* \var output input llr segment array
|
||||
* \var c Pointers to code blocks before LDPC encoding (38.212 V15.4.0 section 5.2.2)
|
||||
* flag indicating that the decoding of the segment was successful
|
||||
* IT MUST BE FILLED BY THE IMPLEMENTATION
|
||||
* \var ts_interleave interleaving time stats
|
||||
* \var ts_rate_match rate matching time stats
|
||||
@@ -150,7 +148,6 @@ typedef struct nrLDPC_slot_decoding_parameters_s{
|
||||
*/
|
||||
typedef struct nrLDPC_segment_encoding_parameters_s{
|
||||
int E;
|
||||
unsigned char *output;
|
||||
uint8_t *c;
|
||||
time_stats_t ts_interleave;
|
||||
time_stats_t ts_rate_match;
|
||||
@@ -179,6 +176,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{
|
||||
* \var F filler bits size
|
||||
* \var C number of segments
|
||||
* \var segments array of segments parameters
|
||||
* \var output input llr TB array
|
||||
*/
|
||||
typedef struct nrLDPC_TB_encoding_parameters_s{
|
||||
|
||||
@@ -202,6 +200,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{
|
||||
|
||||
uint32_t C;
|
||||
nrLDPC_segment_encoding_parameters_t *segments;
|
||||
unsigned char *output;
|
||||
} nrLDPC_TB_encoding_parameters_t;
|
||||
|
||||
/**
|
||||
|
||||
+378
-100
@@ -41,12 +41,184 @@
|
||||
|
||||
#include <syscall.h>
|
||||
|
||||
// #define DEBUG_LDPC_ENCODING
|
||||
// #define DEBUG_LDPC_ENCODING_FREE 1
|
||||
#define DEBUG_LDPC_ENCODING
|
||||
//#define DEBUG_LDPC_ENCODING_FREE 1
|
||||
|
||||
/**
|
||||
* \brief write nrLDPC_coding_segment_encoder output with interleaver output
|
||||
* the interleaver output has 8 bits from 8 different segments per byte
|
||||
* nrLDPC_coding_segment_encoder is made of concatenated packed segments
|
||||
* \param f input interleaved segment, 8 bits from 8 segments per byte
|
||||
* \param E number of bits per segment in f
|
||||
* \param f2 input interleaved segment, 8 bits from 8 segments per byte
|
||||
* interleaved with E2 if there was a E shift in the segment group
|
||||
* \param E2 number of bits per segment in f2
|
||||
* \param Eshift flag indicating if there was a E shift within the segment group
|
||||
* \param E2_first_segment index of the first segment of size E2 if there is a E shift in the segment group
|
||||
* \param nb_segments number of segments in the group
|
||||
* \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits
|
||||
* \param Eoffset offset in number of bits of the first segment of the segment group within output
|
||||
*/
|
||||
static void write_task_output(uint8_t *f,
|
||||
uint32_t E,
|
||||
uint8_t *f2,
|
||||
uint32_t E2,
|
||||
bool Eshift,
|
||||
uint32_t E2_first_segment,
|
||||
uint32_t nb_segments,
|
||||
uint8_t *output,
|
||||
uint32_t Eoffset)
|
||||
{
|
||||
|
||||
#if defined(__AVX512VBMI__)
|
||||
uint64_t *output_p = (uint64_t*)output;
|
||||
__m512i inc = _mm512_set1_epi8(0x1);
|
||||
|
||||
for (int i=0;i<E2;i+=64) {
|
||||
uint32_t Eoffset2 = Eoffset;
|
||||
__m512i bitperm = _mm512_set1_epi64(0x3830282018100800);
|
||||
if (i<E) {
|
||||
for (int j=0; j < E2_first_segment; j++) {
|
||||
// Note: Here and below for AVX2, we are using the 64-bit SIMD instruction
|
||||
// instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the <<
|
||||
// and >> operations are undefined and in fact don't give "0" which is
|
||||
// what we want here. The SIMD version do give 0 when the shift is 64
|
||||
uint32_t Eoffset2_byte = Eoffset2 >> 6;
|
||||
uint32_t Eoffset2_bit = Eoffset2 & 63;
|
||||
__m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i >> 6],bitperm);
|
||||
*(__m64*)(output_p + Eoffset2_byte) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit));
|
||||
*(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit)));
|
||||
Eoffset2 += E;
|
||||
bitperm = _mm512_add_epi8(bitperm ,inc);
|
||||
}
|
||||
} else {
|
||||
for (int j=0; j < E2_first_segment; j++) {
|
||||
Eoffset2 += E;
|
||||
bitperm = _mm512_add_epi8(bitperm ,inc);
|
||||
}
|
||||
}
|
||||
for (int j=E2_first_segment; j < nb_segments; j++) {
|
||||
uint32_t Eoffset2_byte = Eoffset2 >> 6;
|
||||
uint32_t Eoffset2_bit = Eoffset2 & 63;
|
||||
__m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i >> 6],bitperm);
|
||||
*(__m64*)(output_p + Eoffset2_byte) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit));
|
||||
*(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit)));
|
||||
Eoffset2 += E2;
|
||||
bitperm = _mm512_add_epi8(bitperm ,inc);
|
||||
}
|
||||
output_p++;
|
||||
}
|
||||
|
||||
#elif defined(__aarch64__)
|
||||
uint16_t *output_p = (uint16_t*)output;
|
||||
const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = {
|
||||
{0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0
|
||||
{-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1
|
||||
{-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2
|
||||
{-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3
|
||||
{-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4
|
||||
{-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5
|
||||
{-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6
|
||||
{-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7
|
||||
const uint8_t __attribute__ ((aligned (16))) masks[16] =
|
||||
{0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80};
|
||||
int8x16_t vshift[8];
|
||||
for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]);
|
||||
uint8x16_t vmask = vld1q_u8(masks);
|
||||
|
||||
for (int i=0;i<E2;i+=16) {
|
||||
uint32_t Eoffset2 = Eoffset;
|
||||
if (i<E) {
|
||||
for (int j=0; j < E2_first_segment; j++) {
|
||||
uint32_t Eoffset2_byte = Eoffset2 >> 4;
|
||||
uint32_t Eoffset2_bit = Eoffset2 & 15;
|
||||
uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[i >> 4],vshift[j]),vmask);
|
||||
int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift));
|
||||
tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8);
|
||||
*(output_p + Eoffset2_byte) |= (uint16_t)(tmp<<Eoffset2_bit);
|
||||
*(output_p + Eoffset2_byte+1) |= (uint16_t)(tmp>>(16-Eoffset2_bit));
|
||||
Eoffset2 += E;
|
||||
}
|
||||
} else {
|
||||
for (int j=0; j < E2_first_segment; j++) {
|
||||
Eoffset2 += E;
|
||||
}
|
||||
}
|
||||
for (int j=E2_first_segment; j < nb_segments; j++) {
|
||||
uint32_t Eoffset2_byte = Eoffset2 >> 4;
|
||||
uint32_t Eoffset2_bit = Eoffset2 & 15;
|
||||
uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i >> 4],vshift[j]),vmask);
|
||||
int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift));
|
||||
tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8);
|
||||
*(output_p + Eoffset2_byte) |= (uint16_t)(tmp<<Eoffset2_bit);
|
||||
*(output_p + Eoffset2_byte+1) |= (uint16_t)(tmp>>(16-Eoffset2_bit));
|
||||
Eoffset2 += E2;
|
||||
}
|
||||
output_p++;
|
||||
}
|
||||
|
||||
#else
|
||||
uint32_t *output_p = (uint32_t*)output;
|
||||
|
||||
for (int i=0; i < E2; i += 32) {
|
||||
uint32_t Eoffset2 = Eoffset;
|
||||
if (i < E) {
|
||||
for (int j = 0; j < E2_first_segment; j++) {
|
||||
// Note: Here and below, we are using the 64-bit SIMD instruction
|
||||
// instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the <<
|
||||
// and >> operations are undefined and in fact don't give "0" which is
|
||||
// what we want here. The SIMD version do give 0 when the shift is 64
|
||||
uint32_t Eoffset2_byte = Eoffset2 >> 5;
|
||||
uint32_t Eoffset2_bit = Eoffset2 & 31;
|
||||
int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i >> 5], 7 - j));
|
||||
__m64 tmp64 = _mm_set1_pi32(tmp);
|
||||
__m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte));
|
||||
__m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit));
|
||||
__m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit)));
|
||||
*(output_p + Eoffset2_byte) = _m_to_int(tmp64b);
|
||||
*(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32));
|
||||
Eoffset2 += E;
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < E2_first_segment; j++) {
|
||||
Eoffset2 += E;
|
||||
}
|
||||
}
|
||||
for (int j = E2_first_segment; j < nb_segments; j++) {
|
||||
uint32_t Eoffset2_byte = Eoffset2 >> 5;
|
||||
uint32_t Eoffset2_bit = Eoffset2 & 31;
|
||||
int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i >> 5], 7 - j));
|
||||
__m64 tmp64 = _mm_set1_pi32(tmp);
|
||||
__m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte));
|
||||
__m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit));
|
||||
__m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit)));
|
||||
*(output_p + Eoffset2_byte) = _m_to_int(tmp64b);
|
||||
*(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32));
|
||||
Eoffset2 += E2;
|
||||
}
|
||||
output_p++;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* \typedef ldpc8blocks_args_t
|
||||
* \struct ldpc8blocks_args_s
|
||||
* \brief Arguments of an encoding task
|
||||
* encode up to 8 code blocks
|
||||
* \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
|
||||
* \var impp encoder implementation specific parameters for the task
|
||||
* \var f first interleaver output to be filled by the task
|
||||
* \var f2 second interleaver output to be filled by the task
|
||||
* in case of a shift of E in the code blocks group processed by the task
|
||||
*/
|
||||
typedef struct ldpc8blocks_args_s {
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters;
|
||||
encoder_implemparams_t impp;
|
||||
uint8_t *f;
|
||||
uint8_t *f2;
|
||||
} ldpc8blocks_args_t;
|
||||
|
||||
static void ldpc8blocks(void *p)
|
||||
@@ -64,156 +236,214 @@ static void ldpc8blocks(void *p)
|
||||
|
||||
// nrLDPC_encoder output is in "d"
|
||||
// let's make this interface happy!
|
||||
uint8_t tmp[8][68 * 384] __attribute__((aligned(32)));
|
||||
uint8_t *d[impp->n_segments];
|
||||
for (int rr = impp->macro_num * 8, i = 0; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++, i++)
|
||||
d[rr] = tmp[i];
|
||||
|
||||
uint8_t d[68 * 384] __attribute__((aligned(64)));
|
||||
uint8_t *c[nrLDPC_TB_encoding_parameters->C];
|
||||
unsigned int macro_segment, macro_segment_end;
|
||||
|
||||
|
||||
macro_segment = impp->first_seg;
|
||||
macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments;
|
||||
for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++)
|
||||
c[r] = nrLDPC_TB_encoding_parameters->segments[r].c;
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode);
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode);
|
||||
LDPCencoder(c, d, impp);
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode);
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode);
|
||||
// Compute where to place in output buffer that is concatenation of all segments
|
||||
|
||||
uint32_t r_offset = 0;
|
||||
for (int i = 0; i < impp->macro_num * 8; i++)
|
||||
r_offset += nrLDPC_TB_encoding_parameters->segments[i].E;
|
||||
for (int rr = impp->macro_num * 8; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++) {
|
||||
if (impp->F > 0) {
|
||||
// writing into positions d[r][k-2Zc] as in clause 5.3.2 step 2) in 38.212
|
||||
memset(&d[rr][impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F);
|
||||
}
|
||||
|
||||
#ifdef DEBUG_LDPC_ENCODING
|
||||
LOG_D(PHY, "rvidx in encoding = %d\n", rel15->rvIndex[0]);
|
||||
LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
|
||||
#endif
|
||||
uint32_t E = nrLDPC_TB_encoding_parameters->segments[rr].E;
|
||||
LOG_D(NR_PHY,
|
||||
"Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, Filler bits %d, Filler offset %d mod_order %d, nb_rb "
|
||||
const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E;
|
||||
uint32_t E2=E;
|
||||
bool Eshift=false;
|
||||
uint32_t Emax = E;
|
||||
for (int s=macro_segment;s<macro_segment_end;s++)
|
||||
if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
|
||||
E2=nrLDPC_TB_encoding_parameters->segments[s].E;
|
||||
Eshift=true;
|
||||
if(E2 > Emax)
|
||||
Emax = E2;
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
LOG_D(NR_PHY,
|
||||
"Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
|
||||
"%d,nrOfLayer %d)...\n",
|
||||
rr,
|
||||
macro_segment,
|
||||
macro_segment_end,
|
||||
impp->n_segments,
|
||||
G,
|
||||
E,E2,
|
||||
impp->F,
|
||||
impp->K - impp->F - 2 * impp->Zc,
|
||||
mod_order,
|
||||
nb_rb,
|
||||
nrLDPC_TB_encoding_parameters->nb_layers);
|
||||
/*
|
||||
printf("Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
|
||||
"%d,nrOfLayer %d)...\n",
|
||||
macro_segment,
|
||||
macro_segment_end,
|
||||
impp->n_segments,
|
||||
G,
|
||||
E,E2,
|
||||
impp->F,
|
||||
impp->K - impp->F - 2 * impp->Zc,
|
||||
mod_order,
|
||||
nb_rb,
|
||||
nrLDPC_TB_encoding_parameters->nb_layers);
|
||||
*/
|
||||
uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
|
||||
|
||||
uint8_t e[Emax] __attribute__((aligned(64)));
|
||||
// Interleaver outputs are stored in the output arrays
|
||||
uint8_t *f = args->f;
|
||||
uint8_t *f2 = args->f2;
|
||||
bzero(e, Emax);
|
||||
bzero(f, ceil_mod(E, 64));
|
||||
if (Eshift) {
|
||||
bzero(f2, ceil_mod(E2, 64));
|
||||
}
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match);
|
||||
nr_rate_matching_ldpc(Tbslbrm,
|
||||
impp->BG,
|
||||
impp->Zc,
|
||||
d,
|
||||
e,
|
||||
impp->n_segments,
|
||||
impp->F,
|
||||
impp->K - impp->F - 2 * impp->Zc,
|
||||
nrLDPC_TB_encoding_parameters->rv_index,
|
||||
Emax);
|
||||
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match);
|
||||
if (impp->K - impp->F - 2 * impp->Zc > E) {
|
||||
LOG_E(PHY,
|
||||
"dlsch coding A %d Kr %d G %d (nb_rb %d, mod_order %d)\n",
|
||||
A,
|
||||
impp->K,
|
||||
G,
|
||||
nb_rb,
|
||||
(int)mod_order);
|
||||
|
||||
LOG_E(NR_PHY,
|
||||
"Rate Matching, Code segments %d...%d/%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
|
||||
"nb_rb %d)...\n",
|
||||
macro_segment,
|
||||
macro_segment_end,
|
||||
impp->n_segments,
|
||||
G,
|
||||
E,
|
||||
impp->K,
|
||||
impp->F,
|
||||
impp->K - impp->F - 2 * impp->Zc,
|
||||
mod_order,
|
||||
nb_rb,
|
||||
nrLDPC_TB_encoding_parameters->nb_layers);
|
||||
|
||||
uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
|
||||
|
||||
uint8_t e[E];
|
||||
bzero(e, E);
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match);
|
||||
nr_rate_matching_ldpc(Tbslbrm,
|
||||
impp->BG,
|
||||
impp->Zc,
|
||||
d[rr],
|
||||
e,
|
||||
impp->n_segments,
|
||||
impp->F,
|
||||
impp->K - impp->F - 2 * impp->Zc,
|
||||
nrLDPC_TB_encoding_parameters->rv_index,
|
||||
E);
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match);
|
||||
if (impp->K - impp->F - 2 * impp->Zc > E) {
|
||||
LOG_E(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp->K, G, nb_rb, (int)mod_order);
|
||||
|
||||
LOG_E(NR_PHY,
|
||||
"Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, K %d, Filler bits %d, Filler offset %d mod_order %d, "
|
||||
"nb_rb %d)...\n",
|
||||
rr,
|
||||
impp->n_segments,
|
||||
G,
|
||||
E,
|
||||
impp->K,
|
||||
impp->F,
|
||||
impp->K - impp->F - 2 * impp->Zc,
|
||||
mod_order,
|
||||
nb_rb);
|
||||
}
|
||||
#ifdef DEBUG_LDPC_ENCODING
|
||||
|
||||
for (int i = 0; i < 16; i++)
|
||||
printf("output ratematching e[%d]= %d r_offset %u\n", i, e[i], r_offset);
|
||||
|
||||
#endif
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave);
|
||||
nr_interleaving_ldpc(E, mod_order, e, impp->output + r_offset);
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave);
|
||||
#ifdef DEBUG_LDPC_ENCODING
|
||||
|
||||
for (int i = 0; i < 16; i++)
|
||||
printf("output interleaving f[%d]= %d r_offset %u\n", i, impp->output[i + r_offset], r_offset);
|
||||
|
||||
if (r == impp->n_segments - 1)
|
||||
write_output("enc_output.m", "enc", impp->output, G, 1, 4);
|
||||
|
||||
#endif
|
||||
r_offset += E;
|
||||
nb_rb);
|
||||
}
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave);
|
||||
nr_interleaving_ldpc(E,
|
||||
mod_order,
|
||||
e,
|
||||
f);
|
||||
if (Eshift)
|
||||
nr_interleaving_ldpc(E2,
|
||||
mod_order,
|
||||
e,
|
||||
f2);
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave);
|
||||
|
||||
// Task running in // completed
|
||||
completed_task_ans(impp->ans);
|
||||
}
|
||||
|
||||
static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters,
|
||||
static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters,
|
||||
int dlsch_id,
|
||||
thread_info_tm_t *t_info)
|
||||
thread_info_tm_t *t_info,
|
||||
uint8_t **f,
|
||||
uint8_t **f2)
|
||||
{
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
|
||||
encoder_implemparams_t impp = {0};
|
||||
encoder_implemparams_t common_segment_params = {
|
||||
.n_segments = nrLDPC_TB_encoding_parameters->C,
|
||||
.tinput = nrLDPC_slot_encoding_parameters->tinput,
|
||||
.tprep = nrLDPC_slot_encoding_parameters->tprep,
|
||||
.tparity = nrLDPC_slot_encoding_parameters->tparity,
|
||||
.toutput = nrLDPC_slot_encoding_parameters->toutput,
|
||||
.Kb = nrLDPC_TB_encoding_parameters->Kb,
|
||||
.Zc = nrLDPC_TB_encoding_parameters->Z,
|
||||
.BG = nrLDPC_TB_encoding_parameters->BG,
|
||||
.output = nrLDPC_TB_encoding_parameters->output,
|
||||
.K = nrLDPC_TB_encoding_parameters->K,
|
||||
.F = nrLDPC_TB_encoding_parameters->F,
|
||||
};
|
||||
|
||||
impp.n_segments = nrLDPC_TB_encoding_parameters->C;
|
||||
impp.tinput = nrLDPC_slot_encoding_parameters->tinput;
|
||||
impp.tprep = nrLDPC_slot_encoding_parameters->tprep;
|
||||
impp.tparity = nrLDPC_slot_encoding_parameters->tparity;
|
||||
impp.toutput = nrLDPC_slot_encoding_parameters->toutput;
|
||||
impp.Kb = nrLDPC_TB_encoding_parameters->Kb;
|
||||
impp.Zc = nrLDPC_TB_encoding_parameters->Z;
|
||||
impp.BG = nrLDPC_TB_encoding_parameters->BG;
|
||||
impp.output = nrLDPC_TB_encoding_parameters->segments->output;
|
||||
impp.K = nrLDPC_TB_encoding_parameters->K;
|
||||
impp.F = nrLDPC_TB_encoding_parameters->F;
|
||||
const size_t n_tasks = (common_segment_params.n_segments + 7) >> 3;
|
||||
|
||||
size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1));
|
||||
|
||||
for (int j = 0; j < n_seg; j++) {
|
||||
for (int j = 0; j < n_tasks; j++) {
|
||||
ldpc8blocks_args_t *perJobImpp = &((ldpc8blocks_args_t *)t_info->buf)[t_info->len];
|
||||
DevAssert(t_info->len < t_info->cap);
|
||||
impp.ans = t_info->ans;
|
||||
t_info->len += 1;
|
||||
|
||||
impp.macro_num = j;
|
||||
perJobImpp->impp = impp;
|
||||
perJobImpp->impp = common_segment_params;
|
||||
perJobImpp->impp.first_seg = j * 8;
|
||||
perJobImpp->impp.ans = t_info->ans;
|
||||
perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters;
|
||||
perJobImpp->f = f[j];
|
||||
perJobImpp->f2 = f2[j];
|
||||
|
||||
task_t t = {.func = ldpc8blocks, .args = perJobImpp};
|
||||
pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t);
|
||||
}
|
||||
return n_seg;
|
||||
return n_tasks;
|
||||
}
|
||||
|
||||
int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
|
||||
{
|
||||
int nbTasks = 0;
|
||||
|
||||
uint32_t Emax = 0;
|
||||
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
|
||||
// Compute number of tasks to encode TB
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
|
||||
nbTasks += n_seg;
|
||||
|
||||
// Search for maximum E for sizing encoder output f and f2
|
||||
for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
|
||||
uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
|
||||
Emax = E > Emax ? E : Emax;
|
||||
}
|
||||
}
|
||||
|
||||
// Create f and f2 to old encoding tasks outputs
|
||||
uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
|
||||
uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
|
||||
uint8_t f2[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
|
||||
|
||||
ldpc8blocks_args_t arr[nbTasks];
|
||||
task_ans_t ans;
|
||||
init_task_ans(&ans, nbTasks);
|
||||
thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbTasks, .ans = &ans};
|
||||
|
||||
int nbEncode = 0;
|
||||
nbTasks = 0;
|
||||
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
|
||||
nbEncode += nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info);
|
||||
// For easier indexing we store the pointers to sub arrays of f and f2 in pointer arrays
|
||||
// Then a function to which we pass the pointer arrays can directly use f_2d[j] ans f2_2d[j]
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
|
||||
uint8_t *f_2d[n_seg];
|
||||
uint8_t *f2_2d[n_seg];
|
||||
for (int j = 0; j < n_seg; j++) {
|
||||
f_2d[j] = &f[nbTasks + j][0];
|
||||
f2_2d[j] = &f2[nbTasks + j][0];
|
||||
}
|
||||
|
||||
// Launch tasks encoding TB
|
||||
nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info, f_2d, f2_2d);
|
||||
|
||||
nbTasks += n_seg;
|
||||
}
|
||||
if (nbEncode < nbTasks) {
|
||||
completed_many_task_ans(&ans, nbTasks - nbEncode);
|
||||
@@ -221,5 +451,53 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
|
||||
// Execute thread pool tasks
|
||||
join_task_ans(&ans);
|
||||
|
||||
// Write output
|
||||
nbTasks = 0;
|
||||
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
uint32_t C = nrLDPC_TB_encoding_parameters->C;
|
||||
size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1));
|
||||
|
||||
time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput;
|
||||
|
||||
for (int j = 0; j < n_seg; j++) {
|
||||
unsigned int macro_segment = j * 8;
|
||||
unsigned int macro_segment_end = (C > macro_segment + 8) ? macro_segment + 8 : C;
|
||||
const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E;
|
||||
uint32_t E2 = E, E2_first_segment = macro_segment_end - macro_segment;
|
||||
bool Eshift = false;
|
||||
uint32_t Emax = E;
|
||||
for (int s = macro_segment; s < macro_segment_end; s++) {
|
||||
if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
|
||||
E2 = nrLDPC_TB_encoding_parameters->segments[s].E;
|
||||
Eshift = true;
|
||||
E2_first_segment = s-macro_segment;
|
||||
if(E2 > Emax)
|
||||
Emax = E2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(toutput != NULL) start_meas(toutput);
|
||||
|
||||
uint32_t Eoffset=0;
|
||||
for (int s=0; s<macro_segment; s++)
|
||||
Eoffset += (nrLDPC_TB_encoding_parameters->segments[s].E);
|
||||
|
||||
write_task_output(&f[nbTasks + j][0],
|
||||
E,
|
||||
&f2[nbTasks + j][0],
|
||||
E2,
|
||||
Eshift,
|
||||
E2_first_segment,
|
||||
macro_segment_end - macro_segment,
|
||||
nrLDPC_TB_encoding_parameters->output,
|
||||
Eoffset);
|
||||
|
||||
if(toutput != NULL) stop_meas(toutput);
|
||||
}
|
||||
nbTasks += n_seg;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -29,274 +29,419 @@ static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
|
||||
|
||||
void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
|
||||
{
|
||||
uint32_t EQm;
|
||||
|
||||
EQm = E / Qm;
|
||||
const uint32_t EQm = E / Qm;
|
||||
memset(f, 0, E * sizeof(uint8_t));
|
||||
uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7;
|
||||
uint8_t *fp;
|
||||
#if 0 // def __WASAVX2__
|
||||
simde__m256i tmp0,tmp1,tmp2,tmp0b,tmp1b,tmp3,tmp4,tmp5;
|
||||
simde__m256i *e0_256,*e1_256,*e2_256,*e3_256,*e4_256,*e5_256,*e6_256,*e7_256;
|
||||
|
||||
simde__m256i *f_256=(simde__m256i *)f;
|
||||
|
||||
uint8_t *fp2;
|
||||
switch(Qm) {
|
||||
case 2:
|
||||
e0=e;
|
||||
e1=e0+EQm;
|
||||
e0_256=(simde__m256i *)e0;
|
||||
e1_256=(simde__m256i *)e1;
|
||||
for (int k=0,j=0;j<EQm>>5;j++,k+=2) {
|
||||
f_256[k] = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]);
|
||||
f_256[k+1] = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]);
|
||||
}
|
||||
break;
|
||||
case 4:
|
||||
e0=e;
|
||||
e1=e0+EQm;
|
||||
e2=e1+EQm;
|
||||
e3=e2+EQm;
|
||||
e0_256=(simde__m256i *)e0;
|
||||
e1_256=(simde__m256i *)e1;
|
||||
e2_256=(simde__m256i *)e2;
|
||||
e3_256=(simde__m256i *)e3;
|
||||
for (int k=0,j=0;j<EQm>>5;j++,k+=4) {
|
||||
tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
|
||||
f_256[k] = simde_mm256_unpacklo_epi8(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
f_256[k+1] = simde_mm256_unpackhi_epi8(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
|
||||
tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
|
||||
f_256[k+2] = simde_mm256_unpacklo_epi8(tmp0,tmp1);
|
||||
f_256[k+3] = simde_mm256_unpackhi_epi8(tmp0,tmp1);
|
||||
}
|
||||
break;
|
||||
case 6:
|
||||
e0=e;
|
||||
e1=e0+EQm;
|
||||
e2=e1+EQm;
|
||||
e3=e2+EQm;
|
||||
e4=e3+EQm;
|
||||
e5=e4+EQm;
|
||||
e0_256=(simde__m256i *)e0;
|
||||
e1_256=(simde__m256i *)e1;
|
||||
e2_256=(simde__m256i *)e2;
|
||||
e3_256=(simde__m256i *)e3;
|
||||
e4_256=(simde__m256i *)e4;
|
||||
e5_256=(simde__m256i *)e5;
|
||||
|
||||
for (int j=0,k=0;j<EQm>>5;j++,k+=192) {
|
||||
fp = f+k;
|
||||
fp2 = fp+96;
|
||||
|
||||
tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
|
||||
tmp0b = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
tmp1b = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
tmp0 = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
|
||||
*((uint32_t*)fp) = simde_mm256_extract_epi32(tmp0b,0);
|
||||
*((uint16_t*)(fp+4)) = simde_mm256_extract_epi16(tmp0,0);
|
||||
*((uint32_t*)(fp+6)) = simde_mm256_extract_epi32(tmp0b,1);
|
||||
*((uint16_t*)(fp+10)) = simde_mm256_extract_epi16(tmp0,1);
|
||||
*((uint32_t*)(fp+12)) = simde_mm256_extract_epi32(tmp0b,2);
|
||||
*((uint16_t*)(fp+16)) = simde_mm256_extract_epi16(tmp0,2);
|
||||
*((uint32_t*)(fp+18)) = simde_mm256_extract_epi32(tmp0b,3);
|
||||
*((uint16_t*)(fp+22)) = simde_mm256_extract_epi16(tmp0,3);
|
||||
*((uint32_t*)(fp+24)) = simde_mm256_extract_epi32(tmp0b,4);
|
||||
*((uint16_t*)(fp+26)) = simde_mm256_extract_epi16(tmp0,4);
|
||||
*((uint32_t*)(fp+30)) = simde_mm256_extract_epi32(tmp0b,5);
|
||||
*((uint16_t*)(fp+34)) = simde_mm256_extract_epi16(tmp0,5);
|
||||
*((uint32_t*)(fp+36)) = simde_mm256_extract_epi32(tmp0,6);
|
||||
*((uint16_t*)(fp+40)) = simde_mm256_extract_epi16(tmp0,6);
|
||||
*((uint32_t*)(fp+42)) = simde_mm256_extract_epi32(tmp0b,7);
|
||||
*((uint16_t*)(fp+46)) = simde_mm256_extract_epi16(tmp0,7);
|
||||
|
||||
*((uint32_t*)(fp+48)) = simde_mm256_extract_epi32(tmp1b,0);
|
||||
*((uint16_t*)(fp+52)) = simde_mm256_extract_epi16(tmp0,8);
|
||||
*((uint32_t*)(fp+56)) = simde_mm256_extract_epi32(tmp1b,1);
|
||||
*((uint16_t*)(fp+60)) = simde_mm256_extract_epi16(tmp0,9);
|
||||
*((uint32_t*)(fp+62)) = simde_mm256_extract_epi32(tmp1b,2);
|
||||
*((uint16_t*)(fp+66)) = simde_mm256_extract_epi16(tmp0,10);
|
||||
*((uint32_t*)(fp+68)) = simde_mm256_extract_epi32(tmp1b,3);
|
||||
*((uint16_t*)(fp+72)) = simde_mm256_extract_epi16(tmp0,11);
|
||||
*((uint32_t*)(fp+74)) = simde_mm256_extract_epi32(tmp1b,4);
|
||||
*((uint16_t*)(fp+76)) = simde_mm256_extract_epi16(tmp0,12);
|
||||
*((uint32_t*)(fp+80)) = simde_mm256_extract_epi32(tmp1b,5);
|
||||
*((uint16_t*)(fp+82)) = simde_mm256_extract_epi16(tmp0,13);
|
||||
*((uint32_t*)(fp+86)) = simde_mm256_extract_epi32(tmp1b,6);
|
||||
*((uint16_t*)(fp+90)) = simde_mm256_extract_epi16(tmp0,14);
|
||||
*((uint32_t*)(fp+92)) = simde_mm256_extract_epi32(tmp1b,7);
|
||||
*((uint16_t*)(fp+94)) = simde_mm256_extract_epi16(tmp0,15);
|
||||
|
||||
tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
|
||||
tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
|
||||
tmp0b = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
|
||||
tmp1b = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
|
||||
tmp0 = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
|
||||
*((uint32_t*)fp2) = simde_mm256_extract_epi32(tmp0b,0);
|
||||
*((uint16_t*)(fp2+4)) = simde_mm256_extract_epi16(tmp0,0);
|
||||
*((uint32_t*)(fp2+6)) = simde_mm256_extract_epi32(tmp0b,1);
|
||||
*((uint16_t*)(fp2+10)) = simde_mm256_extract_epi16(tmp0,1);
|
||||
*((uint32_t*)(fp2+12)) = simde_mm256_extract_epi32(tmp0b,2);
|
||||
*((uint16_t*)(fp2+16)) = simde_mm256_extract_epi16(tmp0,2);
|
||||
*((uint32_t*)(fp2+18)) = simde_mm256_extract_epi32(tmp0b,3);
|
||||
*((uint16_t*)(fp2+22)) = simde_mm256_extract_epi16(tmp0,3);
|
||||
*((uint32_t*)(fp2+24)) = simde_mm256_extract_epi32(tmp0b,4);
|
||||
*((uint16_t*)(fp2+26)) = simde_mm256_extract_epi16(tmp0,4);
|
||||
*((uint32_t*)(fp2+30)) = simde_mm256_extract_epi32(tmp0b,5);
|
||||
*((uint16_t*)(fp2+34)) = simde_mm256_extract_epi16(tmp0,5);
|
||||
*((uint32_t*)(fp2+36)) = simde_mm256_extract_epi32(tmp0,6);
|
||||
*((uint16_t*)(fp2+40)) = simde_mm256_extract_epi16(tmp0,6);
|
||||
*((uint32_t*)(fp2+42)) = simde_mm256_extract_epi32(tmp0b,7);
|
||||
*((uint16_t*)(fp2+46)) = simde_mm256_extract_epi16(tmp0,7);
|
||||
|
||||
*((uint32_t*)(fp2+48)) = simde_mm256_extract_epi32(tmp1b,0);
|
||||
*((uint16_t*)(fp2+52)) = simde_mm256_extract_epi16(tmp0,8);
|
||||
*((uint32_t*)(fp2+56)) = simde_mm256_extract_epi32(tmp1b,1);
|
||||
*((uint16_t*)(fp2+60)) = simde_mm256_extract_epi16(tmp0,9);
|
||||
*((uint32_t*)(fp2+62)) = simde_mm256_extract_epi32(tmp1b,2);
|
||||
*((uint16_t*)(fp2+66)) = simde_mm256_extract_epi16(tmp0,10);
|
||||
*((uint32_t*)(fp2+68)) = simde_mm256_extract_epi32(tmp1b,3);
|
||||
*((uint16_t*)(fp2+72)) = simde_mm256_extract_epi16(tmp0,11);
|
||||
*((uint32_t*)(fp2+74)) = simde_mm256_extract_epi32(tmp1b,4);
|
||||
*((uint16_t*)(fp2+76)) = simde_mm256_extract_epi16(tmp0,12);
|
||||
*((uint32_t*)(fp2+80)) = simde_mm256_extract_epi32(tmp1b,5);
|
||||
*((uint16_t*)(fp2+82)) = simde_mm256_extract_epi16(tmp0,13);
|
||||
*((uint32_t*)(fp2+86)) = simde_mm256_extract_epi32(tmp1b,6);
|
||||
*((uint16_t*)(fp2+90)) = simde_mm256_extract_epi16(tmp0,14);
|
||||
*((uint32_t*)(fp2+92)) = simde_mm256_extract_epi32(tmp1b,7);
|
||||
*((uint16_t*)(fp2+94)) = simde_mm256_extract_epi16(tmp0,15);
|
||||
}
|
||||
break;
|
||||
case 8:
|
||||
e0=e;
|
||||
e1=e0+EQm;
|
||||
e2=e1+EQm;
|
||||
e3=e2+EQm;
|
||||
e4=e3+EQm;
|
||||
e5=e4+EQm;
|
||||
e6=e5+EQm;
|
||||
e7=e6+EQm;
|
||||
|
||||
e0_256=(simde__m256i *)e0;
|
||||
e1_256=(simde__m256i *)e1;
|
||||
e2_256=(simde__m256i *)e2;
|
||||
e3_256=(simde__m256i *)e3;
|
||||
e4_256=(simde__m256i *)e4;
|
||||
e5_256=(simde__m256i *)e5;
|
||||
e6_256=(simde__m256i *)e6;
|
||||
e7_256=(simde__m256i *)e7;
|
||||
for (int k=0,j=0;j<EQm>>5;j++,k+=8) {
|
||||
tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
|
||||
tmp2 = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
|
||||
tmp3 = simde_mm256_unpacklo_epi8(e6_256[j],e7_256[j]); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
|
||||
tmp4 = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
tmp5 = simde_mm256_unpacklo_epi16(tmp2,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
|
||||
f_256[k] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
|
||||
f_256[k+1] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
|
||||
|
||||
tmp4 = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
tmp5 = simde_mm256_unpackhi_epi16(tmp2,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
|
||||
f_256[k+2] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
|
||||
f_256[k+3] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
|
||||
|
||||
tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
|
||||
tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
|
||||
tmp2 = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
|
||||
tmp3 = simde_mm256_unpackhi_epi8(e6_256[j],e7_256[j]); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
|
||||
tmp4 = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
|
||||
tmp5 = simde_mm256_unpacklo_epi16(tmp2,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
|
||||
f_256[k+4] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
|
||||
f_256[k+5] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
|
||||
|
||||
tmp4 = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
|
||||
tmp5 = simde_mm256_unpackhi_epi16(tmp2,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
|
||||
f_256[k+6] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
|
||||
f_256[k+7] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
|
||||
}
|
||||
break;
|
||||
default: AssertFatal(1==0,"Should be here!\n");
|
||||
}
|
||||
|
||||
#else
|
||||
// original unoptimized loops
|
||||
/*
|
||||
for (int j = 0; j< EQm; j++,j2+=2){
|
||||
for (int i = 0; i< Qm; i++){
|
||||
f[(i+j*Qm)] = e[(i*EQm + j)];
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
fp = f;
|
||||
switch (Qm) {
|
||||
case 2:
|
||||
e0 = e;
|
||||
e1 = e0 + EQm;
|
||||
for (int j = 0, j2 = 0; j < EQm; j++, j2 += 2) {
|
||||
fp = &f[j2];
|
||||
fp[0] = e0[j];
|
||||
fp[1] = e1[j];
|
||||
case 2: {
|
||||
uint8_t *e0 = e;
|
||||
uint8_t *e1 = e0 + EQm;
|
||||
int i = 0;
|
||||
#ifdef __AVX512VBMI__
|
||||
// clang-format off
|
||||
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24,
|
||||
87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16,
|
||||
79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8,
|
||||
71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
|
||||
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120,
|
||||
56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112,
|
||||
48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104,
|
||||
40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
|
||||
// clang-format on
|
||||
__m512i *f_512 = (__m512i *)f;
|
||||
__m512i *e0_512 = (__m512i *)e0;
|
||||
__m512i *e1_512 = (__m512i *)e1;
|
||||
for (; i < (EQm & ~63); i += 64) {
|
||||
__m512i e0j = _mm512_loadu_si512(e0_512++);
|
||||
__m512i e1j = _mm512_loadu_si512(e1_512++);
|
||||
// e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j));
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j));
|
||||
}
|
||||
break;
|
||||
case 4:
|
||||
e0 = e;
|
||||
e1 = e0 + EQm;
|
||||
e2 = e1 + EQm;
|
||||
e3 = e2 + EQm;
|
||||
for (int j = 0, j2 = 0; j < EQm; j++, j2 += 4) {
|
||||
fp = &f[j2];
|
||||
fp[0] = e0[j];
|
||||
fp[1] = e1[j];
|
||||
fp[2] = e2[j];
|
||||
fp[3] = e3[j];
|
||||
}
|
||||
break;
|
||||
case 6:
|
||||
e0 = e;
|
||||
e1 = e0 + EQm;
|
||||
e2 = e1 + EQm;
|
||||
e3 = e2 + EQm;
|
||||
e4 = e3 + EQm;
|
||||
e5 = e4 + EQm;
|
||||
fp = f;
|
||||
for (int j = 0; j < EQm; j++) {
|
||||
*fp++ = e0[j];
|
||||
*fp++ = e1[j];
|
||||
*fp++ = e2[j];
|
||||
*fp++ = e3[j];
|
||||
*fp++ = e4[j];
|
||||
*fp++ = e5[j];
|
||||
}
|
||||
break;
|
||||
case 8:
|
||||
e0 = e;
|
||||
e1 = e0 + EQm;
|
||||
e2 = e1 + EQm;
|
||||
e3 = e2 + EQm;
|
||||
e4 = e3 + EQm;
|
||||
e5 = e4 + EQm;
|
||||
e6 = e5 + EQm;
|
||||
e7 = e6 + EQm;
|
||||
for (int j = 0, j2 = 0; j < EQm; j++, j2 += 8) {
|
||||
fp = &f[j2];
|
||||
fp[0] = e0[j];
|
||||
fp[1] = e1[j];
|
||||
fp[2] = e2[j];
|
||||
fp[3] = e3[j];
|
||||
fp[4] = e4[j];
|
||||
fp[5] = e5[j];
|
||||
fp[6] = e6[j];
|
||||
fp[7] = e7[j];
|
||||
}
|
||||
break;
|
||||
default:
|
||||
AssertFatal(1 == 0, "Should never be here!\n");
|
||||
}
|
||||
e0 = (uint8_t *)e0_512;
|
||||
e1 = (uint8_t *)e1_512;
|
||||
f = (uint8_t *)f_512;
|
||||
#endif
|
||||
#ifdef USE128BIT
|
||||
simde__m128i *f_128 = (simde__m128i *)f;
|
||||
simde__m128i *e0_128 = (simde__m128i *)e0;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
for (; i < (EQm & ~15); i += 64) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j));
|
||||
}
|
||||
e0 = (uint8_t *)e0_128;
|
||||
e1 = (uint8_t *)e1_128;
|
||||
f = (uint8_t *)f_128;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
}
|
||||
} break;
|
||||
case 4: {
|
||||
uint8_t *e0 = e;
|
||||
uint8_t *e1 = e0 + EQm;
|
||||
uint8_t *e2 = e1 + EQm;
|
||||
uint8_t *e3 = e2 + EQm;
|
||||
int i = 0;
|
||||
#ifdef __AVX512VBMI__
|
||||
// clang-format off
|
||||
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
|
||||
22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
|
||||
76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
|
||||
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
|
||||
55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47,
|
||||
110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102,
|
||||
38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
|
||||
const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
|
||||
37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
|
||||
const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
|
||||
22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
|
||||
// clang-format on
|
||||
__m512i *e0_512 = (__m512i *)e0;
|
||||
__m512i *e1_512 = (__m512i *)e1;
|
||||
__m512i *e2_512 = (__m512i *)e2;
|
||||
__m512i *e3_512 = (__m512i *)e3;
|
||||
__m512i *f_512 = (__m512i *)f;
|
||||
for (; i < (EQm & ~63); i += 64) {
|
||||
__m512i e0j = _mm512_loadu_si512(e0_512++);
|
||||
__m512i e1j = _mm512_loadu_si512(e1_512++);
|
||||
__m512i e2j = _mm512_loadu_si512(e2_512++);
|
||||
__m512i e3j = _mm512_loadu_si512(e3_512++);
|
||||
__m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
__m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
|
||||
// e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
|
||||
tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
|
||||
// e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
|
||||
}
|
||||
e0 = (uint8_t *)e0_512;
|
||||
e1 = (uint8_t *)e1_512;
|
||||
e2 = (uint8_t *)e2_512;
|
||||
e3 = (uint8_t *)e3_512;
|
||||
f = (uint8_t *)f_512;
|
||||
#endif
|
||||
#ifdef USE128BIT
|
||||
simde__m128i *e0_128 = (simde__m128i *)e0;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *e2_128 = (simde__m128i *)e2;
|
||||
simde__m128i *e3_128 = (simde__m128i *)e3;
|
||||
simde__m128i *f_128 = (simde__m128i *)f;
|
||||
for (; i < (EQm & ~15); i += 16) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
|
||||
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
|
||||
simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
|
||||
simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
|
||||
simde_mm_storeu_si128(f_128++,
|
||||
simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
|
||||
simde_mm_storeu_si128(
|
||||
f_128++,
|
||||
simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
|
||||
tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1));
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1));
|
||||
}
|
||||
e0 = (uint8_t *)e0_128;
|
||||
e1 = (uint8_t *)e1_128;
|
||||
e2 = (uint8_t *)e2_128;
|
||||
e3 = (uint8_t *)e3_128;
|
||||
f = (uint8_t *)f_128;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
*f++ = *e2++;
|
||||
*f++ = *e3++;
|
||||
}
|
||||
} break;
|
||||
case 6: {
|
||||
uint8_t *e0 = e;
|
||||
uint8_t *e1 = e0 + EQm;
|
||||
uint8_t *e2 = e1 + EQm;
|
||||
uint8_t *e3 = e2 + EQm;
|
||||
uint8_t *e4 = e3 + EQm;
|
||||
uint8_t *e5 = e4 + EQm;
|
||||
int i = 0;
|
||||
#ifdef __AVX512VBMI__
|
||||
// clang-format off
|
||||
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
|
||||
86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
|
||||
77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
|
||||
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
|
||||
55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110,
|
||||
46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
|
||||
const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
|
||||
const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
|
||||
const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0);
|
||||
const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42);
|
||||
const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0,
|
||||
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
|
||||
const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11);
|
||||
// clang-format on
|
||||
__m512i *e0_512 = (__m512i *)e0;
|
||||
__m512i *e1_512 = (__m512i *)e1;
|
||||
__m512i *e2_512 = (__m512i *)e2;
|
||||
__m512i *e3_512 = (__m512i *)e3;
|
||||
__m512i *e4_512 = (__m512i *)e4;
|
||||
__m512i *e5_512 = (__m512i *)e5;
|
||||
__m512i *f_512 = (__m512i *)f;
|
||||
for (; i < (EQm & ~63); i += 64) {
|
||||
__m512i e0j = _mm512_loadu_si512(e0_512++);
|
||||
__m512i e1j = _mm512_loadu_si512(e1_512++);
|
||||
__m512i e2j = _mm512_loadu_si512(e2_512++);
|
||||
__m512i e3j = _mm512_loadu_si512(e3_512++);
|
||||
__m512i e4j = _mm512_loadu_si512(e4_512++);
|
||||
__m512i e5j = _mm512_loadu_si512(e5_512++);
|
||||
__m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31)
|
||||
__m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31)
|
||||
__m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31)
|
||||
// e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
__m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
|
||||
// e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
|
||||
__m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
|
||||
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
|
||||
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15)
|
||||
// e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
|
||||
__m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
|
||||
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
// e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20)
|
||||
// e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
|
||||
// e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
|
||||
|
||||
tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63)
|
||||
tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63)
|
||||
tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63)
|
||||
tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
// e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
|
||||
tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
|
||||
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
|
||||
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
// e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
|
||||
tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
|
||||
// e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
// e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20)
|
||||
// e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
|
||||
// e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
|
||||
}
|
||||
e0 = (uint8_t *)e0_512;
|
||||
e1 = (uint8_t *)e1_512;
|
||||
e2 = (uint8_t *)e2_512;
|
||||
e3 = (uint8_t *)e3_512;
|
||||
e4 = (uint8_t *)e4_512;
|
||||
e5 = (uint8_t *)e5_512;
|
||||
f = (uint8_t *)f_512;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
*f++ = *e2++;
|
||||
*f++ = *e3++;
|
||||
*f++ = *e4++;
|
||||
*f++ = *e5++;
|
||||
}
|
||||
} break;
|
||||
case 8: {
|
||||
uint8_t *e0 = e;
|
||||
uint8_t *e1 = e0 + EQm;
|
||||
uint8_t *e2 = e1 + EQm;
|
||||
uint8_t *e3 = e2 + EQm;
|
||||
uint8_t *e4 = e3 + EQm;
|
||||
uint8_t *e5 = e4 + EQm;
|
||||
uint8_t *e6 = e5 + EQm;
|
||||
uint8_t *e7 = e6 + EQm;
|
||||
|
||||
int i = 0;
|
||||
#ifdef __AVX512VBMI__
|
||||
// clang-format off
|
||||
const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
|
||||
86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
|
||||
76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
|
||||
const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55,
|
||||
118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46,
|
||||
109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
|
||||
|
||||
const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
|
||||
const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
|
||||
|
||||
const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
|
||||
const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
|
||||
// clang-format on
|
||||
__m512i *e0_512 = (__m512i *)e0;
|
||||
__m512i *e1_512 = (__m512i *)e1;
|
||||
__m512i *e2_512 = (__m512i *)e2;
|
||||
__m512i *e3_512 = (__m512i *)e3;
|
||||
__m512i *e4_512 = (__m512i *)e4;
|
||||
__m512i *e5_512 = (__m512i *)e5;
|
||||
__m512i *e6_512 = (__m512i *)e6;
|
||||
__m512i *e7_512 = (__m512i *)e7;
|
||||
__m512i *f_512 = (__m512i *)f;
|
||||
for (; i < (EQm & ~63); i += 64) {
|
||||
__m512i e0j = _mm512_loadu_si512(e0_512++);
|
||||
__m512i e1j = _mm512_loadu_si512(e1_512++);
|
||||
__m512i e2j = _mm512_loadu_si512(e2_512++);
|
||||
__m512i e3j = _mm512_loadu_si512(e3_512++);
|
||||
__m512i e4j = _mm512_loadu_si512(e4_512++);
|
||||
__m512i e5j = _mm512_loadu_si512(e5_512++);
|
||||
__m512i e6j = _mm512_loadu_si512(e6_512++);
|
||||
__m512i e7j = _mm512_loadu_si512(e7_512++);
|
||||
__m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
|
||||
__m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
|
||||
__m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
|
||||
__m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
|
||||
__m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
// e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
|
||||
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3)
|
||||
__m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
|
||||
// e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
|
||||
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)...
|
||||
// e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
|
||||
// e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
|
||||
tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
|
||||
// e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
|
||||
tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
|
||||
// e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11)
|
||||
// e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
|
||||
// e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15)
|
||||
// e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
|
||||
|
||||
tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
|
||||
tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
|
||||
tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
|
||||
tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
|
||||
// e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
|
||||
tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
|
||||
// e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
|
||||
tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
|
||||
// e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19)
|
||||
// e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
|
||||
// e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23)
|
||||
// e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
|
||||
// e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
|
||||
tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
|
||||
// e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
|
||||
tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
|
||||
// e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27)
|
||||
// e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
|
||||
// e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31)
|
||||
// e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
|
||||
_mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
|
||||
}
|
||||
e0 = (uint8_t *)e0_512;
|
||||
e1 = (uint8_t *)e1_512;
|
||||
e2 = (uint8_t *)e2_512;
|
||||
e3 = (uint8_t *)e3_512;
|
||||
e4 = (uint8_t *)e4_512;
|
||||
e5 = (uint8_t *)e5_512;
|
||||
e6 = (uint8_t *)e6_512;
|
||||
e7 = (uint8_t *)e7_512;
|
||||
f = (uint8_t *)f_512;
|
||||
#endif
|
||||
#ifdef USE128BIT
|
||||
e0_128 = (simde__m128i *)e0;
|
||||
e1_128 = (simde__m128i *)e1;
|
||||
e2_128 = (simde__m128i *)e2;
|
||||
e3_128 = (simde__m128i *)e3;
|
||||
e4_128 = (simde__m128i *)e4;
|
||||
e5_128 = (simde__m128i *)e5;
|
||||
e6_128 = (simde__m128i *)e6;
|
||||
e7_128 = (simde__m128i *)e7;
|
||||
for (; i < (EQm & ~15); i += 16) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
|
||||
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
|
||||
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
|
||||
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
|
||||
simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
|
||||
simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
|
||||
simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
|
||||
simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
|
||||
// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
|
||||
simde__m128i tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1);
|
||||
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
simde__m128i tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1);
|
||||
simde__m128i tmp4 = simde_mm_unpacklo_epi8(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7)
|
||||
simde__m128i tmp5 = simde_mm_unpacklo_epi8(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7)
|
||||
// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
|
||||
simde__m128i tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5);
|
||||
// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
|
||||
simde__m128i tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5);
|
||||
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6));
|
||||
// e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6));
|
||||
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7));
|
||||
// e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7));
|
||||
|
||||
tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
|
||||
tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15)
|
||||
// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
|
||||
tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1);
|
||||
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1);
|
||||
tmp4 = simde_mm_unpackhi_epi8(e4j, e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7)
|
||||
tmp5 = simde_mm_unpackhi_epi8(e6j, e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15)
|
||||
// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
|
||||
tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5);
|
||||
// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
|
||||
tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5);
|
||||
// e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6));
|
||||
// e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6));
|
||||
// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7));
|
||||
// e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7));
|
||||
}
|
||||
e0 = (uint8_t *)e0_128;
|
||||
e1 = (uint8_t *)e1_128;
|
||||
e2 = (uint8_t *)e2_128;
|
||||
e3 = (uint8_t *)e3_128;
|
||||
e4 = (uint8_t *)e4_128;
|
||||
e5 = (uint8_t *)e5_128;
|
||||
e6 = (uint8_t *)e6_128;
|
||||
e7 = (uint8_t *)e7_128;
|
||||
f = (uint8_t *)f_128;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
*f++ = *e2++;
|
||||
*f++ = *e3++;
|
||||
*f++ = *e4++;
|
||||
*f++ = *e5++;
|
||||
*f++ = *e6++;
|
||||
*f++ = *e7++;
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
AssertFatal(false, "Should be here!\n");
|
||||
}
|
||||
}
|
||||
|
||||
void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
|
||||
@@ -454,8 +599,11 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
|
||||
printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
|
||||
#endif
|
||||
|
||||
if (d[ind] != NR_NULL)
|
||||
e[k++] = d[ind];
|
||||
if (ind == Foffset)
|
||||
ind = F + Foffset; // skip filler bits
|
||||
|
||||
e[k++] = d[ind];
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,8 +24,6 @@
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define NR_NULL 2
|
||||
|
||||
/**
|
||||
* \brief interleave a code segment after encoding and rate matching
|
||||
* \param E size of the code segment in bits
|
||||
|
||||
@@ -650,21 +650,44 @@ static int
|
||||
retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops,
|
||||
nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
|
||||
{
|
||||
uint8_t *p_out = NULL;
|
||||
unsigned int j = 0;
|
||||
for (unsigned int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){
|
||||
for (unsigned int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) {
|
||||
int E_sum = 0;
|
||||
int bit_offset = 0;
|
||||
int byte_offset = 0;
|
||||
p_out = nrLDPC_slot_encoding_parameters->TBs[h].output;
|
||||
for (unsigned int r = 0; r < nrLDPC_slot_encoding_parameters->TBs[h].C; ++r) {
|
||||
struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output;
|
||||
struct rte_mbuf *m = output->data;
|
||||
uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset;
|
||||
uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].segments[i].output;
|
||||
const char *data = m->buf_addr + m->data_off;
|
||||
const char *end = data + data_len;
|
||||
while (data < end) {
|
||||
uint8_t byte = *data++; // get the current byte
|
||||
for (int bit = 7; bit >= 0; --bit) {
|
||||
*out++ = (byte >> bit) & 1; // extract each bit
|
||||
uint8_t *data = m->buf_addr + m->data_off;
|
||||
reverse_bits_u8(data, data_len, data);
|
||||
if (bit_offset == 0) {
|
||||
memcpy(&p_out[byte_offset], data, data_len);
|
||||
} else {
|
||||
size_t i = 0;
|
||||
for (; i < (data_len & ~0x7); i += 8) {
|
||||
uint8_t carry = *data << bit_offset;
|
||||
p_out[byte_offset + i - 1] |= carry;
|
||||
|
||||
simde__m64 current = *((simde__m64 *)data);
|
||||
data += 8;
|
||||
current = simde_mm_srli_si64(current, 8 - bit_offset);
|
||||
*(simde__m64 *)&p_out[byte_offset + i] = current;
|
||||
}
|
||||
for (; i < data_len; i++) {
|
||||
uint8_t current = *data++;
|
||||
|
||||
uint8_t carry = current << bit_offset;
|
||||
p_out[byte_offset + i - 1] |= carry;
|
||||
|
||||
p_out[byte_offset + i] = (current >> (8 - bit_offset));
|
||||
}
|
||||
}
|
||||
E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[r].E;
|
||||
byte_offset = (E_sum + 7) / 8;
|
||||
bit_offset = E_sum % 8;
|
||||
rte_pktmbuf_free(m);
|
||||
rte_pktmbuf_free(ops[j]->ldpc_enc.input.data);
|
||||
++j;
|
||||
@@ -815,6 +838,10 @@ static int pmd_lcore_ldpc_enc(void *arg)
|
||||
AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments);
|
||||
|
||||
set_ldpc_enc_op(ops_enq, 0, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters);
|
||||
if (nrLDPC_slot_encoding_parameters->tprep != NULL)
|
||||
stop_meas(nrLDPC_slot_encoding_parameters->tprep);
|
||||
if (nrLDPC_slot_encoding_parameters->tparity != NULL)
|
||||
start_meas(nrLDPC_slot_encoding_parameters->tparity);
|
||||
for (enq = 0, deq = 0; enq < num_segments;) {
|
||||
num_to_enq = num_segments;
|
||||
if (unlikely(num_segments - enq < num_to_enq))
|
||||
@@ -828,10 +855,14 @@ static int pmd_lcore_ldpc_enc(void *arg)
|
||||
time_out++;
|
||||
DevAssert(time_out <= TIME_OUT_POLL);
|
||||
}
|
||||
|
||||
if (nrLDPC_slot_encoding_parameters->tparity != NULL)
|
||||
stop_meas(nrLDPC_slot_encoding_parameters->tparity);
|
||||
if (nrLDPC_slot_encoding_parameters->toutput != NULL)
|
||||
start_meas(nrLDPC_slot_encoding_parameters->toutput);
|
||||
ret = retrieve_ldpc_enc_op(ops_deq, nrLDPC_slot_encoding_parameters);
|
||||
AssertFatal(ret == 0, "Failed to retrieve LDPC encoding op!");
|
||||
|
||||
if (nrLDPC_slot_encoding_parameters->toutput != NULL)
|
||||
stop_meas(nrLDPC_slot_encoding_parameters->toutput);
|
||||
rte_bbdev_enc_op_free_bulk(ops_enq, num_segments);
|
||||
rte_free(ops_enq);
|
||||
rte_free(ops_deq);
|
||||
@@ -1074,6 +1105,8 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
|
||||
int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
|
||||
{
|
||||
pthread_mutex_lock(&encode_mutex);
|
||||
if (nrLDPC_slot_encoding_parameters->tprep != NULL)
|
||||
start_meas(nrLDPC_slot_encoding_parameters->tprep);
|
||||
// hardcoded to use the first found board
|
||||
struct active_device *ad = active_devs;
|
||||
int ret;
|
||||
|
||||
+83
-9
@@ -24,7 +24,9 @@
|
||||
#include <stdint.h>
|
||||
#include "../../nrLDPCdecoder_defs.h"
|
||||
|
||||
#define AVOID_MM256_SIGN 1
|
||||
#ifdef __aarch64__
|
||||
#define AVOID_SIGN 1
|
||||
#endif
|
||||
#define DROP_MAXLLR 1
|
||||
void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
{
|
||||
@@ -126,7 +128,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[0]>>4)+lut_idxCnProcG3[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -138,7 +144,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
// Store result
|
||||
// min = simde_mm_min_epu8(min, maxLLR); // 128 in epi8 is -127
|
||||
#ifndef DROP_MAXLLR
|
||||
@@ -185,7 +195,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[1]>>4)+lut_idxCnProcG4[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -199,7 +213,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -252,7 +270,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[2]>>4)+lut_idxCnProcG5[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -266,7 +288,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -319,7 +345,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[3]>>4)+lut_idxCnProcG6[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -333,7 +363,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -389,7 +423,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[4]>>4)+lut_idxCnProcG7[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -403,7 +441,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -460,7 +502,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[5]>>4)+lut_idxCnProcG8[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -474,7 +520,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -531,7 +581,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[6]>>4)+lut_idxCnProcG9[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -545,7 +599,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -603,7 +661,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[7]>>4)+lut_idxCnProcG10[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -617,7 +679,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
@@ -678,7 +744,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
// ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
|
||||
fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[8]>>4)+lut_idxCnProcG19[j][0]*2);
|
||||
// sgn = simde_mm_sign_epi8(ones, ymm0);
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n");
|
||||
#endif
|
||||
// min = simde_mm_abs_epi8(ymm0);
|
||||
fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n");
|
||||
|
||||
@@ -692,7 +762,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
|
||||
fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
|
||||
|
||||
// sgn = simde_mm_sign_epi8(sgn, ymm0);
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#ifdef AVOID_SIGN
|
||||
fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n");
|
||||
#else
|
||||
fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store result
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
/**
|
||||
\brief LDPC encoder parameter structure
|
||||
\var n_segments number of segments in the transport block
|
||||
\var macro_num index of the subset of segments to encode
|
||||
\var first_seg index of the first segment of the subset to encode
|
||||
within the transport block
|
||||
\var gen_code flag to generate parity check code
|
||||
0 -> encoding
|
||||
@@ -52,7 +52,7 @@
|
||||
*/
|
||||
typedef struct {
|
||||
unsigned int n_segments; // optim8seg
|
||||
unsigned int macro_num; // optim8segmulti
|
||||
unsigned int first_seg; // optim8segmulti
|
||||
unsigned char gen_code; //orig
|
||||
time_stats_t *tinput;
|
||||
time_stats_t *tprep;
|
||||
@@ -92,6 +92,6 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams,
|
||||
int8_t *p_out,
|
||||
t_nrLDPC_time_stats *,
|
||||
decode_abort_t *ab);
|
||||
typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t **, encoder_implemparams_t *);
|
||||
typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
#include "PHY/sse_intrin.h"
|
||||
// generated code for Zc=384, byte encoding
|
||||
static inline void ldpc384_byte(uint8_t *c,uint8_t *d) {
|
||||
__m512i *csimd=(__m512i *)c,*dsimd=(__m512i *)d;
|
||||
|
||||
__m512i *c2,*d2;
|
||||
|
||||
int i2;
|
||||
for (i2=0; i2<6; i2++) {
|
||||
c2=&csimd[i2];
|
||||
d2=&dsimd[i2];
|
||||
|
||||
//row: 0
|
||||
d2[0]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[3169],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[3193],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[2413],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[8500],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[4286],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2993],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[13310],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[10432],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[5957],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[6771],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[9421],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[9173],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13405],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[3137],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[12925],c2[10025]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 1
|
||||
d2[6]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],c2[10289])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 2
|
||||
d2[12]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 3
|
||||
d2[18]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 4
|
||||
d2[24]=_mm512_xor_si512(c2[3173],c2[14006]);
|
||||
|
||||
//row: 5
|
||||
d2[30]=_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[10827],_mm512_xor_si512(c2[11093],_mm512_xor_si512(c2[12936],_mm512_xor_si512(c2[795],_mm512_xor_si512(c2[12686],_mm512_xor_si512(c2[6084],_mm512_xor_si512(c2[13743],_mm512_xor_si512(c2[3708],_mm512_xor_si512(c2[3987],_mm512_xor_si512(c2[10851],_mm512_xor_si512(c2[9793],_mm512_xor_si512(c2[3734],_mm512_xor_si512(c2[10071],_mm512_xor_si512(c2[7694],_mm512_xor_si512(c2[13501],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[10873],_mm512_xor_si512(c2[12720],_mm512_xor_si512(c2[4817],_mm512_xor_si512(c2[11944],_mm512_xor_si512(c2[7716],_mm512_xor_si512(c2[14069],_mm512_xor_si512(c2[16444],_mm512_xor_si512(c2[9581],_mm512_xor_si512(c2[10645],_mm512_xor_si512(c2[10644],_mm512_xor_si512(c2[14344],_mm512_xor_si512(c2[10657],_mm512_xor_si512(c2[16204],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[6973],_mm512_xor_si512(c2[4073],_mm512_xor_si512(c2[7503],_mm512_xor_si512(c2[16224],_mm512_xor_si512(c2[8044],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[2776],_mm512_xor_si512(c2[1189],_mm512_xor_si512(c2[3829],_mm512_xor_si512(c2[12290],_mm512_xor_si512(c2[13609],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[10178],_mm512_xor_si512(c2[686],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[13357],_mm512_xor_si512(c2[14429],_mm512_xor_si512(c2[14163],_mm512_xor_si512(c2[13109],_mm512_xor_si512(c2[13913],_mm512_xor_si512(c2[184],_mm512_xor_si512(c2[4935],_mm512_xor_si512(c2[2044],_mm512_xor_si512(c2[16825],_mm512_xor_si512(c2[3625],_mm512_xor_si512(c2[13131],_mm512_xor_si512(c2[4168],_mm512_xor_si512(c2[8656],_mm512_xor_si512(c2[2581],_mm512_xor_si512(c2[4176],_mm512_xor_si512(c2[3648],_mm512_xor_si512(c2[11832],_mm512_xor_si512(c2[4721],_mm512_xor_si512(c2[10789],_mm512_xor_si512(c2[11321],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[11330],_mm512_xor_si512(c2[1299],_mm512_xor_si512(c2[14773],_mm512_xor_si512(c2[3688],_mm512_xor_si512(c2[782],c2[13716]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 6
|
||||
d2[36]=_mm512_xor_si512(c2[5812],_mm512_xor_si512(c2[340],_mm512_xor_si512(c2[384],_mm512_xor_si512(c2[8321],_mm512_xor_si512(c2[7549],_mm512_xor_si512(c2[16311],_mm512_xor_si512(c2[4968],c2[8955])))))));
|
||||
|
||||
//row: 7
|
||||
d2[42]=_mm512_xor_si512(c2[2376],_mm512_xor_si512(c2[16380],_mm512_xor_si512(c2[15892],_mm512_xor_si512(c2[3521],_mm512_xor_si512(c2[9076],c2[13369])))));
|
||||
|
||||
//row: 8
|
||||
d2[48]=_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[7921],_mm512_xor_si512(c2[3962],_mm512_xor_si512(c2[14523],_mm512_xor_si512(c2[4228],_mm512_xor_si512(c2[14525],_mm512_xor_si512(c2[14789],_mm512_xor_si512(c2[6077],_mm512_xor_si512(c2[16368],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[5821],_mm512_xor_si512(c2[16382],_mm512_xor_si512(c2[16120],_mm512_xor_si512(c2[9516],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[280],_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[13478],_mm512_xor_si512(c2[14017],_mm512_xor_si512(c2[7683],_mm512_xor_si512(c2[3986],_mm512_xor_si512(c2[14547],_mm512_xor_si512(c2[2928],_mm512_xor_si512(c2[13225],_mm512_xor_si512(c2[13489],_mm512_xor_si512(c2[13764],_mm512_xor_si512(c2[7430],_mm512_xor_si512(c2[3206],_mm512_xor_si512(c2[13767],_mm512_xor_si512(c2[829],_mm512_xor_si512(c2[11126],_mm512_xor_si512(c2[11390],_mm512_xor_si512(c2[9806],_mm512_xor_si512(c2[9293],_mm512_xor_si512(c2[2953],_mm512_xor_si512(c2[4008],_mm512_xor_si512(c2[14305],_mm512_xor_si512(c2[14569],_mm512_xor_si512(c2[5861],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[16416],_mm512_xor_si512(c2[14847],_mm512_xor_si512(c2[8513],_mm512_xor_si512(c2[5079],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[857],_mm512_xor_si512(c2[11148],_mm512_xor_si512(c2[11412],_mm512_xor_si512(c2[7204],_mm512_xor_si512(c2[864],_mm512_xor_si512(c2[9579],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[2716],_mm512_xor_si512(c2[13013],_mm512_xor_si512(c2[13277],_mm512_xor_si512(c2[3780],_mm512_xor_si512(c2[14341],_mm512_xor_si512(c2[3785],_mm512_xor_si512(c2[14076],_mm512_xor_si512(c2[14340],_mm512_xor_si512(c2[7479],_mm512_xor_si512(c2[881],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[3792],_mm512_xor_si512(c2[14353],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[15673],_mm512_xor_si512(c2[9075],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[108],_mm512_xor_si512(c2[10669],_mm512_xor_si512(c2[14103],_mm512_xor_si512(c2[7769],_mm512_xor_si512(c2[638],_mm512_xor_si512(c2[10935],_mm512_xor_si512(c2[11199],_mm512_xor_si512(c2[9365],_mm512_xor_si512(c2[3025],_mm512_xor_si512(c2[1179],_mm512_xor_si512(c2[11476],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[1971],_mm512_xor_si512(c2[12268],_mm512_xor_si512(c2[12532],_mm512_xor_si512(c2[12806],_mm512_xor_si512(c2[6472],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[4885],_mm512_xor_si512(c2[13865],_mm512_xor_si512(c2[7261],_mm512_xor_si512(c2[7525],_mm512_xor_si512(c2[5425],_mm512_xor_si512(c2[15986],_mm512_xor_si512(c2[6744],_mm512_xor_si512(c2[410],_mm512_xor_si512(c2[11236],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[10716],_mm512_xor_si512(c2[4382],_mm512_xor_si512(c2[3064],_mm512_xor_si512(c2[13361],_mm512_xor_si512(c2[13625],_mm512_xor_si512(c2[6492],_mm512_xor_si512(c2[16789],_mm512_xor_si512(c2[158],_mm512_xor_si512(c2[7564],_mm512_xor_si512(c2[1224],_mm512_xor_si512(c2[7298],_mm512_xor_si512(c2[700],_mm512_xor_si512(c2[964],_mm512_xor_si512(c2[6244],_mm512_xor_si512(c2[16541],_mm512_xor_si512(c2[16805],_mm512_xor_si512(c2[7048],_mm512_xor_si512(c2[708],_mm512_xor_si512(c2[10214],_mm512_xor_si512(c2[3880],_mm512_xor_si512(c2[14965],_mm512_xor_si512(c2[8367],_mm512_xor_si512(c2[8631],_mm512_xor_si512(c2[12074],_mm512_xor_si512(c2[5740],_mm512_xor_si512(c2[9960],_mm512_xor_si512(c2[3626],_mm512_xor_si512(c2[13661],_mm512_xor_si512(c2[7057],_mm512_xor_si512(c2[7321],_mm512_xor_si512(c2[10488],_mm512_xor_si512(c2[14198],_mm512_xor_si512(c2[7864],_mm512_xor_si512(c2[1791],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[12352],_mm512_xor_si512(c2[12617],_mm512_xor_si512(c2[6013],_mm512_xor_si512(c2[6277],_mm512_xor_si512(c2[14212],_mm512_xor_si512(c2[7872],_mm512_xor_si512(c2[13684],_mm512_xor_si512(c2[7080],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[4973],_mm512_xor_si512(c2[15264],_mm512_xor_si512(c2[15528],_mm512_xor_si512(c2[14751],_mm512_xor_si512(c2[8417],_mm512_xor_si512(c2[3924],_mm512_xor_si512(c2[14485],_mm512_xor_si512(c2[4456],_mm512_xor_si512(c2[14753],_mm512_xor_si512(c2[15017],_mm512_xor_si512(c2[8679],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[4465],_mm512_xor_si512(c2[14762],_mm512_xor_si512(c2[15026],_mm512_xor_si512(c2[11329],_mm512_xor_si512(c2[4731],_mm512_xor_si512(c2[4995],_mm512_xor_si512(c2[7908],_mm512_xor_si512(c2[1574],_mm512_xor_si512(c2[13718],_mm512_xor_si512(c2[7384],_mm512_xor_si512(c2[10812],_mm512_xor_si512(c2[4214],_mm512_xor_si512(c2[4478],c2[12929]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 9
|
||||
d2[54]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[10575],_mm512_xor_si512(c2[389],_mm512_xor_si512(c2[1454],_mm512_xor_si512(c2[15204],_mm512_xor_si512(c2[12616],_mm512_xor_si512(c2[16848],c2[4993])))))));
|
||||
|
||||
//row: 10
|
||||
d2[60]=_mm512_xor_si512(c2[9781],_mm512_xor_si512(c2[5045],_mm512_xor_si512(c2[4804],_mm512_xor_si512(c2[12493],_mm512_xor_si512(c2[16733],c2[9149])))));
|
||||
|
||||
//row: 11
|
||||
d2[66]=_mm512_xor_si512(c2[268],_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[6864],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[8979],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[12672],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[2127],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[10045],_mm512_xor_si512(c2[24],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[6888],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[5836],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[16672],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[6108],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[3737],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[12195],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[6916],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[8763],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[7981],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[3759],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[10106],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[12481],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5618],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[6688],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[10381],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[6700],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[12241],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[3016],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[110],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[12267],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[4081],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[4873],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[15708],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[14127],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[16767],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[8333],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[9652],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[14138],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[2256],_mm512_xor_si512(c2[13624],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[5966],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[10200],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[9146],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[9950],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[13116],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[14976],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[16563],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[12600],_mm512_xor_si512(c2[205],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[4693],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[15519],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[219],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[16586],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[7875],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[758],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[6832],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[7358],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[6580],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[7373],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[14237],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[10816],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[16620],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[13720],_mm512_xor_si512(c2[6853],c2[16094])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 12
|
||||
d2[72]=_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[15326],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[10695],_mm512_xor_si512(c2[13356],c2[2857])))));
|
||||
|
||||
//row: 13
|
||||
d2[78]=_mm512_xor_si512(c2[9772],_mm512_xor_si512(c2[10036],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[1852],_mm512_xor_si512(c2[15052],_mm512_xor_si512(c2[1332],_mm512_xor_si512(c2[1596],_mm512_xor_si512(c2[11895],_mm512_xor_si512(c2[2653],_mm512_xor_si512(c2[9528],_mm512_xor_si512(c2[9792],_mm512_xor_si512(c2[16656],_mm512_xor_si512(c2[15604],_mm512_xor_si512(c2[9281],_mm512_xor_si512(c2[9545],_mm512_xor_si512(c2[15876],_mm512_xor_si512(c2[13505],_mm512_xor_si512(c2[12974],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[16684],_mm512_xor_si512(c2[1636],_mm512_xor_si512(c2[10358],_mm512_xor_si512(c2[10622],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[13527],_mm512_xor_si512(c2[2715],_mm512_xor_si512(c2[2979],_mm512_xor_si512(c2[5354],_mm512_xor_si512(c2[15386],_mm512_xor_si512(c2[16456],_mm512_xor_si512(c2[16455],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[2728],_mm512_xor_si512(c2[16468],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[11448],_mm512_xor_si512(c2[12520],_mm512_xor_si512(c2[12784],_mm512_xor_si512(c2[9878],_mm512_xor_si512(c2[13308],_mm512_xor_si512(c2[4876],_mm512_xor_si512(c2[5140],_mm512_xor_si512(c2[13849],_mm512_xor_si512(c2[14641],_mm512_xor_si512(c2[8317],_mm512_xor_si512(c2[8581],_mm512_xor_si512(c2[7000],_mm512_xor_si512(c2[9640],_mm512_xor_si512(c2[936],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[2525],_mm512_xor_si512(c2[7011],_mm512_xor_si512(c2[6233],_mm512_xor_si512(c2[6497],_mm512_xor_si512(c2[15734],_mm512_xor_si512(c2[2273],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[3073],_mm512_xor_si512(c2[2019],_mm512_xor_si512(c2[2559],_mm512_xor_si512(c2[2823],_mm512_xor_si512(c2[5989],_mm512_xor_si512(c2[10740],_mm512_xor_si512(c2[7585],_mm512_xor_si512(c2[7849],_mm512_xor_si512(c2[5741],_mm512_xor_si512(c2[9436],_mm512_xor_si512(c2[9973],_mm512_xor_si512(c2[14461],_mm512_xor_si512(c2[8392],_mm512_xor_si512(c2[9723],_mm512_xor_si512(c2[9987],_mm512_xor_si512(c2[9459],_mm512_xor_si512(c2[748],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10526],_mm512_xor_si512(c2[16600],_mm512_xor_si512(c2[231],_mm512_xor_si512(c2[16084],_mm512_xor_si512(c2[16348],_mm512_xor_si512(c2[240],_mm512_xor_si512(c2[7104],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[3425],_mm512_xor_si512(c2[3689],_mm512_xor_si512(c2[9493],c2[6593])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 14
|
||||
d2[84]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[14931],_mm512_xor_si512(c2[2558],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[7337],c2[3420])))));
|
||||
|
||||
//row: 15
|
||||
d2[90]=_mm512_xor_si512(c2[267],_mm512_xor_si512(c2[6869],_mm512_xor_si512(c2[7129],_mm512_xor_si512(c2[8714],_mm512_xor_si512(c2[8978],_mm512_xor_si512(c2[12939],_mm512_xor_si512(c2[8728],_mm512_xor_si512(c2[2126],_mm512_xor_si512(c2[9521],_mm512_xor_si512(c2[9785],_mm512_xor_si512(c2[540],_mm512_xor_si512(c2[29],_mm512_xor_si512(c2[6893],_mm512_xor_si512(c2[5835],_mm512_xor_si512(c2[16671],_mm512_xor_si512(c2[6113],_mm512_xor_si512(c2[3472],_mm512_xor_si512(c2[3736],_mm512_xor_si512(c2[12194],_mm512_xor_si512(c2[6915],_mm512_xor_si512(c2[8498],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[853],_mm512_xor_si512(c2[7980],_mm512_xor_si512(c2[3758],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[12480],_mm512_xor_si512(c2[5353],_mm512_xor_si512(c2[5617],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[10380],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[12240],_mm512_xor_si512(c2[1421],_mm512_xor_si512(c2[1685],_mm512_xor_si512(c2[3015],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[3545],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[4080],_mm512_xor_si512(c2[4608],_mm512_xor_si512(c2[4872],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[15713],_mm512_xor_si512(c2[14126],_mm512_xor_si512(c2[16502],_mm512_xor_si512(c2[16766],_mm512_xor_si512(c2[8332],_mm512_xor_si512(c2[9651],_mm512_xor_si512(c2[13873],_mm512_xor_si512(c2[14137],_mm512_xor_si512(c2[13623],_mm512_xor_si512(c2[5965],_mm512_xor_si512(c2[9135],_mm512_xor_si512(c2[9399],_mm512_xor_si512(c2[16528],_mm512_xor_si512(c2[10465],_mm512_xor_si512(c2[10205],_mm512_xor_si512(c2[8881],_mm512_xor_si512(c2[9145],_mm512_xor_si512(c2[9949],_mm512_xor_si512(c2[13121],_mm512_xor_si512(c2[977],_mm512_xor_si512(c2[14981],_mm512_xor_si512(c2[12867],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[16562],_mm512_xor_si512(c2[204],_mm512_xor_si512(c2[4692],_mm512_xor_si512(c2[15254],_mm512_xor_si512(c2[15518],_mm512_xor_si512(c2[218],_mm512_xor_si512(c2[16585],_mm512_xor_si512(c2[7610],_mm512_xor_si512(c2[7874],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[757],_mm512_xor_si512(c2[6831],_mm512_xor_si512(c2[7357],_mm512_xor_si512(c2[6579],_mm512_xor_si512(c2[7372],_mm512_xor_si512(c2[13972],_mm512_xor_si512(c2[14236],_mm512_xor_si512(c2[10815],_mm512_xor_si512(c2[16625],_mm512_xor_si512(c2[13455],c2[13719]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 16
|
||||
d2[96]=_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[1322],_mm512_xor_si512(c2[1588],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3181],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[4238],_mm512_xor_si512(c2[3444],_mm512_xor_si512(c2[11377],_mm512_xor_si512(c2[1346],_mm512_xor_si512(c2[288],_mm512_xor_si512(c2[11124],_mm512_xor_si512(c2[566],_mm512_xor_si512(c2[15084],_mm512_xor_si512(c2[4793],_mm512_xor_si512(c2[6653],_mm512_xor_si512(c2[1368],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[12207],_mm512_xor_si512(c2[2439],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[4564],_mm512_xor_si512(c2[6939],_mm512_xor_si512(c2[76],_mm512_xor_si512(c2[1140],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[4839],_mm512_xor_si512(c2[1152],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[13033],_mm512_xor_si512(c2[14369],_mm512_xor_si512(c2[11463],_mm512_xor_si512(c2[14893],_mm512_xor_si512(c2[6725],_mm512_xor_si512(c2[15434],_mm512_xor_si512(c2[16226],_mm512_xor_si512(c2[10166],_mm512_xor_si512(c2[8585],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[15180],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[8596],_mm512_xor_si512(c2[8076],_mm512_xor_si512(c2[424],_mm512_xor_si512(c2[3852],_mm512_xor_si512(c2[4924],_mm512_xor_si512(c2[4658],_mm512_xor_si512(c2[3604],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[7574],_mm512_xor_si512(c2[12325],_mm512_xor_si512(c2[9434],_mm512_xor_si512(c2[7320],_mm512_xor_si512(c2[11021],_mm512_xor_si512(c2[11558],_mm512_xor_si512(c2[16046],_mm512_xor_si512(c2[9977],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[11044],_mm512_xor_si512(c2[2333],_mm512_xor_si512(c2[12111],_mm512_xor_si512(c2[1284],_mm512_xor_si512(c2[1816],_mm512_xor_si512(c2[1032],_mm512_xor_si512(c2[1825],_mm512_xor_si512(c2[8689],_mm512_xor_si512(c2[8956],_mm512_xor_si512(c2[5268],_mm512_xor_si512(c2[11078],c2[8172]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 17
|
||||
d2[102]=_mm512_xor_si512(c2[1060],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[4681],_mm512_xor_si512(c2[10241],c2[14777]))));
|
||||
|
||||
//row: 18
|
||||
d2[108]=_mm512_xor_si512(c2[542],_mm512_xor_si512(c2[9386],_mm512_xor_si512(c2[6496],_mm512_xor_si512(c2[1274],c2[1284]))));
|
||||
|
||||
//row: 19
|
||||
d2[114]=_mm512_xor_si512(c2[4490],_mm512_xor_si512(c2[5559],_mm512_xor_si512(c2[6425],_mm512_xor_si512(c2[13299],c2[1443]))));
|
||||
|
||||
//row: 20
|
||||
d2[120]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[1850],_mm512_xor_si512(c2[2116],_mm512_xor_si512(c2[3965],_mm512_xor_si512(c2[15578],_mm512_xor_si512(c2[3709],_mm512_xor_si512(c2[14008],_mm512_xor_si512(c2[4766],_mm512_xor_si512(c2[11905],_mm512_xor_si512(c2[1874],_mm512_xor_si512(c2[816],_mm512_xor_si512(c2[11652],_mm512_xor_si512(c2[1094],_mm512_xor_si512(c2[15612],_mm512_xor_si512(c2[3735],_mm512_xor_si512(c2[7181],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[3749],_mm512_xor_si512(c2[12735],_mm512_xor_si512(c2[2967],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[5092],_mm512_xor_si512(c2[7467],_mm512_xor_si512(c2[604],_mm512_xor_si512(c2[1668],_mm512_xor_si512(c2[1673],_mm512_xor_si512(c2[5367],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[7227],_mm512_xor_si512(c2[13561],_mm512_xor_si512(c2[14897],_mm512_xor_si512(c2[11991],_mm512_xor_si512(c2[15421],_mm512_xor_si512(c2[2224],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[15962],_mm512_xor_si512(c2[16754],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[9113],_mm512_xor_si512(c2[11753],_mm512_xor_si512(c2[5681],_mm512_xor_si512(c2[3313],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[9124],_mm512_xor_si512(c2[8604],_mm512_xor_si512(c2[952],_mm512_xor_si512(c2[4380],_mm512_xor_si512(c2[5452],_mm512_xor_si512(c2[5186],_mm512_xor_si512(c2[4132],_mm512_xor_si512(c2[4936],_mm512_xor_si512(c2[8102],_mm512_xor_si512(c2[12853],_mm512_xor_si512(c2[9962],_mm512_xor_si512(c2[7848],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[10505],_mm512_xor_si512(c2[12100],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[12639],_mm512_xor_si512(c2[1812],_mm512_xor_si512(c2[2344],_mm512_xor_si512(c2[1560],_mm512_xor_si512(c2[2353],_mm512_xor_si512(c2[9217],_mm512_xor_si512(c2[5796],_mm512_xor_si512(c2[11606],c2[8700]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 21
|
||||
d2[126]=_mm512_xor_si512(c2[3447],_mm512_xor_si512(c2[10093],_mm512_xor_si512(c2[2309],_mm512_xor_si512(c2[5787],c2[8965]))));
|
||||
|
||||
//row: 22
|
||||
d2[132]=_mm512_xor_si512(c2[7920],_mm512_xor_si512(c2[3048],_mm512_xor_si512(c2[10983],c2[6012])));
|
||||
|
||||
//row: 23
|
||||
d2[138]=_mm512_xor_si512(c2[6348],_mm512_xor_si512(c2[6625],_mm512_xor_si512(c2[16224],c2[7344])));
|
||||
|
||||
//row: 24
|
||||
d2[144]=_mm512_xor_si512(c2[7658],_mm512_xor_si512(c2[14260],_mm512_xor_si512(c2[14520],_mm512_xor_si512(c2[16369],_mm512_xor_si512(c2[11092],_mm512_xor_si512(c2[16119],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[281],_mm512_xor_si512(c2[7420],_mm512_xor_si512(c2[14284],_mm512_xor_si512(c2[13226],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[13504],_mm512_xor_si512(c2[11127],_mm512_xor_si512(c2[7958],_mm512_xor_si512(c2[2690],_mm512_xor_si512(c2[14306],_mm512_xor_si512(c2[16153],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[8244],_mm512_xor_si512(c2[15377],_mm512_xor_si512(c2[11149],_mm512_xor_si512(c2[601],_mm512_xor_si512(c2[2976],_mm512_xor_si512(c2[13008],_mm512_xor_si512(c2[14078],_mm512_xor_si512(c2[14077],_mm512_xor_si512(c2[876],_mm512_xor_si512(c2[14090],_mm512_xor_si512(c2[2736],_mm512_xor_si512(c2[9076],_mm512_xor_si512(c2[10406],_mm512_xor_si512(c2[7500],_mm512_xor_si512(c2[10936],_mm512_xor_si512(c2[2762],_mm512_xor_si512(c2[11477],_mm512_xor_si512(c2[12269],_mm512_xor_si512(c2[6209],_mm512_xor_si512(c2[4622],_mm512_xor_si512(c2[7262],_mm512_xor_si512(c2[5153],_mm512_xor_si512(c2[15723],_mm512_xor_si512(c2[147],_mm512_xor_si512(c2[4633],_mm512_xor_si512(c2[4119],_mm512_xor_si512(c2[13356],_mm512_xor_si512(c2[16790],_mm512_xor_si512(c2[961],_mm512_xor_si512(c2[701],_mm512_xor_si512(c2[16536],_mm512_xor_si512(c2[445],_mm512_xor_si512(c2[3617],_mm512_xor_si512(c2[8368],_mm512_xor_si512(c2[5477],_mm512_xor_si512(c2[3363],_mm512_xor_si512(c2[7058],_mm512_xor_si512(c2[7601],_mm512_xor_si512(c2[12089],_mm512_xor_si512(c2[6014],_mm512_xor_si512(c2[7609],_mm512_xor_si512(c2[7081],_mm512_xor_si512(c2[15265],_mm512_xor_si512(c2[8148],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[14748],_mm512_xor_si512(c2[13970],_mm512_xor_si512(c2[14763],_mm512_xor_si512(c2[4732],_mm512_xor_si512(c2[1311],_mm512_xor_si512(c2[7121],c2[4215]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 25
|
||||
d2[150]=_mm512_xor_si512(c2[2125],_mm512_xor_si512(c2[4560],_mm512_xor_si512(c2[16721],c2[14956])));
|
||||
|
||||
//row: 26
|
||||
d2[156]=_mm512_xor_si512(c2[1849],_mm512_xor_si512(c2[4513],_mm512_xor_si512(c2[3217],c2[2294])));
|
||||
|
||||
//row: 27
|
||||
d2[162]=_mm512_xor_si512(c2[543],_mm512_xor_si512(c2[603],c2[9865]));
|
||||
|
||||
//row: 28
|
||||
d2[168]=_mm512_xor_si512(c2[7923],_mm512_xor_si512(c2[5064],_mm512_xor_si512(c2[13959],c2[5008])));
|
||||
|
||||
//row: 29
|
||||
d2[174]=_mm512_xor_si512(c2[528],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[7396],_mm512_xor_si512(c2[8981],_mm512_xor_si512(c2[9245],_mm512_xor_si512(c2[8989],_mm512_xor_si512(c2[2393],_mm512_xor_si512(c2[9782],_mm512_xor_si512(c2[10046],_mm512_xor_si512(c2[15855],_mm512_xor_si512(c2[290],_mm512_xor_si512(c2[7154],_mm512_xor_si512(c2[6096],_mm512_xor_si512(c2[37],_mm512_xor_si512(c2[6374],_mm512_xor_si512(c2[3733],_mm512_xor_si512(c2[3997],_mm512_xor_si512(c2[12461],_mm512_xor_si512(c2[7176],_mm512_xor_si512(c2[8765],_mm512_xor_si512(c2[9029],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[8247],_mm512_xor_si512(c2[4025],_mm512_xor_si512(c2[10372],_mm512_xor_si512(c2[12747],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5884],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6953],_mm512_xor_si512(c2[10383],_mm512_xor_si512(c2[10647],_mm512_xor_si512(c2[6960],_mm512_xor_si512(c2[12507],_mm512_xor_si512(c2[1682],_mm512_xor_si512(c2[1946],_mm512_xor_si512(c2[3276],_mm512_xor_si512(c2[376],_mm512_xor_si512(c2[3806],_mm512_xor_si512(c2[12533],_mm512_xor_si512(c2[4347],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[5139],_mm512_xor_si512(c2[15974],_mm512_xor_si512(c2[14393],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[132],_mm512_xor_si512(c2[8593],_mm512_xor_si512(c2[9912],_mm512_xor_si512(c2[14140],_mm512_xor_si512(c2[14404],_mm512_xor_si512(c2[13884],_mm512_xor_si512(c2[6232],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[9660],_mm512_xor_si512(c2[10732],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[9148],_mm512_xor_si512(c2[9412],_mm512_xor_si512(c2[1488],_mm512_xor_si512(c2[10216],_mm512_xor_si512(c2[13382],_mm512_xor_si512(c2[1238],_mm512_xor_si512(c2[15242],_mm512_xor_si512(c2[13128],_mm512_xor_si512(c2[16565],_mm512_xor_si512(c2[16829],_mm512_xor_si512(c2[471],_mm512_xor_si512(c2[4959],_mm512_xor_si512(c2[15521],_mm512_xor_si512(c2[15785],_mm512_xor_si512(c2[485],_mm512_xor_si512(c2[16852],_mm512_xor_si512(c2[7877],_mm512_xor_si512(c2[8141],_mm512_xor_si512(c2[5234],_mm512_xor_si512(c2[1024],_mm512_xor_si512(c2[7092],_mm512_xor_si512(c2[7624],_mm512_xor_si512(c2[6840],_mm512_xor_si512(c2[7633],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[14497],_mm512_xor_si512(c2[11076],_mm512_xor_si512(c2[16886],_mm512_xor_si512(c2[13716],c2[13980]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 30
|
||||
d2[180]=_mm512_xor_si512(c2[3696],_mm512_xor_si512(c2[10298],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10564],_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[12413],_mm512_xor_si512(c2[8186],_mm512_xor_si512(c2[12157],_mm512_xor_si512(c2[5297],_mm512_xor_si512(c2[5561],_mm512_xor_si512(c2[12950],_mm512_xor_si512(c2[13214],_mm512_xor_si512(c2[3458],_mm512_xor_si512(c2[10322],_mm512_xor_si512(c2[9000],_mm512_xor_si512(c2[9264],_mm512_xor_si512(c2[3205],_mm512_xor_si512(c2[9542],_mm512_xor_si512(c2[6901],_mm512_xor_si512(c2[7165],_mm512_xor_si512(c2[15629],_mm512_xor_si512(c2[10080],_mm512_xor_si512(c2[10344],_mm512_xor_si512(c2[11933],_mm512_xor_si512(c2[12197],_mm512_xor_si512(c2[4288],_mm512_xor_si512(c2[11415],_mm512_xor_si512(c2[6929],_mm512_xor_si512(c2[7193],_mm512_xor_si512(c2[13540],_mm512_xor_si512(c2[15651],_mm512_xor_si512(c2[15915],_mm512_xor_si512(c2[8788],_mm512_xor_si512(c2[9052],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[9857],_mm512_xor_si512(c2[10121],_mm512_xor_si512(c2[13551],_mm512_xor_si512(c2[13815],_mm512_xor_si512(c2[10128],_mm512_xor_si512(c2[15411],_mm512_xor_si512(c2[15675],_mm512_xor_si512(c2[4850],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[6444],_mm512_xor_si512(c2[3544],_mm512_xor_si512(c2[6710],_mm512_xor_si512(c2[6974],_mm512_xor_si512(c2[15701],_mm512_xor_si512(c2[7251],_mm512_xor_si512(c2[7515],_mm512_xor_si512(c2[8043],_mm512_xor_si512(c2[8307],_mm512_xor_si512(c2[9891],_mm512_xor_si512(c2[2247],_mm512_xor_si512(c2[660],_mm512_xor_si512(c2[3036],_mm512_xor_si512(c2[3300],_mm512_xor_si512(c2[11761],_mm512_xor_si512(c2[13080],_mm512_xor_si512(c2[413],_mm512_xor_si512(c2[677],_mm512_xor_si512(c2[157],_mm512_xor_si512(c2[9136],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[12564],_mm512_xor_si512(c2[12828],_mm512_xor_si512(c2[1216],_mm512_xor_si512(c2[13900],_mm512_xor_si512(c2[13370],_mm512_xor_si512(c2[13634],_mm512_xor_si512(c2[12316],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[13384],_mm512_xor_si512(c2[16550],_mm512_xor_si512(c2[4142],_mm512_xor_si512(c2[4406],_mm512_xor_si512(c2[1515],_mm512_xor_si512(c2[16296],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[3096],_mm512_xor_si512(c2[3639],_mm512_xor_si512(c2[7863],_mm512_xor_si512(c2[8127],_mm512_xor_si512(c2[1788],_mm512_xor_si512(c2[2052],_mm512_xor_si512(c2[3653],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[3125],_mm512_xor_si512(c2[11045],_mm512_xor_si512(c2[11309],_mm512_xor_si512(c2[4192],_mm512_xor_si512(c2[10260],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[10008],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[10801],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[770],_mm512_xor_si512(c2[14244],_mm512_xor_si512(c2[3159],_mm512_xor_si512(c2[16884],c2[253])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 31
|
||||
d2[186]=_mm512_xor_si512(c2[13994],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[3701],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3961],_mm512_xor_si512(c2[3697],_mm512_xor_si512(c2[5810],_mm512_xor_si512(c2[5282],_mm512_xor_si512(c2[5546],_mm512_xor_si512(c2[5560],_mm512_xor_si512(c2[5296],_mm512_xor_si512(c2[15853],_mm512_xor_si512(c2[15589],_mm512_xor_si512(c2[6617],_mm512_xor_si512(c2[6089],_mm512_xor_si512(c2[6353],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[13756],_mm512_xor_si512(c2[13492],_mm512_xor_si512(c2[3725],_mm512_xor_si512(c2[3461],_mm512_xor_si512(c2[2667],_mm512_xor_si512(c2[2403],_mm512_xor_si512(c2[13503],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[2945],_mm512_xor_si512(c2[2681],_mm512_xor_si512(c2[568],_mm512_xor_si512(c2[40],_mm512_xor_si512(c2[304],_mm512_xor_si512(c2[9026],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[3747],_mm512_xor_si512(c2[3483],_mm512_xor_si512(c2[5594],_mm512_xor_si512(c2[5066],_mm512_xor_si512(c2[5330],_mm512_xor_si512(c2[14580],_mm512_xor_si512(c2[14316],_mm512_xor_si512(c2[4812],_mm512_xor_si512(c2[4548],_mm512_xor_si512(c2[590],_mm512_xor_si512(c2[326],_mm512_xor_si512(c2[6937],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[9312],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[2449],_mm512_xor_si512(c2[1921],_mm512_xor_si512(c2[2185],_mm512_xor_si512(c2[3519],_mm512_xor_si512(c2[3255],_mm512_xor_si512(c2[3518],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[7212],_mm512_xor_si512(c2[6684],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6159],_mm512_xor_si512(c2[3531],_mm512_xor_si512(c2[3267],_mm512_xor_si512(c2[9072],_mm512_xor_si512(c2[8808],_mm512_xor_si512(c2[15412],_mm512_xor_si512(c2[14884],_mm512_xor_si512(c2[15148],_mm512_xor_si512(c2[16742],_mm512_xor_si512(c2[16478],_mm512_xor_si512(c2[13836],_mm512_xor_si512(c2[13572],_mm512_xor_si512(c2[377],_mm512_xor_si512(c2[113],_mm512_xor_si512(c2[9098],_mm512_xor_si512(c2[8834],_mm512_xor_si512(c2[912],_mm512_xor_si512(c2[648],_mm512_xor_si512(c2[1704],_mm512_xor_si512(c2[1176],_mm512_xor_si512(c2[1440],_mm512_xor_si512(c2[12545],_mm512_xor_si512(c2[12281],_mm512_xor_si512(c2[10958],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[13598],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[13334],_mm512_xor_si512(c2[5164],_mm512_xor_si512(c2[4900],_mm512_xor_si512(c2[6483],_mm512_xor_si512(c2[6219],_mm512_xor_si512(c2[10969],_mm512_xor_si512(c2[10441],_mm512_xor_si512(c2[10705],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[10191],_mm512_xor_si512(c2[2797],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[6231],_mm512_xor_si512(c2[5703],_mm512_xor_si512(c2[5967],_mm512_xor_si512(c2[7297],_mm512_xor_si512(c2[7033],_mm512_xor_si512(c2[7037],_mm512_xor_si512(c2[6773],_mm512_xor_si512(c2[5977],_mm512_xor_si512(c2[5449],_mm512_xor_si512(c2[5713],_mm512_xor_si512(c2[6781],_mm512_xor_si512(c2[6517],_mm512_xor_si512(c2[9953],_mm512_xor_si512(c2[9689],_mm512_xor_si512(c2[14704],_mm512_xor_si512(c2[14440],_mm512_xor_si512(c2[11813],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[9699],_mm512_xor_si512(c2[9435],_mm512_xor_si512(c2[13394],_mm512_xor_si512(c2[12866],_mm512_xor_si512(c2[13130],_mm512_xor_si512(c2[13937],_mm512_xor_si512(c2[13673],_mm512_xor_si512(c2[1524],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[12350],_mm512_xor_si512(c2[11822],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[13945],_mm512_xor_si512(c2[13681],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[13153],_mm512_xor_si512(c2[4706],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[14484],_mm512_xor_si512(c2[14220],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3399],_mm512_xor_si512(c2[4189],_mm512_xor_si512(c2[3925],_mm512_xor_si512(c2[3411],_mm512_xor_si512(c2[3147],_mm512_xor_si512(c2[4204],_mm512_xor_si512(c2[3940],_mm512_xor_si512(c2[11068],_mm512_xor_si512(c2[10540],_mm512_xor_si512(c2[10804],_mm512_xor_si512(c2[7647],_mm512_xor_si512(c2[7383],_mm512_xor_si512(c2[13457],_mm512_xor_si512(c2[13193],_mm512_xor_si512(c2[10551],_mm512_xor_si512(c2[10023],c2[10287]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 32
|
||||
d2[192]=_mm512_xor_si512(c2[5019],_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[11617],_mm512_xor_si512(c2[11881],_mm512_xor_si512(c2[13466],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[10033],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[6614],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[14273],_mm512_xor_si512(c2[14537],_mm512_xor_si512(c2[4781],_mm512_xor_si512(c2[11645],_mm512_xor_si512(c2[10323],_mm512_xor_si512(c2[10587],_mm512_xor_si512(c2[4528],_mm512_xor_si512(c2[10865],_mm512_xor_si512(c2[8224],_mm512_xor_si512(c2[8488],_mm512_xor_si512(c2[51],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[11667],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[13514],_mm512_xor_si512(c2[5605],_mm512_xor_si512(c2[12732],_mm512_xor_si512(c2[8246],_mm512_xor_si512(c2[8510],_mm512_xor_si512(c2[14857],_mm512_xor_si512(c2[73],_mm512_xor_si512(c2[337],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[10369],_mm512_xor_si512(c2[11439],_mm512_xor_si512(c2[11174],_mm512_xor_si512(c2[11438],_mm512_xor_si512(c2[14868],_mm512_xor_si512(c2[15132],_mm512_xor_si512(c2[11451],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[97],_mm512_xor_si512(c2[6173],_mm512_xor_si512(c2[6437],_mm512_xor_si512(c2[7767],_mm512_xor_si512(c2[4861],_mm512_xor_si512(c2[8033],_mm512_xor_si512(c2[8297],_mm512_xor_si512(c2[123],_mm512_xor_si512(c2[8568],_mm512_xor_si512(c2[8832],_mm512_xor_si512(c2[9360],_mm512_xor_si512(c2[9624],_mm512_xor_si512(c2[3564],_mm512_xor_si512(c2[1983],_mm512_xor_si512(c2[4359],_mm512_xor_si512(c2[4623],_mm512_xor_si512(c2[13084],_mm512_xor_si512(c2[14403],_mm512_xor_si512(c2[1730],_mm512_xor_si512(c2[1994],_mm512_xor_si512(c2[2523],_mm512_xor_si512(c2[1480],_mm512_xor_si512(c2[10453],_mm512_xor_si512(c2[10717],_mm512_xor_si512(c2[13887],_mm512_xor_si512(c2[14151],_mm512_xor_si512(c2[15217],_mm512_xor_si512(c2[14693],_mm512_xor_si512(c2[14957],_mm512_xor_si512(c2[13633],_mm512_xor_si512(c2[13897],_mm512_xor_si512(c2[12578],_mm512_xor_si512(c2[14701],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[5465],_mm512_xor_si512(c2[5729],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[724],_mm512_xor_si512(c2[4155],_mm512_xor_si512(c2[4419],_mm512_xor_si512(c2[4956],_mm512_xor_si512(c2[9180],_mm512_xor_si512(c2[9444],_mm512_xor_si512(c2[3111],_mm512_xor_si512(c2[3375],_mm512_xor_si512(c2[4970],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[12362],_mm512_xor_si512(c2[12626],_mm512_xor_si512(c2[5509],_mm512_xor_si512(c2[11583],_mm512_xor_si512(c2[11845],_mm512_xor_si512(c2[12109],_mm512_xor_si512(c2[11331],_mm512_xor_si512(c2[11860],_mm512_xor_si512(c2[12124],_mm512_xor_si512(c2[1829],_mm512_xor_si512(c2[2093],_mm512_xor_si512(c2[15567],_mm512_xor_si512(c2[4476],_mm512_xor_si512(c2[1312],c2[1576])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 33
|
||||
d2[198]=_mm512_xor_si512(c2[809],_mm512_xor_si512(c2[2136],_mm512_xor_si512(c2[10961],c2[11077])));
|
||||
|
||||
//row: 34
|
||||
d2[204]=_mm512_xor_si512(c2[10035],_mm512_xor_si512(c2[5366],_mm512_xor_si512(c2[2823],c2[14992])));
|
||||
|
||||
//row: 35
|
||||
d2[210]=_mm512_xor_si512(c2[16106],_mm512_xor_si512(c2[5813],_mm512_xor_si512(c2[6073],_mm512_xor_si512(c2[7922],_mm512_xor_si512(c2[7672],_mm512_xor_si512(c2[1070],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[17],_mm512_xor_si512(c2[15868],_mm512_xor_si512(c2[5837],_mm512_xor_si512(c2[4779],_mm512_xor_si512(c2[15615],_mm512_xor_si512(c2[5057],_mm512_xor_si512(c2[2680],_mm512_xor_si512(c2[11138],_mm512_xor_si512(c2[5859],_mm512_xor_si512(c2[7706],_mm512_xor_si512(c2[16692],_mm512_xor_si512(c2[6924],_mm512_xor_si512(c2[2702],_mm512_xor_si512(c2[9049],_mm512_xor_si512(c2[11424],_mm512_xor_si512(c2[4561],_mm512_xor_si512(c2[4037],_mm512_xor_si512(c2[5631],_mm512_xor_si512(c2[5630],_mm512_xor_si512(c2[9324],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[11184],_mm512_xor_si512(c2[629],_mm512_xor_si512(c2[1959],_mm512_xor_si512(c2[15948],_mm512_xor_si512(c2[2489],_mm512_xor_si512(c2[11210],_mm512_xor_si512(c2[3024],_mm512_xor_si512(c2[3816],_mm512_xor_si512(c2[14657],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[15710],_mm512_xor_si512(c2[7276],_mm512_xor_si512(c2[8595],_mm512_xor_si512(c2[13081],_mm512_xor_si512(c2[672],_mm512_xor_si512(c2[12567],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[8343],_mm512_xor_si512(c2[9409],_mm512_xor_si512(c2[9149],_mm512_xor_si512(c2[8089],_mm512_xor_si512(c2[8893],_mm512_xor_si512(c2[12065],_mm512_xor_si512(c2[16816],_mm512_xor_si512(c2[13925],_mm512_xor_si512(c2[11811],_mm512_xor_si512(c2[15506],_mm512_xor_si512(c2[16049],_mm512_xor_si512(c2[3636],_mm512_xor_si512(c2[14462],_mm512_xor_si512(c2[16057],_mm512_xor_si512(c2[15529],_mm512_xor_si512(c2[6818],_mm512_xor_si512(c2[16596],_mm512_xor_si512(c2[5775],_mm512_xor_si512(c2[6301],_mm512_xor_si512(c2[5523],_mm512_xor_si512(c2[6316],_mm512_xor_si512(c2[13180],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[15569],c2[12663])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 36
|
||||
d2[216]=_mm512_xor_si512(c2[4755],_mm512_xor_si512(c2[15220],_mm512_xor_si512(c2[11008],c2[5760])));
|
||||
|
||||
//row: 37
|
||||
d2[222]=_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[3448],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[9759],c2[6853])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 38
|
||||
d2[228]=_mm512_xor_si512(c2[15050],_mm512_xor_si512(c2[13046],_mm512_xor_si512(c2[8836],c2[5955])));
|
||||
|
||||
//row: 39
|
||||
d2[234]=_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[7693],_mm512_xor_si512(c2[6953],c2[11056])));
|
||||
|
||||
//row: 40
|
||||
d2[240]=_mm512_xor_si512(c2[12410],_mm512_xor_si512(c2[9864],c2[14992]));
|
||||
|
||||
//row: 41
|
||||
d2[246]=_mm512_xor_si512(c2[13740],_mm512_xor_si512(c2[15352],_mm512_xor_si512(c2[3014],c2[8668])));
|
||||
|
||||
//row: 42
|
||||
d2[252]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[10560],_mm512_xor_si512(c2[12145],_mm512_xor_si512(c2[12409],_mm512_xor_si512(c2[12937],_mm512_xor_si512(c2[12159],_mm512_xor_si512(c2[5293],_mm512_xor_si512(c2[5557],_mm512_xor_si512(c2[12952],_mm512_xor_si512(c2[13216],_mm512_xor_si512(c2[3460],_mm512_xor_si512(c2[10324],_mm512_xor_si512(c2[9002],_mm512_xor_si512(c2[9266],_mm512_xor_si512(c2[3207],_mm512_xor_si512(c2[9544],_mm512_xor_si512(c2[6903],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[15625],_mm512_xor_si512(c2[10082],_mm512_xor_si512(c2[10346],_mm512_xor_si512(c2[11929],_mm512_xor_si512(c2[12193],_mm512_xor_si512(c2[3744],_mm512_xor_si512(c2[4284],_mm512_xor_si512(c2[11417],_mm512_xor_si512(c2[6925],_mm512_xor_si512(c2[7189],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[15653],_mm512_xor_si512(c2[15917],_mm512_xor_si512(c2[8784],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[10118],_mm512_xor_si512(c2[9853],_mm512_xor_si512(c2[10117],_mm512_xor_si512(c2[13553],_mm512_xor_si512(c2[13817],_mm512_xor_si512(c2[10130],_mm512_xor_si512(c2[15413],_mm512_xor_si512(c2[15677],_mm512_xor_si512(c2[4852],_mm512_xor_si512(c2[5116],_mm512_xor_si512(c2[6446],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[6712],_mm512_xor_si512(c2[6976],_mm512_xor_si512(c2[15697],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[7517],_mm512_xor_si512(c2[8045],_mm512_xor_si512(c2[8309],_mm512_xor_si512(c2[2249],_mm512_xor_si512(c2[662],_mm512_xor_si512(c2[3038],_mm512_xor_si512(c2[3302],_mm512_xor_si512(c2[11763],_mm512_xor_si512(c2[13082],_mm512_xor_si512(c2[409],_mm512_xor_si512(c2[673],_mm512_xor_si512(c2[159],_mm512_xor_si512(c2[9132],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[12566],_mm512_xor_si512(c2[12830],_mm512_xor_si512(c2[13896],_mm512_xor_si512(c2[13372],_mm512_xor_si512(c2[13636],_mm512_xor_si512(c2[12312],_mm512_xor_si512(c2[12576],_mm512_xor_si512(c2[13380],_mm512_xor_si512(c2[16552],_mm512_xor_si512(c2[4144],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[1517],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[2834],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[3641],_mm512_xor_si512(c2[7865],_mm512_xor_si512(c2[8129],_mm512_xor_si512(c2[1790],_mm512_xor_si512(c2[2054],_mm512_xor_si512(c2[3649],_mm512_xor_si512(c2[2857],_mm512_xor_si512(c2[3121],_mm512_xor_si512(c2[11041],_mm512_xor_si512(c2[11305],_mm512_xor_si512(c2[4188],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10524],_mm512_xor_si512(c2[10788],_mm512_xor_si512(c2[10010],_mm512_xor_si512(c2[10539],_mm512_xor_si512(c2[10803],_mm512_xor_si512(c2[508],_mm512_xor_si512(c2[772],_mm512_xor_si512(c2[14246],_mm512_xor_si512(c2[3161],_mm512_xor_si512(c2[16886],c2[255]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 43
|
||||
d2[258]=_mm512_xor_si512(c2[7393],_mm512_xor_si512(c2[13995],_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[15840],_mm512_xor_si512(c2[16104],_mm512_xor_si512(c2[15854],_mm512_xor_si512(c2[9252],_mm512_xor_si512(c2[16647],_mm512_xor_si512(c2[16],_mm512_xor_si512(c2[12949],_mm512_xor_si512(c2[7155],_mm512_xor_si512(c2[14019],_mm512_xor_si512(c2[12961],_mm512_xor_si512(c2[6902],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[10598],_mm512_xor_si512(c2[10862],_mm512_xor_si512(c2[2425],_mm512_xor_si512(c2[14041],_mm512_xor_si512(c2[15624],_mm512_xor_si512(c2[15888],_mm512_xor_si512(c2[7985],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[10884],_mm512_xor_si512(c2[336],_mm512_xor_si512(c2[2717],_mm512_xor_si512(c2[12485],_mm512_xor_si512(c2[12749],_mm512_xor_si512(c2[13813],_mm512_xor_si512(c2[13812],_mm512_xor_si512(c2[353],_mm512_xor_si512(c2[617],_mm512_xor_si512(c2[13825],_mm512_xor_si512(c2[2477],_mm512_xor_si512(c2[8547],_mm512_xor_si512(c2[8811],_mm512_xor_si512(c2[10141],_mm512_xor_si512(c2[7241],_mm512_xor_si512(c2[10671],_mm512_xor_si512(c2[2497],_mm512_xor_si512(c2[11212],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[12004],_mm512_xor_si512(c2[5944],_mm512_xor_si512(c2[4357],_mm512_xor_si512(c2[6733],_mm512_xor_si512(c2[6997],_mm512_xor_si512(c2[15458],_mm512_xor_si512(c2[16777],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[4368],_mm512_xor_si512(c2[3854],_mm512_xor_si512(c2[13097],_mm512_xor_si512(c2[16261],_mm512_xor_si512(c2[16525],_mm512_xor_si512(c2[696],_mm512_xor_si512(c2[436],_mm512_xor_si512(c2[16013],_mm512_xor_si512(c2[16277],_mm512_xor_si512(c2[180],_mm512_xor_si512(c2[3352],_mm512_xor_si512(c2[8103],_mm512_xor_si512(c2[5212],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[6529],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[1250],_mm512_xor_si512(c2[7336],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[5485],_mm512_xor_si512(c2[5749],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[6816],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[15000],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[7889],_mm512_xor_si512(c2[13957],_mm512_xor_si512(c2[14489],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[14498],_mm512_xor_si512(c2[4203],_mm512_xor_si512(c2[4467],_mm512_xor_si512(c2[1046],_mm512_xor_si512(c2[6856],_mm512_xor_si512(c2[3686],c2[3950]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 44
|
||||
d2[264]=_mm512_xor_si512(c2[1323],_mm512_xor_si512(c2[7925],_mm512_xor_si512(c2[8185],_mm512_xor_si512(c2[10034],_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[9784],_mm512_xor_si512(c2[3182],_mm512_xor_si512(c2[10841],_mm512_xor_si512(c2[1085],_mm512_xor_si512(c2[7949],_mm512_xor_si512(c2[6891],_mm512_xor_si512(c2[832],_mm512_xor_si512(c2[7169],_mm512_xor_si512(c2[4792],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[7971],_mm512_xor_si512(c2[9818],_mm512_xor_si512(c2[1909],_mm512_xor_si512(c2[9036],_mm512_xor_si512(c2[4814],_mm512_xor_si512(c2[11161],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[7743],_mm512_xor_si512(c2[7742],_mm512_xor_si512(c2[11436],_mm512_xor_si512(c2[3781],_mm512_xor_si512(c2[7755],_mm512_xor_si512(c2[13296],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[4071],_mm512_xor_si512(c2[1165],_mm512_xor_si512(c2[4601],_mm512_xor_si512(c2[9350],_mm512_xor_si512(c2[13322],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5928],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[15182],_mm512_xor_si512(c2[927],_mm512_xor_si512(c2[9388],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[15193],_mm512_xor_si512(c2[14679],_mm512_xor_si512(c2[7021],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[11521],_mm512_xor_si512(c2[11261],_mm512_xor_si512(c2[10201],_mm512_xor_si512(c2[11005],_mm512_xor_si512(c2[14177],_mm512_xor_si512(c2[2033],_mm512_xor_si512(c2[16037],_mm512_xor_si512(c2[13923],_mm512_xor_si512(c2[723],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[5748],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[1274],_mm512_xor_si512(c2[746],_mm512_xor_si512(c2[8930],_mm512_xor_si512(c2[1813],_mm512_xor_si512(c2[7887],_mm512_xor_si512(c2[8413],_mm512_xor_si512(c2[7635],_mm512_xor_si512(c2[8428],_mm512_xor_si512(c2[15292],_mm512_xor_si512(c2[11871],_mm512_xor_si512(c2[780],c2[14775])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
|
||||
|
||||
//row: 45
|
||||
d2[270]=_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[5618],c2[4080]));
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,11 @@
|
||||
#include "common/utils/LOG/log.h"
|
||||
|
||||
|
||||
#ifdef __AVX512F__
|
||||
#include "ldpc384_simd512_byte.c"
|
||||
#else
|
||||
#include "ldpc384_byte.c"
|
||||
#endif
|
||||
#include "ldpc384_byte_128.c"
|
||||
#include "ldpc352_byte.c"
|
||||
#include "ldpc352_byte_128.c"
|
||||
@@ -87,7 +91,7 @@
|
||||
|
||||
static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols)
|
||||
{
|
||||
unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(32))); //double size matrix of c
|
||||
unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(64))); //double size matrix of c
|
||||
for (int i1 = 0; i1 < ncols; i1++) {
|
||||
memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
|
||||
memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
|
||||
@@ -95,7 +99,6 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,sho
|
||||
for (int i1 = 1; i1 < simd_size; i1++) {
|
||||
memcpy(&c[(2 * ncols * Zc * i1)], &c[i1], (2 * ncols * Zc * sizeof(unsigned char)) - i1);
|
||||
}
|
||||
|
||||
if (BG == 1) {
|
||||
switch (Zc) {
|
||||
case 176:
|
||||
|
||||
@@ -41,11 +41,11 @@
|
||||
#include "openair1/PHY/CODING/nrLDPC_extern.h"
|
||||
#include "ldpc_generate_coefficient.c"
|
||||
|
||||
int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder_implemparams_t *impp)
|
||||
int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_implemparams_t *impp)
|
||||
{
|
||||
const unsigned char *input = inputArray[0];
|
||||
// channel input is the output of this function!
|
||||
unsigned char *output = outputArray[0];
|
||||
unsigned char *output = outputArray;
|
||||
const int Zc = impp->Zc;
|
||||
const int Kb = impp->Kb;
|
||||
const short block_length = impp->K;
|
||||
@@ -114,7 +114,13 @@ int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder
|
||||
fprintf(fd,"#include \"PHY/sse_intrin.h\"\n");
|
||||
fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n");
|
||||
|
||||
if (gen_code == 1 && (Zc&31)==0) {
|
||||
if (gen_code == 1 && (Zc&63)==0) {
|
||||
shift=6;
|
||||
mask=63;
|
||||
strcpy(data_type,"__m512i");
|
||||
strcpy(xor_command,"_mm512_xor_si512");
|
||||
}
|
||||
else if (gen_code == 1 && (Zc&31)==0) {
|
||||
shift=5; // AVX2 - 256-bit SIMD
|
||||
mask=31;
|
||||
strcpy(data_type,"simde__m256i");
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
/*! \file ldpc_encoder_optim.c
|
||||
* \brief Defines the optimized LDPC encoder
|
||||
* \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
|
||||
* \email openair_tech@eurecom.fr
|
||||
* \date 27-03-2018
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "assertions.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#include "time_meas.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_defs.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_extern.h"
|
||||
#include "ldpc_encode_parity_check.c"
|
||||
#include "ldpc_generate_coefficient.c"
|
||||
|
||||
int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp)
|
||||
{
|
||||
int Zc = impp->Zc;
|
||||
int Kb = impp->Kb;
|
||||
int block_length = impp->K;
|
||||
int BG = impp->BG;
|
||||
int nrows=0,ncols=0;
|
||||
int rate=3;
|
||||
int no_punctured_columns,removed_bit;
|
||||
|
||||
int simd_size;
|
||||
|
||||
//determine number of bits in codeword
|
||||
//if (block_length>3840)
|
||||
if (BG==1)
|
||||
{
|
||||
//BG=1;
|
||||
nrows=46; //parity check bits
|
||||
ncols=22; //info bits
|
||||
rate=3;
|
||||
}
|
||||
//else if (block_length<=3840)
|
||||
else if (BG==2)
|
||||
{
|
||||
//BG=2;
|
||||
nrows=42; //parity check bits
|
||||
ncols=10; // info bits
|
||||
rate=5;
|
||||
|
||||
}
|
||||
|
||||
|
||||
#ifdef DEBUG_LDPC
|
||||
LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d\n",BG,Zc,Kb,block_length);
|
||||
LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]);
|
||||
#endif
|
||||
|
||||
if ((Zc&31) > 0) simd_size = 16;
|
||||
else simd_size = 32;
|
||||
|
||||
uint8_t c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size
|
||||
uint8_t d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size
|
||||
|
||||
// calculate number of punctured bits
|
||||
no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
|
||||
removed_bit = (nrows - no_punctured_columns - 2) * Zc + block_length - block_length * rate;
|
||||
// printf("%d\n",no_punctured_columns);
|
||||
// printf("%d\n",removed_bit);
|
||||
// unpack input
|
||||
memset(c, 0, sizeof(c));
|
||||
memset(d, 0, sizeof(d));
|
||||
|
||||
if(impp->tinput != NULL) start_meas(impp->tinput);
|
||||
for (int i=0; i<block_length; i++) {
|
||||
c[i] = (test_input[0][i/8]&(128>>(i&7)))>>(7-(i&7));
|
||||
//printf("c(%d,%d)=%d\n",j,i,temp);
|
||||
}
|
||||
|
||||
if(impp->tinput != NULL) stop_meas(impp->tinput);
|
||||
|
||||
if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
|
||||
// extend matrix
|
||||
if(impp->tprep != NULL) start_meas(impp->tprep);
|
||||
if(impp->tprep != NULL) stop_meas(impp->tprep);
|
||||
//parity check part
|
||||
if(impp->tparity != NULL) start_meas(impp->tparity);
|
||||
encode_parity_check_part_optim(c, d, BG, Zc, Kb,simd_size, ncols);
|
||||
if(impp->tparity != NULL) stop_meas(impp->tparity);
|
||||
}
|
||||
else {
|
||||
if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) {
|
||||
printf("Problem with encoder\n");
|
||||
return(-1);
|
||||
}
|
||||
}
|
||||
if(impp->toutput != NULL) start_meas(impp->toutput);
|
||||
// information part and puncture columns
|
||||
memcpy(&channel_input[0][0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char));
|
||||
memcpy(&channel_input[0][block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char));
|
||||
|
||||
if(impp->toutput != NULL) stop_meas(impp->toutput);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
/*! \file ldpc_encoder_optim8seg.c
|
||||
* \brief Defines the optimized LDPC encoder
|
||||
* \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
|
||||
* \email openair_tech@eurecom.fr
|
||||
* \date 27-03-2018
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "assertions.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#include "time_meas.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_defs.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_extern.h"
|
||||
#include "ldpc_encode_parity_check.c"
|
||||
#include "ldpc_generate_coefficient.c"
|
||||
#include "PHY/sse_intrin.h"
|
||||
|
||||
int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp)
|
||||
{
|
||||
int Zc = impp->Zc;
|
||||
int Kb = impp->Kb;
|
||||
short block_length = impp->K;
|
||||
short BG = impp->BG;
|
||||
|
||||
short nrows=0,ncols=0;
|
||||
int i,i1,j,rate=3;
|
||||
int no_punctured_columns,removed_bit;
|
||||
char temp;
|
||||
int simd_size;
|
||||
|
||||
simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
|
||||
simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
|
||||
simde__m256i zero256 = simde_mm256_setzero_si256();
|
||||
simde__m256i masks[8];
|
||||
register simde__m256i c256;
|
||||
masks[0] = simde_mm256_set1_epi8(0x1);
|
||||
masks[1] = simde_mm256_set1_epi8(0x2);
|
||||
masks[2] = simde_mm256_set1_epi8(0x4);
|
||||
masks[3] = simde_mm256_set1_epi8(0x8);
|
||||
masks[4] = simde_mm256_set1_epi8(0x10);
|
||||
masks[5] = simde_mm256_set1_epi8(0x20);
|
||||
masks[6] = simde_mm256_set1_epi8(0x40);
|
||||
masks[7] = simde_mm256_set1_epi8(0x80);
|
||||
|
||||
AssertFatal((impp->n_segments>0&&impp->n_segments<=8),"0 < n_segments %d <= 8\n",impp->n_segments);
|
||||
|
||||
//determine number of bits in codeword
|
||||
//if (block_length>3840)
|
||||
if (BG==1)
|
||||
{
|
||||
nrows=46; //parity check bits
|
||||
ncols=22; //info bits
|
||||
rate=3;
|
||||
}
|
||||
//else if (block_length<=3840)
|
||||
else if (BG==2)
|
||||
{
|
||||
//BG=2;
|
||||
nrows=42; //parity check bits
|
||||
ncols=10; // info bits
|
||||
rate=5;
|
||||
|
||||
}
|
||||
|
||||
#ifdef DEBUG_LDPC
|
||||
LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,n_segments);
|
||||
LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU (seg 0) %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]);
|
||||
#endif
|
||||
|
||||
AssertFatal(Zc>0,"no valid Zc found for block length %d\n",block_length);
|
||||
|
||||
if ((Zc&31) > 0) simd_size = 16;
|
||||
else simd_size = 32;
|
||||
|
||||
unsigned char c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size
|
||||
unsigned char d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size
|
||||
|
||||
|
||||
// calculate number of punctured bits
|
||||
no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
|
||||
removed_bit=(nrows-no_punctured_columns-2) * Zc+block_length-(int)(block_length*rate);
|
||||
// printf("%d\n",no_punctured_columns);
|
||||
// printf("%d\n",removed_bit);
|
||||
// unpack input
|
||||
memset(c,0,sizeof(unsigned char) * ncols * Zc);
|
||||
memset(d,0,sizeof(unsigned char) * nrows * Zc);
|
||||
|
||||
if(impp->tinput != NULL) start_meas(impp->tinput);
|
||||
#if 0
|
||||
for (i=0; i<block_length; i++) {
|
||||
for (j=0; j<n_segments; j++) {
|
||||
|
||||
temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
|
||||
//printf("c(%d,%d)=%d\n",j,i,temp);
|
||||
c[i] |= (temp << j);
|
||||
}
|
||||
}
|
||||
#else
|
||||
for (i=0; i<block_length>>5; i++) {
|
||||
c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[0])[i]), shufmask),andmask),zero256),masks[0]);
|
||||
for (j=1; j<impp->n_segments; j++) {
|
||||
c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[j])[i]), shufmask),andmask),zero256),masks[j]),c256);
|
||||
}
|
||||
((simde__m256i *)c)[i] = c256;
|
||||
}
|
||||
|
||||
for (i=(block_length>>5)<<5;i<block_length;i++) {
|
||||
for (j=0; j<impp->n_segments; j++) {
|
||||
|
||||
temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
|
||||
//printf("c(%d,%d)=%d\n",j,i,temp);
|
||||
c[i] |= (temp << j);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if(impp->tinput != NULL) stop_meas(impp->tinput);
|
||||
|
||||
if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
|
||||
// extend matrix
|
||||
if(impp->tprep != NULL) start_meas(impp->tprep);
|
||||
if(impp->tprep != NULL) stop_meas(impp->tprep);
|
||||
//parity check part
|
||||
if(impp->tparity != NULL) start_meas(impp->tparity);
|
||||
encode_parity_check_part_optim(c, d, BG, Zc, Kb, simd_size, ncols);
|
||||
if(impp->tparity != NULL) stop_meas(impp->tparity);
|
||||
}
|
||||
else {
|
||||
if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) {
|
||||
printf("Problem with encoder\n");
|
||||
return(-1);
|
||||
}
|
||||
}
|
||||
if(impp->toutput != NULL) start_meas(impp->toutput);
|
||||
// information part and puncture columns
|
||||
/*
|
||||
memcpy(&channel_input[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char));
|
||||
memcpy(&channel_input[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char));
|
||||
*/
|
||||
if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) {
|
||||
//AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n");
|
||||
//AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n");
|
||||
uint32_t l1 = (block_length-(2*Zc))>>5;
|
||||
uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5;
|
||||
simde__m256i *c256p = (simde__m256i *)&c[2*Zc];
|
||||
simde__m256i *d256p = (simde__m256i *)&d[0];
|
||||
// if (((block_length-(2*Zc))&31)>0) l1++;
|
||||
|
||||
for (i=0;i<l1;i++)
|
||||
for (j=0;j<impp->n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]);
|
||||
|
||||
// if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++;
|
||||
|
||||
for (i1=0;i1<l2;i1++,i++)
|
||||
for (j=0;j<impp->n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]);
|
||||
}
|
||||
else {
|
||||
#ifdef DEBUG_LDPC
|
||||
LOG_W(PHY,"using non-optimized version\n");
|
||||
#endif
|
||||
// do non-SIMD version
|
||||
for (i=0;i<(block_length-2*Zc);i++)
|
||||
for (j=0; j<impp->n_segments; j++)
|
||||
channel_input[j][i] = (c[2*Zc+i]>>j)&1;
|
||||
for (i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++)
|
||||
for (j=0; j<impp->n_segments; j++)
|
||||
channel_input[j][block_length-2*Zc+i] = (d[i]>>j)&1;
|
||||
}
|
||||
|
||||
if(impp->toutput != NULL) stop_meas(impp->toutput);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
#include "ldpc_encode_parity_check.c"
|
||||
#include "ldpc_generate_coefficient.c"
|
||||
|
||||
int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
|
||||
int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
|
||||
{
|
||||
//set_log(PHY, 4);
|
||||
|
||||
@@ -61,27 +61,11 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
|
||||
unsigned int macro_segment, macro_segment_end;
|
||||
|
||||
|
||||
macro_segment = 8*impp->macro_num;
|
||||
macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments;
|
||||
macro_segment = impp->first_seg;
|
||||
macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments;
|
||||
///printf("macro_segment: %d\n", macro_segment);
|
||||
///printf("macro_segment_end: %d\n", macro_segment_end );
|
||||
|
||||
simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
|
||||
simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
|
||||
simde__m256i zero256 = simde_mm256_setzero_si256();
|
||||
simde__m256i masks[8];
|
||||
register simde__m256i c256;
|
||||
masks[0] = simde_mm256_set1_epi8(0x1);
|
||||
masks[1] = simde_mm256_set1_epi8(0x2);
|
||||
masks[2] = simde_mm256_set1_epi8(0x4);
|
||||
masks[3] = simde_mm256_set1_epi8(0x8);
|
||||
masks[4] = simde_mm256_set1_epi8(0x10);
|
||||
masks[5] = simde_mm256_set1_epi8(0x20);
|
||||
masks[6] = simde_mm256_set1_epi8(0x40);
|
||||
masks[7] = simde_mm256_set1_epi8(0x80);
|
||||
|
||||
|
||||
|
||||
//determine number of bits in codeword
|
||||
if (BG==1)
|
||||
{
|
||||
@@ -103,10 +87,12 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
|
||||
|
||||
AssertFatal(Zc > 0, "no valid Zc found for block length %d\n", block_length);
|
||||
if ((Zc&31) > 0) simd_size = 16;
|
||||
else
|
||||
simd_size = 32;
|
||||
unsigned char cc[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size
|
||||
unsigned char dd[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size
|
||||
#ifdef __AVX512F__
|
||||
else if ((BG==1) && Zc==384) simd_size =64;
|
||||
#endif
|
||||
else simd_size=32;
|
||||
unsigned char cc[22*Zc] __attribute__((aligned(64))); //padded input, unpacked, max size
|
||||
unsigned char dd[46*Zc] __attribute__((aligned(64))); //coded parity part output, unpacked, max size
|
||||
|
||||
// calculate number of punctured bits
|
||||
no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
|
||||
@@ -114,40 +100,95 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
|
||||
//printf("%d\n",no_punctured_columns);
|
||||
//printf("%d\n",removed_bit);
|
||||
// unpack input
|
||||
memset(cc,0,sizeof(unsigned char) * ncols * Zc);
|
||||
memset(dd,0,sizeof(unsigned char) * nrows * Zc);
|
||||
memset(cc,0,sizeof(cc));
|
||||
memset(dd,0,sizeof(dd));
|
||||
|
||||
if(impp->tinput != NULL) start_meas(impp->tinput);
|
||||
#if 0
|
||||
for (i=0; i<block_length; i++) {
|
||||
//for (j=0; j<n_segments; j++) {
|
||||
for (j=macro_segment; j < macro_segment_end; j++) {
|
||||
//interleave up to 8 transport-block segements at a time
|
||||
|
||||
temp = (input[j][i/8]&(1<<(i&7)))>>(i&7);
|
||||
//printf("c(%d,%d)=%d\n",j,i,temp);
|
||||
c[i] |= (temp << (j-macro_segment));
|
||||
unsigned int i_byte = 0;
|
||||
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
|
||||
const __m512i masks5[8] = { _mm512_set1_epi8(0x1), _mm512_set1_epi8(0x2),
|
||||
_mm512_set1_epi8(0x4), _mm512_set1_epi8(0x8),
|
||||
_mm512_set1_epi8(0x10), _mm512_set1_epi8(0x20),
|
||||
_mm512_set1_epi8(0x40), _mm512_set1_epi8(0x80)};
|
||||
const __m512i zero512 = _mm512_setzero_si512();
|
||||
const uint8_t perm[64]__attribute__((aligned(64))) = {7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8,
|
||||
23,22,21,20,19,18,17,16, 31,30,29,28,27,26,25,24,
|
||||
39,38,37,36,35,34,33,32, 47,46,45,44,43,42,41,40,
|
||||
55,54,53,52,51,50,49,48, 63,62,61,60,59,58,57,56};
|
||||
register __m512i c512;
|
||||
|
||||
for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) {
|
||||
unsigned int i = i_byte >> 6;
|
||||
c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]);
|
||||
for (int j = macro_segment + 1; j < macro_segment_end; j++) {
|
||||
c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment]));
|
||||
}
|
||||
c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512);
|
||||
((__m512i *)cc)[i] = c512;
|
||||
}
|
||||
#else
|
||||
for (int i=0; i<block_length>>5; i++) {
|
||||
#endif
|
||||
|
||||
#ifndef __aarch64__
|
||||
simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
|
||||
simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
|
||||
simde__m256i zero256 = simde_mm256_setzero_si256();
|
||||
simde__m256i masks[8];
|
||||
register simde__m256i c256;
|
||||
masks[0] = simde_mm256_set1_epi8(0x1);
|
||||
masks[1] = simde_mm256_set1_epi8(0x2);
|
||||
masks[2] = simde_mm256_set1_epi8(0x4);
|
||||
masks[3] = simde_mm256_set1_epi8(0x8);
|
||||
masks[4] = simde_mm256_set1_epi8(0x10);
|
||||
masks[5] = simde_mm256_set1_epi8(0x20);
|
||||
masks[6] = simde_mm256_set1_epi8(0x40);
|
||||
masks[7] = simde_mm256_set1_epi8(0x80);
|
||||
|
||||
for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) {
|
||||
unsigned int i = i_byte >> 5;
|
||||
c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]);
|
||||
//for (j=1; j<n_segments; j++) {
|
||||
for (int j=macro_segment+1; j < macro_segment_end; j++) {
|
||||
c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[j])[i]), shufmask),andmask),zero256),masks[j-macro_segment]),c256);
|
||||
}
|
||||
((simde__m256i *)cc)[i] = c256;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (int i=(block_length>>5)<<5;i<block_length;i++) {
|
||||
//for (j=0; j<n_segments; j++) {
|
||||
for (int j=macro_segment; j < macro_segment_end; j++) {
|
||||
#ifdef __aarch64__
|
||||
simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
|
||||
simde__m128i andmask = simde_mm_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats.
|
||||
simde__m128i zero128 = simde_mm_setzero_si128();
|
||||
simde__m128i masks[8];
|
||||
register simde__m128i c128;
|
||||
masks[0] = simde_mm_set1_epi8(0x1);
|
||||
masks[1] = simde_mm_set1_epi8(0x2);
|
||||
masks[2] = simde_mm_set1_epi8(0x4);
|
||||
masks[3] = simde_mm_set1_epi8(0x8);
|
||||
masks[4] = simde_mm_set1_epi8(0x10);
|
||||
masks[5] = simde_mm_set1_epi8(0x20);
|
||||
masks[6] = simde_mm_set1_epi8(0x40);
|
||||
masks[7] = simde_mm_set1_epi8(0x80);
|
||||
|
||||
temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
|
||||
//printf("c(%d,%d)=%d\n",j,i,temp);
|
||||
for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) {
|
||||
unsigned int i = i_byte >> 4;
|
||||
c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]);
|
||||
for (int j=macro_segment+1; j < macro_segment_end; j++) {
|
||||
c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128);
|
||||
}
|
||||
((simde__m128i *)cc)[i] = c128;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (; i_byte < block_length; i_byte++) {
|
||||
unsigned int i = i_byte;
|
||||
for (int j = macro_segment; j < macro_segment_end; j++) {
|
||||
|
||||
temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
|
||||
cc[i] |= (temp << (j-macro_segment));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if(impp->tinput != NULL) stop_meas(impp->tinput);
|
||||
|
||||
@@ -166,48 +207,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
|
||||
return(-1);
|
||||
}
|
||||
}
|
||||
if(impp->toutput != NULL) start_meas(impp->toutput);
|
||||
// information part and puncture columns
|
||||
/*
|
||||
memcpy(&output[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char));
|
||||
memcpy(&output[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char));
|
||||
*/
|
||||
if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) {
|
||||
//AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n");
|
||||
//AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n");
|
||||
uint32_t l1 = (block_length-(2*Zc))>>5;
|
||||
uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5;
|
||||
simde__m256i *c256p = (simde__m256i *)&cc[2*Zc];
|
||||
simde__m256i *d256p = (simde__m256i *)&dd[0];
|
||||
// if (((block_length-(2*Zc))&31)>0) l1++;
|
||||
|
||||
for (int i=0;i<l1;i++)
|
||||
//for (j=0;j<n_segments;j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]);
|
||||
for (int j=macro_segment; j < macro_segment_end; j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j-macro_segment),masks[0]);
|
||||
|
||||
|
||||
// if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++;
|
||||
|
||||
for (int i1=0, i=l1;i1<l2;i1++,i++)
|
||||
//for (j=0;j<n_segments;j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]);
|
||||
for (int j=macro_segment; j < macro_segment_end; j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j-macro_segment),masks[0]);
|
||||
}
|
||||
else {
|
||||
#ifdef DEBUG_LDPC
|
||||
LOG_W(PHY,"using non-optimized version\n");
|
||||
#endif
|
||||
// do non-SIMD version
|
||||
for (int i=0;i<(block_length-2*Zc);i++)
|
||||
//for (j=0; j<n_segments; j++)
|
||||
for (int j=macro_segment; j < macro_segment_end; j++)
|
||||
output[j][i] = (cc[2*Zc+i]>>(j-macro_segment))&1;
|
||||
for (int i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++)
|
||||
//for (j=0; j<n_segments; j++)
|
||||
for (int j=macro_segment; j < macro_segment_end; j++)
|
||||
output[j][block_length-2*Zc+i] = (dd[i]>>(j-macro_segment))&1;
|
||||
}
|
||||
|
||||
if(impp->toutput != NULL) stop_meas(impp->toutput);
|
||||
memcpy(output,&cc[2*Zc],(block_length-(2*Zc)));
|
||||
memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -151,13 +151,8 @@ else
|
||||
for (r=0; r<*C; r++) {
|
||||
|
||||
k = 0;
|
||||
|
||||
while (k<((Kprime - L)>>3)) {
|
||||
output_buffers[r][k] = input_buffer[s];
|
||||
//printf("encoding segment %d : byte %d (%d) => %d\n",r,k,(Kprime-L)>>3,input_buffer[s]);
|
||||
k++;
|
||||
s++;
|
||||
}
|
||||
memcpy(output_buffers[r],input_buffer+s,(Kprime-L)>>3);
|
||||
s+=(Kprime-L)>>3;
|
||||
|
||||
if (*C > 1) { // add CRC
|
||||
crc = crc24b(output_buffers[r],Kprime-L)>>8;
|
||||
|
||||
@@ -600,14 +600,14 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
|
||||
printf("\nEncoded payload:\n");
|
||||
for (int i = 0; i < encoded_length; i += 8) {
|
||||
for (int j = 0; j < 8; j++)
|
||||
printf("%d", input_ptr[i + j]);
|
||||
printf("%d", (input_ptr[i >> 3] >> j) & 1);
|
||||
printf("\t");
|
||||
}
|
||||
printf("\n");
|
||||
#endif
|
||||
|
||||
if (IS_SOFTMODEM_DLSIM)
|
||||
memcpy(harq->f, input_ptr, encoded_length);
|
||||
memcpy(harq->f, input_ptr, (encoded_length + 7) >> 3);
|
||||
|
||||
c16_t mod_symbs[rel15->NrOfCodewords][encoded_length] __attribute__((aligned(64)));
|
||||
for (int codeWord = 0; codeWord < rel15->NrOfCodewords; codeWord++) {
|
||||
@@ -763,11 +763,11 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
|
||||
}
|
||||
|
||||
stop_meas(&gNB->dlsch_precoding_stats);
|
||||
/* output and its parts for each dlsch should be aligned on 64 bytes
|
||||
* should remain a multiple of 64 with enough offset to fit each dlsch
|
||||
/* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)
|
||||
* should remain a multiple of 8 * 64 with enough offset to fit each dlsch
|
||||
*/
|
||||
uint32_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers;
|
||||
return ceil_mod(size_output_tb,64);
|
||||
return ((size_output_tb + 511) >> 9) << 6;
|
||||
}
|
||||
|
||||
void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
|
||||
@@ -817,15 +817,15 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
|
||||
/// CRC, coding, interleaving and rate matching
|
||||
AssertFatal(harq->pdu != NULL, "%4d.%2d no HARQ PDU for PDSCH generation\n", msgTx->frame, msgTx->slot);
|
||||
|
||||
/* output and its parts for each dlsch should be aligned on 64 bytes
|
||||
* => size_output is a sum of parts sizes rounded up to a multiple of 64
|
||||
/* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)
|
||||
* => size_output is a sum of parts sizes rounded up to a multiple of 8 * 64
|
||||
*/
|
||||
size_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers;
|
||||
size_output += ceil_mod(size_output_tb, 64);
|
||||
size_output += ceil_mod(size_output_tb, 8 * 64);
|
||||
}
|
||||
|
||||
unsigned char output[size_output] __attribute__((aligned(64)));
|
||||
bzero(output, size_output);
|
||||
unsigned char output[size_output >> 3] __attribute__((aligned(64)));
|
||||
bzero(output, sizeof(output));
|
||||
|
||||
start_meas(dlsch_encoding_stats);
|
||||
if (nr_dlsch_encoding(gNB,
|
||||
|
||||
@@ -265,15 +265,13 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
|
||||
|
||||
TB_parameters->tbslbrm = rel15->maintenance_parms_v3.tbSizeLbrmBytes;
|
||||
|
||||
TB_parameters->output = &output[dlsch_offset >> 3];
|
||||
TB_parameters->segments = &segments[segments_offset];
|
||||
|
||||
size_t r_offset = 0;
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
|
||||
segment_parameters->c = harq->c[r];
|
||||
segment_parameters->E = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, rel15->nrOfLayers, r);
|
||||
segment_parameters->output = &output[dlsch_offset + r_offset];
|
||||
r_offset += segment_parameters->E;
|
||||
|
||||
reset_meas(&segment_parameters->ts_interleave);
|
||||
reset_meas(&segment_parameters->ts_rate_match);
|
||||
@@ -282,11 +280,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
|
||||
|
||||
segments_offset += TB_parameters->C;
|
||||
|
||||
/* output and its parts for each dlsch should be aligned on 64 bytes
|
||||
* => dlsch_offset should remain a multiple of 64 with enough offset to fit each dlsch
|
||||
/* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)
|
||||
* => dlsch_offset should remain a multiple of 8 * 64 with enough offset to fit each dlsch
|
||||
*/
|
||||
const size_t dlsch_size = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * rel15->qamModOrder[0] * rel15->nrOfLayers;
|
||||
dlsch_offset += ceil_mod(dlsch_size, 64);
|
||||
dlsch_offset += ceil_mod(dlsch_size, 8 * 64);
|
||||
}
|
||||
|
||||
gNB->nrLDPC_coding_interface.nrLDPC_coding_encoder(&slot_parameters);
|
||||
|
||||
@@ -33,11 +33,26 @@ void nr_codeword_scrambling(uint8_t *in,
|
||||
{
|
||||
const int roundedSz = (size + 31) / 32;
|
||||
uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz);
|
||||
for (int i = 0; i < roundedSz; i++) {
|
||||
simde__m256i c = ((simde__m256i*)in)[i];
|
||||
uint32_t in32 = simde_mm256_movemask_epi8(simde_mm256_slli_epi16(c, 7));
|
||||
out[i] = in32 ^ seq[i];
|
||||
DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i]));
|
||||
unsigned int i_32 = 0;
|
||||
#if defined(__AVX512F__) && defined(__AVX512VL__)
|
||||
for (; i_32 < ((roundedSz >> 3) << 3); i_32 += 8) {
|
||||
__m256i in_256 = _mm256_load_epi32(&((uint32_t *)in)[i_32]);
|
||||
__m256i seq_256 = _mm256_load_epi32(&seq[i_32]);
|
||||
_mm256_storeu_epi32(&out[i_32], _mm256_xor_si256(in_256, seq_256));
|
||||
}
|
||||
#endif
|
||||
#if defined(__aarch64__)
|
||||
for (; i_32 < ((roundedSz >> 2) << 2); i_32 += 4) {
|
||||
unsigned int i = i_32 >> 2;
|
||||
uint32x4_t in_32x4 = vld1q_u32(&((uint32_t *)in)[i_32]);
|
||||
uint32x4_t seq_32x4 = vld1q_u32(&seq[i_32]);
|
||||
((uint32x4_t *)out)[i] = veorq_u32(in_32x4, seq_32x4);
|
||||
}
|
||||
#endif
|
||||
for (; i_32 < roundedSz; i_32++) {
|
||||
unsigned int i = i_32;
|
||||
out[i] = ((uint32_t *)in)[i] ^ seq[i];
|
||||
DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, ((uint32_t*)in)[i], out[i]));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -156,7 +156,11 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue,
|
||||
NR_UL_UE_HARQ_t *harq_process = &ue->ul_harq_processes[harq_pid];
|
||||
TB_parameters->segments = segments[pusch_id];
|
||||
|
||||
int r_offset = 0;
|
||||
const nfapi_nr_ue_pusch_pdu_t *pusch_pdu = &ulsch->pusch_pdu;
|
||||
uint16_t nb_rb = pusch_pdu->rb_size;
|
||||
memset(harq_process->f, 0, 14 * nb_rb * 12 * 16);
|
||||
TB_parameters->output = harq_process->f;
|
||||
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
|
||||
segment_parameters->c = harq_process->c[r];
|
||||
@@ -165,8 +169,6 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue,
|
||||
TB_parameters->Qm,
|
||||
TB_parameters->nb_layers,
|
||||
r);
|
||||
segment_parameters->output = harq_process->f + r_offset;
|
||||
r_offset += segment_parameters->E;
|
||||
|
||||
reset_meas(&segment_parameters->ts_interleave);
|
||||
reset_meas(&segment_parameters->ts_rate_match);
|
||||
|
||||
@@ -512,8 +512,8 @@ int main(int argc, char **argv)
|
||||
|
||||
//printf("crc32: [0]->0x%08x\n",crc24c(test_input, 32));
|
||||
// generate signal
|
||||
unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(32)));
|
||||
bzero(output,rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS);
|
||||
unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * NR_MAX_NB_LAYERS] __attribute__((aligned(64)));
|
||||
bzero(output, sizeof(output));
|
||||
if (input_fd == NULL) {
|
||||
msgDataTx.num_pdsch_slot = 1;
|
||||
nr_dlsch_encoding(gNB, &msgDataTx, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
|
||||
@@ -533,12 +533,12 @@ int main(int argc, char **argv)
|
||||
//if (i<16)
|
||||
// printf("encoder output f[%d] = %d\n",i,dlsch->harq_processes[0]->f[i]);
|
||||
|
||||
if (output[i] == 0)
|
||||
if ((output[i >> 3] & (1 << (i & 7))) == 0)
|
||||
modulated_input[i] = 1.0; ///sqrt(2); //QPSK
|
||||
else
|
||||
modulated_input[i] = -1.0; ///sqrt(2);
|
||||
|
||||
//if (i<16) printf("modulated_input[%d] = %d\n",i,modulated_input[i]);
|
||||
//if (i<16) printf("modulated_input[%d] = %f (%d)\n",i,modulated_input[i],output[i]);
|
||||
//SNR =10;
|
||||
SNR_lin = pow(10, SNR / 10.0);
|
||||
sigma = 1.0 / sqrt(2 * SNR_lin);
|
||||
|
||||
@@ -542,7 +542,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
*/
|
||||
|
||||
if (harq_process_ul_ue->f[i] == 0)
|
||||
if ((harq_process_ul_ue->f[i >> 3] & (1 << (i & 7))) == 0)
|
||||
modulated_input[i] = 1.0; ///sqrt(2); //QPSK
|
||||
else
|
||||
modulated_input[i] = -1.0; ///sqrt(2);
|
||||
|
||||
@@ -248,7 +248,9 @@ int main(int argc, char *argv[])
|
||||
InitSinLUT();
|
||||
|
||||
int c;
|
||||
while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:")) != -1) {
|
||||
bool setAffinity=false;
|
||||
char gNBthreads[128]="n";
|
||||
while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:Y:")) != -1) {
|
||||
/* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
|
||||
/* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
|
||||
if (c == 1 || c == '-' || c == 'O')
|
||||
@@ -493,6 +495,13 @@ int main(int argc, char *argv[])
|
||||
AssertFatal(filename_csv != NULL, "strdup() error: errno %d\n", errno);
|
||||
break;
|
||||
|
||||
case 'Y':
|
||||
threadCnt = sizeof(gNBthreads)-1;
|
||||
strncpy(gNBthreads, optarg, threadCnt);
|
||||
gNBthreads[threadCnt]=0;
|
||||
setAffinity=true;
|
||||
break;
|
||||
|
||||
case 'Z':
|
||||
transform_precoding = transformPrecoder_enabled;
|
||||
num_dmrs_cdm_grps_no_data = 2;
|
||||
@@ -591,7 +600,11 @@ int main(int argc, char *argv[])
|
||||
gNB->RU_list[0] = calloc(1, sizeof(**gNB->RU_list));
|
||||
gNB->RU_list[0]->rfdevice.openair0_cfg = openair0_cfg;
|
||||
|
||||
initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool");
|
||||
if (setAffinity == false)
|
||||
initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool");
|
||||
else
|
||||
initNamedTpool(gNBthreads, &gNB->threadPool, true, "gNB-tpool");
|
||||
|
||||
initNotifiedFIFO(&gNB->respDecode);
|
||||
|
||||
initNotifiedFIFO(&gNB->respPuschSymb);
|
||||
|
||||
Reference in New Issue
Block a user