maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mod: refactor PHY multiplications

Browse Source
This commit is contained in:
HHI
2025-04-25 14:47:59 +02:00
committed by Laurent THOMAS
parent 4da30019be
commit f470ea5946
29 changed files with 1070 additions and 2407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openair1/PHY/TOOLS/cdot_prod.c
+39 -24
View File
@@ -33,33 +33,48 @@
c32_t dot_product(const c16_t *x, const c16_t *y, const uint32_t N, const int output_shift)
{
const int16_t reflip[32] __attribute__((aligned(32))) = {1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1};
const int8_t imshuffle[64] __attribute__((aligned(32))) = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13, 18, 19, 16, 17, 22, 23, 20, 21, 26, 27, 24, 25, 30, 31, 28, 29};
const c16_t *end = x + N;
simde__m256i cumul_re = {0}, cumul_im = {0};
while (x < end) {
const simde__m256i in1 = simde_mm256_loadu_si256((simde__m256i *)x);
const simde__m256i in2 = simde_mm256_loadu_si256((simde__m256i *)y);
const simde__m256i tmpRe = simde_mm256_madd_epi16(in1, in2);
cumul_re = simde_mm256_add_epi32(cumul_re, simde_mm256_srai_epi32(tmpRe, output_shift));
const simde__m256i tmp1 = simde_mm256_shuffle_epi8(in2, *(simde__m256i *)imshuffle);
const simde__m256i tmp2 = simde_mm256_sign_epi16(tmp1, *(simde__m256i *)reflip);
const simde__m256i tmpIm = simde_mm256_madd_epi16(in1, tmp2);
cumul_im = simde_mm256_add_epi32(cumul_im, simde_mm256_srai_epi32(tmpIm, output_shift));
x += 8;
y += 8;
}
// this gives Re Re Im Im Re Re Im Im
const simde__m256i cumulTmp = simde_mm256_hadd_epi32(cumul_re, cumul_im);
const simde__m256i cumul = simde_mm256_hadd_epi32(cumulTmp, cumulTmp);
c32_t ret;
ret.r = simde_mm256_extract_epi32(cumul, 0) + simde_mm256_extract_epi32(cumul, 4);
ret.i = simde_mm256_extract_epi32(cumul, 1) + simde_mm256_extract_epi32(cumul, 5);
const c16_t *end = x + N;
#if defined(__x86_64__) || defined(__i386__)
if (__builtin_cpu_supports("avx2")) {
simde__m256i cumul_re = simde_mm256_setzero_si256();
simde__m256i cumul_im = simde_mm256_setzero_si256();
simde__m256i *x256 = (simde__m256i *)x;
simde__m256i *y256 = (simde__m256i *)y;
for (int i = 0; i < N >> 3; i++ ) {
const simde__m256i in1 = simde_mm256_loadu_si256(&x256[i]);
const simde__m256i in2 = simde_mm256_loadu_si256(&y256[i]);
const simde__m256i tmpRe = oai_mm256_smadd(in1, in2, output_shift);
const simde__m256i tmpIm = oai_mm256_smadd(oai_mm256_swap(oai_mm256_conj(in1)), in2, output_shift);
//const simde__m256i tmpIm = oai_mm256_smadd(in1, oai_mm256_conj(oai_mm256_swap(in2)), output_shift); // alternative way
cumul_re = simde_mm256_add_epi32(cumul_re, tmpRe);
cumul_im = simde_mm256_add_epi32(cumul_im, tmpIm);
}
// this gives Re Re Im Im Re Re Im Im
const simde__m256i cumulTmp = simde_mm256_hadd_epi32(cumul_re, cumul_im);
const simde__m256i cumul = simde_mm256_hadd_epi32(cumulTmp, cumulTmp);
ret.r = simde_mm256_extract_epi32(cumul, 0) + simde_mm256_extract_epi32(cumul, 4);
ret.i = simde_mm256_extract_epi32(cumul, 1) + simde_mm256_extract_epi32(cumul, 5);
// update pointers
x += (N & ~7);
y += (N & ~7);
}
#endif
// tail processing
if (x!=end) {
x-=8;
y-=8;
for ( ; x <end; x++,y++ ) {
ret.r += ((x->r*y->r)>>output_shift) + ((x->i*y->i)>>output_shift);
ret.i += ((x->r*y->i)>>output_shift) - ((x->i*y->r)>>output_shift);
openair1/PHY/TOOLS/cmult_vv.c
+46 -121
View File
@@ -19,23 +19,14 @@
* contact@openairinterface.org
*/
#include "PHY/defs_common.h"
#include "tools_defs.h"
#include <stdio.h>
#include "PHY/sse_intrin.h"
static const int16_t conjug[8]__attribute__((aligned(16))) = {-1,1,-1,1,-1,1,-1,1} ;
static const int16_t conjug2[8]__attribute__((aligned(16))) = {1,-1,1,-1,1,-1,1,-1} ;
#define simd_q15_t simde__m128i
#define set1_int16(a) simde_mm_set1_epi16(a)
#define setr_int16(a0, a1, a2, a3, a4, a5, a6, a7) simde_mm_setr_epi16(a0, a1, a2, a3, a4, a5, a6, a7 )
int mult_cpx_conj_vector(int16_t *x1,
void mult_cpx_conj_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint32_t N,
int output_shift,
int madd)
int madd)
{
// Multiply elementwise the complex conjugate of x1 with x2.
// x1 - input 1 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
@@ -52,49 +43,29 @@ int mult_cpx_conj_vector(int16_t *x1,
//
// madd - add the output to y
uint32_t i; // loop counter
simde__m128i *x1_128 = (simde__m128i *)x1;
simde__m128i *x2_128 = (simde__m128i *)x2;
simde__m128i *y_128 = (simde__m128i *)y;
simd_q15_t *x1_128;
simd_q15_t *x2_128;
simd_q15_t *y_128;
simd_q15_t tmp_re,tmp_im;
simd_q15_t tmpy0,tmpy1;
// SSE compute 4 cpx multiply for each loop
for(uint32_t i = 0; i < (N >> 2); i++) {
simde__m128i result = oai_mm_cpx_mult_conja(x1_128[i], x2_128[i], output_shift);
if (madd==1)
result = simde_mm_adds_epi16(y_128[i], result);
x1_128 = (simd_q15_t *)&x1[0];
x2_128 = (simd_q15_t *)&x2[0];
y_128 = (simd_q15_t *)&y[0];
// we compute 4 cpx multiply for each loop
for(i=0; i<(N>>2); i++) {
tmp_re = simde_mm_madd_epi16(*x1_128,*x2_128);
tmp_im = simde_mm_shufflelo_epi16(*x1_128, SIMDE_MM_SHUFFLE(2,3,0,1));
tmp_im = simde_mm_shufflehi_epi16(tmp_im, SIMDE_MM_SHUFFLE(2,3,0,1));
tmp_im = simde_mm_sign_epi16(tmp_im,*(simde__m128i*)&conjug[0]);
tmp_im = simde_mm_madd_epi16(tmp_im,*x2_128);
tmp_re = simde_mm_srai_epi32(tmp_re,output_shift);
tmp_im = simde_mm_srai_epi32(tmp_im,output_shift);
tmpy0 = simde_mm_unpacklo_epi32(tmp_re,tmp_im);
tmpy1 = simde_mm_unpackhi_epi32(tmp_re,tmp_im);
if (madd==0)
*y_128 = simde_mm_packs_epi32(tmpy0,tmpy1);
else
*y_128 += simde_mm_packs_epi32(tmpy0,tmpy1);
x1_128++;
x2_128++;
y_128++;
y_128[i] = result;
}
return(0);
}
int mult_cpx_vector(int16_t *x1, //Q15
int16_t *x2,//Q13
int16_t *y,
uint32_t N,
int output_shift)
void mult_cpx_vector(int16_t *x1, //Q15
int16_t *x2, //Q13
int16_t *y,
uint32_t N,
int output_shift)
{
// Multiply elementwise x1 with x2.
// x1 - input 1 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
@@ -108,54 +79,25 @@ int mult_cpx_vector(int16_t *x1, //Q15
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// output_shift - shift to be applied to generate output
uint32_t i; // loop counter
simd_q15_t *x1_128;
simd_q15_t *x2_128;
simd_q15_t *y_128;
simd_q15_t tmp_re,tmp_im;
simd_q15_t tmpy0,tmpy1;
x1_128 = (simd_q15_t *)&x1[0];
x2_128 = (simd_q15_t *)&x2[0];
y_128 = (simd_q15_t *)&y[0];
//print_shorts("x1_128:",&x1_128[0]);
// print_shorts("x2_128:",&x2_128[0]);
simde__m128i *x1_128 = (simde__m128i *)x1;
simde__m128i *x2_128 = (simde__m128i *)x2;
simde__m128i *y_128 = (simde__m128i *)y;
//right shift by 13 while p_a * x0 and 15 while
// we compute 4 cpx multiply for each loop
for(i=0; i<(N>>2); i++) {
tmp_re = simde_mm_sign_epi16(*x1_128,*(simde__m128i*)&conjug2[0]);// Q15
//print_shorts("tmp_re1:",&tmp_re[i]);
tmp_re = simde_mm_madd_epi16(tmp_re,*x2_128); //Q28
//print_ints("tmp_re2:",&tmp_re[i]);
tmp_im = simde_mm_shufflelo_epi16(*x1_128, SIMDE_MM_SHUFFLE(2,3,0,1)); //Q15
//print_shorts("tmp_im1:",&tmp_im[i]);
tmp_im = simde_mm_shufflehi_epi16(tmp_im, SIMDE_MM_SHUFFLE(2,3,0,1)); //Q15
//print_shorts("tmp_im2:",&tmp_im[i]);
tmp_im = simde_mm_madd_epi16(tmp_im, *x2_128); //Q28
//print_ints("tmp_im3:",&tmp_im[i]);
tmp_re = simde_mm_srai_epi32(tmp_re,output_shift);//Q(28-shift)
//print_ints("tmp_re shifted:",&tmp_re[i]);
tmp_im = simde_mm_srai_epi32(tmp_im,output_shift); //Q(28-shift)
//print_ints("tmp_im shifted:",&tmp_im[i]);
tmpy0 = simde_mm_unpacklo_epi32(tmp_re,tmp_im); //Q(28-shift)
//print_ints("unpack lo :",&tmpy0[i]);
tmpy1 = simde_mm_unpackhi_epi32(tmp_re,tmp_im); //Q(28-shift)
//print_ints("mrc rho0:",&tmpy1[i]);
*y_128 = simde_mm_packs_epi32(tmpy0,tmpy1); //must be Q15
//print_shorts("*y_128:",&y_128[i]);
x1_128++;
x2_128++;
y_128++;
// SSE compute 4 cpx multiply for each loop
for(uint32_t i=0; i < (N >> 2); i++) {
y_128[i] = oai_mm_cpx_mult(x1_128[i], x2_128[i], output_shift);
}
return(0);
}
int multadd_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint8_t zero_flag,
uint32_t N,
int output_shift)
void multadd_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint8_t zero_flag,
uint32_t N,
int output_shift)
{
// Multiply elementwise the complex conjugate of x1 with x2.
// x1 - input 1 in the format |Re0 Im0 Re1 Im1|,......,|Re(N-2) Im(N-2) Re(N-1) Im(N-1)|
@@ -171,36 +113,19 @@ int multadd_cpx_vector(int16_t *x1,
// N - the size f the vectors (this function does N cpx mpy. WARNING: N>=4;
//
// output_shift - shift to be applied to generate output
uint32_t i; // loop counter
simd_q15_t *x1_128;
simd_q15_t *x2_128;
simd_q15_t *y_128;
simd_q15_t tmp_re,tmp_im;
simd_q15_t tmpy0,tmpy1;
x1_128 = (simd_q15_t *)&x1[0];
x2_128 = (simd_q15_t *)&x2[0];
y_128 = (simd_q15_t *)&y[0];
// we compute 4 cpx multiply for each loop
for(i=0; i<(N>>2); i++) {
tmp_re = simde_mm_sign_epi16(*x1_128,*(simde__m128i*)&conjug2[0]);
tmp_re = simde_mm_madd_epi16(tmp_re,*x2_128);
tmp_im = simde_mm_shufflelo_epi16(*x1_128, SIMDE_MM_SHUFFLE(2,3,0,1));
tmp_im = simde_mm_shufflehi_epi16(tmp_im, SIMDE_MM_SHUFFLE(2,3,0,1));
tmp_im = simde_mm_madd_epi16(tmp_im,*x2_128);
tmp_re = simde_mm_srai_epi32(tmp_re,output_shift);
tmp_im = simde_mm_srai_epi32(tmp_im,output_shift);
tmpy0 = simde_mm_unpacklo_epi32(tmp_re,tmp_im);
//print_ints("unpack lo:",&tmpy0[i]);
tmpy1 = simde_mm_unpackhi_epi32(tmp_re,tmp_im);
//print_ints("unpack hi:",&tmpy1[i]);
if (zero_flag == 1)
*y_128 = simde_mm_packs_epi32(tmpy0,tmpy1);
else
*y_128 = simde_mm_adds_epi16(*y_128,simde_mm_packs_epi32(tmpy0,tmpy1));
//print_shorts("*y_128:",&y_128[i]);
x1_128++;
x2_128++;
y_128++;
simde__m128i *x1_128 = (simde__m128i *)x1;
simde__m128i *x2_128 = (simde__m128i *)x2;
simde__m128i *y_128 = (simde__m128i *)y;
// SSE compute 4 cpx multiply for each loop
for(uint32_t i = 0; i < (N >> 2); i++) {
simde__m128i result = oai_mm_cpx_mult(x1_128[i], x2_128[i], output_shift);
if (zero_flag == 0)
result = simde_mm_adds_epi16(y_128[i], result);
y_128[i] = result;
}
return(0);
}
openair1/PHY/TOOLS/simde_operations.c → openair1/PHY/TOOLS/oai_arith_operations.c
+33 -19
View File
@@ -21,40 +21,54 @@
#include <simde/x86/avx2.h>
void simde_mm128_separate_real_imag_parts(simde__m128i *out_re, simde__m128i *out_im, simde__m128i in0, simde__m128i in1)
void simde_mm_separate_real_imag_parts(simde__m128i *out_re, simde__m128i *out_im, simde__m128i in0, simde__m128i in1)
{
#define SHUFFLE_MASK SIMDE_MM_SHUFFLE(3, 1, 2, 0) // 0xD8
// Put in0 = [Re(0,1) Re(2,3) Im(0,1) Im(2,3)]
in0 = simde_mm_shufflelo_epi16(in0, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in0 = simde_mm_shufflehi_epi16(in0, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in0 = simde_mm_shuffle_epi32(in0, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in0 = simde_mm_shufflelo_epi16(in0, SHUFFLE_MASK);
in0 = simde_mm_shufflehi_epi16(in0, SHUFFLE_MASK);
in0 = simde_mm_shuffle_epi32(in0, SHUFFLE_MASK);
// Put xmm1 = [Re(4,5) Re(6,7) Im(4,5) Im(6,7)]
in1 = simde_mm_shufflelo_epi16(in1, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in1 = simde_mm_shufflehi_epi16(in1, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in1 = simde_mm_shuffle_epi32(in1, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in1 = simde_mm_shufflelo_epi16(in1, SHUFFLE_MASK);
in1 = simde_mm_shufflehi_epi16(in1, SHUFFLE_MASK);
in1 = simde_mm_shuffle_epi32(in1, SHUFFLE_MASK);
if (out_re != NULL)
*out_re = simde_mm_unpacklo_epi64(in0, in1);
*out_re = simde_mm_unpacklo_epi64(in0, in1);
*out_im = simde_mm_unpackhi_epi64(in0, in1);
if (out_im != NULL)
*out_im = simde_mm_unpackhi_epi64(in0, in1);
#undef SHUFFLE_MASK
}
void simde_mm256_separate_real_imag_parts(simde__m256i *out_re, simde__m256i *out_im, simde__m256i in0, simde__m256i in1)
{
// Put in0 = [Re(0,1,2,3) Im(0,1,2,3) Re(4,5,6,7) Im(4,5,6,7)]
in0 = simde_mm256_shufflelo_epi16(in0, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in0 = simde_mm256_shufflehi_epi16(in0, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in0 = simde_mm256_shuffle_epi32(in0, 0xd8); //_MM_SHUFFLE(0,2,1,3));
#define SHUFFLE_MASK SIMDE_MM_SHUFFLE(3, 1, 2, 0) // 0xD8
// Put in0 = [Re(0,1,2,3) Im(0,1,2,3) Re(4,5,6,7) Im(4,5,6,7)]
in0 = simde_mm256_shufflelo_epi16(in0, SHUFFLE_MASK);
in0 = simde_mm256_shufflehi_epi16(in0, SHUFFLE_MASK);
in0 = simde_mm256_shuffle_epi32(in0, SHUFFLE_MASK);
// Put in1 = [Re(8,9,10,11) Im(8,9,10,11) Re(12,13,14,15) Im(12,13,14,15)]
in1 = simde_mm256_shufflelo_epi16(in1, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in1 = simde_mm256_shufflehi_epi16(in1, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in1 = simde_mm256_shuffle_epi32(in1, 0xd8); //_MM_SHUFFLE(0,2,1,3));
in1 = simde_mm256_shufflelo_epi16(in1, SHUFFLE_MASK);
in1 = simde_mm256_shufflehi_epi16(in1, SHUFFLE_MASK);
in1 = simde_mm256_shuffle_epi32(in1, SHUFFLE_MASK);
// Put tmp0 =[Re(0,1,2,3) Re(8,9,10,11) Re(4,5,6,7) Re(12,13,14,15)]
// Put tmp0 = [Re(0,1,2,3) Re(8,9,10,11) Re(4,5,6,7) Re(12,13,14,15)]
simde__m256i tmp0 = simde_mm256_unpacklo_epi64(in0, in1);
// Put tmp1 = [Im(0,1,2,3) Im(8,9,10,11) Im(4,5,6,7) Im(12,13,14,15)]
simde__m256i tmp1 = simde_mm256_unpackhi_epi64(in0, in1);
if (out_re != NULL)
*out_re = simde_mm256_permute4x64_epi64(tmp0, SHUFFLE_MASK);
*out_re = simde_mm256_permute4x64_epi64(tmp0, 0xd8);
*out_im = simde_mm256_permute4x64_epi64(tmp1, 0xd8);
if (out_im != NULL)
*out_im = simde_mm256_permute4x64_epi64(tmp1, SHUFFLE_MASK);
#undef SHUFFLE_MASK
}
openair1/PHY/TOOLS/tools_defs.h
+16 -17
View File
@@ -32,12 +32,11 @@
#include <stdio.h>
#include <stdint.h>
#include <assert.h>
#include "PHY/sse_intrin.h"
#include "common/utils/assertions.h"
#include "common/utils/utils.h"
#include <simde/simde-common.h>
#include <simde/x86/sse.h>
#include <simde/x86/avx2.h>
#include "common/utils/LOG/log.h"
#define simd_q15_t simde__m128i
@@ -299,7 +298,7 @@ extern "C" {
const simde__m256i x_mul_alpha_shift15 =simde_mm256_mulhrs_epi16(alpha256, x_duplicate_ordered);
// Existing multiplication normalization is weird, constant table in alpha need to be doubled
const simde__m256i x_mul_alpha_x2= simde_mm256_adds_epi16(x_mul_alpha_shift15,x_mul_alpha_shift15);
*y128= simde_mm_adds_epi16(simde_mm256_extracti128_si256(x_mul_alpha_x2,0),*y128);
*y128 = simde_mm_adds_epi16(simde_mm256_extracti128_si256(x_mul_alpha_x2,0),*y128);
y128++;
*y128= simde_mm_adds_epi16(simde_mm256_extracti128_si256(x_mul_alpha_x2,1),*y128);
y128++;
@@ -461,7 +460,7 @@ void multadd_complex_vector_real_scalar(int16_t *x,
@param madd - if not zero result is added to output
*/
int mult_cpx_conj_vector(int16_t *x1,
void mult_cpx_conj_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint32_t N,
@@ -480,18 +479,18 @@ int mult_cpx_conj_vector(int16_t *x1,
@param output_shift - shift to be applied to generate output
*/
int multadd_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint8_t zero_flag,
uint32_t N,
int output_shift);
void multadd_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint8_t zero_flag,
uint32_t N,
int output_shift);
int mult_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint32_t N,
int output_shift);
void mult_cpx_vector(int16_t *x1,
int16_t *x2,
int16_t *y,
uint32_t N,
int output_shift);
// lte_dfts.c
void init_fft(uint16_t size,
@@ -789,7 +788,7 @@ c32_t dot_product(const c16_t *x,
double interp(double x, double *xs, double *ys, int count);
void simde_mm128_separate_real_imag_parts(simde__m128i *out_re, simde__m128i *out_im, simde__m128i in0, simde__m128i in1);
void simde_mm_separate_real_imag_parts(simde__m128i *out_re, simde__m128i *out_im, simde__m128i in0, simde__m128i in1);
void simde_mm256_separate_real_imag_parts(simde__m256i *out_re, simde__m256i *out_im, simde__m256i in0, simde__m256i in1);
void InitSinLUT(void);