SIMD optimizations for Neon/AVX2/AVX512 in the PDSCH transmit path

openair1/PHY/MODULATION/nr_modulation.c

@@ -24,7 +24,9 @@
 #include "executables/softmodem-common.h"
 
 // #define DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // TODO: For debug, to be removed if want to merge to develop
-
+// #define DEBUG_LAYER_MAPPING
+#define USE_NEON
+// #define USE_GATHER
 //  Table 6.3.1.5-1 Precoding Matrix W 1 layer 2 antenna ports 'n' = -1 and 'o' = -j
 const char nr_W_1l_2p[6][2][1] = {
     {{'1'}, {'0'}}, // pmi 0
@@ -112,10 +114,7 @@ const char nr_W_4l_4p[5][4][4] = {
     {{'1', '1', '1', '1'}, {'1', 'n', '1', 'n'}, {'j', 'j', 'o', 'o'}, {'j', 'o', 'o', 'j'}} // pmi 4
 };
 
-void nr_modulation(const uint32_t *in,
+void nr_modulation(const uint32_t *in, uint32_t length, uint16_t mod_order, int16_t *out)
-                   uint32_t length,
-                   uint16_t mod_order,
-                   int16_t *out)
 {
   uint16_t mask = ((1 << mod_order) - 1);
   int32_t *nr_mod_table32;
@@ -133,7 +132,6 @@ void nr_modulation(const uint32_t *in,
   LOG_D(PHY, "nr_modulation: length %d, mod_order %d\n", length, mod_order);
 
   switch (mod_order) {
-
 #if defined(__SSE2__)
     case 2:
       nr_mod_table128 = (simde__m128i *)nr_qpsk_byte_mod_table;
@@ -251,36 +249,296 @@ void nr_layer_mapping(int nbCodes,
                       uint8_t n_layers,
                       int layerSz,
                       uint32_t n_symbs,
-                      c16_t tx_layer[layerSz],
+                      c16_t tx_layers[][layerSz])
-                      int layer)
 {
   LOG_D(PHY, "Doing layer mapping for %d layers, %d symbols\n", n_layers, n_symbs);
 
   switch (n_layers) {
-
     case 1:
-      memcpy(tx_layer, mod_symbs[0], n_symbs * sizeof(**mod_symbs));
+      memcpy(tx_layers[0], mod_symbs[0], n_symbs * sizeof(**mod_symbs));
       break;
 
-    case 2:
+    case 2: {
-    case 3:
+#if defined(__AVX512BW__)
-    case 4:
+      int i = 0;
-    for (int i = 0; i < n_symbs / n_layers; i++) {
+      __m512i perm2a = _mm512_set_epi32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-      const c16_t *base = mod_symbs[0] + n_layers * i;
+      __m512i perm2b = _mm512_set_epi32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1);
-      tx_layer[i] = base[layer];
+#ifndef USE_GATHER
+      __m512i a, b;
+      int j;
+      for (i = 0, j = 0; i < n_symbs >> 4; i += 2, j++) {
+        a = ((__m512i *)mod_symbs[0])[i];
+        b = ((__m512i *)mod_symbs[0])[i + 1];
+        ((__m512i *)tx_layers[0])[j] = _mm512_permutex2var_epi32(a, perm2a, b);
+        ((__m512i *)tx_layers[1])[j] = _mm512_permutex2var_epi32(a, perm2b, b);
       }
-      break;
+      if (i << 4 != n_symbs) {
+        for (int i2 = ((n_symbs >> 4) << 4); i2 < n_symbs; i2 += 2) {
+          tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
+        }
+      }
+#else
+      for (i = 0; i < n_symbs; i += 32) {
+        *(__m512i *)(tx_layers[0] + i / 2) = _mm512_i32gather_epi32(perm2a, mod_symbs[0] + i, 4);
+        *(__m512i *)(tx_layers[1] + i / 2) = _mm512_i32gather_epi32(perm2b, mod_symbs[0] + i, 4);
+      }
+      for (; i < n_symbs; i += 2) {
+        tx_layers[0][i >> 1] = mod_symbs[0][i];
+        tx_layers[1][i >> 1] = mod_symbs[0][i + 1];
+      }
+#endif
+#elif defined(__aarch64__)
+      int i;
+#ifdef USE_NEON
+      // SIMDe doesn't handle this properly, gcc up to 14.2 neither
+      uint8_t const perm0[16] = {0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15};
+      uint8x16_t perm = vld1q_u8(perm0);
+      uint8x16_t d;
+      for (i = 0; i < n_symbs >> 2; i++) {
+        d = vqtbl1q_u8(((uint8x16_t *)mod_symbs[0])[i], perm);
+        ((int64_t *)tx_layers[0])[i] = vgetq_lane_u64((uint64x2_t)d, 0);
+        ((int64_t *)tx_layers[1])[i] = vgetq_lane_u64((uint64x2_t)d, 1);
+      }
+      if (i << 2 != n_symbs) {
+        for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 2) {
+          tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
+        }
+      }
+#else
+      for (i = 0; i < n_symbs; i += 2) {
+        tx_layers[0][i >> 1] = mod_symbs[0][i];
+        tx_layers[1][i >> 1] = mod_symbs[0][i + 1];
+      }
+#endif
+#else
+      int i;
+      simde__m256i perm2 = simde_mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);
+      simde__m256i d;
+      for (i = 0; i < n_symbs >> 3; i++) {
+        d = simde_mm256_permutevar8x32_epi32(((simde__m256i *)mod_symbs[0])[i], perm2);
+        ((simde__m128i *)tx_layers[0])[i] = simde_mm256_extractf128_si256(d, 0);
+        ((simde__m128i *)tx_layers[1])[i] = simde_mm256_extractf128_si256(d, 1);
+      }
+      if (i << 3 != n_symbs) {
+        for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 2) {
+          tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
+        }
+      }
+#endif
+    } break;
+    case 3: {
+#if defined(__AVX512BW__)
+      __m512i i0, i1, i2, d0;
+      __m512i perm3_0 =
+          _mm512_set_epi32(13 + 16, 10 + 16, 7 + 16, 4 + 16, 1 + 16, 14 + 16, 11 + 16, 8 + 16, 5 + 16, 2 + 16, 15, 12, 9, 6, 3, 0);
+      __m512i perm3_0b = _mm512_set_epi32(13 + 16, 10 + 16, 7 + 16, 4 + 16, 1 + 16, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+      __m512i perm3_1 = _mm512_set_epi32(14 + 16,
+                                         11 + 16,
+                                         8 + 16,
+                                         5 + 16,
+                                         2 + 16,
+                                         15 + 16,
+                                         12 + 16,
+                                         9 + 16,
+                                         6 + 16,
+                                         3 + 16,
+                                         0 + 16,
+                                         13,
+                                         10,
+                                         7,
+                                         4,
+                                         1);
+      __m512i perm3_1b = _mm512_set_epi32(14 + 16, 11 + 16, 8 + 16, 5 + 16, 2 + 16, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+      __m512i perm3_2 = _mm512_set_epi32(15 + 16,
+                                         12 + 16,
+                                         9 + 16,
+                                         6 + 16,
+                                         3 + 16,
+                                         0 + 16,
+                                         13 + 16,
+                                         10 + 16,
+                                         7 + 16,
+                                         4 + 16,
+                                         1 + 16,
+                                         14,
+                                         11,
+                                         8,
+                                         5,
+                                         2);
+      __m512i perm3_2b = _mm512_set_epi32(15 + 16, 12 + 16, 9 + 16, 6 + 16, 3 + 16, 0 + 16, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+      int i, n;
+      for (i = 0, n = 0; i < n_symbs >> 4; i += 3, n++) {
+        i0 = ((__m512i *)mod_symbs[0])[i];
+        i1 = ((__m512i *)mod_symbs[0])[i + 1];
+        i2 = ((__m512i *)mod_symbs[0])[i + 2];
+        d0 = _mm512_permutex2var_epi32(i0, perm3_0, i1);
+        ((__m512i *)tx_layers[0])[n] = _mm512_permutex2var_epi32(d0, perm3_0b, i2); // 11000000
+        d0 = _mm512_permutex2var_epi32(i0, perm3_1, i1);
+        ((__m512i *)tx_layers[1])[n] = _mm512_permutex2var_epi32(d0, perm3_1b, i2); // 11000000
+        d0 = _mm512_permutex2var_epi32(i0, perm3_2, i1);
+        ((__m512i *)tx_layers[2])[n] = _mm512_permutex2var_epi32(d0, perm3_2b, i2); // 11000000
+      }
+#elif defined(__aarch64__)
+      for (int i = 0; i < n_symbs; i += 3) {
+        tx_layers[0][i / 3] = mod_symbs[0][i];
+        tx_layers[1][i / 3] = mod_symbs[0][i + 1];
+        tx_layers[2][i / 3] = mod_symbs[0][i + 2];
+      }
+
+#else
+      simde__m256i i0, i1, i2, d0, d1, d2, d3;
+      simde__m256i perm3_0 = simde_mm256_set_epi32(5, 2, 7, 4, 1, 6, 3, 0);
+      simde__m256i perm3_1 = simde_mm256_set_epi32(6, 3, 0, 5, 2, 7, 4, 1);
+      simde__m256i perm3_2 = simde_mm256_set_epi32(7, 4, 1, 6, 3, 0, 5, 2);
+      int i, n;
+      for (i = 0, n = 0; i < n_symbs >> 3; i += 3, n++) {
+        i0 = ((simde__m256i *)mod_symbs[0])[i];
+        i1 = ((simde__m256i *)mod_symbs[0])[i + 1];
+        i2 = ((simde__m256i *)mod_symbs[0])[i + 2];
+        d0 = simde_mm256_permutevar8x32_epi32(i0, perm3_0);
+        d1 = simde_mm256_permutevar8x32_epi32(i1, perm3_0);
+        d2 = simde_mm256_permutevar8x32_epi32(i2, perm3_0);
+        d3 = simde_mm256_blend_epi32(d0, d1, 0x38); // 00111000
+        ((simde__m256i *)tx_layers[0])[n] = simde_mm256_blend_epi32(d3, d2, 0xc0); // 11000000
+        d0 = simde_mm256_permutevar8x32_epi32(i0, perm3_1);
+        d1 = simde_mm256_permutevar8x32_epi32(i1, perm3_1);
+        d2 = simde_mm256_permutevar8x32_epi32(i2, perm3_1);
+        d3 = simde_mm256_blend_epi32(d0, d1, 0x18); // 00011000
+        ((simde__m256i *)tx_layers[1])[n] = simde_mm256_blend_epi32(d3, d2, 0xe0); // 11100000
+        d0 = simde_mm256_permutevar8x32_epi32(i0, perm3_2);
+        d1 = simde_mm256_permutevar8x32_epi32(i1, perm3_2);
+        d2 = simde_mm256_permutevar8x32_epi32(i2, perm3_2);
+        d3 = simde_mm256_blend_epi32(d0, d1, 0x1c); // 00011100
+        ((simde__m256i *)tx_layers[2])[n] = simde_mm256_blend_epi32(d3, d2, 0xe0); // 11100000
+#ifdef DEBUG_LAYER_MAPPING
+        printf("\nsymb %d/%d\n", i << 3, n_symbs);
+        printf(" layer 0:\t");
+        for (int j = 0; j < 8 * 6; j += 6) {
+          printf("%d %d ", ((int16_t *)&mod_symbs[0][i << 3])[j], ((int16_t *)&mod_symbs[0][i << 3])[j + 1]);
+        }
+        printf("\n layer 1:\t");
+        for (int j = 2; j < 8 * 6; j += 6) {
+          printf("%d %d ", ((int16_t *)&mod_symbs[0][i << 3])[j], ((int16_t *)&mod_symbs[0][i << 3])[j + 1]);
+        }
+        printf("\n layer 2:\t");
+        for (int j = 4; j < 8 * 6; j += 6) {
+          printf("%d %d ", ((int16_t *)&mod_symbs[0][i << 3])[j], ((int16_t *)&mod_symbs[0][i << 3])[j + 1]);
+        }
+        printf("\n Mapping layer 0:\t");
+        for (int j = 0; j < 16; j++) {
+          printf("%d ", ((int16_t *)&tx_layers[0][n << 3])[j]);
+        }
+        printf("\n Mapping layer 1:\t");
+        for (int j = 0; j < 16; j++) {
+          printf("%d ", ((int16_t *)&tx_layers[1][n << 3])[j]);
+        }
+        printf("\n Mapping layer 2:\t");
+        for (int j = 0; j < 16; j++) {
+          printf("%d ", ((int16_t *)&tx_layers[2][n << 3])[j]);
+        }
+#endif
+      }
+      if (i << 3 != n_symbs) {
+        for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 3) {
+          tx_layers[0][i2 / 3] = mod_symbs[0][i2];
+          tx_layers[1][i2 / 3] = mod_symbs[0][i2 + 1];
+          tx_layers[2][i2 / 3] = mod_symbs[0][i2 + 2];
+        }
+      }
+#endif
+    } break;
+
+    case 4: {
+#if defined(__AVX512BW__)
+      __m512i perm4 = _mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);
+      __m512i e;
+      int i;
+      for (i = 0; i < n_symbs >> 4; i++) {
+        e = _mm512_permutexvar_epi32(perm4, ((__m512i *)mod_symbs[0])[i]);
+        ((__m128i *)tx_layers[0])[i] = _mm512_extracti64x2_epi64(e, 0);
+        ((__m128i *)tx_layers[1])[i] = _mm512_extracti64x2_epi64(e, 1);
+        ((__m128i *)tx_layers[2])[i] = _mm512_extracti64x2_epi64(e, 2);
+        ((__m128i *)tx_layers[3])[i] = _mm512_extracti64x2_epi64(e, 3);
+      }
+      if (i << 4 != n_symbs) {
+        for (int i2 = ((n_symbs >> 4) << 4); i2 < n_symbs; i2 += 4) {
+          tx_layers[0][i2 >> 2] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 2] = mod_symbs[0][i2 + 1];
+          tx_layers[2][i2 >> 2] = mod_symbs[0][i2 + 2];
+          tx_layers[3][i2 >> 2] = mod_symbs[0][i2 + 3];
+        }
+      }
+#elif defined(__aarch64__)
+      int i;
+#ifdef USE_NEON
+      // SIMDe doesn't handle this properly, gcc up to 14.2 neither
+      uint32x4_t d4;
+      for (i = 0; i < n_symbs >> 2; i++) {
+        d4 = ((uint32x4_t *)mod_symbs[0])[i];
+        ((uint32_t *)tx_layers[0])[i] = vgetq_lane_u32(d4, 0);
+        ((uint32_t *)tx_layers[1])[i] = vgetq_lane_u32(d4, 1);
+        ((uint32_t *)tx_layers[2])[i] = vgetq_lane_u32(d4, 0);
+        ((uint32_t *)tx_layers[3])[i] = vgetq_lane_u32(d4, 1);
+      }
+      if (i << 2 != n_symbs) {
+        for (int i2 = ((n_symbs >> 2) << 2); i2 < n_symbs; i2 += 4) {
+          tx_layers[0][i2 >> 2] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 2] = mod_symbs[0][i2 + 1];
+          tx_layers[0][i2 >> 2] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 2] = mod_symbs[0][i2 + 1];
+        }
+      }
+#else
+      for (int i = 0; i < n_symbs; i += 4) {
+        tx_layers[0][i >> 2] = mod_symbs[0][i];
+        tx_layers[1][i >> 2] = mod_symbs[0][i + 1];
+        tx_layers[2][i >> 2] = mod_symbs[0][i + 2];
+        tx_layers[3][i >> 2] = mod_symbs[0][i + 3];
+      }
+#endif
+#else
+      simde__m256i perm4 = simde_mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
+      simde__m256i e;
+      int i;
+      for (i = 0; i < n_symbs >> 3; i++) {
+        e = simde_mm256_permutevar8x32_epi32(((simde__m256i *)mod_symbs[0])[i], perm4);
+        ((uint64_t *)tx_layers[0])[i] = simde_mm256_extract_epi64(e, 0);
+        ((uint64_t *)tx_layers[1])[i] = simde_mm256_extract_epi64(e, 1);
+        ((uint64_t *)tx_layers[2])[i] = simde_mm256_extract_epi64(e, 2);
+        ((uint64_t *)tx_layers[3])[i] = simde_mm256_extract_epi64(e, 3);
+      }
+      if (i << 3 != n_symbs) {
+        for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 3) {
+          tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
+          tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
+          tx_layers[2][i2 >> 1] = mod_symbs[0][i2 + 2];
+          tx_layers[3][i2 >> 1] = mod_symbs[0][i2 + 2];
+        }
+      }
+#endif
+    } break;
 
     case 5:
-      if (layer < 2)
+    case 6:
+    case 7:
+    case 8:
+      /*
+      // Layer 0,1
       for (int i = 0; i < n_symbs; i += 2) {
           const int txIdx = i / 2;
-          tx_layer[txIdx] = mod_symbs[0][i + layer];
+          tx_layer[0][txIdx] = mod_symbs[0][i];
+          tx_layer[1][txIdx] = mod_symbs[0][i + 1];
       }
+      // layers 2,3,4
       else
         for (int i = 0; i < n_symbs; i += 3) {
           const int txIdx = i / 3;
-          tx_layer[txIdx] = mod_symbs[1][i + layer];
+          tx_layer[2][txIdx] = mod_symbs[1][i + 2];
+          tx_layer[3][txIdx] = mod_symbs[1][i + 3];
+          tx_layer[4][txIdx] = mod_symbs[1][i + 4];
         }
       break;
 
@@ -288,7 +546,12 @@ void nr_layer_mapping(int nbCodes,
       for (int q=0; q<2; q++)
         for (int i = 0; i < n_symbs; i += 3) {
           const int txIdx = i / 3;
-          tx_layer[txIdx] = mod_symbs[q][i + layer];
+          tx_layer[0][txIdx] = mod_symbs[q][i + layer];
+          tx_layer[1][txIdx] = mod_symbs[q][i + layer];
+          tx_layer[2][txIdx] = mod_symbs[q][i + layer];
+          tx_layer[3][txIdx] = mod_symbs[q][i + layer];
+          tx_layer[4][txIdx] = mod_symbs[q][i + layer];
+          tx_layer[5][txIdx] = mod_symbs[q][i + layer];
         }
       break;
 
@@ -312,7 +575,7 @@ void nr_layer_mapping(int nbCodes,
         tx_layer[txIdx] = mod_symbs[q][i + layer];
       }
       break;
-
+*/
     default:
       AssertFatal(0, "Invalid number of layers %d\n", n_layers);
   }
@@ -360,7 +623,6 @@ void nr_dft(c16_t *z, c16_t *d, uint32_t Msc_PUSCH)
     for (i = 0, ip = 0; i < Msc_PUSCH; i++, ip += 4)
       z[i] = dft_out0[ip];
   }
-
 }
 
 void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation)
@@ -430,7 +692,9 @@ void init_timeshift_rotation(NR_DL_FRAME_PARMS *fp)
     fp->timeshift_symbol_rotation[i].i = (int16_t)round(exp_im * 32767);
 
     if (i < 10)
-      LOG_D(PHY,"Timeshift symbol rotation %d => (%d,%d) %f\n",i,
+      LOG_D(PHY,
+            "Timeshift symbol rotation %d => (%d,%d) %f\n",
+            i,
             fp->timeshift_symbol_rotation[i].r,
             fp->timeshift_symbol_rotation[i].i,
             poff);
@@ -524,8 +788,11 @@ void nr_layer_precoder_simd(const int n_layers,
       const simde__m256i w_s = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w
 
       // Multiplication and shift
-      const simde__m256i reals = simde_mm256_srai_epi32(simde_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15
+      const simde__m256i reals =
-      const simde__m256i imags = simde_mm256_slli_epi32(simde_mm256_madd_epi16(x, w_s), 1);  // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16
+          simde_mm256_srai_epi32(simde_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15
+      const simde__m256i imags = simde_mm256_slli_epi32(
+          simde_mm256_madd_epi16(x, w_s),
+          1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16
 
       // Re-arrange to match c16_t format
       const simde__m256i produ = simde_mm256_blend_epi16(reals, imags, 0xAA);
@@ -544,14 +811,7 @@ void nr_layer_precoder_simd(const int n_layers,
 #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Get result with trivial solution, TODO: To be removed
     c16_t y_triv[4];
     for (int i = 0; i < 4; i++)
-        y_triv[i] = nr_layer_precoder_cm(n_layers,
+      y_triv[i] = nr_layer_precoder_cm(n_layers, NR_SYMBOLS_PER_SLOT, symSz, txdataF_res_mapped, ant, pmi_pdu, symbol, sc + i);
-                                         NR_SYMBOLS_PER_SLOT,
-                                         symSz,
-                                         txdataF_res_mapped,
-                                         ant,
-                                         pmi_pdu,
-                                         symbol,
-                                         sc + i);
     memcpy(&txdataF_precoded[sc], y_triv, sizeof(y_triv));
 #endif
 
@@ -568,15 +828,19 @@ void nr_layer_precoder_simd(const int n_layers,
       const simde__m128i w_s = simde_mm_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w
 
       // Multiplication and shift
-      const simde__m128i reals = simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15
+      const simde__m128i reals =
-      const simde__m128i imags = simde_mm_slli_epi32(simde_mm_madd_epi16(x, w_s), 1);  // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16
+          simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15
+      const simde__m128i imags = simde_mm_slli_epi32(
+          simde_mm_madd_epi16(x, w_s),
+          1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16
 
       /* Re-arrange to match c16_t format
-        bit index: 0            | 16              | 32           | 48              | 64           | 80              | 96           | 112
+        bit index: 0            | 16              | 32           | 48              | 64           | 80              | 96           |
-        reals =   {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 | R3.r[15..30] | R3.r[31] (0)*15}
+        112 reals =   {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 |
-        imags =   {0 R0.i[0..14]| R0.i[15..30]    | 0 R1.i[0..14]| R1.i[15..30]    | 0 R2.i[0..14]| R2.i[15..30]    | 0 R3.i[0..14]| R3.i[15..30]   }
+        R3.r[15..30] | R3.r[31] (0)*15} imags =   {0 R0.i[0..14]| R0.i[15..30]    | 0 R1.i[0..14]| R1.i[15..30]    | 0 R2.i[0..14]|
-        16b from  {reals        | imags           | reals        | imags           | reals        | imags           | reals        | imags          }
+        R2.i[15..30]    | 0 R3.i[0..14]| R3.i[15..30]   } 16b from  {reals        | imags           | reals        | imags | reals
-        produ =   {R0.r[15..30] | R0.i[15..30]    | R1.r[15..30] | R1.i[15..30]    | R2.r[15..30] | R2.i[15..30]    | R3.r[15..30] | R3.i[15..30]   }
+        | imags           | reals        | imags          } produ =   {R0.r[15..30] | R0.i[15..30]    | R1.r[15..30] | R1.i[15..30]
+        | R2.r[15..30] | R2.i[15..30]    | R3.r[15..30] | R3.i[15..30]   }
       */
       const simde__m128i produ = simde_mm_blend_epi16(reals, imags, 0xAA);
 

openair1/PHY/MODULATION/nr_modulation.h

@@ -60,8 +60,7 @@ void nr_layer_mapping(int nbCodes,
                       uint8_t n_layers,
                       int layerSz,
                       uint32_t n_symbs,
-                      c16_t tx_layers[layerSz],
+                      c16_t tx_layers[][layerSz]);
-                      int l);
 
 /*! \brief Perform NR layer mapping. TS 38.211 V15.4.0 subclause 7.3.1.3
   @param[in] ulsch_ue, double Pointer to NR_UE_ULSCH_t struct

openair1/PHY/NR_TRANSPORT/nr_dci.c

@@ -248,9 +248,11 @@ void nr_generate_dci_top(processingData_L1tx_t *msgTx, int slot, int txdataF_off
 {
   PHY_VARS_gNB *gNB = msgTx->gNB;
   NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms;
+  start_meas(&gNB->dci_generation_stats);
   for (int i = 0; i < msgTx->num_ul_pdcch; i++)
     nr_generate_dci(msgTx->gNB, &msgTx->ul_pdcch_pdu[i].pdcch_pdu.pdcch_pdu_rel15, txdataF_offset, frame_parms, slot);
   for (int i = 0; i < msgTx->num_dl_pdcch; i++)
     nr_generate_dci(msgTx->gNB, &msgTx->pdcch_pdu[i].pdcch_pdu_rel15, txdataF_offset, frame_parms, slot);
+  stop_meas(&gNB->dci_generation_stats);
 }
 

openair1/PHY/defs_gNB.h

@@ -520,6 +520,8 @@ typedef struct PHY_VARS_gNB_s {
   time_stats_t dlsch_interleaving_stats;
   time_stats_t dlsch_segmentation_stats;
 
+  time_stats_t dci_generation_stats;
+  time_stats_t phase_comp_stats;
   time_stats_t rx_pusch_stats;
   time_stats_t rx_pusch_init_stats;
   time_stats_t rx_pusch_symbol_processing_stats;

openair1/SCHED_NR/phy_procedures_nr_gNB.c

@@ -290,6 +290,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
 
   //apply the OFDM symbol rotation here
   if (gNB->phase_comp) {
+    start_meas(&gNB->phase_comp_stats);
     for(int i = 0; i < gNB->common_vars.num_beams_period; ++i) {
       for (int aa = 0; aa < cfg->carrier_config.num_tx_ant.value; aa++) {
         apply_nr_rotation_TX(fp,
@@ -304,6 +305,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
           T_INT(aa), T_BUFFER(&gNB->common_vars.txdataF[aa][txdataF_offset], fp->samples_per_slot_wCP*sizeof(int32_t)));
       }
     }
+    stop_meas(&gNB->phase_comp_stats);
   }
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_TX + gNB->CC_id, 0);

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -960,10 +960,12 @@ printf("%d\n", slot);
     reset_meas(&gNB->dlsch_segmentation_stats);
     reset_meas(&gNB->dlsch_modulation_stats);
     reset_meas(&gNB->dlsch_encoding_stats);
+    reset_meas(&gNB->dci_generation_stats);
     reset_meas(&gNB->tinput);
     reset_meas(&gNB->tprep);
     reset_meas(&gNB->tparity);
     reset_meas(&gNB->toutput);
+    reset_meas(&gNB->phase_comp_stats);
 
     uint32_t errors_scrambling[16] = {0};
     int n_errors[16] = {0};
@@ -1259,6 +1261,7 @@ printf("%d\n", slot);
              UE->dl_harq_processes[0][slot].C,
              msgDataTx->dlsch[0][0].harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0] << 3);
       printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx");
+      printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");
       printStatIndent2(&gNB->dlsch_encoding_stats,"DLSCH encoding time");
       printStatIndent3(&gNB->dlsch_segmentation_stats,"DLSCH segmentation time");
       printStatIndent3(&gNB->tinput,"DLSCH LDPC input processing time");
@@ -1271,6 +1274,8 @@ printf("%d\n", slot);
       printStatIndent2(&gNB->dlsch_scrambling_stats, "DLSCH scrambling time");
       printStatIndent2(&gNB->dlsch_resource_mapping_stats, "DLSCH Resource Mapping time");
       printStatIndent2(&gNB->dlsch_precoding_stats,"DLSCH Layer Precoding time");
+      if (gNB->phase_comp)
+        printStatIndent2(&gNB->phase_comp_stats, "Phase Compensation");
 
       printf("\nUE function statistics (per %d us slot)\n", 1000 >> *scc->ssbSubcarrierSpacing);
       for (int i = RX_PDSCH_STATS; i <= DLSCH_PROCEDURES_STATS; i++) {