From 66cbe490e1ccd286b1c662c11d91517be1361073 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <sakthivel.velumani@eurecom.fr>
Date: Wed, 30 Jun 2021 16:08:05 +0200
Subject: [PATCH 01/14] Prevent buffer under/overflow in ULSCH decoding through
 alignment and sufficient array size

---
 openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c
index 3aad4980bc..d152c9432a 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c
@@ -308,8 +308,8 @@ void nr_processULSegment(void* arg) {
   int max_ldpc_iterations = p_decoderParms->numMaxIter;
   int8_t llrProcBuf[OAI_UL_LDPC_MAX_NUM_LLR] __attribute__ ((aligned(32)));
 
-  int16_t  z [68*384];
-  int8_t   l [68*384];
+  int16_t  z [68*384 + 16] __attribute__ ((aligned(16)));
+  int8_t   l [68*384 + 16] __attribute__ ((aligned(16)));
 
   __m128i *pv = (__m128i*)&z;
   __m128i *pl = (__m128i*)&l;

From 4d05f4b288ef204a42cd1eb6b6d5ffe877aee7ac Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Thu, 1 Jul 2021 15:22:10 +0200
Subject: [PATCH 02/14] Blocking nr_scheduler_response untill L1 tx finishes

---
 openair1/SCHED_NR/fapi_nr_l1.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index f04401aac5..bac52845ff 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -178,6 +178,13 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
 
   gNB         = RC.gNB[Mod_id];
 
+  // we wait for the L1 Tx processing to finish before copying MAC PDUs to PHY
+  {
+    notifiedFIFO_elt_t *res;
+    res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
+    pushNotifiedFIFO(gNB->resp_L1_tx,res);
+  }
+
   uint8_t number_dl_pdu             = (DL_req==NULL) ? 0 : DL_req->dl_tti_request_body.nPDUs;
   uint8_t number_ul_dci_pdu         = (UL_dci_req==NULL) ? 0 : UL_dci_req->numPdus;
   uint8_t number_ul_tti_pdu         = (UL_tti_req==NULL) ? 0 : UL_tti_req->n_pdus;

From cb68456357e4e97f298fb7e81af0cc4ab72eb1a0 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Sat, 3 Jul 2021 01:15:10 +0200
Subject: [PATCH 03/14] Enabled multi-threading for gNB L1 tx A second thread
 message is created for L1 tx processing and pushed into the threaPool. The
 message contain the nfapi rel15 PDUs for SSB, PDCCH and PDSCH. The memory of
 these PDUs are reused for subsequent slots. Need to fix the nfapi mode VNF
 PNF implementation to adapt to this memory scheme.

---
 executables/nr-gnb.c                       | 24 +++++++----
 executables/nr-ru.c                        | 23 +++++++++--
 openair1/PHY/INIT/nr_init.c                | 24 ++++++-----
 openair1/PHY/NR_TRANSPORT/nr_dlsch.c       |  8 ++--
 openair1/PHY/NR_TRANSPORT/nr_dlsch.h       |  4 +-
 openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c | 38 +++---------------
 openair1/PHY/defs_gNB.h                    | 21 ++++++++--
 openair1/SCHED_NR/fapi_nr_l1.c             | 45 ++++++++-------------
 openair1/SCHED_NR/fapi_nr_l1.h             |  4 +-
 openair1/SCHED_NR/phy_procedures_nr_gNB.c  | 46 ++++++++++------------
 10 files changed, 116 insertions(+), 121 deletions(-)

diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c
index e9319f9d3c..2bbabf3fe0 100644
--- a/executables/nr-gnb.c
+++ b/executables/nr-gnb.c
@@ -113,15 +113,25 @@ time_stats_t softmodem_stats_rx_sf; // total rx time
 
 void tx_func(void *param) {
 
-  processingData_L1_t *info = (processingData_L1_t *) param;
+  processingData_L1tx_t *info = (processingData_L1tx_t *) param;
   PHY_VARS_gNB *gNB = info->gNB;
   int frame_tx = info->frame_tx;
   int slot_tx = info->slot_tx;
 
-  phy_procedures_gNB_TX(gNB, frame_tx,slot_tx, 1);
+  phy_procedures_gNB_TX(info,
+                        frame_tx,
+                        slot_tx,
+                        1);
+  if ((frame_tx&127) == 0) dump_pdsch_stats(gNB);
 
-  // start FH TX processing
+  // If the later of the 2 L1 tx thread finishes first,
+  // we wait for the earlier one to finish and start the RU thread
+  // to avoid realtime issues with USRP
   notifiedFIFO_elt_t *res; 
+  res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
+  pushNotifiedFIFO(gNB->resp_L1_tx,res);
+
+  // Start RU TX processing.
   res = pullTpool(gNB->resp_RU_tx, gNB->threadPool);
   processingData_RU_t *syncMsg = (processingData_RU_t *)NotifiedFifoData(res);
   syncMsg->frame_tx = frame_tx;
@@ -257,7 +267,6 @@ void rx_func(void *param) {
   LOG_D(PHY,"%s() Exit proc[rx:%d%d tx:%d%d]\n", __FUNCTION__, frame_rx, slot_rx, frame_tx, slot_tx);
 
   // Call the scheduler
-
   start_meas(&gNB->ul_indication_stats);
   pthread_mutex_lock(&gNB->UL_INFO_mutex);
   gNB->UL_INFO.frame     = frame_rx;
@@ -269,10 +278,9 @@ void rx_func(void *param) {
   stop_meas(&gNB->ul_indication_stats);
   
   notifiedFIFO_elt_t *res; 
-
   if (tx_slot_type == NR_DOWNLINK_SLOT || tx_slot_type == NR_MIXED_SLOT) {
     res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
-    processingData_L1_t *syncMsg = (processingData_L1_t *)NotifiedFifoData(res);
+    processingData_L1tx_t *syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res);
     syncMsg->gNB = gNB;
     syncMsg->frame_rx = frame_rx;
     syncMsg->slot_rx = slot_rx;
@@ -327,7 +335,6 @@ static void *process_stats_thread(void *param) {
 
   PHY_VARS_gNB *gNB  = (PHY_VARS_gNB *)param;
 
-  reset_meas(&gNB->phy_proc_tx);
   reset_meas(&gNB->dlsch_encoding_stats);
   reset_meas(&gNB->phy_proc_rx);
   reset_meas(&gNB->ul_indication_stats);
@@ -339,7 +346,8 @@ static void *process_stats_thread(void *param) {
   while(!oai_exit)
   {
     sleep(1);
-    print_meas(&gNB->phy_proc_tx, "L1 Tx processing", NULL, NULL);
+    print_meas(gNB->phy_proc_tx_0, "L1 Tx processing thread 0", NULL, NULL);
+    print_meas(gNB->phy_proc_tx_1, "L1 Tx processing thread 1", NULL, NULL);
     print_meas(&gNB->dlsch_encoding_stats, "DLSCH encoding", NULL, NULL);
     print_meas(&gNB->phy_proc_rx, "L1 Rx processing", NULL, NULL);
     print_meas(&gNB->ul_indication_stats, "UL Indication", NULL, NULL);
diff --git a/executables/nr-ru.c b/executables/nr-ru.c
index be6d13daad..a01be1ebaa 100644
--- a/executables/nr-ru.c
+++ b/executables/nr-ru.c
@@ -1270,12 +1270,27 @@ void *ru_thread( void *param ) {
   wait_sync("ru_thread");
 
   notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_L1,rx_func);
-  notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_L1_tx,tx_func);
-  notifiedFIFO_elt_t *msgRUTx = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_RU_tx,ru_tx_func);
-  processingData_L1_t *syncMsg;
-  notifiedFIFO_elt_t *res; 
   pushNotifiedFIFO(gNB->resp_L1,msg); // to unblock the process in the beginning
+
+  // we create 2 threads for L1 tx processing
+  notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
+  processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+  init_DLSCH_struct(gNB, msgDataTx);
+  reset_meas(&msgDataTx->phy_proc_tx);
+  gNB->phy_proc_tx_0 = &msgDataTx->phy_proc_tx;
   pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
+
+  msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
+  msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+  init_DLSCH_struct(gNB, msgDataTx);
+  reset_meas(&msgTx->phy_proc_tx);
+  gNB->phy_proc_tx_1 = &msgDataTx->phy_proc_tx;
+  pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
+
+  processingData_L1_t *syncMsg;
+  notifiedFIFO_elt_t *res;
+
+  notifiedFIFO_elt_t *msgRUTx = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_RU_tx,ru_tx_func);
   pushNotifiedFIFO(gNB->resp_RU_tx,msgRUTx); // to unblock the process in the beginning
 
   if(!emulate_rf) {
diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c
index 1c4b9bd09c..af878cb3df 100644
--- a/openair1/PHY/INIT/nr_init.c
+++ b/openair1/PHY/INIT/nr_init.c
@@ -546,6 +546,20 @@ void nr_phy_config_request(NR_PHY_Config_t *phy_config) {
   LOG_I(PHY,"gNB %d configured\n",Mod_id);
 }
 
+void init_DLSCH_struct(PHY_VARS_gNB *gNB, processingData_L1tx_t *msg) {
+  NR_DL_FRAME_PARMS *fp = &msg->gNB->frame_parms;
+  nfapi_nr_config_request_scf_t *cfg = &msg->gNB->gNB_config;
+  uint16_t grid_size = cfg->carrier_config.dl_grid_size[fp->numerology_index].value;
+  memset(msg->num_pdsch_rnti, 0, sizeof(uint16_t)*80);
+
+  for (i=0; i<NUMBER_OF_NR_DLSCH_MAX; i++) {
+    LOG_I(PHY,"Allocating Transport Channel Buffers for DLSCH %d/%d\n",i,NUMBER_OF_NR_DLSCH_MAX);
+    for (j=0; j<2; j++) {
+      msg->dlsch[i][j] = new_gNB_dlsch(fp,1,16,NSOFT,0,grid_size);
+      AssertFatal(msg->dlsch[i][j]!=NULL,"Can't initialize dlsch %d \n", i);
+    }
+  }
+}
 
 void init_nr_transport(PHY_VARS_gNB *gNB) {
   int i;
@@ -570,16 +584,6 @@ void init_nr_transport(PHY_VARS_gNB *gNB) {
     AssertFatal(gNB->pucch[i]!=NULL,"Can't initialize pucch %d \n", i);
   }
 
-  for (i=0; i<gNB->number_of_nr_dlsch_max; i++) {
-
-    LOG_I(PHY,"Allocating Transport Channel Buffers for DLSCH %d/%d\n",i,gNB->number_of_nr_dlsch_max);
-
-    for (j=0; j<2; j++) {
-      gNB->dlsch[i][j] = new_gNB_dlsch(fp,1,16,NSOFT,0,grid_size);
-      AssertFatal(gNB->dlsch[i][j]!=NULL,"Can't initialize dlsch %d \n", i);
-    }
-  }
-
   for (i=0; i<gNB->number_of_nr_ulsch_max; i++) {
 
     LOG_I(PHY,"Allocating Transport Channel Buffer for ULSCH  %d/%d\n",i,gNB->number_of_nr_ulsch_max);
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c
index 03bd0ddf6c..f043ce6551 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c
@@ -111,10 +111,11 @@ void nr_pdsch_codeword_scrambling_optim(uint8_t *in,
 }
 
 
-uint8_t nr_generate_pdsch(PHY_VARS_gNB *gNB,
+uint8_t nr_generate_pdsch(processingData_L1tx_t *msgTx,
 			  int frame,
 			  int slot) {
 
+  PHY_VARS_gNB *gNB = msgTx->gNB;
   NR_gNB_DLSCH_t *dlsch;
   uint32_t ***pdsch_dmrs = gNB->nr_gold_pdsch_dmrs[slot];
   int32_t** txdataF = gNB->common_vars.txdataF;
@@ -132,9 +133,8 @@ uint8_t nr_generate_pdsch(PHY_VARS_gNB *gNB,
   time_stats_t *dlsch_interleaving_stats=&gNB->dlsch_interleaving_stats;
   time_stats_t *dlsch_segmentation_stats=&gNB->dlsch_segmentation_stats;
 
-  for (int dlsch_id=0;dlsch_id<gNB->number_of_nr_dlsch_max;dlsch_id++) {
-    dlsch = gNB->dlsch[dlsch_id][0];
-    if (dlsch->slot_tx[slot] == 0) continue;
+  for (int dlsch_id=0; dlsch_id<msgTx->num_pdsch_slot; dlsch_id++) {
+    dlsch = msgTx->dlsch[dlsch_id][0];
 
     NR_DL_gNB_HARQ_t *harq = &dlsch->harq_process;
     nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15 = &harq->pdsch_pdu.pdsch_pdu_rel15;
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.h b/openair1/PHY/NR_TRANSPORT/nr_dlsch.h
index 6af8f28eac..9f03d34261 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.h
+++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.h
@@ -63,9 +63,7 @@ void nr_pdsch_codeword_scrambling(uint8_t *in,
                                   uint32_t n_RNTI,
                                   uint32_t* out);
 
-void nr_fill_dlsch(PHY_VARS_gNB *gNB,
-                   int frame,
-                   int slot,
+void nr_fill_dlsch(processingData_L1tx_t *msgTx,
                    nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                    unsigned char *sdu); 
 
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
index 543002827d..dbf764324d 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
@@ -254,48 +254,20 @@ void nr_emulate_dlsch_payload(uint8_t* pdu, uint16_t size) {
     *(pdu+i) = (uint8_t)rand();
 }
 
-int16_t find_nr_dlsch(uint16_t rnti, PHY_VARS_gNB *gNB,find_type_t type) {
- 
-   uint16_t i;
-   int16_t first_free_index=-1;
- 
-   AssertFatal(gNB!=NULL,"gNB is null\n");
-   for (i=0; i<gNB->number_of_nr_dlsch_max; i++) {
-     AssertFatal(gNB->dlsch[i]!=NULL,"gNB->dlsch[%d] is null\n",i);
-     AssertFatal(gNB->dlsch[i][0]!=NULL,"gNB->dlsch[%d][0] is null\n",i);
-     LOG_D(PHY,"searching for rnti %x : dlsch_index %d=> harq_mask %x, rnti %x, first_free_index %d\n", rnti,i,
-	   gNB->dlsch[i][0]->harq_mask,gNB->dlsch[i][0]->rnti,first_free_index);
-     if ((gNB->dlsch[i][0]->harq_mask >0) &&
-	 (gNB->dlsch[i][0]->rnti==rnti))       return i;
-     else if ((gNB->dlsch[i][0]->harq_mask == 0) && (first_free_index==-1)) first_free_index=i;
-   }
-   if (type == SEARCH_EXIST) return -1;
-   if (first_free_index != -1)
-     gNB->dlsch[first_free_index][0]->rnti = 0;
-   return first_free_index;
-   
-}
-
-
-void nr_fill_dlsch(PHY_VARS_gNB *gNB,
-                   int frame,
-                   int slot,
+void nr_fill_dlsch(processingData_L1tx_t *msgTx,
                    nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                    uint8_t *sdu) {
 
   nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15 = &pdsch_pdu->pdsch_pdu_rel15;
  
-  int dlsch_id = find_nr_dlsch(rel15->rnti,gNB,SEARCH_EXIST);
-  AssertFatal( (dlsch_id>=0) && (dlsch_id<gNB->number_of_nr_dlsch_max),
-              "illegal or no dlsch_id found!!! rnti %04x dlsch_id %d\n",rel15->rnti,dlsch_id);
-  NR_gNB_DLSCH_t  *dlsch = gNB->dlsch[dlsch_id][0];
+  AssertFatal(msgTx->num_pdsch_slot < NUMBER_OF_NR_DLSCH_MAX,
+              "Number of PDSCH PDUs %d exceeded the limit\n",msgTx->num_pdsch_slot);
+  NR_gNB_DLSCH_t  *dlsch = msgTx->dlsch[msgTx->num_pdsch_slot][0];
   NR_DL_gNB_HARQ_t *harq  = &dlsch->harq_process;
   /// DLSCH struct
   memcpy((void*)&harq->pdsch_pdu, (void*)pdsch_pdu, sizeof(nfapi_nr_dl_tti_pdsch_pdu));
-  gNB->num_pdsch_rnti[slot]++;
+  msgTx->num_pdsch_slot++;
   AssertFatal(sdu!=NULL,"sdu is null\n");
   harq->pdu = sdu;
-
-
 }
 
diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h
index d1fb886341..9aea468a0d 100644
--- a/openair1/PHY/defs_gNB.h
+++ b/openair1/PHY/defs_gNB.h
@@ -759,7 +759,6 @@ typedef struct PHY_VARS_gNB_s {
   //  nfapi_nr_dl_tti_pdcch_pdu    *pdcch_pdu;
   //  nfapi_nr_ul_dci_request_pdus_t  *ul_dci_pdu;
   uint16_t num_pdsch_rnti[80];
-  NR_gNB_SSB_t       ssb[64];
   NR_gNB_PBCH        pbch;
   nr_cce_t           cce_list[MAX_DCI_CORESET][NR_MAX_PDCCH_AGG_LEVEL];
   NR_gNB_COMMON      common_vars;
@@ -850,7 +849,8 @@ typedef struct PHY_VARS_gNB_s {
   /*
   time_stats_t phy_proc;
   */
-  time_stats_t phy_proc_tx;
+  time_stats_t *phy_proc_tx_0;
+  time_stats_t *phy_proc_tx_1;
   time_stats_t phy_proc_rx;
   time_stats_t rx_prach;
   /*
@@ -940,6 +940,21 @@ typedef struct processingData_L1 {
   int slot_tx;
   openair0_timestamp timestamp_tx;
   PHY_VARS_gNB *gNB;
-} processingData_L1_t;
+};
+
+typedef struct processingData_L1tx {
+  int frame_rx;
+  int frame_tx;
+  int slot_rx;
+  int slot_tx;
+  openair0_timestamp timestamp_tx;
+  PHY_VARS_gNB *gNB;
+  nfapi_nr_dl_tti_pdcch_pdu pdcch_pdu;
+  nfapi_nr_ul_dci_request_pdus_t ul_pdcch_pdu;
+  NR_gNB_DLSCH_t *dlsch[NUMBER_OF_NR_DLSCH_MAX][2];
+  NR_gNB_SSB_t ssb[64];
+  uint16_t num_pdsch_slot;
+  time_stats_t phy_proc_tx;
+} processingData_L1tx_t;
 
 #endif
diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index bac52845ff..99eaedd1e0 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -43,7 +43,7 @@ extern int oai_nfapi_ul_tti_req(nfapi_nr_ul_tti_request_t *ul_tti_req);
 
 extern uint8_t nfapi_mode;
 
-void handle_nr_nfapi_ssb_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
+void handle_nr_nfapi_ssb_pdu(processingData_L1tx_t *msgTx,int frame,int slot,
                              nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu)
 {
 
@@ -53,11 +53,11 @@ void handle_nr_nfapi_ssb_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
   uint8_t i_ssb = dl_tti_pdu->ssb_pdu.ssb_pdu_rel15.SsbBlockIndex;
 
   LOG_D(PHY,"%d.%d : ssb index %d pbch_pdu: %x\n",frame,slot,i_ssb,dl_tti_pdu->ssb_pdu.ssb_pdu_rel15.bchPayload);
-  if (gNB->ssb[i_ssb].active)
+  if (msgTx->ssb[i_ssb].active)
     AssertFatal(1==0,"SSB PDU with index %d already active\n",i_ssb);
   else {
-    gNB->ssb[i_ssb].active = true;
-    memcpy((void*)&gNB->ssb[i_ssb].ssb_pdu,&dl_tti_pdu->ssb_pdu,sizeof(dl_tti_pdu->ssb_pdu));
+    msgTx->ssb[i_ssb].active = true;
+    memcpy((void*)&msgTx->ssb[i_ssb].ssb_pdu,&dl_tti_pdu->ssb_pdu,sizeof(dl_tti_pdu->ssb_pdu));
   }
 }
 
@@ -127,7 +127,6 @@ void handle_nfapi_nr_ul_dci_pdu(PHY_VARS_gNB *gNB,
 
 }
 
-
 void handle_nfapi_nr_csirs_pdu(PHY_VARS_gNB *gNB,
 			       int frame, int slot,
 			       nfapi_nr_dl_tti_csi_rs_pdu *csirs_pdu) {
@@ -151,13 +150,13 @@ void handle_nfapi_nr_csirs_pdu(PHY_VARS_gNB *gNB,
 }
 
 
-void handle_nr_nfapi_pdsch_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
+void handle_nr_nfapi_pdsch_pdu(processingData_L1tx_t *msgTx,
                             nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                             uint8_t *sdu)
 {
 
 
-  nr_fill_dlsch(gNB,frame,slot,pdsch_pdu,sdu);
+  nr_fill_dlsch(msgTx,pdsch_pdu,sdu);
 
 }
 
@@ -178,12 +177,9 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
 
   gNB         = RC.gNB[Mod_id];
 
-  // we wait for the L1 Tx processing to finish before copying MAC PDUs to PHY
-  {
-    notifiedFIFO_elt_t *res;
-    res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
-    pushNotifiedFIFO(gNB->resp_L1_tx,res);
-  }
+  notifiedFIFO_elt_t *res;
+  res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
+  processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
 
   uint8_t number_dl_pdu             = (DL_req==NULL) ? 0 : DL_req->dl_tti_request_body.nPDUs;
   uint8_t number_ul_dci_pdu         = (UL_dci_req==NULL) ? 0 : UL_dci_req->numPdus;
@@ -197,11 +193,7 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
 	  number_ul_dci_pdu,number_ul_tti_pdu);
 
   int pdcch_received=0;
-  gNB->num_pdsch_rnti[slot]=0;
-  for (int i=0; i<gNB->number_of_nr_dlsch_max; i++) {
-    gNB->dlsch[i][0]->rnti=0;
-    gNB->dlsch[i][0]->harq_mask=0;
-  }
+  msgTx->num_pdsch_slot=0;
 
   for (int i=0;i<number_dl_pdu;i++) {
     nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu = &DL_req->dl_tti_request_body.dl_tti_pdu_list[i];
@@ -210,7 +202,7 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
       case NFAPI_NR_DL_TTI_SSB_PDU_TYPE:
 
         if(NFAPI_MODE != NFAPI_MODE_VNF)
-        handle_nr_nfapi_ssb_pdu(gNB,frame,slot,
+        handle_nr_nfapi_ssb_pdu(msgTx,frame,slot,
                                 dl_tti_pdu);
 
       break;
@@ -218,11 +210,9 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
       case NFAPI_NR_DL_TTI_PDCCH_PDU_TYPE:
 	AssertFatal(pdcch_received == 0, "pdcch_received is not 0, we can only handle one PDCCH PDU per slot\n");
         if(NFAPI_MODE != NFAPI_MODE_VNF)
-        handle_nfapi_nr_pdcch_pdu(gNB,
-				  frame, slot,
-				  &dl_tti_pdu->pdcch_pdu);
+          msgTx->pdcch_pdu = dl_tti_pdu->pdcch_pdu;
  
-	pdcch_received = 1;
+        pdcch_received = 1;
 
       break;
       case NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE:
@@ -241,16 +231,15 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
 		    pduIndex,TX_req->pdu_list[pduIndex].num_TLV);
         uint8_t *sdu = (uint8_t *)TX_req->pdu_list[pduIndex].TLVs[0].value.direct;
         if(NFAPI_MODE != NFAPI_MODE_VNF)
-        handle_nr_nfapi_pdsch_pdu(gNB,frame,slot,&dl_tti_pdu->pdsch_pdu, sdu);
+        handle_nr_nfapi_pdsch_pdu(msgTx,&dl_tti_pdu->pdsch_pdu, sdu);
       }
     }
   }
 
-  //  if (UL_tti_req!=NULL) memcpy(&gNB->UL_tti_req,UL_tti_req,sizeof(nfapi_nr_ul_tti_request_t));
   if(NFAPI_MODE != NFAPI_MODE_VNF)
-    for (int i=0;i<number_ul_dci_pdu;i++) {
-      handle_nfapi_nr_ul_dci_pdu(gNB, frame, slot, &UL_dci_req->ul_dci_pdu_list[i]);
-    }
+    msgTx->ul_pdcch_pdu = UL_dci_req->ul_dci_pdu_list[number_ul_dci_pdu-1]; // copy the last pdu
+
+  pushNotifiedFIFO(gNB->resp_L1_tx,res);
 
   if(NFAPI_MODE != NFAPI_MODE_VNF)
     for (int i = 0; i < number_ul_tti_pdu; i++) {
diff --git a/openair1/SCHED_NR/fapi_nr_l1.h b/openair1/SCHED_NR/fapi_nr_l1.h
index 19c98e4ec0..3dbbf5f450 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.h
+++ b/openair1/SCHED_NR/fapi_nr_l1.h
@@ -37,7 +37,7 @@
 #include "nfapi_nr_interface_scf.h"
 
 // added
-void handle_nr_nfapi_ssb_pdu(PHY_VARS_gNB *gNB,
+void handle_nr_nfapi_ssb_pdu(processingData_L1tx_t *msgTx,
 						int frame,int slot,
 						nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu);
 
@@ -51,7 +51,7 @@ void handle_nfapi_nr_pdcch_pdu(PHY_VARS_gNB *gNB,
 			       int frame, int subframe,
 			       nfapi_nr_dl_tti_pdcch_pdu *dcl_dl_pdu);
 
-void handle_nr_nfapi_pdsch_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
+void handle_nr_nfapi_pdsch_pdu(processingData_L1tx_t *msgTx,
 			       nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                             uint8_t *sdu);
 
diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
index 6fa37fcd99..3ba66db73d 100644
--- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c
+++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
@@ -124,10 +124,12 @@ void nr_common_signal_procedures (PHY_VARS_gNB *gNB,int frame,int slot,nfapi_nr_
 }
 
 
-void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
-                           int frame,int slot,
+void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
+                           int frame,
+                           int slot,
                            int do_meas) {
   int aa;
+  PHY_VARS_gNB *gNB = msgTx->gNB,
   NR_DL_FRAME_PARMS *fp=&gNB->frame_parms;
   nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
   int offset = gNB->CC_id;
@@ -138,7 +140,7 @@ void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_TX+offset,1);
 
-  if (do_meas==1) start_meas(&gNB->phy_proc_tx);
+  if (do_meas==1) start_meas(gNB->phy_proc_tx);
 
   // clear the transmit data array and beam index for the current slot
   for (aa=0; aa<cfg->carrier_config.num_tx_ant.value; aa++) {
@@ -149,56 +151,48 @@ void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_COMMON_TX,1);
   if (NFAPI_MODE == NFAPI_MONOLITHIC || NFAPI_MODE == NFAPI_MODE_PNF) { 
     for (int i=0; i<fp->Lmax; i++) {
-      if (gNB->ssb[i].active) {
-        nr_common_signal_procedures(gNB,frame,slot,gNB->ssb[i].ssb_pdu);
-        gNB->ssb[i].active = false;
+      if (msgTx->ssb[i].active) {
+        nr_common_signal_procedures(gNB,frame,slot,msgTx->ssb[i].ssb_pdu);
+        msgTx->ssb[i].active = false;
       }
     }
   }
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_COMMON_TX,0);
 
-  int pdcch_pdu_id=find_nr_pdcch(frame,slot,gNB,SEARCH_EXIST);
-  int ul_pdcch_pdu_id=find_nr_ul_dci(frame,slot,gNB,SEARCH_EXIST);
-
   LOG_D(PHY,"[gNB %d] Frame %d slot %d, pdcch_pdu_id %d, ul_pdcch_pdu_id %d\n",
 	gNB->Mod_id,frame,slot,pdcch_pdu_id,ul_pdcch_pdu_id);
 
+  int num_ul_dci = msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci;
+  int num_dl_dci = msgTx->pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci;
+
   if (pdcch_pdu_id >= 0 || ul_pdcch_pdu_id >= 0) {
     LOG_D(PHY, "[gNB %d] Frame %d slot %d Calling nr_generate_dci_top (number of UL/DL DCI %d/%d)\n",
-	  gNB->Mod_id, frame, slot,
-	  gNB->ul_pdcch_pdu[ul_pdcch_pdu_id].pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci,
-	  gNB->pdcch_pdu[pdcch_pdu_id].pdcch_pdu.pdcch_pdu_rel15.numDlDci);
+	  gNB->Mod_id, frame, slot, num_dl_dci, num_ul_dci);
   
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,1);
 
     nr_generate_dci_top(gNB,
-			pdcch_pdu_id>=0 ? &gNB->pdcch_pdu[pdcch_pdu_id].pdcch_pdu : NULL,
-			ul_pdcch_pdu_id>=0 ? &gNB->ul_pdcch_pdu[ul_pdcch_pdu_id].pdcch_pdu.pdcch_pdu : NULL,
+			num_dl_dci > 0 ? &msgTx->pdcch_pdu.pdcch_pdu : NULL,
+			num_ul_dci > 0 ? &msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu : NULL,
 			gNB->nr_gold_pdcch_dmrs[slot],
 			&gNB->common_vars.txdataF[0][txdataF_offset],
 			AMP, fp);
 
-    // free up entry in pdcch tables
-    if (pdcch_pdu_id>=0) gNB->pdcch_pdu[pdcch_pdu_id].frame = -1;
-    if (ul_pdcch_pdu_id>=0) gNB->ul_pdcch_pdu[ul_pdcch_pdu_id].frame = -1;
-
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,0);
-    if (pdcch_pdu_id >= 0) gNB->pdcch_pdu[pdcch_pdu_id].frame = -1;
-    if (ul_pdcch_pdu_id >= 0) gNB->ul_pdcch_pdu[ul_pdcch_pdu_id].frame = -1;
   }
  
-  for (int i=0; i<gNB->num_pdsch_rnti[slot]; i++) {
+  for (int i=0; i<msgTx->num_pdsch_slot; i++) {
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,1);
-    LOG_D(PHY, "PDSCH generation started (%d) in frame %d.%d\n", gNB->num_pdsch_rnti[slot],frame,slot);
-    nr_generate_pdsch(gNB,frame, slot);
+    LOG_D(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
+    nr_generate_pdsch(msgTx, frame, slot);
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
   }
 
   for (int i=0;i<NUMBER_OF_NR_CSIRS_MAX;i++){
     NR_gNB_CSIRS_t *csirs = &gNB->csirs_pdu[i];
     if ((csirs->active == 1) &&
-	(csirs->frame == frame) &&
-	(csirs->slot == slot) ) {
+        (csirs->frame == frame) &&
+        (csirs->slot == slot) ) {
       LOG_D(PHY, "CSI-RS generation started in frame %d.%d\n",frame,slot);
       nfapi_nr_dl_tti_csi_rs_pdu_rel15_t csi_params = csirs->csirs_pdu.csi_rs_pdu_rel15;
       nr_generate_csi_rs(gNB, AMP, csi_params, gNB->gNB_config.cell_config.phy_cell_id.value, slot);
@@ -206,7 +200,7 @@ void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
     }
   }
 
-  if (do_meas==1) stop_meas(&gNB->phy_proc_tx);
+  if (do_meas==1) stop_meas(gNB->phy_proc_tx);
 
   if ((frame&127) == 0) dump_pdsch_stats(gNB);
 

From 902d6b65505724137542b3dc72439bff92c9faaf Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Tue, 6 Jul 2021 08:40:16 +0200
Subject: [PATCH 04/14] correcting the dlsch array size

---
 openair1/PHY/INIT/nr_init.c                |  4 ++--
 openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c |  2 --
 openair1/SCHED_NR/fapi_nr_l1.c             |  2 ++
 openair1/SCHED_NR/phy_procedures_nr_gNB.c  | 10 ++++------
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c
index af878cb3df..bdbd2df92c 100644
--- a/openair1/PHY/INIT/nr_init.c
+++ b/openair1/PHY/INIT/nr_init.c
@@ -552,8 +552,8 @@ void init_DLSCH_struct(PHY_VARS_gNB *gNB, processingData_L1tx_t *msg) {
   uint16_t grid_size = cfg->carrier_config.dl_grid_size[fp->numerology_index].value;
   memset(msg->num_pdsch_rnti, 0, sizeof(uint16_t)*80);
 
-  for (i=0; i<NUMBER_OF_NR_DLSCH_MAX; i++) {
-    LOG_I(PHY,"Allocating Transport Channel Buffers for DLSCH %d/%d\n",i,NUMBER_OF_NR_DLSCH_MAX);
+  for (i=0; i<gNB->number_of_nr_dlsch_max; i++) {
+    LOG_I(PHY,"Allocating Transport Channel Buffers for DLSCH %d/%d\n",i,number_of_nr_dlsch_max);
     for (j=0; j<2; j++) {
       msg->dlsch[i][j] = new_gNB_dlsch(fp,1,16,NSOFT,0,grid_size);
       AssertFatal(msg->dlsch[i][j]!=NULL,"Can't initialize dlsch %d \n", i);
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
index dbf764324d..7f1f0d0d96 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
@@ -260,8 +260,6 @@ void nr_fill_dlsch(processingData_L1tx_t *msgTx,
 
   nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15 = &pdsch_pdu->pdsch_pdu_rel15;
  
-  AssertFatal(msgTx->num_pdsch_slot < NUMBER_OF_NR_DLSCH_MAX,
-              "Number of PDSCH PDUs %d exceeded the limit\n",msgTx->num_pdsch_slot);
   NR_gNB_DLSCH_t  *dlsch = msgTx->dlsch[msgTx->num_pdsch_slot][0];
   NR_DL_gNB_HARQ_t *harq  = &dlsch->harq_process;
   /// DLSCH struct
diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index 99eaedd1e0..4457cfe642 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -231,6 +231,8 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
 		    pduIndex,TX_req->pdu_list[pduIndex].num_TLV);
         uint8_t *sdu = (uint8_t *)TX_req->pdu_list[pduIndex].TLVs[0].value.direct;
         if(NFAPI_MODE != NFAPI_MODE_VNF)
+        AssertFatal(msgTx->num_pdsch_slot < gNB->number_of_nr_dlsch_max,
+                    "Number of PDSCH PDUs %d exceeded the limit %d\n",msgTx->num_pdsch_slot,gNB->number_of_nr_dlsch_max);
         handle_nr_nfapi_pdsch_pdu(msgTx,&dl_tti_pdu->pdsch_pdu, sdu);
       }
     }
diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
index 3ba66db73d..185521d5df 100644
--- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c
+++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
@@ -181,12 +181,10 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,0);
   }
  
-  for (int i=0; i<msgTx->num_pdsch_slot; i++) {
-    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,1);
-    LOG_D(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
-    nr_generate_pdsch(msgTx, frame, slot);
-    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
-  }
+  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,1);
+  LOG_D(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
+  nr_generate_pdsch(msgTx, frame, slot);
+  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
 
   for (int i=0;i<NUMBER_OF_NR_CSIRS_MAX;i++){
     NR_gNB_CSIRS_t *csirs = &gNB->csirs_pdu[i];

From bc6f39370c1ca1f73ca863e16ca832e37367fc60 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Wed, 7 Jul 2021 09:19:14 +0200
Subject: [PATCH 05/14] Added checks if tx RU processing start in order

---
 executables/nr-gnb.c                      |  49 +++++++---
 executables/nr-ru.c                       |  32 ++-----
 openair1/PHY/INIT/nr_init.c               |  27 ++----
 openair1/PHY/NR_TRANSPORT/nr_dci_tools.c  | 104 ----------------------
 openair1/PHY/NR_TRANSPORT/nr_dlsch.h      |   2 +-
 openair1/PHY/defs_RU.h                    |   1 +
 openair1/PHY/defs_gNB.h                   |  12 ++-
 openair1/SCHED_NR/fapi_nr_l1.c            |  40 ++-------
 openair1/SCHED_NR/phy_frame_config_nr.c   |  16 ++++
 openair1/SCHED_NR/phy_frame_config_nr.h   |   2 +
 openair1/SCHED_NR/phy_procedures_nr_gNB.c |  27 +++---
 openair1/SCHED_NR/sched_nr.h              |   2 +-
 12 files changed, 101 insertions(+), 213 deletions(-)

diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c
index 2bbabf3fe0..ef1d4e22fa 100644
--- a/executables/nr-gnb.c
+++ b/executables/nr-gnb.c
@@ -122,20 +122,27 @@ void tx_func(void *param) {
                         frame_tx,
                         slot_tx,
                         1);
+  info->status = NOT_FILLED;
   if ((frame_tx&127) == 0) dump_pdsch_stats(gNB);
 
   // If the later of the 2 L1 tx thread finishes first,
   // we wait for the earlier one to finish and start the RU thread
   // to avoid realtime issues with USRP
-  notifiedFIFO_elt_t *res; 
-  res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
-  pushNotifiedFIFO(gNB->resp_L1_tx,res);
 
   // Start RU TX processing.
+  notifiedFIFO_elt_t *res;
   res = pullTpool(gNB->resp_RU_tx, gNB->threadPool);
   processingData_RU_t *syncMsg = (processingData_RU_t *)NotifiedFifoData(res);
+  LOG_D(PHY,"waiting for previous tx to finish, next slot %d,%d\n",syncMsg->next_slot,slot_tx);
+  while (syncMsg->next_slot != slot_tx) {
+    pushNotifiedFIFO(gNB->resp_RU_tx, res);
+    res = pullTpool(gNB->resp_RU_tx, gNB->threadPool);
+    syncMsg = (processingData_RU_t *)NotifiedFifoData(res);
+  }
+  LOG_D(PHY,"previous tx to finished, next slot %d,%d\n",syncMsg->next_slot,slot_tx);
   syncMsg->frame_tx = frame_tx;
   syncMsg->slot_tx = slot_tx;
+  syncMsg->next_slot = get_next_downlink_slot(gNB, &gNB->gNB_config, frame_tx, slot_tx);
   syncMsg->timestamp_tx = info->timestamp_tx;
   syncMsg->ru = gNB->RU_list[0];
   res->key = slot_tx;
@@ -213,13 +220,6 @@ void rx_func(void *param) {
         }
         up_removed++;
       }
-    for (j = 0; j < NUMBER_OF_NR_DLSCH_MAX; j++)
-      if (gNB->dlsch[j][0]->rnti == rnti_to_remove[i]) {
-        gNB->dlsch[j][0]->rnti = 0;
-        gNB->dlsch[j][0]->harq_mask = 0;
-        //clean_gNB_dlsch(gNB->dlsch[j][0]);
-        down_removed++;
-      }
     for (j = 0; j < NUMBER_OF_NR_PUCCH_MAX; j++)
       if (gNB->pucch[j]->active > 0 &&
           gNB->pucch[j]->pucch_pdu.rnti == rnti_to_remove[i]) {
@@ -277,10 +277,15 @@ void rx_func(void *param) {
   pthread_mutex_unlock(&gNB->UL_INFO_mutex);
   stop_meas(&gNB->ul_indication_stats);
   
-  notifiedFIFO_elt_t *res; 
   if (tx_slot_type == NR_DOWNLINK_SLOT || tx_slot_type == NR_MIXED_SLOT) {
+    notifiedFIFO_elt_t *res;
     res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
     processingData_L1tx_t *syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res);
+    while (syncMsg->status == NOT_FILLED) {
+      pushNotifiedFIFO(gNB->resp_L1_tx, res);
+      res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
+      syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res);
+    }
     syncMsg->gNB = gNB;
     syncMsg->frame_rx = frame_rx;
     syncMsg->slot_rx = slot_rx;
@@ -400,14 +405,36 @@ void init_gNB_Tpool(int inst) {
   // L1 RX result FIFO 
   gNB->resp_L1 = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
   initNotifiedFIFO(gNB->resp_L1);
+  notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_L1,rx_func);
+  pushNotifiedFIFO(gNB->resp_L1,msg); // to unblock the process in the beginning
 
   // L1 TX result FIFO 
   gNB->resp_L1_tx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
   initNotifiedFIFO(gNB->resp_L1_tx);
+  // we create 2 threads for L1 tx processing
+  notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
+  processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+  init_DLSCH_struct(gNB, msgDataTx);
+  memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
+  reset_meas(&msgDataTx->phy_proc_tx);
+  gNB->phy_proc_tx_0 = &msgDataTx->phy_proc_tx;
+  pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
+
+  msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
+  msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+  init_DLSCH_struct(gNB, msgDataTx);
+  memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
+  reset_meas(&msgDataTx->phy_proc_tx);
+  gNB->phy_proc_tx_1 = &msgDataTx->phy_proc_tx;
+  pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
 
   // RU TX result FIFO 
   gNB->resp_RU_tx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
   initNotifiedFIFO(gNB->resp_RU_tx);
+  notifiedFIFO_elt_t *msgRUTx = newNotifiedFIFO_elt(sizeof(processingData_RU_t),0,gNB->resp_RU_tx,ru_tx_func);
+  processingData_RU_t *msgData = (processingData_RU_t*)msgRUTx->msgData;
+  msgData->next_slot = sf_ahead*gNB->frame_parms.slots_per_subframe; // first Tx slot
+  pushNotifiedFIFO(gNB->resp_RU_tx,msgRUTx); // to unblock the process in the beginning
 
   // Stats measurement thread
   if(opp_enabled == 1) threadCreate(&proc->process_stats_thread, process_stats_thread,(void *)gNB, "time_meas", -1, OAI_PRIORITY_RT_LOW);
diff --git a/executables/nr-ru.c b/executables/nr-ru.c
index a01be1ebaa..40acf97285 100644
--- a/executables/nr-ru.c
+++ b/executables/nr-ru.c
@@ -1269,30 +1269,9 @@ void *ru_thread( void *param ) {
   pthread_mutex_unlock(&RC.ru_mutex);
   wait_sync("ru_thread");
 
-  notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_L1,rx_func);
-  pushNotifiedFIFO(gNB->resp_L1,msg); // to unblock the process in the beginning
-
-  // we create 2 threads for L1 tx processing
-  notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
-  processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
-  init_DLSCH_struct(gNB, msgDataTx);
-  reset_meas(&msgDataTx->phy_proc_tx);
-  gNB->phy_proc_tx_0 = &msgDataTx->phy_proc_tx;
-  pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
-
-  msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
-  msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
-  init_DLSCH_struct(gNB, msgDataTx);
-  reset_meas(&msgTx->phy_proc_tx);
-  gNB->phy_proc_tx_1 = &msgDataTx->phy_proc_tx;
-  pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
-
   processingData_L1_t *syncMsg;
   notifiedFIFO_elt_t *res;
 
-  notifiedFIFO_elt_t *msgRUTx = newNotifiedFIFO_elt(sizeof(processingData_L1_t),0,gNB->resp_RU_tx,ru_tx_func);
-  pushNotifiedFIFO(gNB->resp_RU_tx,msgRUTx); // to unblock the process in the beginning
-
   if(!emulate_rf) {
     // Start RF device if any
     if (ru->start_rf) {
@@ -1423,9 +1402,14 @@ void *ru_thread( void *param ) {
     else LOG_I(PHY,"RU %d rf device stopped\n",ru->idx);
   }
 
-  delNotifiedFIFO_elt(msg);
-  delNotifiedFIFO_elt(msgL1Tx);
-  delNotifiedFIFO_elt(msgRUTx);
+  res = pullNotifiedFIFO(gNB->resp_L1);
+  delNotifiedFIFO_elt(res);
+  res = pullNotifiedFIFO(gNB->resp_L1_tx);
+  delNotifiedFIFO_elt(res);
+  res = pullNotifiedFIFO(gNB->resp_L1_tx);
+  delNotifiedFIFO_elt(res);
+  res = pullNotifiedFIFO(gNB->resp_RU_tx);
+  delNotifiedFIFO_elt(res);
 
   ru_thread_status = 0;
   return &ru_thread_status;
diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c
index bdbd2df92c..d2347bd3df 100644
--- a/openair1/PHY/INIT/nr_init.c
+++ b/openair1/PHY/INIT/nr_init.c
@@ -547,14 +547,14 @@ void nr_phy_config_request(NR_PHY_Config_t *phy_config) {
 }
 
 void init_DLSCH_struct(PHY_VARS_gNB *gNB, processingData_L1tx_t *msg) {
-  NR_DL_FRAME_PARMS *fp = &msg->gNB->frame_parms;
-  nfapi_nr_config_request_scf_t *cfg = &msg->gNB->gNB_config;
+  NR_DL_FRAME_PARMS *fp = &gNB->frame_parms;
+  nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
   uint16_t grid_size = cfg->carrier_config.dl_grid_size[fp->numerology_index].value;
-  memset(msg->num_pdsch_rnti, 0, sizeof(uint16_t)*80);
+  msg->num_pdsch_slot = 0;
 
-  for (i=0; i<gNB->number_of_nr_dlsch_max; i++) {
-    LOG_I(PHY,"Allocating Transport Channel Buffers for DLSCH %d/%d\n",i,number_of_nr_dlsch_max);
-    for (j=0; j<2; j++) {
+  for (int i=0; i<gNB->number_of_nr_dlsch_max; i++) {
+    LOG_I(PHY,"Allocating Transport Channel Buffers for DLSCH %d/%d\n",i,gNB->number_of_nr_dlsch_max);
+    for (int j=0; j<2; j++) {
       msg->dlsch[i][j] = new_gNB_dlsch(fp,1,16,NSOFT,0,grid_size);
       AssertFatal(msg->dlsch[i][j]!=NULL,"Can't initialize dlsch %d \n", i);
     }
@@ -562,8 +562,6 @@ void init_DLSCH_struct(PHY_VARS_gNB *gNB, processingData_L1tx_t *msg) {
 }
 
 void init_nr_transport(PHY_VARS_gNB *gNB) {
-  int i;
-  int j;
   NR_DL_FRAME_PARMS *fp = &gNB->frame_parms;
   nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
   LOG_I(PHY, "Initialise nr transport\n");
@@ -571,24 +569,17 @@ void init_nr_transport(PHY_VARS_gNB *gNB) {
 
   memset(gNB->num_pdsch_rnti, 0, sizeof(uint16_t)*80);
 
-  for (i=0; i <NUMBER_OF_NR_PDCCH_MAX; i++) {
-    LOG_I(PHY,"Initializing PDCCH list for PDCCH %d/%d\n",i,NUMBER_OF_NR_PDCCH_MAX);
-    gNB->pdcch_pdu[i].frame=-1;
-    LOG_I(PHY,"Initializing UL PDCCH list for UL PDCCH %d/%d\n",i,NUMBER_OF_NR_PDCCH_MAX);
-    gNB->ul_pdcch_pdu[i].frame=-1;
-  }
-    
-  for (i=0; i<NUMBER_OF_NR_PUCCH_MAX; i++) {
+  for (int i=0; i<NUMBER_OF_NR_PUCCH_MAX; i++) {
     LOG_I(PHY,"Allocating Transport Channel Buffers for PUCCH %d/%d\n",i,NUMBER_OF_NR_PUCCH_MAX);
     gNB->pucch[i] = new_gNB_pucch();
     AssertFatal(gNB->pucch[i]!=NULL,"Can't initialize pucch %d \n", i);
   }
 
-  for (i=0; i<gNB->number_of_nr_ulsch_max; i++) {
+  for (int i=0; i<gNB->number_of_nr_ulsch_max; i++) {
 
     LOG_I(PHY,"Allocating Transport Channel Buffer for ULSCH  %d/%d\n",i,gNB->number_of_nr_ulsch_max);
 
-    for (j=0; j<2; j++) {
+    for (int j=0; j<2; j++) {
       // ULSCH for data
       gNB->ulsch[i][j] = new_gNB_ulsch(MAX_LDPC_ITERATIONS, fp->N_RB_UL, 0);
 
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dci_tools.c b/openair1/PHY/NR_TRANSPORT/nr_dci_tools.c
index c562dfaa41..770c35105f 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dci_tools.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_dci_tools.c
@@ -202,107 +202,3 @@ void nr_fill_cce_list(PHY_VARS_gNB *gNB, uint8_t m,  nfapi_nr_dl_tti_pdcch_pdu_r
     ret |= ((field>>i)&1)<<(size-i-1);
   return ret;
 }*/
-int16_t find_nr_pdcch(int frame,int slot, PHY_VARS_gNB *gNB,find_type_t type) {
-
-  uint16_t i;
-  int16_t first_free_index=-1;
-
-  AssertFatal(gNB!=NULL,"gNB is null\n");
-  for (i=0; i<NUMBER_OF_NR_PDCCH_MAX; i++) {
-    LOG_D(PHY,"searching for frame.slot %d.%d : pdcch_index %d frame.slot %d.%d, first_free_index %d\n", frame,slot,i,gNB->pdcch_pdu[i].frame,gNB->pdcch_pdu[i].slot,first_free_index);
-    if ((gNB->pdcch_pdu[i].frame == frame) &&
-        (gNB->pdcch_pdu[i].slot==slot))       return i;
-    else if ( gNB->pdcch_pdu[i].frame==-1 && first_free_index==-1) first_free_index=i;
-  }
-  if (type == SEARCH_EXIST) return -1;
-
-  return first_free_index;
-}
-
-
-void nr_fill_dci(PHY_VARS_gNB *gNB,
-                 int frame,
-                 int slot,
-		 nfapi_nr_dl_tti_pdcch_pdu *pdcch_pdu) {
-
-  nfapi_nr_dl_tti_pdcch_pdu_rel15_t *pdcch_pdu_rel15 = &pdcch_pdu->pdcch_pdu_rel15;
-  NR_gNB_DLSCH_t *dlsch; 
-
-  int pdcch_id = find_nr_pdcch(frame,slot,gNB,SEARCH_EXIST_OR_FREE);
-  AssertFatal(pdcch_id>=0 && pdcch_id<NUMBER_OF_NR_PDCCH_MAX,"Cannot find space for PDCCH, exiting\n");
-  memcpy((void*)&gNB->pdcch_pdu[pdcch_id].pdcch_pdu,(void*)pdcch_pdu,sizeof(*pdcch_pdu));
-  gNB->pdcch_pdu[pdcch_id].frame = frame;
-  gNB->pdcch_pdu[pdcch_id].slot  = slot;
-
-  for (int i=0;i<pdcch_pdu_rel15->numDlDci;i++) {
-
-    //uint64_t *dci_pdu = (uint64_t*)pdcch_pdu_rel15->dci_pdu[i].Payload;
-
-    int dlsch_id = find_nr_dlsch(pdcch_pdu_rel15->dci_pdu[i].RNTI,gNB,SEARCH_EXIST_OR_FREE);
-    if( (dlsch_id<0) || (dlsch_id>=gNB->number_of_nr_dlsch_max) ){
-      LOG_E(PHY,"illegal dlsch_id found!!! rnti %04x dlsch_id %d\n",(unsigned int)pdcch_pdu_rel15->dci_pdu[i].RNTI,dlsch_id);
-      return;
-    }
-    
-    dlsch = gNB->dlsch[dlsch_id][0];
-    int harq_pid = 0;
-
-    dlsch->slot_tx[slot]             = 1;
-    dlsch->harq_ids[frame % 2][slot] = 0;
-    AssertFatal(harq_pid < 8 && harq_pid >= 0,
-		"illegal harq_pid %d\n",harq_pid);
-    
-    dlsch->harq_mask                |= (1<<harq_pid);
-    dlsch->rnti                      = pdcch_pdu_rel15->dci_pdu[i].RNTI;
-    
-    //    nr_fill_cce_list(gNB,0);
-  }
-
-}
-
-
-int16_t find_nr_ul_dci(int frame,int slot, PHY_VARS_gNB *gNB,find_type_t type) {
-
-  uint16_t i;
-  int16_t first_free_index=-1;
-
-  AssertFatal(gNB!=NULL,"gNB is null\n");
-  for (i=0; i<NUMBER_OF_NR_PDCCH_MAX; i++) {
-    LOG_D(PHY,"searching for frame.slot %d.%d : ul_pdcch_index %d frame.slot %d.%d, first_free_index %d\n", frame,slot,i,gNB->ul_pdcch_pdu[i].frame,gNB->ul_pdcch_pdu[i].slot,first_free_index);
-    if ((gNB->ul_pdcch_pdu[i].frame == frame) &&
-        (gNB->ul_pdcch_pdu[i].slot==slot))       return i;
-    else if (gNB->ul_pdcch_pdu[i].frame==-1 && first_free_index==-1) first_free_index=i;
-  }
-  if (type == SEARCH_EXIST) return -1;
-
-  return first_free_index;
-}
-
-
-void nr_fill_ul_dci(PHY_VARS_gNB *gNB,
-		    int frame,
-		    int slot,
-		    nfapi_nr_ul_dci_request_pdus_t *pdcch_pdu) {
-
-  nfapi_nr_dl_tti_pdcch_pdu_rel15_t *pdcch_pdu_rel15 = &pdcch_pdu->pdcch_pdu.pdcch_pdu_rel15;
-
-  int pdcch_id = find_nr_ul_dci(frame,slot,gNB,SEARCH_EXIST_OR_FREE);
-  AssertFatal(pdcch_id>=0 && pdcch_id<NUMBER_OF_NR_PDCCH_MAX,"Cannot find space for UL PDCCH, exiting\n");
-  memcpy((void*)&gNB->ul_pdcch_pdu[pdcch_id].pdcch_pdu,(void*)pdcch_pdu,sizeof(*pdcch_pdu));
-  gNB->ul_pdcch_pdu[pdcch_id].frame = frame;
-  gNB->ul_pdcch_pdu[pdcch_id].slot  = slot;
-
-  for (int i=0;i<pdcch_pdu_rel15->numDlDci;i++) {
-
-    //uint64_t *dci_pdu = (uint64_t*)pdcch_pdu_rel15->dci_pdu[i].Payload;
-
-    // if there's no DL DCI then generate CCE list
-    //    nr_fill_cce_list(gNB,0);  
-    /*
-    LOG_D(PHY, "DCI PDU: [0]->0x%lx \t [1]->0x%lx \n",dci_pdu[0], dci_pdu[1]);
-    LOG_D(PHY, "DCI type %d payload (size %d) generated on candidate %d\n", dci_alloc->pdcch_params.dci_format, dci_alloc->size, cand_idx);
-    */
-
-  }
-
-}
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.h b/openair1/PHY/NR_TRANSPORT/nr_dlsch.h
index 9f03d34261..f773b99548 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.h
+++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.h
@@ -67,7 +67,7 @@ void nr_fill_dlsch(processingData_L1tx_t *msgTx,
                    nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                    unsigned char *sdu); 
 
-uint8_t nr_generate_pdsch(PHY_VARS_gNB *gNB,
+uint8_t nr_generate_pdsch(processingData_L1tx_t *msgTx,
 			  int frame,
 			  int slot);
 void free_gNB_dlsch(NR_gNB_DLSCH_t **dlschptr, uint16_t N_RB);
diff --git a/openair1/PHY/defs_RU.h b/openair1/PHY/defs_RU.h
index 66cb51adc6..76dd92ef5d 100644
--- a/openair1/PHY/defs_RU.h
+++ b/openair1/PHY/defs_RU.h
@@ -762,6 +762,7 @@ typedef struct RRU_config_s {
 typedef struct processingData_RU {
   int frame_tx;
   int slot_tx;
+  int next_slot;
   openair0_timestamp timestamp_tx;
   RU_t *ru;
 } processingData_RU_t;
diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h
index 9aea468a0d..b11f902dd2 100644
--- a/openair1/PHY/defs_gNB.h
+++ b/openair1/PHY/defs_gNB.h
@@ -122,8 +122,6 @@ typedef struct {
 
 typedef struct {
   uint8_t active;
-  int frame;
-  int slot;
   nfapi_nr_dl_tti_csi_rs_pdu csirs_pdu;
 } NR_gNB_CSIRS_t;
 
@@ -766,7 +764,6 @@ typedef struct PHY_VARS_gNB_s {
   NR_gNB_PUSCH       *pusch_vars[NUMBER_OF_NR_ULSCH_MAX];
   NR_gNB_PUCCH_t     *pucch[NUMBER_OF_NR_PUCCH_MAX];
   NR_gNB_PDCCH_t     pdcch_pdu[NUMBER_OF_NR_PDCCH_MAX];
-  NR_gNB_CSIRS_t     csirs_pdu[NUMBER_OF_NR_CSIRS_MAX];
   NR_gNB_UL_PDCCH_t  ul_pdcch_pdu[NUMBER_OF_NR_PDCCH_MAX];
   NR_gNB_DLSCH_t     *dlsch[NUMBER_OF_NR_DLSCH_MAX][2];    // Nusers times two spatial streams
   NR_gNB_ULSCH_t     *ulsch[NUMBER_OF_NR_ULSCH_MAX][2];  // [Nusers times][2 codewords] 
@@ -940,7 +937,12 @@ typedef struct processingData_L1 {
   int slot_tx;
   openair0_timestamp timestamp_tx;
   PHY_VARS_gNB *gNB;
-};
+} processingData_L1_t;
+
+typedef enum {
+  FILLED,
+  NOT_FILLED
+} msgStatus_t;
 
 typedef struct processingData_L1tx {
   int frame_rx;
@@ -951,10 +953,12 @@ typedef struct processingData_L1tx {
   PHY_VARS_gNB *gNB;
   nfapi_nr_dl_tti_pdcch_pdu pdcch_pdu;
   nfapi_nr_ul_dci_request_pdus_t ul_pdcch_pdu;
+  NR_gNB_CSIRS_t csirs_pdu[NUMBER_OF_NR_CSIRS_MAX];
   NR_gNB_DLSCH_t *dlsch[NUMBER_OF_NR_DLSCH_MAX][2];
   NR_gNB_SSB_t ssb[64];
   uint16_t num_pdsch_slot;
   time_stats_t phy_proc_tx;
+  msgStatus_t status;
 } processingData_L1tx_t;
 
 #endif
diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index 4457cfe642..e06d66ed02 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -100,45 +100,16 @@ void handle_nr_nfapi_ssb_pdu(processingData_L1tx_t *msgTx,int frame,int slot,
 }*/
 
 
-void handle_nfapi_nr_pdcch_pdu(PHY_VARS_gNB *gNB,
-			       int frame, int slot,
-			       nfapi_nr_dl_tti_pdcch_pdu *pdcch_pdu) {
-
-  LOG_D(PHY,"Frame %d, Slot %d: DCI processing - proc:slot_tx:%d pdcch_pdu_rel15->numDlDci:%d\n",frame,slot, slot, pdcch_pdu->pdcch_pdu_rel15.numDlDci);
-
-  // copy dci configuration into gNB structure
-  //  gNB->pdcch_pdu = pdcch_pdu;
-
-  nr_fill_dci(gNB,frame,slot,pdcch_pdu);
-
-}
-
-
-void handle_nfapi_nr_ul_dci_pdu(PHY_VARS_gNB *gNB,
-			       int frame, int slot,
-			       nfapi_nr_ul_dci_request_pdus_t *ul_dci_request_pdu) {
-
-  LOG_D(PHY,"Frame %d, Slot %d: UL DCI processing - proc:slot_tx:%d pdcch_pdu_rel15->numDlDci:%d\n",frame,slot, slot, ul_dci_request_pdu->pdcch_pdu.pdcch_pdu_rel15.numDlDci);
-
-  // copy dci configuration into gNB structure
-  //  gNB->ul_dci_pdu = ul_dci_request_pdu;
-
-  nr_fill_ul_dci(gNB,frame,slot,ul_dci_request_pdu);
-
-}
-
-void handle_nfapi_nr_csirs_pdu(PHY_VARS_gNB *gNB,
-			       int frame, int slot,
+void handle_nfapi_nr_csirs_pdu(processingData_L1tx_t *msgTx,
+             int frame,int slot,
 			       nfapi_nr_dl_tti_csi_rs_pdu *csirs_pdu) {
 
   int found = 0;
 
   for (int id=0; id<NUMBER_OF_NR_CSIRS_MAX; id++) {
-    NR_gNB_CSIRS_t *csirs = &gNB->csirs_pdu[id];
+    NR_gNB_CSIRS_t *csirs = &msgTx->csirs_pdu[id];
     if (csirs->active == 0) {
       LOG_D(PHY,"Frame %d Slot %d CSI_RS with ID %d is now active\n",frame,slot,id);
-      csirs->frame = frame;
-      csirs->slot = slot;
       csirs->active = 1;
       memcpy((void*)&csirs->csirs_pdu, (void*)csirs_pdu, sizeof(nfapi_nr_dl_tti_csi_rs_pdu));
       found = 1;
@@ -217,8 +188,8 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
       break;
       case NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE:
         LOG_D(PHY,"frame %d, slot %d, Got NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE for %d.%d\n",frame,slot,DL_req->SFN,DL_req->Slot);
-        handle_nfapi_nr_csirs_pdu(gNB,
-				  frame, slot,
+        handle_nfapi_nr_csirs_pdu(msgTx,
+          int frame,int slot,
 				  &dl_tti_pdu->csi_rs_pdu);
       break;
       case NFAPI_NR_DL_TTI_PDSCH_PDU_TYPE:
@@ -241,6 +212,7 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
   if(NFAPI_MODE != NFAPI_MODE_VNF)
     msgTx->ul_pdcch_pdu = UL_dci_req->ul_dci_pdu_list[number_ul_dci_pdu-1]; // copy the last pdu
 
+  msgTx->status = FILLED;
   pushNotifiedFIFO(gNB->resp_L1_tx,res);
 
   if(NFAPI_MODE != NFAPI_MODE_VNF)
diff --git a/openair1/SCHED_NR/phy_frame_config_nr.c b/openair1/SCHED_NR/phy_frame_config_nr.c
index 9be6780597..c817efcd02 100644
--- a/openair1/SCHED_NR/phy_frame_config_nr.c
+++ b/openair1/SCHED_NR/phy_frame_config_nr.c
@@ -30,6 +30,7 @@
 ************************************************************************/
 
 #include "PHY/defs_nr_common.h"
+#include "PHY/defs_gNB.h"
 #include "PHY/defs_nr_UE.h"
 #include "SCHED_NR/phy_frame_config_nr.h"
 
@@ -308,6 +309,21 @@ int set_tdd_configuration_dedicated_nr(NR_DL_FRAME_PARMS *frame_parms) {
 *
 *********************************************************************/
 
+int get_next_downlink_slot(PHY_VARS_gNB *gNB, nfapi_nr_config_request_scf_t *cfg, int nr_frame, int nr_slot) {
+
+  int slot = nr_slot;
+  int frame = nr_frame;
+  int slots_per_frame = gNB->frame_parms.slots_per_frame;
+  while (true) {
+    slot++;
+    frame = (slot/slots_per_frame) ? frame++ : frame;
+    slot %= slots_per_frame;
+    int slot_type = nr_slot_select(cfg, frame, slot);
+    if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) return slot;
+    AssertFatal(frame < nr_frame+2, "Something went worng. This shouldn't happen\n");
+  }
+}
+
 int nr_slot_select(nfapi_nr_config_request_scf_t *cfg, int nr_frame, int nr_slot) {
   /* for FFD all slot can be considered as an uplink */
   int mu = cfg->ssb_config.scs_common.value,check_slot=0;
diff --git a/openair1/SCHED_NR/phy_frame_config_nr.h b/openair1/SCHED_NR/phy_frame_config_nr.h
index 434dc41d1f..aba26283ed 100644
--- a/openair1/SCHED_NR/phy_frame_config_nr.h
+++ b/openair1/SCHED_NR/phy_frame_config_nr.h
@@ -92,5 +92,7 @@ void free_tdd_configuration_nr(NR_DL_FRAME_PARMS *frame_parms);
 
 void free_tdd_configuration_dedicated_nr(NR_DL_FRAME_PARMS *frame_parms);
 
+int get_next_downlink_slot(PHY_VARS_gNB *gNB, nfapi_nr_config_request_scf_t *cfg, int nr_frame, int nr_slot);
+
 #endif  /* PHY_FRAME_CONFIG_NR_H */
 
diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
index 185521d5df..22d97be0c3 100644
--- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c
+++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
@@ -129,7 +129,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
                            int slot,
                            int do_meas) {
   int aa;
-  PHY_VARS_gNB *gNB = msgTx->gNB,
+  PHY_VARS_gNB *gNB = msgTx->gNB;
   NR_DL_FRAME_PARMS *fp=&gNB->frame_parms;
   nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
   int offset = gNB->CC_id;
@@ -140,7 +140,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_TX+offset,1);
 
-  if (do_meas==1) start_meas(gNB->phy_proc_tx);
+  if (do_meas==1) start_meas(&msgTx->phy_proc_tx);
 
   // clear the transmit data array and beam index for the current slot
   for (aa=0; aa<cfg->carrier_config.num_tx_ant.value; aa++) {
@@ -159,24 +159,21 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
   }
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_COMMON_TX,0);
 
-  LOG_D(PHY,"[gNB %d] Frame %d slot %d, pdcch_pdu_id %d, ul_pdcch_pdu_id %d\n",
-	gNB->Mod_id,frame,slot,pdcch_pdu_id,ul_pdcch_pdu_id);
+  int num_dl_dci = msgTx->pdcch_pdu.pdcch_pdu_rel15.numDlDci;
+  int num_ul_dci = msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci;
 
-  int num_ul_dci = msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci;
-  int num_dl_dci = msgTx->pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci;
-
-  if (pdcch_pdu_id >= 0 || ul_pdcch_pdu_id >= 0) {
+  if (num_dl_dci > 0 || num_ul_dci > 0) {
     LOG_D(PHY, "[gNB %d] Frame %d slot %d Calling nr_generate_dci_top (number of UL/DL DCI %d/%d)\n",
 	  gNB->Mod_id, frame, slot, num_dl_dci, num_ul_dci);
   
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,1);
 
     nr_generate_dci_top(gNB,
-			num_dl_dci > 0 ? &msgTx->pdcch_pdu.pdcch_pdu : NULL,
-			num_ul_dci > 0 ? &msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu : NULL,
+			num_dl_dci > 0 ? &msgTx->pdcch_pdu : NULL,
+			num_ul_dci > 0 ? &msgTx->ul_pdcch_pdu.pdcch_pdu : NULL,
 			gNB->nr_gold_pdcch_dmrs[slot],
 			&gNB->common_vars.txdataF[0][txdataF_offset],
-			AMP, fp);
+			AMP, &fp);
 
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,0);
   }
@@ -187,10 +184,8 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
 
   for (int i=0;i<NUMBER_OF_NR_CSIRS_MAX;i++){
-    NR_gNB_CSIRS_t *csirs = &gNB->csirs_pdu[i];
-    if ((csirs->active == 1) &&
-        (csirs->frame == frame) &&
-        (csirs->slot == slot) ) {
+    NR_gNB_CSIRS_t *csirs = &msgTx->csirs_pdu[i];
+    if ((csirs->active == 1)) {
       LOG_D(PHY, "CSI-RS generation started in frame %d.%d\n",frame,slot);
       nfapi_nr_dl_tti_csi_rs_pdu_rel15_t csi_params = csirs->csirs_pdu.csi_rs_pdu_rel15;
       nr_generate_csi_rs(gNB, AMP, csi_params, gNB->gNB_config.cell_config.phy_cell_id.value, slot);
@@ -198,7 +193,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
     }
   }
 
-  if (do_meas==1) stop_meas(gNB->phy_proc_tx);
+  if (do_meas==1) stop_meas(&msgTx->phy_proc_tx);
 
   if ((frame&127) == 0) dump_pdsch_stats(gNB);
 
diff --git a/openair1/SCHED_NR/sched_nr.h b/openair1/SCHED_NR/sched_nr.h
index 5e31c08d38..d7e0332b85 100644
--- a/openair1/SCHED_NR/sched_nr.h
+++ b/openair1/SCHED_NR/sched_nr.h
@@ -36,7 +36,7 @@
 
 void fill_ul_rb_mask(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx);
 void nr_set_ssb_first_subcarrier(nfapi_nr_config_request_scf_t *cfg, NR_DL_FRAME_PARMS *fp);
-void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB, int frame_tx, int slot_tx, int do_meas);
+void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx, int frame_tx, int slot_tx, int do_meas);
 void phy_procedures_gNB_common_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx);
 int  phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx);
 void L1_nr_prach_procedures(PHY_VARS_gNB *gNB,int frame,int slot);

From 3f1c9df35c574c633b6b15a72f6743d9aed66cad Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Wed, 7 Jul 2021 09:20:09 +0200
Subject: [PATCH 06/14] commenting out function calls in nfapi the priority now
 is monolitic. this should we reverted and fixed properly once we have a
 working code in monolitic

---
 nfapi/oai_integration/nfapi_pnf.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/nfapi/oai_integration/nfapi_pnf.c b/nfapi/oai_integration/nfapi_pnf.c
index 5ce66aad89..b8912f0199 100644
--- a/nfapi/oai_integration/nfapi_pnf.c
+++ b/nfapi/oai_integration/nfapi_pnf.c
@@ -88,10 +88,10 @@ extern void handle_nfapi_nr_ul_dci_pdu(PHY_VARS_gNB *gNB,
 extern void handle_nfapi_nr_pdcch_pdu(PHY_VARS_gNB *gNB,
 			       int frame, int slot,
 			       nfapi_nr_dl_tti_pdcch_pdu *pdcch_pdu);
-extern void handle_nr_nfapi_pdsch_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
+extern void handle_nr_nfapi_pdsch_pdu(processingData_L1tx_t *msgTx,
                             nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                             uint8_t *sdu);
-extern void handle_nr_nfapi_ssb_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
+extern void handle_nr_nfapi_ssb_pdu(processingData_L1tx_t *msgTx,int frame,int slot,
                              nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu);
 extern void nr_fill_ulsch(PHY_VARS_gNB *gNB,
                    int frame,
@@ -1157,7 +1157,7 @@ int pnf_phy_ul_dci_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
       //LOG_D(PHY,"[PNF] HI_DCI0_REQ sfn_sf:%d PDU[%d] - NFAPI_HI_DCI0_DCI_PDU_TYPE\n", NFAPI_SFNSF2DEC(req->sfn_sf), i);
        nfapi_nr_ul_dci_request_pdus_t *ul_dci_req_pdu = &req->ul_dci_pdu_list[i]; 
        int SFN=req->SFN+2;
-      handle_nfapi_nr_ul_dci_pdu(gNB, SFN, req->Slot, ul_dci_req_pdu); 
+      //handle_nfapi_nr_ul_dci_pdu(gNB, SFN, req->Slot, ul_dci_req_pdu); 
     } 
     else {
       LOG_E(PHY,"[PNF] UL_DCI_REQ sfn_slot:%d PDU[%d] - unknown pdu type:%d\n", NFAPI_SFNSLOT2DEC(req->SFN, req->Slot), i, req->ul_dci_pdu_list[i].PDUType);
@@ -1241,12 +1241,12 @@ int pnf_phy_dl_tti_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
       nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu=&dl_tti_pdu_list[i];
       memcpy(dl_tti_pdu,&dl_tti_pdu_list[i],sizeof(nfapi_nr_dl_tti_request_pdu_t));
       int SFN=sfn+2;
-      handle_nfapi_nr_pdcch_pdu(gNB, SFN, slot, &dl_tti_pdu->pdcch_pdu);
+      //handle_nfapi_nr_pdcch_pdu(gNB, SFN, slot, &dl_tti_pdu->pdcch_pdu);
       //dl_tti_pdu_list[i].pdcch_pdu.pdcch_pdu_rel15.numDlDci++; // ?
       // NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() pdcch_vars->num_dci:%d\n", __FUNCTION__, pdcch_vars->num_dci);
     } else if (dl_tti_pdu_list[i].PDUType == NFAPI_NR_DL_TTI_SSB_PDU_TYPE) {
         //NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() PDU:%d BCH: pdu_index:%u pdu_length:%d sdu_length:%d BCH_SDU:%x,%x,%x\n", __FUNCTION__, i, pdu_index, bch_pdu->bch_pdu_rel8.length, tx_request_pdu[sfn][sf][pdu_index]->segments[0].segment_length, sdu[0], sdu[1], sdu[2]);
-        handle_nr_nfapi_ssb_pdu(gNB, sfn, slot, &dl_tti_pdu_list[i]);
+        //handle_nr_nfapi_ssb_pdu(gNB, sfn, slot, &dl_tti_pdu_list[i]);
         gNB->pbch_configured=1;
       //} else {
         // NFAPI_TRACE(NFAPI_TRACE_ERROR, "%s() BCH NULL TX PDU SFN/SF:%d PDU_INDEX:%d\n", __FUNCTION__, NFAPI_SFNSF2DEC(req->sfn_sf), pdu_index);
@@ -1258,10 +1258,10 @@ int pnf_phy_dl_tti_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
       nfapi_nr_pdu_t *tx_data = tx_data_request[sfn][slot][0];
 
       if (tx_data != NULL) {
-        int UE_id = find_nr_dlsch(rel15_pdu->rnti,gNB,SEARCH_EXIST_OR_FREE);
+        int UE_id = 0;//find_nr_dlsch(rel15_pdu->rnti,gNB,SEARCH_EXIST_OR_FREE);
         AssertFatal(UE_id!=-1,"no free or exiting dlsch_context\n");
         AssertFatal(UE_id<NUMBER_OF_UE_MAX,"returned UE_id %d >= %d(NUMBER_OF_UE_MAX)\n",UE_id,NUMBER_OF_UE_MAX);
-        NR_gNB_DLSCH_t *dlsch0 = gNB->dlsch[UE_id][0];
+        NR_gNB_DLSCH_t *dlsch0 = NULL;//gNB->dlsch[UE_id][0];
         int harq_pid = dlsch0->harq_ids[sfn%2][slot];
 
         if(harq_pid >= dlsch0->Mdlharq) {
@@ -1272,7 +1272,7 @@ int pnf_phy_dl_tti_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
         uint8_t *dlsch_sdu = nr_tx_pdus[UE_id][harq_pid];
        memcpy(dlsch_sdu, tx_data->TLVs[0].value.direct,tx_data->PDU_length);//TODO: Check if required
         //NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() DLSCH:pdu_index:%d handle_nfapi_dlsch_pdu(eNB, proc_rxtx, dlsch_pdu, transport_blocks:%d sdu:%p) eNB->pdcch_vars[proc->subframe_tx & 1].num_pdcch_symbols:%d\n", __FUNCTION__, rel8_pdu->pdu_index, rel8_pdu->transport_blocks, dlsch_sdu, eNB->pdcch_vars[proc->subframe_tx & 1].num_pdcch_symbols);
-        handle_nr_nfapi_pdsch_pdu(gNB, sfn, slot,pdsch_pdu, dlsch_sdu);
+        //handle_nr_nfapi_pdsch_pdu(gNB, sfn, slot,pdsch_pdu, dlsch_sdu);
        } else {
           NFAPI_TRACE(NFAPI_TRACE_ERROR, "%s() DLSCH NULL TX PDU SFN/SF:%d PDU_INDEX:%d\n", __FUNCTION__, NFAPI_SFNSLOT2DEC(sfn,slot), rel15_pdu->pduIndex);     }
     } else {

From 41380e932e4b3450dd4c8f40062b08a2badfbde9 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <sakthivel.velumani@eurecom.fr>
Date: Tue, 13 Jul 2021 12:31:34 +0200
Subject: [PATCH 07/14] Fixed ssb sync issue

---
 executables/nr-gnb.c                      | 15 +++++++--------
 openair1/PHY/NR_TRANSPORT/nr_dci.c        |  4 ++--
 openair1/PHY/defs_gNB.h                   |  7 ++-----
 openair1/SCHED_NR/fapi_nr_l1.c            |  5 ++++-
 openair1/SCHED_NR/phy_procedures_nr_gNB.c | 12 +++++++-----
 5 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c
index ef1d4e22fa..db8dd4babf 100644
--- a/executables/nr-gnb.c
+++ b/executables/nr-gnb.c
@@ -115,14 +115,14 @@ void tx_func(void *param) {
 
   processingData_L1tx_t *info = (processingData_L1tx_t *) param;
   PHY_VARS_gNB *gNB = info->gNB;
-  int frame_tx = info->frame_tx;
-  int slot_tx = info->slot_tx;
+  int frame_tx = info->frame;
+  int slot_tx = info->slot;
 
   phy_procedures_gNB_TX(info,
                         frame_tx,
                         slot_tx,
                         1);
-  info->status = NOT_FILLED;
+  info->slot = -1;
   if ((frame_tx&127) == 0) dump_pdsch_stats(gNB);
 
   // If the later of the 2 L1 tx thread finishes first,
@@ -281,16 +281,13 @@ void rx_func(void *param) {
     notifiedFIFO_elt_t *res;
     res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
     processingData_L1tx_t *syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res);
-    while (syncMsg->status == NOT_FILLED) {
+    while (syncMsg->slot != slot_tx) {
       pushNotifiedFIFO(gNB->resp_L1_tx, res);
       res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
       syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res);
     }
     syncMsg->gNB = gNB;
-    syncMsg->frame_rx = frame_rx;
-    syncMsg->slot_rx = slot_rx;
-    syncMsg->frame_tx = frame_tx;
-    syncMsg->slot_tx = slot_tx;
+    AssertFatal(syncMsg->slot == slot_tx, "Thread message slot and logical slot number do not match\n");
     syncMsg->timestamp_tx = info->timestamp_tx;
     res->key = slot_tx;
     pushTpool(gNB->threadPool, res);
@@ -415,6 +412,7 @@ void init_gNB_Tpool(int inst) {
   notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
   processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
   init_DLSCH_struct(gNB, msgDataTx);
+  msgDataTx->slot = -1;
   memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
   reset_meas(&msgDataTx->phy_proc_tx);
   gNB->phy_proc_tx_0 = &msgDataTx->phy_proc_tx;
@@ -423,6 +421,7 @@ void init_gNB_Tpool(int inst) {
   msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,tx_func);
   msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
   init_DLSCH_struct(gNB, msgDataTx);
+  msgDataTx->slot = -1;
   memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
   reset_meas(&msgDataTx->phy_proc_tx);
   gNB->phy_proc_tx_1 = &msgDataTx->phy_proc_tx;
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dci.c b/openair1/PHY/NR_TRANSPORT/nr_dci.c
index 54e6b2ac79..13c2f5ce4d 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dci.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_dci.c
@@ -132,8 +132,8 @@ void nr_generate_dci(PHY_VARS_gNB *gNB,
 						   dci_pdu->AggregationLevel,
 						   0,NULL);
     polar_encoder_fast((uint64_t*)dci_pdu->Payload, (void*)encoder_output, n_RNTI,1,currentPtr);
-#ifdef DEBUG_CHANNEL_CODING
-    printf("polar rnti %x,length %d, L %d\n",n_RNTI, dci_pdu->PayloadSizeBits,pdcch_pdu_rel15->dci_pdu.AggregationLevel[d]);
+#if DEBUG_CHANNEL_CODING
+    printf("polar rnti %x,length %d, L %d\n",n_RNTI, dci_pdu->PayloadSizeBits,dci_pdu->AggregationLevel);
     printf("DCI PDU: [0]->0x%lx \t [1]->0x%lx\n",
 	   ((uint64_t*)dci_pdu->Payload)[0], ((uint64_t*)dci_pdu->Payload)[1]);
     printf("Encoded Payload (length:%d dwords):\n", encoded_length>>5);
diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h
index b11f902dd2..c88bcf1c18 100644
--- a/openair1/PHY/defs_gNB.h
+++ b/openair1/PHY/defs_gNB.h
@@ -945,10 +945,8 @@ typedef enum {
 } msgStatus_t;
 
 typedef struct processingData_L1tx {
-  int frame_rx;
-  int frame_tx;
-  int slot_rx;
-  int slot_tx;
+  int frame;
+  int slot;
   openair0_timestamp timestamp_tx;
   PHY_VARS_gNB *gNB;
   nfapi_nr_dl_tti_pdcch_pdu pdcch_pdu;
@@ -958,7 +956,6 @@ typedef struct processingData_L1tx {
   NR_gNB_SSB_t ssb[64];
   uint16_t num_pdsch_slot;
   time_stats_t phy_proc_tx;
-  msgStatus_t status;
 } processingData_L1tx_t;
 
 #endif
diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index e06d66ed02..ddcc6000b6 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -165,6 +165,10 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
 
   int pdcch_received=0;
   msgTx->num_pdsch_slot=0;
+  msgTx->pdcch_pdu.pdcch_pdu_rel15.numDlDci = 0;
+  msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci = 0;
+  msgTx->slot = slot;
+  msgTx->frame = frame;
 
   for (int i=0;i<number_dl_pdu;i++) {
     nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu = &DL_req->dl_tti_request_body.dl_tti_pdu_list[i];
@@ -212,7 +216,6 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
   if(NFAPI_MODE != NFAPI_MODE_VNF)
     msgTx->ul_pdcch_pdu = UL_dci_req->ul_dci_pdu_list[number_ul_dci_pdu-1]; // copy the last pdu
 
-  msgTx->status = FILLED;
   pushNotifiedFIFO(gNB->resp_L1_tx,res);
 
   if(NFAPI_MODE != NFAPI_MODE_VNF)
diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
index 22d97be0c3..140e055655 100644
--- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c
+++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
@@ -164,7 +164,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
 
   if (num_dl_dci > 0 || num_ul_dci > 0) {
     LOG_D(PHY, "[gNB %d] Frame %d slot %d Calling nr_generate_dci_top (number of UL/DL DCI %d/%d)\n",
-	  gNB->Mod_id, frame, slot, num_dl_dci, num_ul_dci);
+	  gNB->Mod_id, frame, slot, num_ul_dci, num_dl_dci);
   
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,1);
 
@@ -178,10 +178,12 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,0);
   }
  
-  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,1);
-  LOG_D(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
-  nr_generate_pdsch(msgTx, frame, slot);
-  VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
+  if (msgTx->num_pdsch_slot > 0) {
+    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,1);
+    LOG_I(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
+    nr_generate_pdsch(msgTx, frame, slot);
+    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
+  }
 
   for (int i=0;i<NUMBER_OF_NR_CSIRS_MAX;i++){
     NR_gNB_CSIRS_t *csirs = &msgTx->csirs_pdu[i];

From 463634365ca423c58d9dd79c39c550939ab54204 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Tue, 13 Jul 2021 21:14:07 +0200
Subject: [PATCH 08/14] Fixed frame parms bug in dci gen

---
 openair1/SCHED_NR/phy_procedures_nr_gNB.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
index 140e055655..b5e36fed72 100644
--- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c
+++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c
@@ -173,14 +173,14 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
 			num_ul_dci > 0 ? &msgTx->ul_pdcch_pdu.pdcch_pdu : NULL,
 			gNB->nr_gold_pdcch_dmrs[slot],
 			&gNB->common_vars.txdataF[0][txdataF_offset],
-			AMP, &fp);
+			AMP, fp);
 
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_PDCCH_TX,0);
   }
  
   if (msgTx->num_pdsch_slot > 0) {
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,1);
-    LOG_I(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
+    LOG_D(PHY, "PDSCH generation started (%d) in frame %d.%d\n", msgTx->num_pdsch_slot,frame,slot);
     nr_generate_pdsch(msgTx, frame, slot);
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_GENERATE_DLSCH,0);
   }

From c706f09e44992c47a67e4716248a073ae5c6733a Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Wed, 14 Jul 2021 00:24:43 +0200
Subject: [PATCH 09/14] fixed phy simulators

---
 openair1/PHY/INIT/nr_init.c           |  2 --
 openair1/PHY/INIT/phy_init.h          |  1 +
 openair1/SCHED_NR/fapi_nr_l1.c        |  3 ++-
 openair1/SIMULATION/NR_PHY/dlschsim.c |  6 +++--
 openair1/SIMULATION/NR_PHY/dlsim.c    | 36 +++++++++++++++++++--------
 openair1/SIMULATION/NR_PHY/pbchsim.c  |  9 ++++---
 6 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c
index d2347bd3df..d3b28e7604 100644
--- a/openair1/PHY/INIT/nr_init.c
+++ b/openair1/PHY/INIT/nr_init.c
@@ -563,9 +563,7 @@ void init_DLSCH_struct(PHY_VARS_gNB *gNB, processingData_L1tx_t *msg) {
 
 void init_nr_transport(PHY_VARS_gNB *gNB) {
   NR_DL_FRAME_PARMS *fp = &gNB->frame_parms;
-  nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
   LOG_I(PHY, "Initialise nr transport\n");
-  uint16_t grid_size = cfg->carrier_config.dl_grid_size[fp->numerology_index].value;
 
   memset(gNB->num_pdsch_rnti, 0, sizeof(uint16_t)*80);
 
diff --git a/openair1/PHY/INIT/phy_init.h b/openair1/PHY/INIT/phy_init.h
index 2ab610739d..eff49fb2ce 100644
--- a/openair1/PHY/INIT/phy_init.h
+++ b/openair1/PHY/INIT/phy_init.h
@@ -409,6 +409,7 @@ void init_nr_transport(PHY_VARS_gNB *gNB);
 void init_dfts(void);
 
 void fill_subframe_mask(PHY_VARS_eNB *eNB);
+void init_DLSCH_struct(PHY_VARS_gNB *gNB, processingData_L1tx_t *msg);
 
 /** @} */
 #endif
diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index ddcc6000b6..6d4d7af89f 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -214,7 +214,8 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
   }
 
   if(NFAPI_MODE != NFAPI_MODE_VNF)
-    msgTx->ul_pdcch_pdu = UL_dci_req->ul_dci_pdu_list[number_ul_dci_pdu-1]; // copy the last pdu
+    if (number_ul_dci_pdu > 0)
+      msgTx->ul_pdcch_pdu = UL_dci_req->ul_dci_pdu_list[number_ul_dci_pdu-1]; // copy the last pdu
 
   pushNotifiedFIFO(gNB->resp_L1_tx,res);
 
diff --git a/openair1/SIMULATION/NR_PHY/dlschsim.c b/openair1/SIMULATION/NR_PHY/dlschsim.c
index 633a214699..a0e0183fd1 100644
--- a/openair1/SIMULATION/NR_PHY/dlschsim.c
+++ b/openair1/SIMULATION/NR_PHY/dlschsim.c
@@ -426,7 +426,9 @@ int main(int argc, char **argv)
 	UE->dlsch_SI[0] = new_nr_ue_dlsch(1, 1, Nsoft, 5, N_RB_DL, 0);
 	UE->dlsch_ra[0] = new_nr_ue_dlsch(1, 1, Nsoft, 5, N_RB_DL, 0);
 	unsigned char harq_pid = 0; //dlsch->harq_ids[subframe];
-	NR_gNB_DLSCH_t *dlsch = gNB->dlsch[0][0];
+  processingData_L1tx_t msgDataTx;
+  init_DLSCH_struct(gNB, &msgDataTx);
+	NR_gNB_DLSCH_t *dlsch = msgDataTx.dlsch[0][0];
 	nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15 = &dlsch->harq_process.pdsch_pdu.pdsch_pdu_rel15;
 	//time_stats_t *rm_stats, *te_stats, *i_stats;
 	uint8_t is_crnti = 0, llr8_flag = 0;
@@ -634,7 +636,7 @@ int main(int argc, char **argv)
 
 	for (i = 0; i < 2; i++) {
 		printf("gNB %d\n", i);
-		free_gNB_dlsch(&(gNB->dlsch[0][i]),N_RB_DL);
+		free_gNB_dlsch(&(msgDataTx.dlsch[0][i]),N_RB_DL);
 		printf("UE %d\n", i);
 		free_nr_ue_dlsch(&(UE->dlsch[0][0][i]),N_RB_DL);
 	}
diff --git a/openair1/SIMULATION/NR_PHY/dlsim.c b/openair1/SIMULATION/NR_PHY/dlsim.c
index b597425ddf..ea79c6d29d 100644
--- a/openair1/SIMULATION/NR_PHY/dlsim.c
+++ b/openair1/SIMULATION/NR_PHY/dlsim.c
@@ -925,8 +925,6 @@ int main(int argc, char **argv)
   // generate signal
   AssertFatal(input_fd==NULL,"Not ready for input signal file\n");
   gNB->pbch_configured = 1;
-  gNB->ssb[0].ssb_pdu.ssb_pdu_rel15.bchPayload=0x001234;
-  gNB->ssb[0].ssb_pdu.ssb_pdu_rel15.SsbBlockIndex = 0;
 
   //Configure UE
   rrc.carrier.MIB = (uint8_t*) malloc(4);
@@ -955,11 +953,25 @@ int main(int argc, char **argv)
   //NR_COMMON_channels_t *cc = RC.nrmac[0]->common_channels;
   snrRun = 0;
 
+  gNB->threadPool = (tpool_t*)malloc(sizeof(tpool_t));
+  char tp_param[] = "n";
+  initTpool(tp_param, gNB->threadPool, true);
+  gNB->resp_L1_tx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
+  initNotifiedFIFO(gNB->resp_L1_tx);
+  // we create 2 threads for L1 tx processing
+  notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,processSlotTX);
+  processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+  init_DLSCH_struct(gNB, msgDataTx);
+  msgDataTx->slot = slot;
+  msgDataTx->frame = frame;
+  memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t));
+  reset_meas(&msgDataTx->phy_proc_tx);
+  gNB->phy_proc_tx_0 = &msgDataTx->phy_proc_tx;
+  pushTpool(gNB->threadPool,msgL1Tx);
 
   for (SNR = snr0; SNR < snr1; SNR += .2) {
 
     varArray_t *table_tx=initVarArray(1000,sizeof(double));
-    reset_meas(&gNB->phy_proc_tx); // total gNB tx
     reset_meas(&gNB->dlsch_scrambling_stats);
     reset_meas(&gNB->dlsch_interleaving_stats);
     reset_meas(&gNB->dlsch_rate_matching_stats);
@@ -1000,7 +1012,7 @@ int main(int argc, char **argv)
       int harq_pid = slot;
       NR_DL_UE_HARQ_t *UE_harq_process = dlsch0->harq_processes[harq_pid];
 
-      NR_gNB_DLSCH_t *gNB_dlsch = gNB->dlsch[0][0];
+      NR_gNB_DLSCH_t *gNB_dlsch = msgDataTx->dlsch[0][0];
       nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15 = &gNB_dlsch->harq_process.pdsch_pdu.pdsch_pdu_rel15;
       
       UE_harq_process->harq_ack.ack = 0;
@@ -1050,10 +1062,14 @@ int main(int argc, char **argv)
           ptrsRePerSymb = ((rel15->rbSize + rel15->PTRSFreqDensity - 1)/rel15->PTRSFreqDensity);
           printf("[DLSIM] PTRS Symbols in a slot: %2u, RE per Symbol: %3u, RE in a slot %4d\n", ptrsSymbPerSlot,ptrsRePerSymb, ptrsSymbPerSlot*ptrsRePerSymb );
         }
+
+        msgDataTx->ssb[0].ssb_pdu.ssb_pdu_rel15.bchPayload=0x001234;
+        msgDataTx->ssb[0].ssb_pdu.ssb_pdu_rel15.SsbBlockIndex = 0;
+        msgDataTx->gNB = gNB;
         if (run_initial_sync)
-          nr_common_signal_procedures(gNB,frame,slot,gNB->ssb[0].ssb_pdu);
+          nr_common_signal_procedures(gNB,frame,slot,msgDataTx->ssb[0].ssb_pdu);
         else
-          phy_procedures_gNB_TX(gNB,frame,slot,1);
+          phy_procedures_gNB_TX(msgDataTx,frame,slot,1);
             
         int txdataF_offset = (slot%2) * frame_parms->samples_per_slot_wCP;
         
@@ -1261,10 +1277,10 @@ int main(int argc, char **argv)
     if (print_perf==1) {
       printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, TBS %d, Kr %d (Zc %d))\n",
 	     1000>>*scc->ssbSubcarrierSpacing, g_rbSize, g_mcsIndex,
-	     gNB->dlsch[0][0]->harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0]<<3,
-	     gNB->dlsch[0][0]->harq_process.K,
-	     gNB->dlsch[0][0]->harq_process.K/((gNB->dlsch[0][0]->harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0]<<3)>3824?22:10));
-      printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx");
+	     msgDataTx->dlsch[0][0]->harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0]<<3,
+	     msgDataTx->dlsch[0][0]->harq_process.K,
+	     msgDataTx->dlsch[0][0]->harq_process.K/((msgDataTx->dlsch[0][0]->harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0]<<3)>3824?22:10));
+      printDistribution(gNB->phy_proc_tx_0,table_tx,"PHY proc tx");
       printStatIndent2(&gNB->dlsch_encoding_stats,"DLSCH encoding time");
       printStatIndent3(&gNB->dlsch_segmentation_stats,"DLSCH segmentation time");
       printStatIndent3(&gNB->tinput,"DLSCH LDPC input processing time");
diff --git a/openair1/SIMULATION/NR_PHY/pbchsim.c b/openair1/SIMULATION/NR_PHY/pbchsim.c
index 448e1e7799..703d2f8d22 100644
--- a/openair1/SIMULATION/NR_PHY/pbchsim.c
+++ b/openair1/SIMULATION/NR_PHY/pbchsim.c
@@ -537,14 +537,15 @@ int main(int argc, char **argv)
 
   nr_gold_pbch(UE);
 
+  processingData_L1tx_t msgDataTx;
   // generate signal
   if (input_fd==NULL) {
 
     for (i=0; i<frame_parms->Lmax; i++) {
       if((SSB_positions >> i) & 0x01) {
 
-        gNB->ssb[i].ssb_pdu.ssb_pdu_rel15.bchPayload = 0x55dd33;
-        gNB->ssb[i].ssb_pdu.ssb_pdu_rel15.SsbBlockIndex = i;
+        msgDataTx.ssb[i].ssb_pdu.ssb_pdu_rel15.bchPayload = 0x55dd33;
+        msgDataTx.ssb[i].ssb_pdu.ssb_pdu_rel15.SsbBlockIndex = i;
 
         start_symbol = nr_get_ssb_start_symbol(frame_parms,i);
         int slot = start_symbol/14;
@@ -552,7 +553,7 @@ int main(int argc, char **argv)
         for (aa=0; aa<gNB->frame_parms.nb_antennas_tx; aa++)
           memset(gNB->common_vars.txdataF[aa],0,frame_parms->samples_per_slot_wCP*sizeof(int32_t));
 
-        nr_common_signal_procedures (gNB,frame,slot,gNB->ssb[i].ssb_pdu);
+        nr_common_signal_procedures (gNB,frame,slot,msgDataTx.ssb[i].ssb_pdu);
 
         for (aa=0; aa<gNB->frame_parms.nb_antennas_tx; aa++) {
           if (cyclic_prefix_type == 1) {
@@ -722,7 +723,7 @@ int main(int argc, char **argv)
  
 	  payload_ret = (UE->pbch_vars[0]->xtra_byte == gNB_xtra_byte);
 	  for (i=0;i<3;i++){
-	    payload_ret += (UE->pbch_vars[0]->decoded_output[i] == ((gNB->ssb[ssb_index].ssb_pdu.ssb_pdu_rel15.bchPayload>>(8*i)) & 0xff));
+	    payload_ret += (UE->pbch_vars[0]->decoded_output[i] == ((msgDataTx.ssb[ssb_index].ssb_pdu.ssb_pdu_rel15.bchPayload>>(8*i)) & 0xff));
 	  } 
 	  //printf("xtra byte gNB: 0x%02x UE: 0x%02x\n",gNB_xtra_byte, UE->pbch_vars[0]->xtra_byte);
 	  //printf("ret %d\n", payload_ret);

From 4ba8ef3a6046ad5ff6e5f47f8340a8d65a625a2f Mon Sep 17 00:00:00 2001
From: magounaki <magounak@eurecom.fr>
Date: Mon, 19 Jul 2021 16:45:44 +0200
Subject: [PATCH 10/14] n310 config file

---
 .../CONF/testing_gnb_nsa_n310.conf            | 296 ++++++++++++++++++
 1 file changed, 296 insertions(+)
 create mode 100644 targets/PROJECTS/GENERIC-LTE-EPC/CONF/testing_gnb_nsa_n310.conf

diff --git a/targets/PROJECTS/GENERIC-LTE-EPC/CONF/testing_gnb_nsa_n310.conf b/targets/PROJECTS/GENERIC-LTE-EPC/CONF/testing_gnb_nsa_n310.conf
new file mode 100644
index 0000000000..59e507ed3e
--- /dev/null
+++ b/targets/PROJECTS/GENERIC-LTE-EPC/CONF/testing_gnb_nsa_n310.conf
@@ -0,0 +1,296 @@
+Active_gNBs = ( "gNB-Eurecom-5GNRBox");
+# Asn1_verbosity, choice in: none, info, annoying
+Asn1_verbosity = "none";
+
+gNBs =
+(
+ {
+    ////////// Identification parameters:
+    gNB_ID    =  0xe00;
+
+    cell_type =  "CELL_MACRO_GNB";
+
+    gNB_name  =  "gNB-Eurecom-5GNRBox";
+
+    // Tracking area code, 0x0000 and 0xfffe are reserved values
+    tracking_area_code  =  1;
+
+    plmn_list = ({mcc = 222; mnc = 01; mnc_length = 2;});	 
+
+    tr_s_preference     = "local_mac"
+
+    ////////// Physical parameters:
+
+    ssb_SubcarrierOffset                                      = 31; //0;
+    pdsch_AntennaPorts                                        = 1;
+    pusch_AntennaPorts                                        = 1;
+    pusch_TargetSNRx10                                        = 200;
+    pucch_TargetSNRx10                                        = 200;
+
+    servingCellConfigCommon = (
+    {
+ #spCellConfigCommon
+
+      physCellId                                                    = 0;
+
+#  downlinkConfigCommon
+    #frequencyInfoDL
+      # this is 3600 MHz + 84 PRBs@30kHz SCS (same as initial BWP)
+      absoluteFrequencySSB                                          = 641272; //641032;      #641968; 641968=start of ssb at 3600MHz + 82 RBs    641032=center of SSB at center of cell
+      dl_frequencyBand                                                 = 78;
+      # this is 3600 MHz
+      dl_absoluteFrequencyPointA                                       = 640000;
+      #scs-SpecificCarrierList
+        dl_offstToCarrier                                              = 0;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120  
+        dl_subcarrierSpacing                                           = 1;
+        dl_carrierBandwidth                                            = 106;
+     #initialDownlinkBWP
+      #genericParameters
+        # this is RBstart=84,L=13 (275*(L-1))+RBstart
+        initialDLBWPlocationAndBandwidth                                        = 6368;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120  
+        initialDLBWPsubcarrierSpacing                                           = 1;
+      #pdcch-ConfigCommon
+        initialDLBWPcontrolResourceSetZero                                      = 0;
+        initialDLBWPsearchSpaceZero                                             = 0;
+      #pdsch-ConfigCommon
+        #pdschTimeDomainAllocationList (up to 16 entries)
+        initialDLBWPk0_0                    = 0;  #for DL slot
+        initialDLBWPmappingType_0           = 0;  #0=typeA,1=typeB
+        initialDLBWPstartSymbolAndLength_0  = 40; #this is SS=1,L=13
+
+        initialDLBWPk0_1                    = 0;  #for mixed slot
+        initialDLBWPmappingType_1           = 0;
+        initialDLBWPstartSymbolAndLength_1  = 57; #this is SS=1,L=5
+
+  #uplinkConfigCommon 
+     #frequencyInfoUL
+      ul_frequencyBand                                                 = 78;
+      #scs-SpecificCarrierList
+      ul_offstToCarrier                                              = 0;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120  
+      ul_subcarrierSpacing                                           = 1;
+      ul_carrierBandwidth                                            = 106;
+      pMax                                                          = 20;
+     #initialUplinkBWP
+      #genericParameters
+        initialULBWPlocationAndBandwidth                                        = 6368;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120  
+        initialULBWPsubcarrierSpacing                                           = 1;
+      #rach-ConfigCommon
+        #rach-ConfigGeneric
+          prach_ConfigurationIndex                                  = 98;
+#prach_msg1_FDM
+#0 = one, 1=two, 2=four, 3=eight
+          prach_msg1_FDM                                            = 0;
+          prach_msg1_FrequencyStart                                 = 0;
+          zeroCorrelationZoneConfig                                 = 13;
+          preambleReceivedTargetPower                               = -100;
+#preamblTransMax (0...10) = (3,4,5,6,7,8,10,20,50,100,200)
+          preambleTransMax                                          = 6;
+#powerRampingStep
+# 0=dB0,1=dB2,2=dB4,3=dB6
+        powerRampingStep                                            = 1;
+#ra_ReponseWindow
+#1,2,4,8,10,20,40,80
+        ra_ResponseWindow                                           = 5;
+#ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR
+#1=oneeighth,2=onefourth,3=half,4=one,5=two,6=four,7=eight,8=sixteen
+        ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR                = 4;
+#oneHalf (0..15) 4,8,12,16,...60,64
+        ssb_perRACH_OccasionAndCB_PreamblesPerSSB                   = 14; //15;
+#ra_ContentionResolutionTimer
+#(0..7) 8,16,24,32,40,48,56,64
+        ra_ContentionResolutionTimer                                = 7;
+        rsrp_ThresholdSSB                                           = 19;
+#prach-RootSequenceIndex_PR
+#1 = 839, 2 = 139
+        prach_RootSequenceIndex_PR                                  = 2;
+        prach_RootSequenceIndex                                     = 1;
+        # SCS for msg1, can only be 15 for 30 kHz < 6 GHz, takes precendence over the one derived from prach-ConfigIndex
+        #  
+        msg1_SubcarrierSpacing                                      = 1,
+
+# restrictedSetConfig
+# 0=unrestricted, 1=restricted type A, 2=restricted type B
+        restrictedSetConfig                                         = 0,
+      # pusch-ConfigCommon (up to 16 elements)
+        initialULBWPk2_0                      = 2;  # used for UL slot
+        initialULBWPmappingType_0             = 1
+        initialULBWPstartSymbolAndLength_0    = 41; # this is SS=0 L=13
+
+        initialULBWPk2_1                      = 2;  # used for mixed slot
+        initialULBWPmappingType_1             = 1;
+        initialULBWPstartSymbolAndLength_1    = 24; # this is SS=10 L=2
+
+        initialULBWPk2_2                      = 7;  # used for Msg.3 during RA
+        initialULBWPmappingType_2             = 1;
+        initialULBWPstartSymbolAndLength_2    = 52; # this is SS=10 L=4
+
+        msg3_DeltaPreamble                                          = 1;
+        p0_NominalWithGrant                                         =-90;
+
+# pucch-ConfigCommon setup :
+# pucchGroupHopping
+# 0 = neither, 1= group hopping, 2=sequence hopping
+        pucchGroupHopping                                           = 0;
+        hoppingId                                                   = 40;
+        p0_nominal                                                  = -90;
+# ssb_PositionsInBurs_BitmapPR
+# 1=short, 2=medium, 3=long
+      ssb_PositionsInBurst_PR                                       = 2;
+      ssb_PositionsInBurst_Bitmap                                   = 1; #0x80;
+
+# ssb_periodicityServingCell
+# 0 = ms5, 1=ms10, 2=ms20, 3=ms40, 4=ms80, 5=ms160, 6=spare2, 7=spare1 
+      ssb_periodicityServingCell                                    = 2;
+
+# dmrs_TypeA_position
+# 0 = pos2, 1 = pos3
+      dmrs_TypeA_Position                                           = 0;
+
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120  
+      subcarrierSpacing                                             = 1;
+
+
+  #tdd-UL-DL-ConfigurationCommon
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120  
+      referenceSubcarrierSpacing                                    = 1;
+      # pattern1 
+      # dl_UL_TransmissionPeriodicity
+      # 0=ms0p5, 1=ms0p625, 2=ms1, 3=ms1p25, 4=ms2, 5=ms2p5, 6=ms5, 7=ms10
+      dl_UL_TransmissionPeriodicity                                 = 6;
+      nrofDownlinkSlots                                             = 7; //8; //7;
+      nrofDownlinkSymbols                                           = 6; //0; //6;
+      nrofUplinkSlots                                               = 2;
+      nrofUplinkSymbols                                             = 4; //0; //4;
+
+  ssPBCH_BlockPower                                             = -25;
+  }
+
+  );
+
+
+    # ------- SCTP definitions
+    SCTP :
+    {
+        # Number of streams to use in input/output
+        SCTP_INSTREAMS  = 2;
+        SCTP_OUTSTREAMS = 2;
+    };
+
+
+    ////////// MME parameters:
+    amf_ip_address      = ( { ipv4       = "192.168.18.99";
+                              ipv6       = "192:168:30::17";
+                              active     = "yes";
+                              preference = "ipv4";
+                            }
+                          );
+
+    ///X2
+    enable_x2 = "yes";
+    t_reloc_prep      = 1000;      /* unit: millisecond */
+    tx2_reloc_overall = 2000;      /* unit: millisecond */
+    t_dc_prep         = 1000;      /* unit: millisecond */
+    t_dc_overall      = 2000;      /* unit: millisecond */
+    target_enb_x2_ip_address      = (
+                                     { ipv4       = "192.168.18.199";
+                                       ipv6       = "192:168:30::17";
+                                       preference = "ipv4";
+                                     }
+                                    );
+
+    NETWORK_INTERFACES :
+    {
+
+        GNB_INTERFACE_NAME_FOR_NG_AMF            = "eth0";
+        GNB_IPV4_ADDRESS_FOR_NG_AMF              = "192.168.18.198/24";
+        GNB_INTERFACE_NAME_FOR_NGU               = "eth0";
+        GNB_IPV4_ADDRESS_FOR_NGU                 = "192.168.18.198/24";
+        GNB_PORT_FOR_S1U                         = 2152; # Spec 2152
+        GNB_IPV4_ADDRESS_FOR_X2C                 = "192.168.18.198/24";
+        GNB_PORT_FOR_X2C                         = 36422; # Spec 36422
+    };
+  }
+);
+
+MACRLCs = (
+	{
+	num_cc = 1;
+	tr_s_preference = "local_L1";
+	tr_n_preference = "local_RRC";
+        }  
+);
+
+L1s = (
+    	{
+	num_cc = 1;
+	tr_n_preference = "local_mac";
+	pusch_proc_threads = 8;
+        }  
+);
+
+RUs = (
+    {		  
+       local_rf       = "yes"
+         nb_tx          = 1
+         nb_rx          = 1
+         att_tx         = 0
+         att_rx         = 0;
+         bands          = [7];
+         max_pdschReferenceSignalPower = -27;
+         max_rxgain                    = 75;
+         eNB_instances  = [0];
+         sdr_addrs = "mgmt_addr=192.168.18.252,addr=192.168.80.50,clock_source=internal,time_source=    internal"
+         #clock_src = "external";
+    }
+);  
+
+THREAD_STRUCT = (
+  {
+    #three config for level of parallelism "PARALLEL_SINGLE_THREAD", "PARALLEL_RU_L1_SPLIT", or "PARALLEL_RU_L1_TRX_SPLIT"
+    parallel_config    = "PARALLEL_RU_L1_TRX_SPLIT";
+    //parallel_config    = "PARALLEL_SINGLE_THREAD";
+    #two option for worker "WORKER_DISABLE" or "WORKER_ENABLE"
+    worker_config      = "WORKER_ENABLE";
+  }
+);
+
+security = {
+  # preferred ciphering algorithms
+  # the first one of the list that an UE supports in chosen
+  # valid values: nea0, nea1, nea2, nea3
+  ciphering_algorithms = ( "nea0", "nea2" );
+
+  # preferred integrity algorithms
+  # the first one of the list that an UE supports in chosen
+  # valid values: nia0, nia1, nia2, nia3
+  integrity_algorithms = ( "nia0" );
+};
+
+     log_config :
+     {
+       global_log_level                      ="info";
+       global_log_verbosity                  ="medium";
+       hw_log_level                          ="info";
+       hw_log_verbosity                      ="medium";
+       phy_log_level                         ="info";
+       phy_log_verbosity                     ="medium";
+       mac_log_level                         ="info";
+       mac_log_verbosity                     ="high";
+       rlc_log_level                         ="info";
+       rlc_log_verbosity                     ="medium";
+       pdcp_log_level                        ="info";
+       pdcp_log_verbosity                    ="medium";
+       rrc_log_level                         ="info";
+       rrc_log_verbosity                     ="medium";
+    };
+

From 9bc29750e33dc7af6686e47e1146c80af44337f0 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Mon, 9 Aug 2021 08:02:24 +0530
Subject: [PATCH 11/14] fixed errors after rebase

---
 executables/nr-gnb.c                       | 1 +
 openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c | 2 --
 openair1/SCHED_NR/fapi_nr_l1.c             | 3 +--
 openair1/SCHED_NR/fapi_nr_l1.h             | 2 +-
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c
index db8dd4babf..4273878ef5 100644
--- a/executables/nr-gnb.c
+++ b/executables/nr-gnb.c
@@ -49,6 +49,7 @@
 #include "SCHED_NR/fapi_nr_l1.h"
 #include "PHY/NR_TRANSPORT/nr_transport_proto.h"
 #include "PHY/MODULATION/nr_modulation.h"
+#include "PHY/NR_TRANSPORT/nr_dlsch.h"
 
 #undef MALLOC //there are two conflicting definitions, so we better make sure we don't use it at all
 //#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all
diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
index 7f1f0d0d96..3ba13d2f44 100644
--- a/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
+++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch_tools.c
@@ -258,8 +258,6 @@ void nr_fill_dlsch(processingData_L1tx_t *msgTx,
                    nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                    uint8_t *sdu) {
 
-  nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15 = &pdsch_pdu->pdsch_pdu_rel15;
- 
   NR_gNB_DLSCH_t  *dlsch = msgTx->dlsch[msgTx->num_pdsch_slot][0];
   NR_DL_gNB_HARQ_t *harq  = &dlsch->harq_process;
   /// DLSCH struct
diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c
index 6d4d7af89f..73d1c13132 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.c
+++ b/openair1/SCHED_NR/fapi_nr_l1.c
@@ -192,8 +192,7 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){
       break;
       case NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE:
         LOG_D(PHY,"frame %d, slot %d, Got NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE for %d.%d\n",frame,slot,DL_req->SFN,DL_req->Slot);
-        handle_nfapi_nr_csirs_pdu(msgTx,
-          int frame,int slot,
+        handle_nfapi_nr_csirs_pdu(msgTx,frame,slot,
 				  &dl_tti_pdu->csi_rs_pdu);
       break;
       case NFAPI_NR_DL_TTI_PDSCH_PDU_TYPE:
diff --git a/openair1/SCHED_NR/fapi_nr_l1.h b/openair1/SCHED_NR/fapi_nr_l1.h
index 3dbbf5f450..6b93b2a3f7 100644
--- a/openair1/SCHED_NR/fapi_nr_l1.h
+++ b/openair1/SCHED_NR/fapi_nr_l1.h
@@ -43,7 +43,7 @@ void handle_nr_nfapi_ssb_pdu(processingData_L1tx_t *msgTx,
 
 void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO);
 
-void handle_nfapi_nr_csirs_pdu(PHY_VARS_gNB *gNB,
+void handle_nfapi_nr_csirs_pdu(processingData_L1tx_t *msgTx,
 			       int frame, int slot,
 			       nfapi_nr_dl_tti_csi_rs_pdu *csirs_pdu);
 

From 6443cedd6a148d5adec987a9f6dbe156020b0ed0 Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Wed, 15 Sep 2021 14:52:24 +0530
Subject: [PATCH 12/14] Updated nfapi PNF to use the thread message buffers

---
 executables/nr-gnb.c                    |  2 +-
 nfapi/oai_integration/nfapi_pnf.c       | 85 ++++++++++++++++++-------
 openair1/PHY/defs_gNB.h                 |  1 +
 openair1/SCHED/nfapi_lte_dummy.c        | 11 ++--
 openair1/SCHED/nfapi_nr_dummy.c         |  9 ++-
 openair1/SCHED_NR/phy_frame_config_nr.c |  4 +-
 6 files changed, 76 insertions(+), 36 deletions(-)

diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c
index 05cccdbada..0e15146314 100644
--- a/executables/nr-gnb.c
+++ b/executables/nr-gnb.c
@@ -141,7 +141,7 @@ void tx_func(void *param) {
     res = pullTpool(gNB->resp_RU_tx, gNB->threadPool);
     syncMsg = (processingData_RU_t *)NotifiedFifoData(res);
   }
-  LOG_D(PHY,"previous tx to finished, next slot %d,%d\n",syncMsg->next_slot,slot_tx);
+  LOG_D(PHY,"previous tx finished, next slot %d,%d\n",syncMsg->next_slot,slot_tx);
   syncMsg->frame_tx = frame_tx;
   syncMsg->slot_tx = slot_tx;
   syncMsg->next_slot = get_next_downlink_slot(gNB, &gNB->gNB_config, frame_tx, slot_tx);
diff --git a/nfapi/oai_integration/nfapi_pnf.c b/nfapi/oai_integration/nfapi_pnf.c
index 68f9630a49..4edbeee342 100644
--- a/nfapi/oai_integration/nfapi_pnf.c
+++ b/nfapi/oai_integration/nfapi_pnf.c
@@ -1104,25 +1104,70 @@ void pnf_phy_deallocate_p7_vendor_ext(nfapi_p7_message_header_t *header) {
   free(header);
 }
 
+notifiedFIFO_elt_t *l1tx_message_extract(PHY_VARS_gNB *gNB, int frame, int slot) {
+  notifiedFIFO_elt_t *res;
+  notifiedFIFO_elt_t *freeRes = NULL;
+
+  // check first message
+  res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
+  processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
+  if (msgTx->slot == slot) {
+    return res;
+  }
+  if (msgTx->slot == -1) {
+    freeRes = res;
+  }
+
+  // check second message
+  pushNotifiedFIFO(gNB->resp_L1_tx,res);
+  res = pullTpool(gNB->resp_L1_tx, gNB->threadPool);
+  msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
+  if (msgTx->slot == slot) {
+    return res;
+  }
+  if (msgTx->slot == -1) {
+    freeRes = res;
+  }
+
+  if (freeRes) {
+    msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
+    msgTx->num_pdsch_slot=0;
+    msgTx->pdcch_pdu.pdcch_pdu_rel15.numDlDci = 0;
+    msgTx->ul_pdcch_pdu.pdcch_pdu.pdcch_pdu_rel15.numDlDci = 0;
+    msgTx->slot = slot;
+    msgTx->frame = frame;
+    return freeRes;
+  }
+  pushNotifiedFIFO(gNB->resp_L1_tx,res);
+  AssertFatal(1==0, "It means both L1 Tx messages are still waiting to be processed. This happens when L1 Tx processing is too slow. Message slot %d, scheduled slot %d\n",
+    msgTx->slot, slot);
+}
 
 int pnf_phy_ul_dci_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7, nfapi_nr_ul_dci_request_t *req) {
   
   //   LOG_D(PHY,"[PNF] HI_DCI0_REQUEST SFN/SF:%05d dci:%d hi:%d\n", NFAPI_SFNSF2DEC(req->sfn_sf), req->hi_dci0_request_body.number_of_dci, req->hi_dci0_request_body.number_of_hi);
 
   struct PHY_VARS_gNB_s *gNB = RC.gNB[0];
+
+  // extract the next available thread message (priority to message with current slot, then free message)
+  notifiedFIFO_elt_t *res;
+  res = l1tx_message_extract(gNB, req->SFN, req->Slot);
+  processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
+
   if (proc ==NULL) 
     proc = &gNB->proc.L1_proc;
 
-  for (int i=0; i<req->numPdus; i++) {
-    if (req->ul_dci_pdu_list[i].PDUType == 0) {
-      nfapi_nr_ul_dci_request_pdus_t *ul_dci_req_pdu = &req->ul_dci_pdu_list[i]; 
-      handle_nfapi_nr_ul_dci_pdu(gNB, req->SFN, req->Slot, ul_dci_req_pdu); 
+  if (req->numPdus > 0) {
+    if (req->ul_dci_pdu_list[req->numPdus-1].PDUType == 0) { // copy only the last PDU (PHY can have only one UL PDCCH pdu)
+      msgTx->ul_pdcch_pdu = req->ul_dci_pdu_list[req->numPdus-1]; // copy the last pdu
     } 
     else {
-      LOG_E(PHY,"[PNF] UL_DCI_REQ sfn_slot:%d PDU[%d] - unknown pdu type:%d\n", NFAPI_SFNSLOT2DEC(req->SFN, req->Slot), i, req->ul_dci_pdu_list[i].PDUType);
+      LOG_E(PHY,"[PNF] UL_DCI_REQ sfn_slot:%d PDU[%d] - unknown pdu type:%d\n", NFAPI_SFNSLOT2DEC(req->SFN, req->Slot), req->numPdus-1, req->ul_dci_pdu_list[req->numPdus-1].PDUType);
     }
   }
 
+  pushNotifiedFIFO(gNB->resp_L1_tx,res);
+
   return 0;
 }
 
@@ -1193,14 +1238,17 @@ int pnf_phy_dl_tti_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
 
   for (int i=0; i<req->dl_tti_request_body.nPDUs; i++) {
     // NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() sfn/sf:%d PDU[%d] size:%d pdcch_vars->num_dci:%d\n", __FUNCTION__, NFAPI_SFNSF2DEC(req->sfn_sf), i, dl_config_pdu_list[i].pdu_size,pdcch_vars->num_dci);
+    notifiedFIFO_elt_t *res;
+    res = l1tx_message_extract(gNB, sfn, slot);
+    processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res);
 
     if (dl_tti_pdu_list[i].PDUType == NFAPI_NR_DL_TTI_PDCCH_PDU_TYPE) {
-      nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu=&dl_tti_pdu_list[i];
-      handle_nfapi_nr_pdcch_pdu(gNB, sfn, slot, &dl_tti_pdu->pdcch_pdu);
+      // we trust the scheduler sends only one PDCCH PDU per slot
+      msgTx->pdcch_pdu = dl_tti_pdu_list[i].pdcch_pdu; // fills the last received PDCCH PDU
     } 
     else if (dl_tti_pdu_list[i].PDUType == NFAPI_NR_DL_TTI_SSB_PDU_TYPE) {
       //NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() PDU:%d BCH: pdu_index:%u pdu_length:%d sdu_length:%d BCH_SDU:%x,%x,%x\n", __FUNCTION__, i, pdu_index, bch_pdu->bch_pdu_rel8.length, tx_request_pdu[sfn][sf][pdu_index]->segments[0].segment_length, sdu[0], sdu[1], sdu[2]);
-      handle_nr_nfapi_ssb_pdu(gNB, sfn, slot, &dl_tti_pdu_list[i]);
+      handle_nr_nfapi_ssb_pdu(msgTx, sfn, slot, &dl_tti_pdu_list[i]);
       gNB->pbch_configured=1;
     } 
     else if (dl_tti_pdu_list[i].PDUType == NFAPI_NR_DL_TTI_PDSCH_PDU_TYPE) {
@@ -1209,21 +1257,11 @@ int pnf_phy_dl_tti_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
       nfapi_nr_pdu_t *tx_data = tx_data_request[sfn][slot][rel15_pdu->pduIndex];
 
       if (tx_data != NULL) {
-        int UE_id = 0;//find_nr_dlsch(rel15_pdu->rnti,gNB,SEARCH_EXIST_OR_FREE);
-        AssertFatal(UE_id!=-1,"no free or exiting dlsch_context\n");
-        AssertFatal(UE_id<NUMBER_OF_UE_MAX,"returned UE_id %d >= %d(NUMBER_OF_UE_MAX)\n",UE_id,NUMBER_OF_UE_MAX);
-        NR_gNB_DLSCH_t *dlsch0 = NULL;//gNB->dlsch[UE_id][0];
-        int harq_pid = dlsch0->harq_ids[sfn%2][slot];
-
-        if(harq_pid >= dlsch0->Mdlharq) {
-          LOG_E(PHY,"pnf_phy_dl_config_req illegal harq_pid %d\n", harq_pid);
-          return(-1);
-        }
-
-        uint8_t *dlsch_sdu = nr_tx_pdus[UE_id][harq_pid];
-        memcpy(dlsch_sdu, tx_data->TLVs[0].value.direct,tx_data->PDU_length);
+        uint8_t *dlsch_sdu = (uint8_t *)tx_data->TLVs[0].value.direct;
         //NFAPI_TRACE(NFAPI_TRACE_INFO, "%s() DLSCH:pdu_index:%d handle_nfapi_dlsch_pdu(eNB, proc_rxtx, dlsch_pdu, transport_blocks:%d sdu:%p) eNB->pdcch_vars[proc->subframe_tx & 1].num_pdcch_symbols:%d\n", __FUNCTION__, rel8_pdu->pdu_index, rel8_pdu->transport_blocks, dlsch_sdu, eNB->pdcch_vars[proc->subframe_tx & 1].num_pdcch_symbols);
-        handle_nr_nfapi_pdsch_pdu(gNB, sfn, slot,pdsch_pdu, dlsch_sdu);
+        AssertFatal(msgTx->num_pdsch_slot < gNB->number_of_nr_dlsch_max,"Number of PDSCH PDUs %d exceeded the limit %d\n",
+          msgTx->num_pdsch_slot,gNB->number_of_nr_dlsch_max);
+        handle_nr_nfapi_pdsch_pdu(msgTx, pdsch_pdu, dlsch_sdu);
       } 
       else {
         NFAPI_TRACE(NFAPI_TRACE_ERROR, "%s() DLSCH NULL TX PDU SFN/SF:%d PDU_INDEX:%d\n", __FUNCTION__, NFAPI_SFNSLOT2DEC(sfn,slot), rel15_pdu->pduIndex);     
@@ -1231,11 +1269,12 @@ int pnf_phy_dl_tti_req(gNB_L1_rxtx_proc_t *proc, nfapi_pnf_p7_config_t *pnf_p7,
     }
     else if (dl_tti_pdu_list[i].PDUType == NFAPI_NR_DL_TTI_CSI_RS_PDU_TYPE) {
       nfapi_nr_dl_tti_csi_rs_pdu *csi_rs_pdu = &dl_tti_pdu_list[i].csi_rs_pdu;
-      handle_nfapi_nr_csirs_pdu(gNB, sfn, slot, csi_rs_pdu);
+      handle_nfapi_nr_csirs_pdu(msgTx, sfn, slot, csi_rs_pdu);
     }
     else {
       NFAPI_TRACE(NFAPI_TRACE_ERROR, "%s() UNKNOWN:%d\n", __FUNCTION__, dl_tti_pdu_list[i].PDUType);
     }
+    pushNotifiedFIFO(gNB->resp_L1_tx,res);
   }
 
   if(req->vendor_extension)
diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h
index 11bb54a720..2f561d0ed0 100644
--- a/openair1/PHY/defs_gNB.h
+++ b/openair1/PHY/defs_gNB.h
@@ -945,6 +945,7 @@ typedef struct processingData_L1 {
 
 typedef enum {
   FILLED,
+  FILLING,
   NOT_FILLED
 } msgStatus_t;
 
diff --git a/openair1/SCHED/nfapi_lte_dummy.c b/openair1/SCHED/nfapi_lte_dummy.c
index 7d1e7e7684..4e82c3ecb3 100644
--- a/openair1/SCHED/nfapi_lte_dummy.c
+++ b/openair1/SCHED/nfapi_lte_dummy.c
@@ -13,11 +13,11 @@ void handle_nr_nfapi_ssb_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
                              nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu){}
 
 int16_t find_nr_dlsch(uint16_t rnti, PHY_VARS_gNB *gNB,find_type_t type){return 0;}
-void handle_nr_nfapi_pdsch_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
-                            nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
+void handle_nr_nfapi_pdsch_pdu(processingData_L1tx_t *msgTx,
+			       nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                             uint8_t *sdu){
-                            }
-void handle_nfapi_nr_csirs_pdu(PHY_VARS_gNB *gNB,
+}
+void handle_nfapi_nr_csirs_pdu(processingData_L1tx_t *msgTx,
 			       int frame, int slot,
 			       nfapi_nr_dl_tti_csi_rs_pdu *csirs_pdu){
                     }                           
@@ -51,4 +51,5 @@ void  nr_phy_config_request(NR_PHY_Config_t *gNB){}
 
 void install_nr_schedule_handlers(NR_IF_Module_t *if_inst){}
 
-void nr_dump_frame_parms(NR_DL_FRAME_PARMS *fp){}
\ No newline at end of file
+void nr_dump_frame_parms(NR_DL_FRAME_PARMS *fp){}
+
diff --git a/openair1/SCHED/nfapi_nr_dummy.c b/openair1/SCHED/nfapi_nr_dummy.c
index 1286a4e48a..511911723f 100644
--- a/openair1/SCHED/nfapi_nr_dummy.c
+++ b/openair1/SCHED/nfapi_nr_dummy.c
@@ -13,11 +13,11 @@ void handle_nr_nfapi_ssb_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
                              nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdu){}
 
 int16_t find_nr_dlsch(uint16_t rnti, PHY_VARS_gNB *gNB,find_type_t type){return 0;}
-void handle_nr_nfapi_pdsch_pdu(PHY_VARS_gNB *gNB,int frame,int slot,
-                            nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
+void handle_nr_nfapi_pdsch_pdu(processingData_L1tx_t *msgTx,
+			       nfapi_nr_dl_tti_pdsch_pdu *pdsch_pdu,
                             uint8_t *sdu){
-                            }
-void handle_nfapi_nr_csirs_pdu(PHY_VARS_gNB *gNB,
+}
+void handle_nfapi_nr_csirs_pdu(processingData_L1tx_t *msgTx,
 			       int frame, int slot,
 			       nfapi_nr_dl_tti_csi_rs_pdu *csirs_pdu){
                     }
@@ -53,4 +53,3 @@ void install_nr_schedule_handlers(NR_IF_Module_t *if_inst){}
 
 //void nr_dump_frame_parms(NR_DL_FRAME_PARMS *fp){}
 
-                   
\ No newline at end of file
diff --git a/openair1/SCHED_NR/phy_frame_config_nr.c b/openair1/SCHED_NR/phy_frame_config_nr.c
index c817efcd02..25c39846fe 100644
--- a/openair1/SCHED_NR/phy_frame_config_nr.c
+++ b/openair1/SCHED_NR/phy_frame_config_nr.c
@@ -316,11 +316,11 @@ int get_next_downlink_slot(PHY_VARS_gNB *gNB, nfapi_nr_config_request_scf_t *cfg
   int slots_per_frame = gNB->frame_parms.slots_per_frame;
   while (true) {
     slot++;
-    frame = (slot/slots_per_frame) ? frame++ : frame;
+    if (slot/slots_per_frame) frame++;
     slot %= slots_per_frame;
     int slot_type = nr_slot_select(cfg, frame, slot);
     if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) return slot;
-    AssertFatal(frame < nr_frame+2, "Something went worng. This shouldn't happen\n");
+    AssertFatal(frame < (nr_frame+2), "Something went worng. This shouldn't happen\n");
   }
 }
 

From 4a93a8220f1e05a8fac9fafc77d3e376f450606f Mon Sep 17 00:00:00 2001
From: Sakthivel Velumani <velumani@eurecom.fr>
Date: Tue, 28 Sep 2021 17:55:56 +0530
Subject: [PATCH 13/14] Fix ulsim segfault

---
 openair1/SIMULATION/NR_PHY/ulsim.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c
index e267c8b6d7..8d6355a6ea 100644
--- a/openair1/SIMULATION/NR_PHY/ulsim.c
+++ b/openair1/SIMULATION/NR_PHY/ulsim.c
@@ -672,6 +672,13 @@ int main(int argc, char **argv)
   char tp_param[] = "n";
   initTpool(tp_param, gNB->threadPool, true);
   initNotifiedFIFO(gNB->respDecode);
+  gNB->resp_L1_tx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
+  initNotifiedFIFO(gNB->resp_L1_tx);
+  notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0,gNB->resp_L1_tx,NULL);
+  processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx);
+  msgDataTx->slot = -1;
+  gNB->phy_proc_tx_0 = &msgDataTx->phy_proc_tx;
+  pushNotifiedFIFO(gNB->resp_L1_tx,msgL1Tx); // to unblock the process in the beginning
   //gNB_config = &gNB->gNB_config;
 
   //memset((void *)&gNB->UL_INFO,0,sizeof(gNB->UL_INFO));

From 5c335eaf336a8249bde7de9428020a5437c7be72 Mon Sep 17 00:00:00 2001
From: hardy <remi.hardy@openairinterface.org>
Date: Tue, 28 Sep 2021 16:37:00 +0200
Subject: [PATCH 14/14] update RT stats config file to capture new metrics

---
 ci-scripts/datalog_rt_stats.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci-scripts/datalog_rt_stats.yaml b/ci-scripts/datalog_rt_stats.yaml
index 548caa0bdd..507ea5d182 100644
--- a/ci-scripts/datalog_rt_stats.yaml
+++ b/ci-scripts/datalog_rt_stats.yaml
@@ -12,7 +12,8 @@ Ref :
   feptx_prec : 8.0
   feptx_ofdm : 50.0
   feptx_total : 75.0
-  L1 Tx processing : 300.0
+  L1 Tx processing thread 0 : 300.0
+  L1 Tx processing thread 1 : 300.0
   DLSCH encoding : 230.0
   L1 Rx processing : 175.0
   PUSCH inner-receiver : 100.0
@@ -24,7 +25,8 @@ Threshold :
   feptx_prec : 1.25
   feptx_ofdm : 1.25
   feptx_total : 1.25
-  L1 Tx processing : 1.25
+  L1 Tx processing thread 0 : 1.25
+  L1 Tx processing thread 1 : 1.25
   DLSCH encoding : 1.25
   L1 Rx processing : 1.25
   PUSCH inner-receiver : 1.25