Merge remote-tracking branch 'origin/telnet-ci' into integration_2022_wk11

ci-scripts/main.py

@@ -429,6 +429,7 @@ def GetParametersFromXML(action):
 	elif action == 'Custom_Command':
 		RAN.node = test.findtext('node')
 		RAN.command = test.findtext('command')
+		RAN.command_fail = test.findtext('command_fail') in ['True', 'true', 'Yes', 'yes']
 
 	else:
 		logging.warning(f"unknown action {action} from option-parsing point-of-view")
@@ -752,6 +753,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 				elif action == 'Custom_Command':
 					logging.info(f"Executing custom command")
 					RAN.CustomCommand(HTML)
+					if RAN.prematureExit:
+						CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS)
 				elif action == 'Initialize_eNB':
 					RAN.InitializeeNB(HTML, EPC)
 					if RAN.prematureExit:

ci-scripts/ran.py

@@ -99,6 +99,7 @@ class RANManagement():
 		self.cmd_prefix = '' # prefix before {lte,nr}-softmodem
 		self.node = ''
 		self.command = ''
+		self.command_fail = False
 
 
 #-----------------------------------------------------------
@@ -259,19 +260,24 @@ class RANManagement():
 		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)], HTML)
 
 	def CustomCommand(self, HTML):
-		if self.node == '' or self.node == "localhost":
-			cmd = cls_cmd.LocalCmd()
-		else:
-			cmd = cls_cmd.RemoteCmd(self.node)
+		cmd = cls_cmd.getConnection(self.node)
 		ret = cmd.run(self.command)
 		cmd.close()
-		logging.debug(f'CustomCommand: {self.command} returnCode: {ret.returncode} output: {ret.stdout}')
-		html_queue = SimpleQueue()
+		logging.debug(f'CustomCommand: {self.command} returnCode: {ret.returncode}')
 		status = 'OK'
-		if ret.returncode != 0:
-			html_queue.put(ret.stdout)
+		message = []
+		if ret.returncode != 0 and not self.command_fail:
+			message = [ret.stdout]
+			logging.warning(f'CustomCommand output: {message}')
 			status = 'Warning'
-		HTML.CreateHtmlTestRow(self.command, status, 1, html_queue)
+		if self.command_fail: # important command since it would make the pipeline fail, so show output in HTML
+			message = [ret.stdout]
+		if ret.returncode != 0 and self.command_fail:
+			message = [ret.stdout]
+			logging.error(f'CustomCommand failed: output: {message}')
+			status = 'KO'
+			self.prematureExit = True
+		HTML.CreateHtmlTestRowQueue(self.command, status, message)
 
 	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML):
 		HTML.testCase_id=testcaseId

ci-scripts/xml_files/container_sa_b200_quectel.xml

@@ -22,7 +22,7 @@
 -->
 <testCaseList>
   <htmlTabRef>TEST-SA-FR1-B200</htmlTabRef>
-  <htmlTabName>40 MHz TDD SA</htmlTabName>
+  <htmlTabName>20 MHz TDD SA</htmlTabName>
   <htmlTabIcon>tasks</htmlTabIcon>
   <repeatCount>1</repeatCount>
   <TestCaseRequestedList>
@@ -36,6 +36,10 @@
     070000
     070001
     070002
+    060001
+    000002
+    060002
+    050000
     010010
     010002
     030201
@@ -136,8 +140,8 @@
 
   <testCase id="070001">
     <class>Iperf</class>
-    <desc>iperf (UL/8Mbps/UDP)(30 sec)(single-ue profile)</desc>
-    <iperf_args>-u -b 8M -t 30 -i 1 -fm</iperf_args>
+    <desc>iperf (UL/3.5Mbps/UDP)(30 sec)(single-ue profile)</desc>
+    <iperf_args>-u -b 3.5M -t 30 -i 1 -fm</iperf_args>
     <direction>UL</direction>
     <id>idefix</id>
     <iperf_packetloss_threshold>1</iperf_packetloss_threshold>
@@ -154,6 +158,21 @@
     <iperf_profile>single-ue</iperf_profile>
   </testCase>
 
+  <testCase id="060001">
+    <class>Custom_Command</class>
+    <desc>Trigger Reestablishment</desc>
+    <node>nepes</node>
+    <command>echo ci force_reestab | nc -N 192.168.68.194 9090 | grep -E 'force-remove UE RNTI [0-9a-f]{4} from RLC to trigger reestablishment'</command>
+    <command_fail>yes</command_fail>
+  </testCase>
+  <testCase id="060002">
+    <class>Custom_Command</class>
+    <desc>Verify Reestablishment</desc>
+    <node>nepes</node>
+    <command>echo ci get_reestab_count | nc -N 192.168.68.194 9090 | grep -E 'UE RNTI [0-9a-f]{4} reestab 1 reconf_after_reestab 1'</command>
+    <command_fail>yes</command_fail>
+  </testCase>
+
   <testCase id="030201">
     <class>Undeploy_Object</class>
     <desc>Undeploy gNB</desc>

ci-scripts/yaml_files/sa_b200_gnb/docker-compose.yml

@@ -20,7 +20,7 @@ services:
             GNB_NGA_IP_ADDRESS: 192.168.68.194
             GNB_NGU_IF_NAME: eth0
             GNB_NGU_IP_ADDRESS: 192.168.68.194
-            USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function
+            USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function
         volumes:
             - /dev:/dev
         networks:

common/utils/telnetsrv/CMakeLists.txt

@@ -54,8 +54,13 @@ add_library(telnetsrv_5Gue MODULE telnetsrv_5Gue_measurements.c)
 add_dependencies(telnetsrv telnetsrv_5Gue)
 target_link_libraries(telnetsrv_5Gue PRIVATE asn1_nr_rrc_hdrs asn1_lte_rrc_hdrs)
 
+message(STATUS "Add CI specific telnet functions in libtelnetsrv_ci.so")
+add_library(telnetsrv_ci MODULE telnetsrv_ci.c)
+target_link_libraries(telnetsrv_ci PRIVATE asn1_nr_rrc_hdrs asn1_lte_rrc_hdrs)
+add_dependencies(telnetsrv telnetsrv_ci)
+
 # all libraries should be written to root build dir
-set_target_properties(telnetsrv telnetsrv_enb telnetsrv_5Gue
+set_target_properties(telnetsrv telnetsrv_enb telnetsrv_5Gue telnetsrv_ci
                       PROPERTIES LIBRARY_OUTPUT_DIRECTORY ../../..
 )
 

common/utils/telnetsrv/DOC/telnetusage.md

@@ -8,7 +8,7 @@ By default the embedded telnet server, which is implemented in a shared library,
  ./build_oai  --build-lib telnetsrv
 ```
 
-This will create the `libtelnetsrv.so` and `libtelnetsrv_<app> file in the `cmake_targets/ran_build/build` subdirectory of the oai repository. <app> can be "enb", "gnb", "4GUE" or "5GUE", each library containing functions specific to a given executable.
+This will create the `libtelnetsrv.so` and `libtelnetsrv_<app>` file in the `cmake_targets/ran_build/build` subdirectory of the oai repository. `<app>` can be "enb", "gnb", "4GUE", "5GUE", or "ci", each library containing functions specific to a given executable.
 
 When starting the softmodem, you must specify the **_\-\-telnetsrv_** option to load and start the telnet server. The telnet server is loaded via the [oai shared library loader](loader).
 
@@ -32,9 +32,15 @@ The telnet server is using the [oai configuration module](Config/Rtusage). Telne
 |:---:|:---:|:---:|:----|:----:|
 | `listenaddr` | `ipV4 address, ascii format` | "0.0.0.0" | local address the server is listening on| N |
 | `listenport` | `integer` | 9090 | port number the server is listening on | N |
+| `listenstdin` | `integer` (bool) | 0 | enable input from stdin via additional thread | N |
+| `policy` | `integer` | 0 | scheduling priority for telnet (0-99) | N |
 | `loopcount` | `integer` | 10 | number of iterations for the loop command  | Y |
 | `loopdelay` | `integer` | 5000 | delay (in ms) between 2 loop command iterations  | Y |
-| `histfile` | `character string` | "oaitelnet.history" | file used for command history persistency | Y |
-| `histfsize` | `integer` | 50 | maximum number of commands saved in the history | Y |
+| `histfile` | `string` | "oaitelnet.history" | file used for command history persistency | Y |
+| `histsize` | `integer` | 50 | maximum number of commands saved in the history | Y |
+| `logfile`  | `string` | `oaisoftmodem.log` | output file to which to redirect
+| `phypbsize`  | `integer` | 65000 | string buffer size to dump phy stats | Y |
+| `staticmod`  | `string` | (empty) | additional internally defined telnet modules (`--telnetsrv.staticmod X`, comma-separated) to load on startup. The modules should define a function `add_X_cmds()` in which the module can register telnet commands | N |
+| `shrmod`  | `string` | (empty) | additional shared object files `telnetsrv_X.so` (`--telnetsrv.shrmod X`, comma-separated) to load on startup. The shared object(s) should define a function `add_X_cmds()` in which the shared object can register telnet commands | N |
 
 [oai telnet server home](telnetsrv.md)

common/utils/telnetsrv/telnetsrv.c

@@ -821,14 +821,14 @@ void poll_telnetcmdq(void *qid, void *arg) {
  *
  *
 */
-void exec_moduleinit(char *modname) {
+static bool exec_moduleinit(char *modname) {
   void (*fptr)(void);
   char initfunc[TELNET_CMD_MAXSIZE+10];
 
   if (strlen(modname) > TELNET_CMD_MAXSIZE) {
     fprintf(stderr,"[TELNETSRV] module %s not loaded, name exceeds the %i size limit\n",
             modname, TELNET_CMD_MAXSIZE);
-    return;
+    return false;
   }
 
   sprintf(initfunc,"add_%s_cmds",modname);
@@ -836,37 +836,35 @@ void exec_moduleinit(char *modname) {
 
   if ( fptr != NULL) {
     fptr();
-  } else {
-    fprintf(stderr,"[TELNETSRV] couldn't find %s for module %s \n",initfunc,modname);
+    return true;
   }
+  fprintf(stderr, "[TELNETSRV] couldn't find %s for module %s \n", initfunc, modname);
+  return false;
 }
 
 int add_embeddedmodules(void) {
   int ret=0;
   int pindex = config_paramidx_fromname(telnetoptions,sizeof(telnetoptions)/sizeof(paramdef_t), TELNETSRV_OPTNAME_STATICMOD); 
   for(int i=0; i<telnetoptions[pindex].numelt; i++) {
-    ret++;
-    exec_moduleinit(telnetoptions[pindex].strlistptr[i]);
+    bool success = exec_moduleinit(telnetoptions[pindex].strlistptr[i]);
+    if (success)
+      ret++;
   }
 
   return ret;
 }
 
 int add_sharedmodules(void) {
-  char initfunc[TELNET_CMD_MAXSIZE+9];
-  void (*fptr)(void);
   int ret=0;
   int pindex = config_paramidx_fromname(telnetoptions,sizeof(telnetoptions)/sizeof(paramdef_t), TELNETSRV_OPTNAME_SHRMOD); 
   for(int i=0; i<telnetoptions[pindex].numelt; i++) {
-    sprintf(initfunc,"add_%s_cmds",telnetoptions[pindex].strlistptr[i]);
-    fptr = dlsym(RTLD_DEFAULT,initfunc);
-
-    if ( fptr != NULL) {
-      fptr();
+    char *name = telnetoptions[pindex].strlistptr[i];
+    char libname[256];
+    snprintf(libname, sizeof(libname), "telnetsrv_%s", name);
+    load_module_shlib(libname, NULL, 0, NULL);
+    bool success = exec_moduleinit(name);
+    if (success)
       ret++;
-    } else {
-      fprintf(stderr,"[TELNETSRV] couldn't find %s for module %s \n",initfunc,telnetoptions[pindex].strlistptr[i]);
-    }
   }
 
   return ret;
@@ -890,6 +888,7 @@ int telnetsrv_autoinit(void) {
 
   add_telnetcmd("telnet", telnet_vardef, telnet_cmdarray);
   add_embeddedmodules();
+  add_sharedmodules();
   if ( telnetparams.listenstdin ) {
     if(pthread_create(&telnetparams.telnetclt_pthread,NULL, (void *(*)(void *))run_telnetclt, NULL) != 0) {
       fprintf(stderr,"[TELNETSRV] Error %s on pthread_create f() run_telnetclt \n",strerror(errno));

common/utils/telnetsrv/telnetsrv_ci.c

@@ -0,0 +1,133 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#include <sys/types.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "openair2/RRC/NR/rrc_gNB_UE_context.h"
+#include "openair2/LAYER2/nr_rlc/nr_rlc_oai_api.h"
+#include "openair2/LAYER2/NR_MAC_gNB/nr_mac_gNB.h"
+
+#define TELNETSERVERCODE
+#include "telnetsrv.h"
+
+#define ERROR_MSG_RET(mSG, aRGS...) do { prnt(mSG, ##aRGS); return 1; } while (0)
+
+static int get_single_ue_rnti(void)
+{
+  NR_UE_info_t *ue = NULL;
+  UE_iterator(RC.nrmac[0]->UE_info.list, it) {
+    if (it && ue)
+      return -1;
+    if (it)
+      ue = it;
+  }
+  if (!ue)
+    return -1;
+
+  // verify it exists in RRC as well
+  rrc_gNB_ue_context_t *rrcue = rrc_gNB_get_ue_context(RC.nrrrc[0], ue->rnti);
+  if (!rrcue)
+    return -1;
+
+  return ue->rnti;
+}
+
+int get_single_rnti(char *buf, int debug, telnet_printfunc_t prnt)
+{
+  if (buf)
+    ERROR_MSG_RET("no parameter allowed\n");
+
+  int rnti = get_single_ue_rnti();
+  if (rnti < 1)
+    ERROR_MSG_RET("different number of UEs\n");
+
+  prnt("single UE RNTI %04x\n", rnti);
+  return 0;
+}
+
+int get_reestab_count(char *buf, int debug, telnet_printfunc_t prnt)
+{
+  int rnti = -1;
+  if (!buf) {
+    rnti = get_single_ue_rnti();
+    if (rnti < 1)
+      ERROR_MSG_RET("no UE found\n");
+  } else {
+    rnti = strtol(buf, NULL, 16);
+    if (rnti < 1 || rnti >= 0xfffe)
+      ERROR_MSG_RET("RNTI needs to be [1,0xfffe]\n");
+  }
+
+  rrc_gNB_ue_context_t *ue = rrc_gNB_get_ue_context(RC.nrrrc[0], rnti);
+  if (!ue)
+    ERROR_MSG_RET("could not find UE with RNTI %04x\n", rnti);
+
+  prnt("UE RNTI %04x reestab %d reconf_after_reestab %d\n",
+       rnti,
+       ue->ue_context.ue_reestablishment_counter,
+       ue->ue_context.ue_reconfiguration_after_reestablishment_counter);
+  return 0;
+}
+
+int trigger_reestab(char *buf, int debug, telnet_printfunc_t prnt)
+{
+  int rnti = -1;
+  if (!buf) {
+    rnti = get_single_ue_rnti();
+    if (rnti < 1)
+      ERROR_MSG_RET("no UE found\n");
+  } else {
+    rnti = strtol(buf, NULL, 16);
+    if (rnti < 1 || rnti >= 0xfffe)
+      ERROR_MSG_RET("RNTI needs to be [1,0xfffe]\n");
+  }
+
+  // verify it exists in RRC as well
+  rrc_gNB_ue_context_t *rrcue = rrc_gNB_get_ue_context(RC.nrrrc[0], rnti);
+  if (!rrcue)
+    ERROR_MSG_RET("could not find UE with RNTI %04x\n", rnti);
+
+  nr_rlc_remove_ue(rnti);
+  prnt("force-remove UE RNTI %04x from RLC to trigger reestablishment\n", rnti);
+  return 0;
+}
+
+static telnetshell_cmddef_t cicmds[] = {
+  {"get_single_rnti", "", get_single_rnti},
+  {"force_reestab", "[rnti(hex,opt)]", trigger_reestab},
+  {"get_reestab_count", "[rnti(hex,opt)]", get_reestab_count},
+  {"", "", NULL},
+};
+
+static telnetshell_vardef_t civars[] = {
+
+  {"", 0, 0, NULL}
+};
+
+void add_ci_cmds(void) {
+  add_telnetcmd("ci", civars, cicmds);
+}

docker/Dockerfile.gNB.rhel8.2

@@ -73,6 +73,7 @@ COPY --from=gnb-build \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \
     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ci.so \
     /usr/local/lib/
 
 COPY --from=gnb-base \

docker/Dockerfile.gNB.ubuntu20

@@ -78,6 +78,7 @@ COPY --from=gnb-build \
     /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \
     /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \
     /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ci.so \
     /usr/local/lib/
 
 # Now we are copying from builder-image the UHD files.

openair2/RRC/NR/nr_rrc_defs.h

@@ -384,7 +384,8 @@ typedef struct gNB_RRC_UE_s {
   uint8_t                            established_pdu_sessions_flag;
   uint32_t                           ue_rrc_inactivity_timer;
   int8_t                             reestablishment_xid;
-  //------------------------------------------------------------------------------//
+  uint32_t                           ue_reestablishment_counter;
+  uint32_t                           ue_reconfiguration_after_reestablishment_counter;
   NR_CellGroupId_t                                      cellGroupId;
   struct NR_SpCellConfig                                *spCellConfig;
   struct NR_CellGroupConfig__sCellToAddModList          *sCellconfig;

openair2/RRC/NR/rrc_gNB.c

@@ -1203,6 +1203,7 @@ rrc_gNB_process_RRCReconfigurationComplete(
     for (int i = 0; i < MAX_MOBILES_PER_GNB; i++) {
       nr_reestablish_rnti_map_t *nr_reestablish_rnti_map = &(RC.nrrrc[ctxt_pP->module_id])->nr_reestablish_rnti_map[i];
       if (nr_reestablish_rnti_map->ue_id == ctxt_pP->rntiMaybeUEid) {
+        ue_context_pP->ue_context.ue_reconfiguration_after_reestablishment_counter++;
         reestablish_ue_id = nr_reestablish_rnti_map[i].c_rnti;
         LOG_D(NR_RRC, "Removing reestablish_rnti_map[%d] UEid %lx, RNTI %04x\n", i, nr_reestablish_rnti_map->ue_id, nr_reestablish_rnti_map->c_rnti);
         // clear current C-RNTI from map
@@ -2616,6 +2617,8 @@ rrc_gNB_decode_dcch(
 
           gNB_MAC_INST *nrmac = RC.nrmac[ctxt_pP->module_id]; // WHAT A BEAUTIFULL RACE CONDITION !!!
           mac_remove_nr_ue(nrmac, reestablish_rnti);
+
+          ue_context_p->ue_context.ue_reestablishment_counter++;
         }
 
         // ue_context_p->ue_context.ue_release_timer = 0;