From db4769bc359e66083cd38b745e59ae93811ed959 Mon Sep 17 00:00:00 2001 From: Xenofon Foukas Date: Mon, 24 May 2021 16:33:12 +0100 Subject: [PATCH 1/7] Fixes to L2NFAPI_NOS1 dataplane traffic forwarding This commit fixes a bug that makes the dataplane traffic of all the emulated UEs to go through the interface of the first UE (oaitun_ue1). The fix uses the last octet of the IP assigned to each UE to map it to its UE_id, which is then used to get its RNTI and the correct PDCP queue. For the emulator to work properly, the routing tables of the emulated UEs must also be set. This patch introduces a script (setup_routes.sh) in cmake_targets/tools that configures the routing tables. This script must be run at the UE host machine once the UEs have successfully attached, passing the number of emulated UEs as the argument. For example, in the case of 16 emulated UEs, one must run "./setup_routes.sh 16". --- cmake_targets/tools/setup_routes.sh | 13 +++++++++++++ openair2/LAYER2/PDCP_v10.1.0/pdcp.c | 8 +++++++- openair2/LAYER2/PDCP_v10.1.0/pdcp_fifo.c | 7 ++++++- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100755 cmake_targets/tools/setup_routes.sh diff --git a/cmake_targets/tools/setup_routes.sh b/cmake_targets/tools/setup_routes.sh new file mode 100755 index 0000000000..67fe250be7 --- /dev/null +++ b/cmake_targets/tools/setup_routes.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +for i in $(seq 1 $1); +do + let table=1000+$i + echo "sudo ip route add 10.0.1.0/24 dev oaitun_ue$i table $table" + sudo ip route add 10.0.1.0/24 dev oaitun_ue$i table $table + echo "sudo ip route add default via 10.0.1.1 dev oaitun_ue$i table $table" + sudo ip route add default via 10.0.1.1 dev oaitun_ue$i table $table + let octet=$i+1 + echo "sudo ip rule add from 10.0.1.$octet table $table" + sudo ip rule add from 10.0.1.$octet table $table +done diff --git a/openair2/LAYER2/PDCP_v10.1.0/pdcp.c b/openair2/LAYER2/PDCP_v10.1.0/pdcp.c index 51d3cfc8f3..cddb666bfd 100644 --- a/openair2/LAYER2/PDCP_v10.1.0/pdcp.c +++ b/openair2/LAYER2/PDCP_v10.1.0/pdcp.c @@ -1081,7 +1081,13 @@ pdcp_data_ind( pdcpHead->inst = 1; } } // nfapi_mode - } + } else { + if (UE_NAS_USE_TUN) { + pdcpHead->inst = ctxt_pP->module_id; + } else if (ENB_NAS_USE_TUN) { + pdcpHead->inst = 0; + } + } } else { pdcpHead->rb_id = rb_id + (ctxt_pP->module_id * LTE_maxDRB); pdcpHead->inst = ctxt_pP->module_id; diff --git a/openair2/LAYER2/PDCP_v10.1.0/pdcp_fifo.c b/openair2/LAYER2/PDCP_v10.1.0/pdcp_fifo.c index 8ff3c06cd4..cc44572abd 100644 --- a/openair2/LAYER2/PDCP_v10.1.0/pdcp_fifo.c +++ b/openair2/LAYER2/PDCP_v10.1.0/pdcp_fifo.c @@ -43,6 +43,7 @@ extern int otg_enabled; #include #include #include +#include #define rtf_put write #define rtf_get read @@ -222,7 +223,11 @@ int pdcp_fifo_read_input_sdus_fromtun (const protocol_ctxt_t *const ctxt_pP) { key = PDCP_COLL_KEY_DEFAULT_DRB_VALUE(ctxt.module_id, ctxt.rnti, ctxt.enb_flag); h_rc = hashtable_get(pdcp_coll_p, key, (void **)&pdcp_p); } else { // => ENB_NAS_USE_TUN - ctxt.rnti=pdcp_eNB_UE_instance_to_rnti[0]; + /* Get the IP from a packet */ + struct ip *ip_pack = (struct ip *) nl_rx_buf; + /* Use last octet of destination IP to get index of UE */ + int ue_indx = ((ip_pack->ip_dst.s_addr >> 24) - 2) % MAX_MOBILES_PER_ENB; + ctxt.rnti=pdcp_eNB_UE_instance_to_rnti[ue_indx]; ctxt.enb_flag=ENB_FLAG_YES; ctxt.module_id=0; key = PDCP_COLL_KEY_VALUE(ctxt.module_id, ctxt.rnti, ctxt.enb_flag, rab_id, SRB_FLAG_NO); From 2632d81d36723801e565ac40231e74eb43077514 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 28 May 2021 10:36:52 +0200 Subject: [PATCH 2/7] CI: adding setting the routes for L2 Simulator in noS1 mode Signed-off-by: Raphael Defosseux --- ci-scripts/runTestOnVM.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci-scripts/runTestOnVM.sh b/ci-scripts/runTestOnVM.sh index fc35370e0a..b9a1adb6af 100755 --- a/ci-scripts/runTestOnVM.sh +++ b/ci-scripts/runTestOnVM.sh @@ -964,6 +964,12 @@ function start_l2_sim_ue { echo "ifconfig" > $1 ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_UE_VM_IP_ADDR < $1 rm $1 + else + echo "Setting Routes for all UEs" + echo "cd /home/ubuntu/tmp/cmake_targets/tools" > $1 + echo "./setup_routes.sh $LOC_NB_UES" >> $1 + ssh -T -o StrictHostKeyChecking=no ubuntu@$LOC_UE_VM_IP_ADDR < $1 + rm $1 fi } From 2a6ebaaaae035f49d9712d935c356f2e4e6f4bcc Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Tue, 8 Jun 2021 11:31:47 +0200 Subject: [PATCH 3/7] Fixes on the array size computation. This work is based on feedback from Thomas Schlichter. His comments and suggestions were made just after the original MR was merged. Signed-off-by: Raphael Defosseux --- common/utils/system.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/common/utils/system.c b/common/utils/system.c index 37622932ff..2ed986fe85 100644 --- a/common/utils/system.c +++ b/common/utils/system.c @@ -96,14 +96,14 @@ static void read_pipe(int p, char *b, int size) { } } -static int baseRunTimeCommand(char* cmd) { +static int baseRunTimeCommand(char* cmd, size_t cmdSize) { FILE *fp; size_t retSize = 0; fp = popen(cmd, "r"); - memset(cmd, 1, sizeof(*cmd)); - retSize = fread(cmd, 1, sizeof(*cmd), fp); + memset(cmd, 0, cmdSize); + retSize = fread(cmd, 1, cmdSize, fp); fclose(fp); if (retSize == 0) { @@ -115,26 +115,26 @@ static int baseRunTimeCommand(char* cmd) { int checkIfFedoraDistribution(void) { char cmd[200]; - memset(cmd, 1, 200); + memset(cmd, 0, 200); sprintf(cmd, "cat /etc/os-release | grep ID_LIKE | grep -ic fedora || true"); - return baseRunTimeCommand(cmd); + return baseRunTimeCommand(cmd, 200); } int checkIfGenericKernelOnFedora(void) { char cmd[200]; - memset(cmd, 1, 200); + memset(cmd, 0, 200); sprintf(cmd, "uname -a | grep -c rt || true"); - return (1 - baseRunTimeCommand(cmd)); + return (1 - baseRunTimeCommand(cmd, 200)); } int checkIfInsideContainer(void) { char cmd[200]; int res = 0; - memset(cmd, 1, 200); + memset(cmd, 0, 200); sprintf(cmd, "cat /proc/self/cgroup | egrep -c 'libpod|podman|kubepods' || true"); - res = baseRunTimeCommand(cmd); + res = baseRunTimeCommand(cmd, 200); if (res > 0) return 1; else From 534a58c5583ea1a49e5e8e75fe0a402f545f580e Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 15 Jun 2021 09:58:43 +0200 Subject: [PATCH 4/7] fix iperf + tune max_rxgain for our setup --- ci-scripts/cls_oaicitest.py | 9 ++++++++- .../conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf | 2 +- .../conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf | 2 +- ci-scripts/xml_files/fr1_nsa_quectel.xml | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 809af4ebec..d3d1d70c13 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2234,7 +2234,7 @@ class OaiCiTest(): SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup iperf -s -i 1 -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' SSH.command(cmd,'\$',5) #client side UE @@ -2252,6 +2252,13 @@ class OaiCiTest(): else : logging.debug("Incorrect or missing IPERF direction in XML") + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() return diff --git a/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf b/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf index 33aae14dac..ca1710ca47 100644 --- a/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf +++ b/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf @@ -264,7 +264,7 @@ RUs = ( att_rx = 0; bands = [7]; max_pdschReferenceSignalPower = -27; - max_rxgain = 118; + max_rxgain = 115; eNB_instances = [0]; # clock_src = "external"; } diff --git a/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf b/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf index f0d5130f83..72e0e19c6a 100644 --- a/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf +++ b/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf @@ -248,7 +248,7 @@ RUs = ( att_rx = 0; bands = [7]; max_pdschReferenceSignalPower = -27; - max_rxgain = 114; + max_rxgain = 111; eNB_instances = [0]; # clock_src = "external"; } diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index fbaa32aea7..2e8c07968d 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -110,7 +110,7 @@ Iperf iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) - -u -b 20M -t 60 -i 1 + -u -b 20M -t 60 DL idefix 50 @@ -120,7 +120,7 @@ Iperf iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) - -u -b 3M -t 60 -i 1 + -u -b 3M -t 60 UL idefix 50 From 5ef64f467a522316c1a9370a0322273ccc5dff5d Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 16 Jun 2021 13:40:08 +0200 Subject: [PATCH 5/7] update att_tx in config to improve 5g attach in our setup --- ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf | 2 +- ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf b/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf index ca1710ca47..056c4bdb6a 100644 --- a/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf +++ b/ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf @@ -260,7 +260,7 @@ RUs = ( local_rf = "yes" nb_tx = 1 nb_rx = 1 - att_tx = 0 + att_tx = 3 att_rx = 0; bands = [7]; max_pdschReferenceSignalPower = -27; diff --git a/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf b/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf index 72e0e19c6a..5838e96999 100644 --- a/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf +++ b/ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf @@ -244,7 +244,7 @@ RUs = ( local_rf = "yes" nb_tx = 1 nb_rx = 1 - att_tx = 0 + att_tx = 3 att_rx = 0; bands = [7]; max_pdschReferenceSignalPower = -27; From 8371109a9369bed0d89dbfe961c2f89d4a021b05 Mon Sep 17 00:00:00 2001 From: Remi Hardy Date: Wed, 16 Jun 2021 22:22:34 +0200 Subject: [PATCH 6/7] adding new class for log mgt/rotation on remote servers --- ci-scripts/cls_log_mgt.py | 103 ++++++++++++++++++++++++++++++++++++ ci-scripts/cls_module_ue.py | 8 +++ 2 files changed, 111 insertions(+) create mode 100644 ci-scripts/cls_log_mgt.py diff --git a/ci-scripts/cls_log_mgt.py b/ci-scripts/cls_log_mgt.py new file mode 100644 index 0000000000..36e7d51a4b --- /dev/null +++ b/ci-scripts/cls_log_mgt.py @@ -0,0 +1,103 @@ +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# +# Required Python Version +# Python 3.x +# +#--------------------------------------------------------------------- + +#USAGE: +# log=Log_Mgt(IPAddress,Password,Path) +# log.LogRotation() + + + + +import re +import subprocess +import logging +import math + +class Log_Mgt: + + def __init__(self,IPAddress,Password,Path): + self.IPAddress=IPAddress + self.Password=Password + self.path=Path + +#-----------------$ +#PRIVATE# Methods$ +#-----------------$ + + + def __CheckAvailSpace(self): + HOST=self.IPAddress + COMMAND="df "+ self.path + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + s=result[1].decode('utf-8').rstrip()#result[1] is the second line with the results we are looking for + tmp=s.split() + return tmp[3] #return avail space from the line + + def __GetOldestFile(self): + HOST=self.IPAddress + COMMAND="ls -rtl "+ self.path #-rtl will bring oldest file on top + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + s=result[1].decode('utf-8').rstrip() + tmp=s.split() + return tmp[8]#return filename from the line + + + def __AvgSize(self): + HOST=self.IPAddress + COMMAND="ls -rtl "+ self.path + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + total_size=0 + for i in range(1,len(result)): + s=result[i].decode('utf-8').rstrip() + tmp=s.split() + total_size+=int(tmp[4]) #get filesize + return math.floor(total_size/(len(result)-1)) #compute average file/artifact size + + +#-----------------$ +#PUBLIC Methods$ +#-----------------$ + + + def LogRotation(self): + avail_space =int(self.__CheckAvailSpace())*1000 #avail space in target folder, initially displayed in Gb + avg_size=self.__AvgSize() #average size of artifacts in the target folder + logging.debug("Avail Space : " + str(avail_space) + " / Artifact Avg Size : " + str(avg_size)) + if avail_space < 2*avg_size: #reserved space is 2x artifact file ; oldest file will be deleted + oldestfile=self.__GetOldestFile() + HOST=self.IPAddress + COMMAND="echo " + self.Password + " | sudo -S rm "+ self.path + "/" + oldestfile + logging.debug(COMMAND) + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + else: + logging.debug("Still some space left for artifacts storage") + + + + diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index f93cb9f036..7f5c81d988 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -39,6 +39,9 @@ import subprocess from datetime import datetime +#for log rotation mgt +import cls_log_mgt + class Module_UE: def __init__(self,Module): @@ -157,6 +160,11 @@ class Module_UE: now_string = now.strftime("%Y%m%d-%H%M") source='ci_qlog' destination='/opt/ci_qlogs/ci_qlog_'+now_string+'.zip' + #qlog artifact is zipped into the target folder mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S zip -r '+destination+' '+source+' &','\$', 10) mySSH.close() + #post action : log cleaning to make sure enough space is reserved for the next run + Log_Mgt=cls_log_mgt.Log_Mgt(self.HostIPAddress, self.HostPassword, "/opt/ci_qlogs") + Log_Mgt.LogRotation() + return destination From 569dfb1420af7f7347b107573f39aa261d19ef13 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 17 Jun 2021 15:25:09 +0200 Subject: [PATCH 7/7] change storage location of zipped qlog --- ci-scripts/cls_module_ue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index 7f5c81d988..cb5c81e271 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -134,7 +134,7 @@ class Module_UE: mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) #delete old artifacts mySSH.command('echo ' + self.HostPassword + ' | sudo -S rm -rf ci_qlog','\$',5) - #start Trace + #start Trace, artifact is created in home dir mySSH.command('echo $USER; nohup sudo -E QLog/QLog -s ci_qlog -f NR5G.cfg &','\$', 5) mySSH.close() @@ -155,16 +155,16 @@ class Module_UE: def LogCollect(self): mySSH = sshconnection.SSHConnection() mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) - #archive qlog to /opt/ci_qlogs with datetime suffix + #archive qlog to USB stick in /media/usb-drive/ci_qlogs with datetime suffix now=datetime.now() now_string = now.strftime("%Y%m%d-%H%M") source='ci_qlog' - destination='/opt/ci_qlogs/ci_qlog_'+now_string+'.zip' + destination='/media/usb-drive/ci_qlogs/ci_qlog_'+now_string+'.zip' #qlog artifact is zipped into the target folder mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S zip -r '+destination+' '+source+' &','\$', 10) mySSH.close() #post action : log cleaning to make sure enough space is reserved for the next run - Log_Mgt=cls_log_mgt.Log_Mgt(self.HostIPAddress, self.HostPassword, "/opt/ci_qlogs") + Log_Mgt=cls_log_mgt.Log_Mgt(self.HostIPAddress, self.HostPassword, "/media/usb-drive/ci_qlogs") Log_Mgt.LogRotation() return destination